diff --git a/src/scheduler/testsuite/meson.build b/src/scheduler/testsuite/meson.build index 5d78725c..e844effe 100644 --- a/src/scheduler/testsuite/meson.build +++ b/src/scheduler/testsuite/meson.build @@ -2747,6 +2747,46 @@ test('scheduler trace error paths test', sched_trace_error_paths_test, suite: 'scheduler', env: ['ASAN_OPTIONS=detect_leaks=0']) +# Scheduler rejoin-hardening plan, Phase 0 (scheduler-hardening-plan.md): +# cross-file contract lock on the three-path UNAVAIL/RECONCILING skip +# symmetry (core_reset_workers / sched_update_account / +# sched_reconcile_host_resource_usage) whose observed asymmetries were +# bug-1799/bug-1801/bug-1814 -- see the test's own header comment for why +# one test drives all three paths. Same separate-compiled-sources +# technique as sched_trace_error_paths_test above: the test #includes +# core.c/core_session.c/resource.c/schedobj.c + the four hooked trace +# sources, and every OTHER scheduler_srcs entry (minus main.c) is its own +# TU -- which is what lets the test call sched_account_jobs() +# (scheduler.c) without white-box including scheduler.c (deliberate: no +# test in this directory #includes scheduler.c, and doing so would drag +# plugin.c/driver.c/the client-facing surface into the test TU). Real +# plugin.c comes in as one of those TUs, so the test defines no plugin_* +# stubs. +sched_res_account_symmetry_included = ['src/main.c', 'src/core.c', + 'src/core_session.c', + 'src/resource.c', 'src/schedobj.c', + 'src/sched_trace.c', + 'src/sched_trace_scan.c', + 'src/sched_trace_capture.c', + 'src/sched_trace_outcome.c'] +sched_res_account_symmetry_test_srcs = ['test_res_account_symmetry.c'] +foreach f : scheduler_srcs + if not sched_res_account_symmetry_included.contains(f) + sched_res_account_symmetry_test_srcs += ['../' + f] + endif +endforeach + +sched_res_account_symmetry_test = executable( + 'sched_res_account_symmetry_test', + sched_res_account_symmetry_test_srcs, + include_directories:[incdir], + dependencies: [scheduler_deps, unity_dep], + install: false) + +test('scheduler res account symmetry test', + sched_res_account_symmetry_test, suite: 'scheduler', + env: ['ASAN_OPTIONS=detect_leaks=0']) + # scheduler-sim-decision-trace task 6.4 (Requirements 9.1, 9.3, 9.5; # design.md "Testing Strategy -> Performance"): a 1000-host, maxTasks = 64 # parallel job traced at full level by the real allocator stays within the diff --git a/src/scheduler/testsuite/test_res_account_symmetry.c b/src/scheduler/testsuite/test_res_account_symmetry.c new file mode 100644 index 00000000..b332b5cf --- /dev/null +++ b/src/scheduler/testsuite/test_res_account_symmetry.c @@ -0,0 +1,542 @@ +/* + * test_res_account_symmetry.c - Phase 0 of the scheduler rejoin-hardening + * plan (scheduler-hardening-plan.md): locks the cross-file contract that + * every w->resInUse/core->sharedInUse-mutating path shares ONE identical + * skip condition for UNAVAIL/RECONCILING workers -- the invariant whose + * three observed violations became bug-1799 (charge without reset-skip), + * bug-1801 (its complement), and bug-1814 (rusage delta without the skip). + * + * Why one test drives all THREE paths instead of relying on the existing + * per-path tests (test_core_session.c covers the reset skip, + * test_resource_accounting.c covers the charge skip and the rusage-delta + * skip individually): the live bug-1799 shape was NOT any single path + * misbehaving -- each path passed its own test -- it was the ASYMMETRY + * between two paths' skip conditions compounding across rounds (reset + * skipped, charge did not -> resInUse grew one full charge per round for + * two weeks before anyone looked at a worker's used= number). A per-path + * test cannot see asymmetry by construction; this file puts all three + * paths in one core and asserts the accounts do not drift across + * consecutive rounds, which is exactly the observable the live incident + * showed. + * + * The three w->resInUse-mutating paths under test (resource-accounting + * steering doc calls out each explicitly): + * (a) core_reset_workers() (core_session.c, bug-1751 skip) + * (b) sched_charge_account() via + * sched_update_account() (resource.c, bug-1801 skip) + * (c) sched_reconcile_host_resource_usage() (resource.c, bug-1814 skip) + * plus the unconditional shared-pool reset (core_reset_shared_resval()) + * paired with sched_update_account()'s UNCONDITIONAL shared charge -- a + * deliberate asymmetry (no per-worker gating exists for the shared pool) + * that is itself part of the locked contract here. + * + * The two-round zero-drift test is bug-1799's minimal reproduction shape: + * seed a RUN job's account, run two full + * reset->account->reconcile->charge cycles with no state change between + * them, and assert the pools read EXACTLY what one charge should leave -- + * any reset/charge asymmetry shows up as +1 charge per round. + * + * White-box #include of ../src/core.c, ../src/core_session.c and + * ../src/resource.c (the three TUs the paths live in), plus + * ../src/schedobj.c for the real kill_sched_* datakillers + * xce_sched_core_init() references. The hooked trace sources compile + * SCHED_TRACE bodies for resource.c per the testsuite-wide convention. + * sched_worker_release_host_ctx() resolves from resource.c itself, not a + * stub. + * + * Link technique (meson registration "scheduler res account symmetry + * test"): the "separate compiled sources" shape of + * sched_trace_error_paths_test -- this file textually includes the five + * sources above, and meson compiles EVERY OTHER scheduler_srcs entry + * (minus main.c) as its own TU into the same executable. That is what + * makes sched_account_jobs() (scheduler.c) resolvable here WITHOUT + * white-box including scheduler.c itself (which would drag plugin.c, + * driver.c and the whole client-facing surface into this TU): it links + * from scheduler.c's own object file. The same TU set supplies real + * plugin.c, so the plugin_unregister/plugin_prepare_allocation/ + * plugin_job_update/plugin_queue_update/plugin_job_rusage_update stubs + * test_resource_accounting.c needs are NOT defined here -- defining them + * would collide with plugin.c's real symbols at link time. sched_init() + * is never called (no client surface is exercised), so scheduler.c's + * undefined xcecli_* references resolve but never execute. + */ +#include +#include +#include + +#include + +#include "resobj.h" +#include "sched/types.h" +#include "scheduler.h" +#include "sim.h" + +/* Pull in the real functions under test (white-box #include, same + * convention as test_resource_accounting.c/test_core_session.c). */ +#include "../src/core.c" +#include "../src/core_session.c" +#include "../src/resource.c" +/* Hooked sources compile SCHED_TRACE bodies (scheduler-sim-decision-trace). */ +#include "../src/sched_trace.c" +#include "../src/sched_trace_scan.c" +#include "../src/sched_trace_capture.c" +#include "../src/sched_trace_outcome.c" +#include "../src/schedobj.c" + +/* NOTE on stubs: unlike test_resource_accounting.c, this file defines NO + * plugin_* stubs. The meson registration compiles the real ../src/plugin.c + * as its own TU (see the link-technique note in the header), so its + * plugin_unregister()/plugin_prepare_allocation()/plugin_job_update()/ + * plugin_queue_update()/plugin_job_rusage_update() definitions satisfy + * resource.c and schedobj.c here -- local stubs would be duplicate symbols. + */ + +void setUp(void) { return; } +void tearDown(void) { return; } + +/* ---------------------------------------------------------------------- */ +/* Fixture */ +/* ---------------------------------------------------------------------- */ + +#define WORKER_A "symm-host-a" /* healthy worker */ +#define WORKER_B "symm-host-b" /* the excluded (bad-status) worker */ +#define RES_SLOTS 0 /* per-host resource under test */ +#define RES_SHARED 1 /* shared resource under test */ +#define RUN_JOBID ((uint64_t)730601ULL) + +/* make_run_job_account - one RUN-state job whose account charges both + * pools and names BOTH hosts: + * shared: RES_SHARED used 3.0 + reserved 1.0 (shared charge = 4.0) + * per-host: WORKER_A and WORKER_B each get their OWN xce_res_account_t + * on RES_SLOTS with used 2.0, reserved 0.0. + * Two host entries (not one) is deliberate: sched_update_account() + * charges only hosts the account NAMES, so if the account named only + * WORKER_A, the charge path would never even look at WORKER_B and its + * skip (bug-1801) would pass vacuously. Naming both hosts forces every + * path's skip on B to be an ACTIVE decision per round. + * + * The used+reserved formula note: resource-accounting.md's charge is + * `used + reserved` where `used` is the OVERAGE component (kept up to + * date by sched_reconcile_host_resource_usage(); 0.0 for a job charging + * only its reservation), NOT the job's total usage. */ +static xce_job_account_t* make_run_job_account(void) +{ + xce_job_account_t* acct = (xce_job_account_t*)calloc(1, sizeof(*acct)); + TEST_ASSERT_NOT_NULL(acct); + + /* shared: 1 entry (RES_SHARED, used 3.0, reserved 1.0) */ + acct->nres_shared = 1; + acct->accounts_shared = (xce_res_account_t**)calloc( + 1, sizeof(*acct->accounts_shared)); + TEST_ASSERT_NOT_NULL(acct->accounts_shared); + acct->accounts_shared[0] + = (xce_res_account_t*)calloc(1, sizeof(xce_res_account_t)); + TEST_ASSERT_NOT_NULL(acct->accounts_shared[0]); + acct->accounts_shared[0]->residx = RES_SHARED; + acct->accounts_shared[0]->used = 3.0; + acct->accounts_shared[0]->reserved = 1.0; + + /* hosts: both, each with one RES_SLOTS entry (used 2.0, reserved + * 0.0 -> per-host charge 2.0 each) */ + acct->nhosts = 2; + acct->accounts = (xce_host_account_t**)calloc( + 2, sizeof(*acct->accounts)); + TEST_ASSERT_NOT_NULL(acct->accounts); + for (int h = 0; h < 2; h++) { + acct->accounts[h] + = (xce_host_account_t*)calloc(1, sizeof(xce_host_account_t)); + TEST_ASSERT_NOT_NULL(acct->accounts[h]); + acct->accounts[h]->host + = strdup(h == 0 ? WORKER_A : WORKER_B); + acct->accounts[h]->naccount = 1; + acct->accounts[h]->accounts = (xce_res_account_t**)calloc( + 1, sizeof(*acct->accounts[h]->accounts)); + TEST_ASSERT_NOT_NULL(acct->accounts[h]->accounts); + acct->accounts[h]->accounts[0] + = (xce_res_account_t*)calloc(1, sizeof(xce_res_account_t)); + TEST_ASSERT_NOT_NULL(acct->accounts[h]->accounts[0]); + acct->accounts[h]->accounts[0]->residx = RES_SLOTS; + acct->accounts[h]->accounts[0]->used = 2.0; + acct->accounts[h]->accounts[0]->reserved = 0.0; + } + + return acct; +} + +/* host_slot_acct - convenience accessor: the RES_SLOTS xce_res_account_t + * entry of `host`'s entry in a make_run_job_account() account (the exact + * pointer reconcile mutates: each host entry is a DISTINCT struct with + * its own `used` baseline). */ +static xce_res_account_t* host_slot_acct(xce_job_account_t* acct, + const char* host) +{ + for (int h = 0; h < acct->nhosts; h++) { + if (strcmp(acct->accounts[h]->host, host) == 0) { + return acct->accounts[h]->accounts[0]; + } + } + TEST_FAIL_MESSAGE("host not found in job account"); + return NULL; +} + +/* add_symm_worker - worker with one per-host resval (RES_SLOTS) and one + * seeded resInUse entry, mirroring test_core_session.c's fixture_worker + * seeding so the reset skip is observable (a skipped reset leaves the + * seeded value standing; a reset zeroes it). */ +static xce_sched_worker_t* add_symm_worker(xce_sched_core_t* core, + const char* name, + uint32_t status, + float seeded_inuse) +{ + xce_sched_worker_t* w = (xce_sched_worker_t*)calloc(1, sizeof(*w)); + xce_val_t* v = (xce_val_t*)calloc(1, sizeof(*v)); + xce_val_t* cap = (xce_val_t*)calloc(1, sizeof(*cap)); + int rc; + + TEST_ASSERT_NOT_NULL(w); + TEST_ASSERT_NOT_NULL(v); + TEST_ASSERT_NOT_NULL(cap); + + w->name = strdup(name); + w->status = status; + w->idx = -1; + + zarray_init(&(w->resvals), 8, (zarray_item_killer_t)kill_sched_resval); + zarray_init(&(w->resInUse), 8, (zarray_item_killer_t)kill_sched_resval); + + /* capacity: slots=4 on both hosts */ + cap->which = Value_u_fvalue; + cap->u.fvalue = 4.0f; + zarray_put(&(w->resvals), RES_SLOTS, cap); + + /* seeded in-use: observable reset-skip signal */ + v->which = Value_u_fvalue; + v->u.fvalue = seeded_inuse; + zarray_put(&(w->resInUse), RES_SLOTS, v); + + rc = sched_add_worker(core, w); + TEST_ASSERT_TRUE(rc >= 0); + + return w; +} + +static float read_inuse(zarray_t* arr, int idx) +{ + float val[1]; + int i[1] = { idx }; + + get_resvals(arr, 1, i, val); + return val[0]; +} + +/* shared capacity for the shared pool (RES_SHARED), set once per core */ +static void seed_shared_capacity(xce_sched_core_t* core) +{ + xce_val_t* cap = (xce_val_t*)calloc(1, sizeof(*cap)); + + TEST_ASSERT_NOT_NULL(cap); + cap->which = Value_u_fvalue; + cap->u.fvalue = 100.0f; + zarray_put(&(core->sharedInst), RES_SHARED, cap); +} + +/* ---------------------------------------------------------------------- */ +/* Test 1: the three-path round cycle on a core with one healthy and one + * excluded worker -- accounts must not drift across rounds. */ +/* ---------------------------------------------------------------------- */ + +/* + * drive_round - one complete per-session cycle exactly as the normal-mode + * round sequence runs it (driver.c's xce_sched_driver_run_normal_mode_ + * round ordering, minus the picker/allocator threads this fixture never + * starts): begin_session (resets + cache clear), then + * sched_account_jobs()'s re-charge of every RUN job, then one + * rusage-delta reconcile call per host -- the three paths under test, in + * the same order the live round uses. + * + * `slots_actual_a`/`slots_actual_b` feed + * sched_reconcile_host_resource_usage() for each host's OWN RES_SLOTS + * account entry (the report each worker would send). The live caller + * (sched_reconcile_job_resource_usage) reports per job; here we drive + * the per-host primitive directly so both hosts' behavior is asserted + * in the same round with no job-level filtering in between. + */ +static void drive_round(xce_sched_core_t* core, xce_sched_job_t* job, + double slots_actual_a, double slots_actual_b) +{ + xce_res_account_t* acct_a; + xce_res_account_t* acct_b; + xce_sched_worker_t* w; + int rc; + + rc = xce_core_begin_session(core); + TEST_ASSERT_EQUAL_INT(0, rc); + + /* Register the job ONCE (key_add() is -EEXIST on a duplicate; + * sched_account_jobs() below iterates core->jobs, where the job is + * already reachable from round 2 on). The real function -- scheduler.c's, + * core-typed since the task-1.1 relocation, declared in core.h -- does + * the charging; no re-implementation of its loop here. */ + rc = sched_add_job(core, job); + TEST_ASSERT_TRUE(rc >= 0 || rc == -EEXIST); + job->state = JOB_STAT_RUN; + TEST_ASSERT_EQUAL_INT(0, sched_account_jobs(core)); + + /* rusage-delta path (bug-1814): one tick per host against that + * host's OWN account entry -- the healthy host applies the delta, + * the excluded one must be skipped wholesale (its entry's used + * baseline advancing on a skipped worker was bug-1814's half of + * the damage). */ + acct_a = host_slot_acct(job->accounts, WORKER_A); + acct_b = host_slot_acct(job->accounts, WORKER_B); + w = sched_get_worker(core, WORKER_A); + TEST_ASSERT_NOT_NULL(w); + TEST_ASSERT_EQUAL_INT(0, + sched_reconcile_host_resource_usage(core, w, acct_a, slots_actual_a)); + + w = sched_get_worker(core, WORKER_B); + TEST_ASSERT_NOT_NULL(w); + TEST_ASSERT_EQUAL_INT(0, + sched_reconcile_host_resource_usage(core, w, acct_b, slots_actual_b)); + + rc = xce_core_end_session(core); + TEST_ASSERT_EQUAL_INT(0, rc); +} + +/* NOTE on job registration: sched_add_job() is idempotent (-EEXIST on a + * duplicate jobid) and the job object lives in core->jobs from its first + * drive_round() call onward -- the account pass reads the registry, so + * registration is registration, not a per-round re-add. Jobs are NOT + * torn down (fixtures in this directory are never freed -- the + * established convention), and core->jobs holds a pointer, not a copy. */ + +/* Two full rounds on a core where WORKER_B is RECONCILING: + * - WORKER_A's resInUse must equal exactly ONE charge after the rounds + * (reset -> re-charge is idempotent per round), + * - WORKER_B's resInUse must still read its seeded value (frozen: + * not reset, not charged, not reconciled), + * - core->sharedInUse must equal exactly ONE shared charge. + * Any reset/charge asymmetry shows up as +1 charge per round (the + * bug-1799 live shape: used=26.00 on a 22-slot worker). */ +static void test_two_rounds_zero_drift_with_reconciling_worker(void) +{ + xce_sched_core_t* core = (xce_sched_core_t*)calloc(1, sizeof(*core)); + xce_sched_worker_t* wa; + xce_sched_worker_t* wb; + xce_sched_job_t* job; + int rc; + + TEST_ASSERT_NOT_NULL(core); + rc = xce_sched_core_init(core); + TEST_ASSERT_EQUAL_INT(0, rc); + + seed_shared_capacity(core); + wa = add_symm_worker(core, WORKER_A, 0, 0.0f); + wb = add_symm_worker(core, WORKER_B, XCE_WORKER_RECONCILING, 2.5f); + + job = (xce_sched_job_t*)calloc(1, sizeof(*job)); + TEST_ASSERT_NOT_NULL(job); + job->jobid = RUN_JOBID; + job->state = JOB_STAT_RUN; + job->accounts = make_run_job_account(); + + /* Round 1: actual 2.0 == the account's charged level on both hosts + * (used 2.0 + reserved 0.0); reconcile computes new_used = + * max(0, 2.0 - 0.0) = 2.0, delta = 2.0 - entry->used(2.0) = 0 -- + * the common "no-op tick" -- and the pools must read exactly one + * charge. */ + drive_round(core, job, 2.0, 2.0); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 2.0f, read_inuse(&(wa->resInUse), RES_SLOTS), + "healthy worker after round 1: exactly one host charge (2.0)"); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 2.5f, read_inuse(&(wb->resInUse), RES_SLOTS), + "RECONCILING worker after round 1: resInUse frozen at its seeded " + "value -- not reset, not charged (bug-1751/1801), even though " + "the job's account NAMES this host"); + + /* Round 2: no state change -- every pool must land on the SAME + * numbers as after round 1. This is the bug-1799 assertion: the + * pre-fix code re-added a full charge here (22-slot worker showing + * used=26.00). */ + drive_round(core, job, 2.0, 2.0); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 2.0f, read_inuse(&(wa->resInUse), RES_SLOTS), + "round 2 must NOT grow the healthy worker's resInUse: " + "reset-then-recharge is idempotent per round"); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 2.5f, read_inuse(&(wb->resInUse), RES_SLOTS), + "round 2 must NOT grow the RECONCILING worker's resInUse either " + "-- if this reads 4.5 (2.5 + one charge), the charge path's skip " + "has lost the reset path's skip (bug-1799's exact shape)"); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 4.0f, read_inuse(&(core->sharedInUse), RES_SHARED), + "shared pool after two rounds: exactly ONE shared charge (3.0 " + "used + 1.0 reserved) -- the shared reset is unconditional and " + "the shared charge must stay symmetric with it"); + + /* and the healthy host's account baseline must be intact: the + * reconcile tick was a no-op, so acct->used stays 2.0 (a skipped + * worker's entry advancing here would be bug-1814's half). */ + TEST_ASSERT_EQUAL_FLOAT( + 2.0, host_slot_acct(job->accounts, WORKER_A)->used); + TEST_ASSERT_EQUAL_FLOAT( + 2.0, host_slot_acct(job->accounts, WORKER_B)->used); +} + +/* Same cycle with WORKER_B UNAVAIL instead of RECONCILING -- the skip + * condition covers BOTH bits on all three paths (they are ORed, never + * ANDed -- the truth-table-completeness argument + * test_begin_session_skips_host_with_both_unavail_and_reconciling in + * test_core_session.c makes, generalized here to all three paths at + * once). Also drives the rusage-delta path with a real usage CHANGE on + * the healthy worker (2.0 -> 5.0 -> 2.0) to prove the delta applies + * there while the excluded worker still does not move. */ +static void test_unavail_worker_and_overage_delta_stay_symmetric(void) +{ + xce_sched_core_t* core = (xce_sched_core_t*)calloc(1, sizeof(*core)); + xce_sched_worker_t* wa; + xce_sched_worker_t* wb; + xce_sched_job_t* job; + int rc; + + TEST_ASSERT_NOT_NULL(core); + rc = xce_sched_core_init(core); + TEST_ASSERT_EQUAL_INT(0, rc); + + seed_shared_capacity(core); + wa = add_symm_worker(core, WORKER_A, 0, 0.0f); + wb = add_symm_worker(core, WORKER_B, XCE_WORKER_UNAVAIL, 1.5f); + + job = (xce_sched_job_t*)calloc(1, sizeof(*job)); + TEST_ASSERT_NOT_NULL(job); + job->jobid = RUN_JOBID; + job->state = JOB_STAT_RUN; + job->accounts = make_run_job_account(); + + /* Round 1: WORKER_A's report climbs to 5.0 (reserved 0.0) -> + * new_used = 5.0, delta = 5.0 - entry->used(2.0) = +3.0 on top of + * the 2.0 charge -> pool reads 5.0 == max(reserved, actual). + * WORKER_B's tick is a full skip: its entry's used baseline must + * NOT advance (2.0 stays), its pool must NOT move. */ + drive_round(core, job, 5.0, 5.0); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 5.0f, read_inuse(&(wa->resInUse), RES_SLOTS), + "healthy worker: charge (2.0) adjusted by the reconcile delta " + "(+3.0) so the pool reads max(reserved, actual) = 5.0 -- the " + "delta path must still APPLY on a healthy host while the skip " + "invariants hold"); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 1.5f, read_inuse(&(wb->resInUse), RES_SLOTS), + "UNAVAIL worker: resInUse frozen at its seeded value across the " + "full cycle -- all three paths skipped it (this test drives all " + "three against the same worker in one round)"); + + /* bug-1814's exact half-damage assertion: the EXCLUDED host's + * account entry must not have had its used baseline advanced by a + * tick that was skipped. */ + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 2.0, host_slot_acct(job->accounts, WORKER_B)->used, + "skipped host's account entry used baseline must stay at the " + "charged 2.0 -- a tick that skips the worker must not mutate " + "the entry it would have fed"); + + TEST_ASSERT_EQUAL_FLOAT( + 5.0, host_slot_acct(job->accounts, WORKER_A)->used); + + /* Round 2 with the usage GONE again (0.0 <= reserved): the healthy + * worker's round-2 reset zeroes the pool, the charge re-applies the + * entry's UPDATED baseline (reconcile set used=5.0 in round 1, so + * the charge is 5.0+0.0 -- NOT the stale 2.0), and the reconcile + * tick drives it back down: new_used = max(0, 0.0 - 0.0) = 0.0, + * delta = 0.0 - 5.0 = -5.0 -> pool = 5.0 - 5.0 = 0.0. The + * used-baseline handoff between charge and reconcile is what makes + * the pool converge to max(reserved, actual) every round with no + * double-apply in either direction. WORKER_B stays frozen. */ + drive_round(core, job, 0.0, 0.0); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 0.0f, read_inuse(&(wa->resInUse), RES_SLOTS), + "round 2 on the healthy worker: pool converges back to " + "max(reserved, actual) = 0.0 -- the charge picked up the " + "reconciled used baseline (5.0) and the delta unwound it " + "exactly; neither stacks nor double-applies"); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 1.5f, read_inuse(&(wb->resInUse), RES_SLOTS), + "round 2: the UNAVAIL worker's pool STILL has not moved -- " + "two full rounds of skips, zero drift"); + + TEST_ASSERT_EQUAL_FLOAT( + 0.0, host_slot_acct(job->accounts, WORKER_A)->used); + TEST_ASSERT_EQUAL_FLOAT( + 2.0, host_slot_acct(job->accounts, WORKER_B)->used); +} + +/* Status CLEARS mid-stream: the excluded worker rejoins the normal + * reset+charge cycle on the very next round and its seeded value is + * finally reset -- no permanent freeze (the complement of the skip + * contract; mirrors test_begin_session_reincludes_host_once_ + * reconciling_clears but through the full three-path cycle). */ +static void test_status_clear_rejoins_normal_cycle_next_round(void) +{ + xce_sched_core_t* core = (xce_sched_core_t*)calloc(1, sizeof(*core)); + xce_sched_worker_t* wb; + xce_sched_job_t* job; + int rc; + + TEST_ASSERT_NOT_NULL(core); + rc = xce_sched_core_init(core); + TEST_ASSERT_EQUAL_INT(0, rc); + + seed_shared_capacity(core); + (void)add_symm_worker(core, WORKER_A, 0, 0.0f); + wb = add_symm_worker(core, WORKER_B, XCE_WORKER_RECONCILING, 9.0f); + + job = (xce_sched_job_t*)calloc(1, sizeof(*job)); + TEST_ASSERT_NOT_NULL(job); + job->jobid = RUN_JOBID; + job->state = JOB_STAT_RUN; + job->accounts = make_run_job_account(); /* names BOTH hosts: the + * charge/reconcile must reach + * for B and be REFUSED */ + + drive_round(core, job, 2.0, 2.0); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 9.0f, read_inuse(&(wb->resInUse), RES_SLOTS), + "round 1: the RECONCILING worker's pool is frozen even though a " + "RUN job's account names it -- the skip wins over the charge"); + + /* Status clears; the next round's reset+charge must treat it as an + * ordinary worker: seeded 9.0 reset to 0, then re-charged 2.0. The + * reconcile tick is a no-op here (actual 2.0 == entry used 2.0). */ + wb->status = 0; + drive_round(core, job, 2.0, 2.0); + + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 2.0f, read_inuse(&(wb->resInUse), RES_SLOTS), + "round 2 after the bit clears: seeded 9.0 was finally reset, " + "then exactly one charge applied -- the freeze is per-round " + "status-gated, never permanent"); +} + +/* ---------------------------------------------------------------------- */ +/* Suite entry */ +/* ---------------------------------------------------------------------- */ + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_two_rounds_zero_drift_with_reconciling_worker); + RUN_TEST(test_unavail_worker_and_overage_delta_stay_symmetric); + RUN_TEST(test_status_clear_rejoins_normal_cycle_next_round); + return UNITY_END(); +}