test(scheduler): lock three-path UNAVAIL/RECONCILING skip symmetry (Phase 0)
Phase 0 of scheduler-hardening-plan.md: a cross-file structural regression test locking the invariant whose three observed violations were bug-1799 (charge without reset-skip), bug-1801 (the complement) and bug-1814 (rusage delta without the skip): every w->resInUse/core->sharedInUse- mutating path shares ONE identical skip condition for UNAVAIL/RECONCILING workers. test_res_account_symmetry.c drives all three paths in one core per round -- core_reset_workers() (core_session.c, bug-1751 skip), sched_charge_account()/sched_update_account() (resource.c, bug-1801 skip) and sched_reconcile_host_resource_usage() (resource.c, bug-1814 skip) -- using the real sched_account_jobs() from scheduler.c, and asserts the accounts do not drift across consecutive rounds. A per-path test cannot see asymmetry by construction; bug-1799's live shape was asymmetry compounding across rounds (resInUse grew one full charge per round for two weeks). Three cases: two-round zero drift with a RECONCILING worker (B frozen at its seeded value, both host entries named by the job's account so the skips are active decisions, not vacuous); UNAVAIL worker plus a real usage climb 2.0->5.0->0.0 proving the reconcile delta still applies on the healthy host while the excluded host's account-entry used baseline never advances (bug-1814's half-damage); and a status-clear round proving the freeze is per-round status-gated, never permanent. Link technique: the separate-compiled-sources shape of sched_trace_error_paths_test -- the test white-box #includes core.c/core_session.c/resource.c/schedobj.c + the four hooked trace sources, and meson compiles every other scheduler_srcs entry (minus main.c) as its own TU, which resolves sched_account_jobs() from scheduler.c's own object without white-box including scheduler.c (and without the plugin_* stubs test_resource_accounting.c needs: real plugin.c is one of those TUs). Red proof: temporarily removing resource.c's bug-1801 skip trips the zero-drift assertion immediately (2.5 -> 4.5, exactly one extra charge per round -- bug-1799's live shape); mutation reverted, suite green. Full scheduler suite: 102/102 OK. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen
parent
78a7bae5ed
commit
b20caf8ddf
2 changed files with 582 additions and 0 deletions
|
|
@ -2747,6 +2747,46 @@ test('scheduler trace error paths test',
|
||||||
sched_trace_error_paths_test, suite: 'scheduler',
|
sched_trace_error_paths_test, suite: 'scheduler',
|
||||||
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
||||||
|
|
||||||
|
# Scheduler rejoin-hardening plan, Phase 0 (scheduler-hardening-plan.md):
|
||||||
|
# cross-file contract lock on the three-path UNAVAIL/RECONCILING skip
|
||||||
|
# symmetry (core_reset_workers / sched_update_account /
|
||||||
|
# sched_reconcile_host_resource_usage) whose observed asymmetries were
|
||||||
|
# bug-1799/bug-1801/bug-1814 -- see the test's own header comment for why
|
||||||
|
# one test drives all three paths. Same separate-compiled-sources
|
||||||
|
# technique as sched_trace_error_paths_test above: the test #includes
|
||||||
|
# core.c/core_session.c/resource.c/schedobj.c + the four hooked trace
|
||||||
|
# sources, and every OTHER scheduler_srcs entry (minus main.c) is its own
|
||||||
|
# TU -- which is what lets the test call sched_account_jobs()
|
||||||
|
# (scheduler.c) without white-box including scheduler.c (deliberate: no
|
||||||
|
# test in this directory #includes scheduler.c, and doing so would drag
|
||||||
|
# plugin.c/driver.c/the client-facing surface into the test TU). Real
|
||||||
|
# plugin.c comes in as one of those TUs, so the test defines no plugin_*
|
||||||
|
# stubs.
|
||||||
|
sched_res_account_symmetry_included = ['src/main.c', 'src/core.c',
|
||||||
|
'src/core_session.c',
|
||||||
|
'src/resource.c', 'src/schedobj.c',
|
||||||
|
'src/sched_trace.c',
|
||||||
|
'src/sched_trace_scan.c',
|
||||||
|
'src/sched_trace_capture.c',
|
||||||
|
'src/sched_trace_outcome.c']
|
||||||
|
sched_res_account_symmetry_test_srcs = ['test_res_account_symmetry.c']
|
||||||
|
foreach f : scheduler_srcs
|
||||||
|
if not sched_res_account_symmetry_included.contains(f)
|
||||||
|
sched_res_account_symmetry_test_srcs += ['../' + f]
|
||||||
|
endif
|
||||||
|
endforeach
|
||||||
|
|
||||||
|
sched_res_account_symmetry_test = executable(
|
||||||
|
'sched_res_account_symmetry_test',
|
||||||
|
sched_res_account_symmetry_test_srcs,
|
||||||
|
include_directories:[incdir],
|
||||||
|
dependencies: [scheduler_deps, unity_dep],
|
||||||
|
install: false)
|
||||||
|
|
||||||
|
test('scheduler res account symmetry test',
|
||||||
|
sched_res_account_symmetry_test, suite: 'scheduler',
|
||||||
|
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
||||||
|
|
||||||
# scheduler-sim-decision-trace task 6.4 (Requirements 9.1, 9.3, 9.5;
|
# scheduler-sim-decision-trace task 6.4 (Requirements 9.1, 9.3, 9.5;
|
||||||
# design.md "Testing Strategy -> Performance"): a 1000-host, maxTasks = 64
|
# design.md "Testing Strategy -> Performance"): a 1000-host, maxTasks = 64
|
||||||
# parallel job traced at full level by the real allocator stays within the
|
# parallel job traced at full level by the real allocator stays within the
|
||||||
|
|
|
||||||
542
src/scheduler/testsuite/test_res_account_symmetry.c
Normal file
542
src/scheduler/testsuite/test_res_account_symmetry.c
Normal file
|
|
@ -0,0 +1,542 @@
|
||||||
|
/*
|
||||||
|
* test_res_account_symmetry.c - Phase 0 of the scheduler rejoin-hardening
|
||||||
|
* plan (scheduler-hardening-plan.md): locks the cross-file contract that
|
||||||
|
* every w->resInUse/core->sharedInUse-mutating path shares ONE identical
|
||||||
|
* skip condition for UNAVAIL/RECONCILING workers -- the invariant whose
|
||||||
|
* three observed violations became bug-1799 (charge without reset-skip),
|
||||||
|
* bug-1801 (its complement), and bug-1814 (rusage delta without the skip).
|
||||||
|
*
|
||||||
|
* Why one test drives all THREE paths instead of relying on the existing
|
||||||
|
* per-path tests (test_core_session.c covers the reset skip,
|
||||||
|
* test_resource_accounting.c covers the charge skip and the rusage-delta
|
||||||
|
* skip individually): the live bug-1799 shape was NOT any single path
|
||||||
|
* misbehaving -- each path passed its own test -- it was the ASYMMETRY
|
||||||
|
* between two paths' skip conditions compounding across rounds (reset
|
||||||
|
* skipped, charge did not -> resInUse grew one full charge per round for
|
||||||
|
* two weeks before anyone looked at a worker's used= number). A per-path
|
||||||
|
* test cannot see asymmetry by construction; this file puts all three
|
||||||
|
* paths in one core and asserts the accounts do not drift across
|
||||||
|
* consecutive rounds, which is exactly the observable the live incident
|
||||||
|
* showed.
|
||||||
|
*
|
||||||
|
* The three w->resInUse-mutating paths under test (resource-accounting
|
||||||
|
* steering doc calls out each explicitly):
|
||||||
|
* (a) core_reset_workers() (core_session.c, bug-1751 skip)
|
||||||
|
* (b) sched_charge_account() via
|
||||||
|
* sched_update_account() (resource.c, bug-1801 skip)
|
||||||
|
* (c) sched_reconcile_host_resource_usage() (resource.c, bug-1814 skip)
|
||||||
|
* plus the unconditional shared-pool reset (core_reset_shared_resval())
|
||||||
|
* paired with sched_update_account()'s UNCONDITIONAL shared charge -- a
|
||||||
|
* deliberate asymmetry (no per-worker gating exists for the shared pool)
|
||||||
|
* that is itself part of the locked contract here.
|
||||||
|
*
|
||||||
|
* The two-round zero-drift test is bug-1799's minimal reproduction shape:
|
||||||
|
* seed a RUN job's account, run two full
|
||||||
|
* reset->account->reconcile->charge cycles with no state change between
|
||||||
|
* them, and assert the pools read EXACTLY what one charge should leave --
|
||||||
|
* any reset/charge asymmetry shows up as +1 charge per round.
|
||||||
|
*
|
||||||
|
* White-box #include of ../src/core.c, ../src/core_session.c and
|
||||||
|
* ../src/resource.c (the three TUs the paths live in), plus
|
||||||
|
* ../src/schedobj.c for the real kill_sched_* datakillers
|
||||||
|
* xce_sched_core_init() references. The hooked trace sources compile
|
||||||
|
* SCHED_TRACE bodies for resource.c per the testsuite-wide convention.
|
||||||
|
* sched_worker_release_host_ctx() resolves from resource.c itself, not a
|
||||||
|
* stub.
|
||||||
|
*
|
||||||
|
* Link technique (meson registration "scheduler res account symmetry
|
||||||
|
* test"): the "separate compiled sources" shape of
|
||||||
|
* sched_trace_error_paths_test -- this file textually includes the five
|
||||||
|
* sources above, and meson compiles EVERY OTHER scheduler_srcs entry
|
||||||
|
* (minus main.c) as its own TU into the same executable. That is what
|
||||||
|
* makes sched_account_jobs() (scheduler.c) resolvable here WITHOUT
|
||||||
|
* white-box including scheduler.c itself (which would drag plugin.c,
|
||||||
|
* driver.c and the whole client-facing surface into this TU): it links
|
||||||
|
* from scheduler.c's own object file. The same TU set supplies real
|
||||||
|
* plugin.c, so the plugin_unregister/plugin_prepare_allocation/
|
||||||
|
* plugin_job_update/plugin_queue_update/plugin_job_rusage_update stubs
|
||||||
|
* test_resource_accounting.c needs are NOT defined here -- defining them
|
||||||
|
* would collide with plugin.c's real symbols at link time. sched_init()
|
||||||
|
* is never called (no client surface is exercised), so scheduler.c's
|
||||||
|
* undefined xcecli_* references resolve but never execute.
|
||||||
|
*/
|
||||||
|
#include <errno.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include <unity/unity.h>
|
||||||
|
|
||||||
|
#include "resobj.h"
|
||||||
|
#include "sched/types.h"
|
||||||
|
#include "scheduler.h"
|
||||||
|
#include "sim.h"
|
||||||
|
|
||||||
|
/* Pull in the real functions under test (white-box #include, same
|
||||||
|
* convention as test_resource_accounting.c/test_core_session.c). */
|
||||||
|
#include "../src/core.c"
|
||||||
|
#include "../src/core_session.c"
|
||||||
|
#include "../src/resource.c"
|
||||||
|
/* Hooked sources compile SCHED_TRACE bodies (scheduler-sim-decision-trace). */
|
||||||
|
#include "../src/sched_trace.c"
|
||||||
|
#include "../src/sched_trace_scan.c"
|
||||||
|
#include "../src/sched_trace_capture.c"
|
||||||
|
#include "../src/sched_trace_outcome.c"
|
||||||
|
#include "../src/schedobj.c"
|
||||||
|
|
||||||
|
/* NOTE on stubs: unlike test_resource_accounting.c, this file defines NO
|
||||||
|
* plugin_* stubs. The meson registration compiles the real ../src/plugin.c
|
||||||
|
* as its own TU (see the link-technique note in the header), so its
|
||||||
|
* plugin_unregister()/plugin_prepare_allocation()/plugin_job_update()/
|
||||||
|
* plugin_queue_update()/plugin_job_rusage_update() definitions satisfy
|
||||||
|
* resource.c and schedobj.c here -- local stubs would be duplicate symbols.
|
||||||
|
*/
|
||||||
|
|
||||||
|
void setUp(void) { return; }
|
||||||
|
void tearDown(void) { return; }
|
||||||
|
|
||||||
|
/* ---------------------------------------------------------------------- */
|
||||||
|
/* Fixture */
|
||||||
|
/* ---------------------------------------------------------------------- */
|
||||||
|
|
||||||
|
#define WORKER_A "symm-host-a" /* healthy worker */
|
||||||
|
#define WORKER_B "symm-host-b" /* the excluded (bad-status) worker */
|
||||||
|
#define RES_SLOTS 0 /* per-host resource under test */
|
||||||
|
#define RES_SHARED 1 /* shared resource under test */
|
||||||
|
#define RUN_JOBID ((uint64_t)730601ULL)
|
||||||
|
|
||||||
|
/* make_run_job_account - one RUN-state job whose account charges both
|
||||||
|
* pools and names BOTH hosts:
|
||||||
|
* shared: RES_SHARED used 3.0 + reserved 1.0 (shared charge = 4.0)
|
||||||
|
* per-host: WORKER_A and WORKER_B each get their OWN xce_res_account_t
|
||||||
|
* on RES_SLOTS with used 2.0, reserved 0.0.
|
||||||
|
* Two host entries (not one) is deliberate: sched_update_account()
|
||||||
|
* charges only hosts the account NAMES, so if the account named only
|
||||||
|
* WORKER_A, the charge path would never even look at WORKER_B and its
|
||||||
|
* skip (bug-1801) would pass vacuously. Naming both hosts forces every
|
||||||
|
* path's skip on B to be an ACTIVE decision per round.
|
||||||
|
*
|
||||||
|
* The used+reserved formula note: resource-accounting.md's charge is
|
||||||
|
* `used + reserved` where `used` is the OVERAGE component (kept up to
|
||||||
|
* date by sched_reconcile_host_resource_usage(); 0.0 for a job charging
|
||||||
|
* only its reservation), NOT the job's total usage. */
|
||||||
|
static xce_job_account_t* make_run_job_account(void)
|
||||||
|
{
|
||||||
|
xce_job_account_t* acct = (xce_job_account_t*)calloc(1, sizeof(*acct));
|
||||||
|
TEST_ASSERT_NOT_NULL(acct);
|
||||||
|
|
||||||
|
/* shared: 1 entry (RES_SHARED, used 3.0, reserved 1.0) */
|
||||||
|
acct->nres_shared = 1;
|
||||||
|
acct->accounts_shared = (xce_res_account_t**)calloc(
|
||||||
|
1, sizeof(*acct->accounts_shared));
|
||||||
|
TEST_ASSERT_NOT_NULL(acct->accounts_shared);
|
||||||
|
acct->accounts_shared[0]
|
||||||
|
= (xce_res_account_t*)calloc(1, sizeof(xce_res_account_t));
|
||||||
|
TEST_ASSERT_NOT_NULL(acct->accounts_shared[0]);
|
||||||
|
acct->accounts_shared[0]->residx = RES_SHARED;
|
||||||
|
acct->accounts_shared[0]->used = 3.0;
|
||||||
|
acct->accounts_shared[0]->reserved = 1.0;
|
||||||
|
|
||||||
|
/* hosts: both, each with one RES_SLOTS entry (used 2.0, reserved
|
||||||
|
* 0.0 -> per-host charge 2.0 each) */
|
||||||
|
acct->nhosts = 2;
|
||||||
|
acct->accounts = (xce_host_account_t**)calloc(
|
||||||
|
2, sizeof(*acct->accounts));
|
||||||
|
TEST_ASSERT_NOT_NULL(acct->accounts);
|
||||||
|
for (int h = 0; h < 2; h++) {
|
||||||
|
acct->accounts[h]
|
||||||
|
= (xce_host_account_t*)calloc(1, sizeof(xce_host_account_t));
|
||||||
|
TEST_ASSERT_NOT_NULL(acct->accounts[h]);
|
||||||
|
acct->accounts[h]->host
|
||||||
|
= strdup(h == 0 ? WORKER_A : WORKER_B);
|
||||||
|
acct->accounts[h]->naccount = 1;
|
||||||
|
acct->accounts[h]->accounts = (xce_res_account_t**)calloc(
|
||||||
|
1, sizeof(*acct->accounts[h]->accounts));
|
||||||
|
TEST_ASSERT_NOT_NULL(acct->accounts[h]->accounts);
|
||||||
|
acct->accounts[h]->accounts[0]
|
||||||
|
= (xce_res_account_t*)calloc(1, sizeof(xce_res_account_t));
|
||||||
|
TEST_ASSERT_NOT_NULL(acct->accounts[h]->accounts[0]);
|
||||||
|
acct->accounts[h]->accounts[0]->residx = RES_SLOTS;
|
||||||
|
acct->accounts[h]->accounts[0]->used = 2.0;
|
||||||
|
acct->accounts[h]->accounts[0]->reserved = 0.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return acct;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* host_slot_acct - convenience accessor: the RES_SLOTS xce_res_account_t
|
||||||
|
* entry of `host`'s entry in a make_run_job_account() account (the exact
|
||||||
|
* pointer reconcile mutates: each host entry is a DISTINCT struct with
|
||||||
|
* its own `used` baseline). */
|
||||||
|
static xce_res_account_t* host_slot_acct(xce_job_account_t* acct,
|
||||||
|
const char* host)
|
||||||
|
{
|
||||||
|
for (int h = 0; h < acct->nhosts; h++) {
|
||||||
|
if (strcmp(acct->accounts[h]->host, host) == 0) {
|
||||||
|
return acct->accounts[h]->accounts[0];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TEST_FAIL_MESSAGE("host not found in job account");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* add_symm_worker - worker with one per-host resval (RES_SLOTS) and one
|
||||||
|
* seeded resInUse entry, mirroring test_core_session.c's fixture_worker
|
||||||
|
* seeding so the reset skip is observable (a skipped reset leaves the
|
||||||
|
* seeded value standing; a reset zeroes it). */
|
||||||
|
static xce_sched_worker_t* add_symm_worker(xce_sched_core_t* core,
|
||||||
|
const char* name,
|
||||||
|
uint32_t status,
|
||||||
|
float seeded_inuse)
|
||||||
|
{
|
||||||
|
xce_sched_worker_t* w = (xce_sched_worker_t*)calloc(1, sizeof(*w));
|
||||||
|
xce_val_t* v = (xce_val_t*)calloc(1, sizeof(*v));
|
||||||
|
xce_val_t* cap = (xce_val_t*)calloc(1, sizeof(*cap));
|
||||||
|
int rc;
|
||||||
|
|
||||||
|
TEST_ASSERT_NOT_NULL(w);
|
||||||
|
TEST_ASSERT_NOT_NULL(v);
|
||||||
|
TEST_ASSERT_NOT_NULL(cap);
|
||||||
|
|
||||||
|
w->name = strdup(name);
|
||||||
|
w->status = status;
|
||||||
|
w->idx = -1;
|
||||||
|
|
||||||
|
zarray_init(&(w->resvals), 8, (zarray_item_killer_t)kill_sched_resval);
|
||||||
|
zarray_init(&(w->resInUse), 8, (zarray_item_killer_t)kill_sched_resval);
|
||||||
|
|
||||||
|
/* capacity: slots=4 on both hosts */
|
||||||
|
cap->which = Value_u_fvalue;
|
||||||
|
cap->u.fvalue = 4.0f;
|
||||||
|
zarray_put(&(w->resvals), RES_SLOTS, cap);
|
||||||
|
|
||||||
|
/* seeded in-use: observable reset-skip signal */
|
||||||
|
v->which = Value_u_fvalue;
|
||||||
|
v->u.fvalue = seeded_inuse;
|
||||||
|
zarray_put(&(w->resInUse), RES_SLOTS, v);
|
||||||
|
|
||||||
|
rc = sched_add_worker(core, w);
|
||||||
|
TEST_ASSERT_TRUE(rc >= 0);
|
||||||
|
|
||||||
|
return w;
|
||||||
|
}
|
||||||
|
|
||||||
|
static float read_inuse(zarray_t* arr, int idx)
|
||||||
|
{
|
||||||
|
float val[1];
|
||||||
|
int i[1] = { idx };
|
||||||
|
|
||||||
|
get_resvals(arr, 1, i, val);
|
||||||
|
return val[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
/* shared capacity for the shared pool (RES_SHARED), set once per core */
|
||||||
|
static void seed_shared_capacity(xce_sched_core_t* core)
|
||||||
|
{
|
||||||
|
xce_val_t* cap = (xce_val_t*)calloc(1, sizeof(*cap));
|
||||||
|
|
||||||
|
TEST_ASSERT_NOT_NULL(cap);
|
||||||
|
cap->which = Value_u_fvalue;
|
||||||
|
cap->u.fvalue = 100.0f;
|
||||||
|
zarray_put(&(core->sharedInst), RES_SHARED, cap);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---------------------------------------------------------------------- */
|
||||||
|
/* Test 1: the three-path round cycle on a core with one healthy and one
|
||||||
|
* excluded worker -- accounts must not drift across rounds. */
|
||||||
|
/* ---------------------------------------------------------------------- */
|
||||||
|
|
||||||
|
/*
|
||||||
|
* drive_round - one complete per-session cycle exactly as the normal-mode
|
||||||
|
* round sequence runs it (driver.c's xce_sched_driver_run_normal_mode_
|
||||||
|
* round ordering, minus the picker/allocator threads this fixture never
|
||||||
|
* starts): begin_session (resets + cache clear), then
|
||||||
|
* sched_account_jobs()'s re-charge of every RUN job, then one
|
||||||
|
* rusage-delta reconcile call per host -- the three paths under test, in
|
||||||
|
* the same order the live round uses.
|
||||||
|
*
|
||||||
|
* `slots_actual_a`/`slots_actual_b` feed
|
||||||
|
* sched_reconcile_host_resource_usage() for each host's OWN RES_SLOTS
|
||||||
|
* account entry (the report each worker would send). The live caller
|
||||||
|
* (sched_reconcile_job_resource_usage) reports per job; here we drive
|
||||||
|
* the per-host primitive directly so both hosts' behavior is asserted
|
||||||
|
* in the same round with no job-level filtering in between.
|
||||||
|
*/
|
||||||
|
static void drive_round(xce_sched_core_t* core, xce_sched_job_t* job,
|
||||||
|
double slots_actual_a, double slots_actual_b)
|
||||||
|
{
|
||||||
|
xce_res_account_t* acct_a;
|
||||||
|
xce_res_account_t* acct_b;
|
||||||
|
xce_sched_worker_t* w;
|
||||||
|
int rc;
|
||||||
|
|
||||||
|
rc = xce_core_begin_session(core);
|
||||||
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||||
|
|
||||||
|
/* Register the job ONCE (key_add() is -EEXIST on a duplicate;
|
||||||
|
* sched_account_jobs() below iterates core->jobs, where the job is
|
||||||
|
* already reachable from round 2 on). The real function -- scheduler.c's,
|
||||||
|
* core-typed since the task-1.1 relocation, declared in core.h -- does
|
||||||
|
* the charging; no re-implementation of its loop here. */
|
||||||
|
rc = sched_add_job(core, job);
|
||||||
|
TEST_ASSERT_TRUE(rc >= 0 || rc == -EEXIST);
|
||||||
|
job->state = JOB_STAT_RUN;
|
||||||
|
TEST_ASSERT_EQUAL_INT(0, sched_account_jobs(core));
|
||||||
|
|
||||||
|
/* rusage-delta path (bug-1814): one tick per host against that
|
||||||
|
* host's OWN account entry -- the healthy host applies the delta,
|
||||||
|
* the excluded one must be skipped wholesale (its entry's used
|
||||||
|
* baseline advancing on a skipped worker was bug-1814's half of
|
||||||
|
* the damage). */
|
||||||
|
acct_a = host_slot_acct(job->accounts, WORKER_A);
|
||||||
|
acct_b = host_slot_acct(job->accounts, WORKER_B);
|
||||||
|
w = sched_get_worker(core, WORKER_A);
|
||||||
|
TEST_ASSERT_NOT_NULL(w);
|
||||||
|
TEST_ASSERT_EQUAL_INT(0,
|
||||||
|
sched_reconcile_host_resource_usage(core, w, acct_a, slots_actual_a));
|
||||||
|
|
||||||
|
w = sched_get_worker(core, WORKER_B);
|
||||||
|
TEST_ASSERT_NOT_NULL(w);
|
||||||
|
TEST_ASSERT_EQUAL_INT(0,
|
||||||
|
sched_reconcile_host_resource_usage(core, w, acct_b, slots_actual_b));
|
||||||
|
|
||||||
|
rc = xce_core_end_session(core);
|
||||||
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* NOTE on job registration: sched_add_job() is idempotent (-EEXIST on a
|
||||||
|
* duplicate jobid) and the job object lives in core->jobs from its first
|
||||||
|
* drive_round() call onward -- the account pass reads the registry, so
|
||||||
|
* registration is registration, not a per-round re-add. Jobs are NOT
|
||||||
|
* torn down (fixtures in this directory are never freed -- the
|
||||||
|
* established convention), and core->jobs holds a pointer, not a copy. */
|
||||||
|
|
||||||
|
/* Two full rounds on a core where WORKER_B is RECONCILING:
|
||||||
|
* - WORKER_A's resInUse must equal exactly ONE charge after the rounds
|
||||||
|
* (reset -> re-charge is idempotent per round),
|
||||||
|
* - WORKER_B's resInUse must still read its seeded value (frozen:
|
||||||
|
* not reset, not charged, not reconciled),
|
||||||
|
* - core->sharedInUse must equal exactly ONE shared charge.
|
||||||
|
* Any reset/charge asymmetry shows up as +1 charge per round (the
|
||||||
|
* bug-1799 live shape: used=26.00 on a 22-slot worker). */
|
||||||
|
static void test_two_rounds_zero_drift_with_reconciling_worker(void)
|
||||||
|
{
|
||||||
|
xce_sched_core_t* core = (xce_sched_core_t*)calloc(1, sizeof(*core));
|
||||||
|
xce_sched_worker_t* wa;
|
||||||
|
xce_sched_worker_t* wb;
|
||||||
|
xce_sched_job_t* job;
|
||||||
|
int rc;
|
||||||
|
|
||||||
|
TEST_ASSERT_NOT_NULL(core);
|
||||||
|
rc = xce_sched_core_init(core);
|
||||||
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||||
|
|
||||||
|
seed_shared_capacity(core);
|
||||||
|
wa = add_symm_worker(core, WORKER_A, 0, 0.0f);
|
||||||
|
wb = add_symm_worker(core, WORKER_B, XCE_WORKER_RECONCILING, 2.5f);
|
||||||
|
|
||||||
|
job = (xce_sched_job_t*)calloc(1, sizeof(*job));
|
||||||
|
TEST_ASSERT_NOT_NULL(job);
|
||||||
|
job->jobid = RUN_JOBID;
|
||||||
|
job->state = JOB_STAT_RUN;
|
||||||
|
job->accounts = make_run_job_account();
|
||||||
|
|
||||||
|
/* Round 1: actual 2.0 == the account's charged level on both hosts
|
||||||
|
* (used 2.0 + reserved 0.0); reconcile computes new_used =
|
||||||
|
* max(0, 2.0 - 0.0) = 2.0, delta = 2.0 - entry->used(2.0) = 0 --
|
||||||
|
* the common "no-op tick" -- and the pools must read exactly one
|
||||||
|
* charge. */
|
||||||
|
drive_round(core, job, 2.0, 2.0);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
2.0f, read_inuse(&(wa->resInUse), RES_SLOTS),
|
||||||
|
"healthy worker after round 1: exactly one host charge (2.0)");
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
2.5f, read_inuse(&(wb->resInUse), RES_SLOTS),
|
||||||
|
"RECONCILING worker after round 1: resInUse frozen at its seeded "
|
||||||
|
"value -- not reset, not charged (bug-1751/1801), even though "
|
||||||
|
"the job's account NAMES this host");
|
||||||
|
|
||||||
|
/* Round 2: no state change -- every pool must land on the SAME
|
||||||
|
* numbers as after round 1. This is the bug-1799 assertion: the
|
||||||
|
* pre-fix code re-added a full charge here (22-slot worker showing
|
||||||
|
* used=26.00). */
|
||||||
|
drive_round(core, job, 2.0, 2.0);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
2.0f, read_inuse(&(wa->resInUse), RES_SLOTS),
|
||||||
|
"round 2 must NOT grow the healthy worker's resInUse: "
|
||||||
|
"reset-then-recharge is idempotent per round");
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
2.5f, read_inuse(&(wb->resInUse), RES_SLOTS),
|
||||||
|
"round 2 must NOT grow the RECONCILING worker's resInUse either "
|
||||||
|
"-- if this reads 4.5 (2.5 + one charge), the charge path's skip "
|
||||||
|
"has lost the reset path's skip (bug-1799's exact shape)");
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
4.0f, read_inuse(&(core->sharedInUse), RES_SHARED),
|
||||||
|
"shared pool after two rounds: exactly ONE shared charge (3.0 "
|
||||||
|
"used + 1.0 reserved) -- the shared reset is unconditional and "
|
||||||
|
"the shared charge must stay symmetric with it");
|
||||||
|
|
||||||
|
/* and the healthy host's account baseline must be intact: the
|
||||||
|
* reconcile tick was a no-op, so acct->used stays 2.0 (a skipped
|
||||||
|
* worker's entry advancing here would be bug-1814's half). */
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT(
|
||||||
|
2.0, host_slot_acct(job->accounts, WORKER_A)->used);
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT(
|
||||||
|
2.0, host_slot_acct(job->accounts, WORKER_B)->used);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Same cycle with WORKER_B UNAVAIL instead of RECONCILING -- the skip
|
||||||
|
* condition covers BOTH bits on all three paths (they are ORed, never
|
||||||
|
* ANDed -- the truth-table-completeness argument
|
||||||
|
* test_begin_session_skips_host_with_both_unavail_and_reconciling in
|
||||||
|
* test_core_session.c makes, generalized here to all three paths at
|
||||||
|
* once). Also drives the rusage-delta path with a real usage CHANGE on
|
||||||
|
* the healthy worker (2.0 -> 5.0 -> 2.0) to prove the delta applies
|
||||||
|
* there while the excluded worker still does not move. */
|
||||||
|
static void test_unavail_worker_and_overage_delta_stay_symmetric(void)
|
||||||
|
{
|
||||||
|
xce_sched_core_t* core = (xce_sched_core_t*)calloc(1, sizeof(*core));
|
||||||
|
xce_sched_worker_t* wa;
|
||||||
|
xce_sched_worker_t* wb;
|
||||||
|
xce_sched_job_t* job;
|
||||||
|
int rc;
|
||||||
|
|
||||||
|
TEST_ASSERT_NOT_NULL(core);
|
||||||
|
rc = xce_sched_core_init(core);
|
||||||
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||||
|
|
||||||
|
seed_shared_capacity(core);
|
||||||
|
wa = add_symm_worker(core, WORKER_A, 0, 0.0f);
|
||||||
|
wb = add_symm_worker(core, WORKER_B, XCE_WORKER_UNAVAIL, 1.5f);
|
||||||
|
|
||||||
|
job = (xce_sched_job_t*)calloc(1, sizeof(*job));
|
||||||
|
TEST_ASSERT_NOT_NULL(job);
|
||||||
|
job->jobid = RUN_JOBID;
|
||||||
|
job->state = JOB_STAT_RUN;
|
||||||
|
job->accounts = make_run_job_account();
|
||||||
|
|
||||||
|
/* Round 1: WORKER_A's report climbs to 5.0 (reserved 0.0) ->
|
||||||
|
* new_used = 5.0, delta = 5.0 - entry->used(2.0) = +3.0 on top of
|
||||||
|
* the 2.0 charge -> pool reads 5.0 == max(reserved, actual).
|
||||||
|
* WORKER_B's tick is a full skip: its entry's used baseline must
|
||||||
|
* NOT advance (2.0 stays), its pool must NOT move. */
|
||||||
|
drive_round(core, job, 5.0, 5.0);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
5.0f, read_inuse(&(wa->resInUse), RES_SLOTS),
|
||||||
|
"healthy worker: charge (2.0) adjusted by the reconcile delta "
|
||||||
|
"(+3.0) so the pool reads max(reserved, actual) = 5.0 -- the "
|
||||||
|
"delta path must still APPLY on a healthy host while the skip "
|
||||||
|
"invariants hold");
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
1.5f, read_inuse(&(wb->resInUse), RES_SLOTS),
|
||||||
|
"UNAVAIL worker: resInUse frozen at its seeded value across the "
|
||||||
|
"full cycle -- all three paths skipped it (this test drives all "
|
||||||
|
"three against the same worker in one round)");
|
||||||
|
|
||||||
|
/* bug-1814's exact half-damage assertion: the EXCLUDED host's
|
||||||
|
* account entry must not have had its used baseline advanced by a
|
||||||
|
* tick that was skipped. */
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
2.0, host_slot_acct(job->accounts, WORKER_B)->used,
|
||||||
|
"skipped host's account entry used baseline must stay at the "
|
||||||
|
"charged 2.0 -- a tick that skips the worker must not mutate "
|
||||||
|
"the entry it would have fed");
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT(
|
||||||
|
5.0, host_slot_acct(job->accounts, WORKER_A)->used);
|
||||||
|
|
||||||
|
/* Round 2 with the usage GONE again (0.0 <= reserved): the healthy
|
||||||
|
* worker's round-2 reset zeroes the pool, the charge re-applies the
|
||||||
|
* entry's UPDATED baseline (reconcile set used=5.0 in round 1, so
|
||||||
|
* the charge is 5.0+0.0 -- NOT the stale 2.0), and the reconcile
|
||||||
|
* tick drives it back down: new_used = max(0, 0.0 - 0.0) = 0.0,
|
||||||
|
* delta = 0.0 - 5.0 = -5.0 -> pool = 5.0 - 5.0 = 0.0. The
|
||||||
|
* used-baseline handoff between charge and reconcile is what makes
|
||||||
|
* the pool converge to max(reserved, actual) every round with no
|
||||||
|
* double-apply in either direction. WORKER_B stays frozen. */
|
||||||
|
drive_round(core, job, 0.0, 0.0);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
0.0f, read_inuse(&(wa->resInUse), RES_SLOTS),
|
||||||
|
"round 2 on the healthy worker: pool converges back to "
|
||||||
|
"max(reserved, actual) = 0.0 -- the charge picked up the "
|
||||||
|
"reconciled used baseline (5.0) and the delta unwound it "
|
||||||
|
"exactly; neither stacks nor double-applies");
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
1.5f, read_inuse(&(wb->resInUse), RES_SLOTS),
|
||||||
|
"round 2: the UNAVAIL worker's pool STILL has not moved -- "
|
||||||
|
"two full rounds of skips, zero drift");
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT(
|
||||||
|
0.0, host_slot_acct(job->accounts, WORKER_A)->used);
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT(
|
||||||
|
2.0, host_slot_acct(job->accounts, WORKER_B)->used);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Status CLEARS mid-stream: the excluded worker rejoins the normal
|
||||||
|
* reset+charge cycle on the very next round and its seeded value is
|
||||||
|
* finally reset -- no permanent freeze (the complement of the skip
|
||||||
|
* contract; mirrors test_begin_session_reincludes_host_once_
|
||||||
|
* reconciling_clears but through the full three-path cycle). */
|
||||||
|
static void test_status_clear_rejoins_normal_cycle_next_round(void)
|
||||||
|
{
|
||||||
|
xce_sched_core_t* core = (xce_sched_core_t*)calloc(1, sizeof(*core));
|
||||||
|
xce_sched_worker_t* wb;
|
||||||
|
xce_sched_job_t* job;
|
||||||
|
int rc;
|
||||||
|
|
||||||
|
TEST_ASSERT_NOT_NULL(core);
|
||||||
|
rc = xce_sched_core_init(core);
|
||||||
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||||
|
|
||||||
|
seed_shared_capacity(core);
|
||||||
|
(void)add_symm_worker(core, WORKER_A, 0, 0.0f);
|
||||||
|
wb = add_symm_worker(core, WORKER_B, XCE_WORKER_RECONCILING, 9.0f);
|
||||||
|
|
||||||
|
job = (xce_sched_job_t*)calloc(1, sizeof(*job));
|
||||||
|
TEST_ASSERT_NOT_NULL(job);
|
||||||
|
job->jobid = RUN_JOBID;
|
||||||
|
job->state = JOB_STAT_RUN;
|
||||||
|
job->accounts = make_run_job_account(); /* names BOTH hosts: the
|
||||||
|
* charge/reconcile must reach
|
||||||
|
* for B and be REFUSED */
|
||||||
|
|
||||||
|
drive_round(core, job, 2.0, 2.0);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
9.0f, read_inuse(&(wb->resInUse), RES_SLOTS),
|
||||||
|
"round 1: the RECONCILING worker's pool is frozen even though a "
|
||||||
|
"RUN job's account names it -- the skip wins over the charge");
|
||||||
|
|
||||||
|
/* Status clears; the next round's reset+charge must treat it as an
|
||||||
|
* ordinary worker: seeded 9.0 reset to 0, then re-charged 2.0. The
|
||||||
|
* reconcile tick is a no-op here (actual 2.0 == entry used 2.0). */
|
||||||
|
wb->status = 0;
|
||||||
|
drive_round(core, job, 2.0, 2.0);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
|
||||||
|
2.0f, read_inuse(&(wb->resInUse), RES_SLOTS),
|
||||||
|
"round 2 after the bit clears: seeded 9.0 was finally reset, "
|
||||||
|
"then exactly one charge applied -- the freeze is per-round "
|
||||||
|
"status-gated, never permanent");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---------------------------------------------------------------------- */
|
||||||
|
/* Suite entry */
|
||||||
|
/* ---------------------------------------------------------------------- */
|
||||||
|
|
||||||
|
int main(void)
|
||||||
|
{
|
||||||
|
UNITY_BEGIN();
|
||||||
|
RUN_TEST(test_two_rounds_zero_drift_with_reconciling_worker);
|
||||||
|
RUN_TEST(test_unavail_worker_and_overage_delta_stay_symmetric);
|
||||||
|
RUN_TEST(test_status_clear_rejoins_normal_cycle_next_round);
|
||||||
|
return UNITY_END();
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue