diff --git a/src/scheduler/src/resource.c b/src/scheduler/src/resource.c index b360f2ae..71dc4089 100644 --- a/src/scheduler/src/resource.c +++ b/src/scheduler/src/resource.c @@ -798,6 +798,27 @@ ensure_host_account_by_resname(xce_sched_core_t* core, * A no-op (returns 0, `used` unchanged) when the computed overage is * unchanged from what's already charged -- the common case, since most * rusage ticks report usage still below the reservation. + * + * bug-1814 follow-up fix (2026-09-15): must skip entirely while the worker + * is IS_WORKER_UNAVAIL/IS_WORKER_RECONCILING, mirroring sched_update_ + * account()'s own bug-1801 guard above -- this is a THIRD independent + * w->resInUse-mutating path (alongside charge/uncharge) that bug-1801 did + * not cover. Without this, a RECONCILING worker's resInUse -- deliberately + * left FROZEN by core_reset_workers() (bug-1751) and now also by sched_ + * update_account() (bug-1801), so a clean per-round rebuild can restore it + * once the worker's real state is confirmed -- keeps getting mutated by + * every rusage tick that arrives during the same window regardless. Live- + * confirmed root cause of a job's "mem used" figure that grows and never + * self-corrects until the job finishes: a JOBS_PUSH resync merge occurring + * in the same RECONCILING window can replace a job's xce_res_account_t + * with a freshly-built one (`.used=0.0`, core_events.c's JOBS_PUSH RUN- + * merge branch) without uncharging the old one first (safe ONLY because + * the next clean reset+resum round is supposed to correct it -- exactly + * the round that stays skipped while RECONCILING). The very next rusage + * tick then computes its delta against that fresh 0.0 baseline and adds + * the full usage on top of whatever is already frozen in the pool, instead + * of leaving the pool alone until the worker's status clears and a normal + * rebuild can reconcile everything at once. */ int sched_reconcile_host_resource_usage(xce_sched_core_t* core, xce_sched_worker_t* w, @@ -813,6 +834,10 @@ int sched_reconcile_host_resource_usage(xce_sched_core_t* core, return -EINVAL; } + if (IS_WORKER_UNAVAIL(w->status) || IS_WORKER_RECONCILING(w->status)) { + return 0; + } + new_used = actual_usage - acct_entry->reserved; if (new_used < 0.0) { new_used = 0.0; diff --git a/src/scheduler/testsuite/test_resource_accounting.c b/src/scheduler/testsuite/test_resource_accounting.c index 4ca749cc..62167917 100644 --- a/src/scheduler/testsuite/test_resource_accounting.c +++ b/src/scheduler/testsuite/test_resource_accounting.c @@ -663,6 +663,74 @@ static void test_reconcile_host_resource_usage_null_args_return_einval(void) &(sched.core), NULL, &acct_entry, 0.0)); } +/* + * test_reconcile_host_resource_usage_skips_reconciling_or_unavail_worker - + * bug-1814 follow-up: sched_reconcile_host_resource_usage() is a THIRD + * w->resInUse-mutating path (alongside sched_charge_account()/sched_ + * uncharge_account(), both already gated by bug-1801) that must respect the + * same "frozen while RECONCILING/UNAVAIL" invariant core_reset_workers() + * (bug-1751) and sched_update_account() (bug-1801) already share -- a live + * replay showed a job's "mem used" growing and never self-correcting until + * the job finished, root-caused to this exact gap: a rusage tick landing + * mid-resync kept mutating resInUse (and advancing acct_entry->used) on top + * of a pool a concurrent per-round rebuild was deliberately leaving alone. + */ +static void +test_reconcile_host_resource_usage_skips_reconciling_or_unavail_worker(void) +{ + xce_scheduler_t sched; + xce_sched_worker_t* w; + xce_res_account_t acct_entry = { 0 }; + int rc; + + memset(&sched, 0, sizeof(sched)); + setup_core_with_worker(&(sched.core), WORKER_NAME); + w = sched_get_worker(&(sched.core), WORKER_NAME); + TEST_ASSERT_NOT_NULL(w); + + acct_entry.residx = MEM_RESIDX; + acct_entry.reserved = MEM_RESERVED_MB; + acct_entry.used = 0.0; + + /* --- RECONCILING: no mutation at all, not to the pool, not to + * acct_entry->used --- */ + w->status = XCE_WORKER_RECONCILING; + rc = sched_reconcile_host_resource_usage(&(sched.core), w, + &acct_entry, 9.0); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 0.0f, get_worker_resinuse(w, MEM_RESIDX), + "bug-1814: a RECONCILING worker's resInUse must not be touched by a " + "rusage tick -- it is deliberately frozen until a clean per-round " + "rebuild can reconcile it, same invariant bug-1801 already " + "restores for the regular charge path"); + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 0.0, (float)acct_entry.used, + "the account's own used baseline must also stay untouched while " + "skipped, so the eventual real reconcile call sees a clean delta"); + + /* --- UNAVAIL: same skip --- */ + w->status = XCE_WORKER_UNAVAIL; + rc = sched_reconcile_host_resource_usage(&(sched.core), w, + &acct_entry, 9.0); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 0.0f, get_worker_resinuse(w, MEM_RESIDX), + "bug-1814: an UNAVAIL worker's resInUse must not be touched either"); + + /* --- status clears: reconciliation resumes normally on the very next + * call, not a permanent block --- */ + w->status = 0; + rc = sched_reconcile_host_resource_usage(&(sched.core), w, + &acct_entry, 9.0); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_FLOAT_MESSAGE( + 5.0, (float)acct_entry.used, + "overage = 9 - 4 reserved = 5, applied on the first call once the " + "worker is no longer RECONCILING/UNAVAIL"); + TEST_ASSERT_EQUAL_FLOAT(5.0, get_worker_resinuse(w, MEM_RESIDX)); +} + /* * test_reconcile_job_resource_usage_* - the rusages-report wrapper: resname * mapping ("memory_rss" on the wire -> "mem" in the account), the single- @@ -1230,6 +1298,8 @@ int main(void) RUN_TEST( test_reconcile_host_resource_usage_repeated_growth_applies_only_delta); RUN_TEST(test_reconcile_host_resource_usage_null_args_return_einval); + RUN_TEST( + test_reconcile_host_resource_usage_skips_reconciling_or_unavail_worker); RUN_TEST(test_reconcile_job_resource_usage_maps_memory_rss_to_mem_account); RUN_TEST(test_reconcile_job_resource_usage_skips_multihost_jobs); RUN_TEST(test_reconcile_job_resource_usage_charges_unreserved_job_full_usage);