From e8aa761ca757c0e0903fe2d50f13168b6472309d Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 8 Sep 2026 10:44:11 +0000 Subject: [PATCH] chore(wolf): update buglog and anatomy/memory index Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01GLfB5uWRdmx2WiZSFC7Fm7 --- .wolf/anatomy-index.json | 100 +++++++++++++++++++++++++++++++++++++-- .wolf/anatomy.md | 26 +++++++++- .wolf/buglog.json | 2 +- .wolf/memory.md | 11 +++++ 4 files changed, 131 insertions(+), 8 deletions(-) diff --git a/.wolf/anatomy-index.json b/.wolf/anatomy-index.json index 52bd6fa6..6b8590a9 100644 --- a/.wolf/anatomy-index.json +++ b/.wolf/anatomy-index.json @@ -1,13 +1,13 @@ { "version": 1, "meta": { - "lastScanned": "2026-09-08T09:10:34.359Z", - "fileCount": 510, + "lastScanned": "2026-09-08T10:36:44.938Z", + "fileCount": 520, "hits": 0, "misses": 0, - "renderedHash": "2a1a6f7d5627269a9beb535944e21f32daa272c537cdc52c200662b960e05d05", - "storeUpdatedAt": "2026-09-08T09:10:34.366Z", - "rootHash": "cdfc2e29b87713ac" + "renderedHash": "4dba218326648adbe71882525a019c6ec002063fe24ba8b92716e1302fcd4b99", + "storeUpdatedAt": "2026-09-08T10:36:44.946Z", + "rootHash": "2cfee791fb2a32dd" }, "files": { ".agent-shell/transcripts/2026-08-11-11-39-59.md": { @@ -9807,6 +9807,96 @@ "mtimeMs": 1788858634325.683, "updatedAt": "2026-09-08T09:10:34.359Z", "source": "hook" + }, + "src/lib/resreq.c": { + "description": "define _XOPEN_SOURCE 700", + "tokens": 6663, + "hash": "22dd648c6322cb56", + "size": 25209, + "mtimeMs": 1788863523979.526, + "updatedAt": "2026-09-08T10:32:04.029Z", + "source": "hook" + }, + "src/include/resreq.h": { + "description": "if !defined(_RESREQ_H_)", + "tokens": 3310, + "hash": "a67af6e6ef6d9eb5", + "size": 12761, + "mtimeMs": 1788863533338.636, + "updatedAt": "2026-09-08T10:32:13.381Z", + "source": "hook" + }, + "src/scheduler/src/plugins/seqjob_allocator.c": { + "description": "include ", + "tokens": 3274, + "hash": "fb40c75e65b78df6", + "size": 12275, + "mtimeMs": 1788863549841.1375, + "updatedAt": "2026-09-08T10:32:29.892Z", + "source": "hook" + }, + "src/scheduler/src/plugins/parjob_allocator.c": { + "description": "include ", + "tokens": 7506, + "hash": "5c36d1b1cd16eb20", + "size": 28145, + "mtimeMs": 1788863559398.071, + "updatedAt": "2026-09-08T10:32:39.432Z", + "source": "hook" + }, + "src/scheduler/src/resource.c": { + "description": "include ", + "tokens": 8324, + "hash": "762e3e3ead330089", + "size": 31218, + "mtimeMs": 1788863596080.0664, + "updatedAt": "2026-09-08T10:33:16.127Z", + "source": "hook" + }, + "src/scheduler/include/scheduler.h": { + "description": "if !defined(_SCHEDULER_H_)", + "tokens": 4716, + "hash": "92201f60362d4928", + "size": 17683, + "mtimeMs": 1788863604566.5034, + "updatedAt": "2026-09-08T10:33:24.601Z", + "source": "hook" + }, + "src/scheduler/src/plugin.c": { + "description": "include ", + "tokens": 4554, + "hash": "995466e26a5e4296", + "size": 17077, + "mtimeMs": 1788863615715.2449, + "updatedAt": "2026-09-08T10:33:35.749Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_resource_accounting.c": { + "description": "Declares pulled", + "tokens": 12310, + "hash": "4863d64f2e4f58cd", + "size": 46159, + "mtimeMs": 1788863722019.7217, + "updatedAt": "2026-09-08T10:35:22.066Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_seqjob_allocator.c": { + "description": "Declares char", + "tokens": 3701, + "hash": "cc07427c68f6ab82", + "size": 13877, + "mtimeMs": 1788863761526.755, + "updatedAt": "2026-09-08T10:36:01.561Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_parjob_combinatorial.c": { + "description": "Declares of", + "tokens": 12038, + "hash": "4d51a4b478d7335a", + "size": 45140, + "mtimeMs": 1788863804898.0203, + "updatedAt": "2026-09-08T10:36:44.938Z", + "source": "hook" } } } \ No newline at end of file diff --git a/.wolf/anatomy.md b/.wolf/anatomy.md index f27a05f5..cd67e1aa 100644 --- a/.wolf/anatomy.md +++ b/.wolf/anatomy.md @@ -1,7 +1,7 @@ # anatomy.md -> Auto-maintained by OpenWolf. Last scanned: 2026-09-08T09:10:34.359Z -> Files: 510 tracked | Anatomy hits: 0 | Misses: 0 +> Auto-maintained by OpenWolf. Last scanned: 2026-09-08T10:36:44.938Z +> Files: 520 tracked | Anatomy hits: 0 | Misses: 0 ## ./ @@ -831,14 +831,36 @@ - `limits.yaml` (~603 tok) +## src/include/ + +- `resreq.h` — if !defined(_RESREQ_H_) (~3310 tok) + ## src/lib/ - `misc.c` — include (~486 tok) +- `resreq.c` — define _XOPEN_SOURCE 700 (~6663 tok) ## src/lib/testsuite/ - `test_xcealloc.c` — include (~4979 tok) +## src/scheduler/include/ + +- `scheduler.h` — if !defined(_SCHEDULER_H_) (~4716 tok) + +## src/scheduler/src/ + +- `plugin.c` — include (~4554 tok) +- `resource.c` — include (~8324 tok) + +## src/scheduler/src/plugins/ + +- `parjob_allocator.c` — include (~7506 tok) +- `seqjob_allocator.c` — include (~3274 tok) + ## src/scheduler/testsuite/ - `test_core_events.c` — Declares and (~18252 tok) +- `test_parjob_combinatorial.c` — Declares of (~12038 tok) +- `test_resource_accounting.c` — Declares pulled (~12310 tok) +- `test_seqjob_allocator.c` — Declares char (~3701 tok) diff --git a/.wolf/buglog.json b/.wolf/buglog.json index f5b6cb84..3d289f4a 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -21449,7 +21449,7 @@ "error_message": "OPEN, NOT A CRASH -- known residual gap explicitly carved out of bug-1789's fix, filed for its own tracking per user request. A hypothetical NEW global limit defined on a per-host resource OTHER than \"slots\" (e.g. a future absolute-value global mem cap, as opposed to the existing percent-based host-scoped mem limits in limits.yaml) would silently under-enforce the same way bug-1789's slots limits did, for a different reason than bug-1789's root cause.", "file": "src/lib/resreq.c", "root_cause": "bug-1789 fixed sched_fill_limit_requests()'s READ side: it now picks whichever of alloc_req->shared/alloc_req->host to read a limit's underlying resource's want from by checking that resource's own XCE_RESF_GLOBAL flag on vres->res (and separately picks the WRITE destination for the limit's own vres entry by checking vres->vres's flag) -- see resource.c:928-955. That fix only works because alloc_req->host already has SOME value for \"slots\" to read, staged by seqjob_prepare_allocation()/parjob_prepare_allocation() via a separate, unconditional step outside the normal rusages loop, in EVERY phase including the global one. No other per-host resource gets this special treatment. The general population path -- calc_wanted_global_resvals() (src/lib/resreq.c:734) and calc_wanted_host_resvals() (src/lib/resreq.c:793), both called from seqjob_allocator.c/parjob_allocator.c's prepare_allocation hooks -- each unconditionally skip any resource whose XCE_RESF_GLOBAL flag doesn't match their own phase (calc_wanted_global_resvals() line 763: `if ((r->flags & XCE_RESF_GLOBAL) == 0) continue;`; calc_wanted_host_resvals() line 822: `if ((r->flags & XCE_RESF_GLOBAL) != 0) continue;`), and only calc_wanted_global_resvals() runs during the ctx->worker==NULL global phase. So for any per-host resource other than \"slots\", alloc_req->host is simply never populated with a want during the global phase -- sched_fill_limit_requests()'s now-correct read-side dispatch would read a genuine zero from an empty vector, not a population bug on its own side, but the net effect (a global limit on a per-host resource never seeing a nonzero want) is the same silent under-enforcement bug-1789 fixed for slots.", - "fix": "FIXED (2026-09-07, follow-up session, user-driven design). Corrected an earlier, wrong candidate fix (calling calc_wanted_host_resvals() during the global phase) identified during design discussion as invalid: that function's PER_HOST formula returns one host's own share (v->value, no multiplier) and, for parjob specifically, its nslots argument is reset PER BUCKET (parjob_allocator.c's parjob_try_plan_(), ctx->nslots = plan->bucket_slots[b], set right before each candidate host is tried) -- neither matches what a GLOBAL limit needs, which is the job-WIDE total, not any one host's/bucket's share.\n\nREAL FIX: added calc_wanted_host_resval_job_totals() (src/lib/resreq.c), a new function for resources NOT flagged XCE_RESF_GLOBAL that computes the JOB-WIDE TOTAL want using the SAME formulas calc_wanted_global_resvals() already uses for GLOBAL resources (PER_TASK: value*nslots, PER_JOB: value, PER_HOST: value*nhosts -- nslots/nhosts being the whole job's totals, set during the global phase, not any bucket's share). Both functions now share a common static helper (calc_wanted_job_total_resvals_()) parameterized only by which flag direction to match, avoiding duplicating the ~30-line loop body. seqjob_prepare_allocation()/parjob_prepare_allocation()'s global-phase branch (ctx->worker == NULL) now calls this new function into ctx->alloc_req.host immediately alongside the existing calc_wanted_global_resvals() call into .shared.\n\nSAFETY, traced precisely rather than assumed: plugin_prepare_allocation() (plugin.c:607-613) resets ONLY .shared at the top of a global-phase call and ONLY .host at the top of a per-host call -- never the other -- so .host is NOT reset when this new write happens. sched_fill_limit_requests() for obj->glimits runs later in that SAME plugin_prepare_allocation() invocation (plugin.c:644-647), reading and transferring the value into the limit's own permanent .shared record BEFORE any later per-host call ever resets .host. The value is therefore fully consumed and irrelevant by the time .host gets correctly wiped and repopulated with real per-host/per-bucket data for the actual capacity check -- two different consumers at two different times, not a conflict, confirmed by reading parjob_allocator.c's own per-bucket reset code directly (not inferred).\n\nTDD: two new regression tests calling seqjob_prepare_allocation()/parjob_prepare_allocation() directly with ctx.worker == NULL and a \"mem\"-named (non-\"slots\", non-global) rusage entry -- test_seqjob_prepare_allocation_global_phase_stores_host_resource_job_total (test_seqjob_allocator.c) and test_parjob_prepare_allocation_global_phase_stores_host_resource_job_total (test_parjob_combinatorial.c, maxTasks=3 specifically to prove the formula uses the job's WHOLE nslots=3, giving 2.0*3=6.0, not any bucket's share). Both assert .shared stays empty (mem correctly excluded, not globally flagged) and .host carries both the new mem entry and the pre-existing unconditional \"slots\" entry. Both PASS standalone with full Unity PASS lines confirmed (not just suite-level OK).\n\nFull `xce-build.sh all asan --incremental`: 92/94 Ok. The 2 failures are both pre-existing and unrelated, confirmed by cross-referencing this exact buglog: test_job_rusage_reconciles_mem_reservation_against_usage (bug-1762) and libxce test's 456-byte/29-allocation leak in test_xcealloc.c's make_alloc()/make_host_alloc() calls (bug-1733's fix note already confirms this exact leak signature -- 456 bytes, 29 allocations -- present byte-for-byte identically via an independent git-stash A/B rebuild in an earlier, unrelated session). LIVE-VERIFIED (2026-09-08, follow-up session). Added ug1_global_mem to limits.yaml -- a permanent, real GLOBAL limit on \"mem\" (a per-host resource, not \"slots\"), capping ug1's total cluster-wide mem usage at 5,000,000,000 -- exactly the previously-hypothetical scenario this bug was about, now real example config instead of untested. Fresh single-node deploy, 3 jobs submitted as user1 (ug1 member) each requesting rusage[mem=2000000000] (2GB) on queue normal: scheduler trace confirms ug1_global_mem matching (rc=1) and, for the first time, correctly computing a nonzero job-wide-total wanted value via calc_wanted_host_resval_job_totals() -- 'unsatisfy_ resource 36 total 5000000000.000000 inuse 4000000000.000000 uncomit 0.000000 wanted 2000000000.000000' / 'unsatisfy_ unmet resource 36 (limit=ug1_global_mem) shortfall 1000000000.000000' -- correctly detecting the 3rd job (4GB inuse + 2GB wanted = 6GB > 5GB cap) and PENDing it until an earlier job's mem freed up. Deploy config validated cleanly with zero new warnings. This closes the residual live-verification gap entirely -- bug-1790 is now proven correct for a resource other than \"slots\", not just unit-tested.", + "fix": "FIXED (2026-09-07, follow-up session, user-driven design). Corrected an earlier, wrong candidate fix (calling calc_wanted_host_resvals() during the global phase) identified during design discussion as invalid: that function's PER_HOST formula returns one host's own share (v->value, no multiplier) and, for parjob specifically, its nslots argument is reset PER BUCKET (parjob_allocator.c's parjob_try_plan_(), ctx->nslots = plan->bucket_slots[b], set right before each candidate host is tried) -- neither matches what a GLOBAL limit needs, which is the job-WIDE total, not any one host's/bucket's share.\n\nREAL FIX: added calc_wanted_host_resval_job_totals() (src/lib/resreq.c), a new function for resources NOT flagged XCE_RESF_GLOBAL that computes the JOB-WIDE TOTAL want using the SAME formulas calc_wanted_global_resvals() already uses for GLOBAL resources (PER_TASK: value*nslots, PER_JOB: value, PER_HOST: value*nhosts -- nslots/nhosts being the whole job's totals, set during the global phase, not any bucket's share). Both functions now share a common static helper (calc_wanted_job_total_resvals_()) parameterized only by which flag direction to match, avoiding duplicating the ~30-line loop body. seqjob_prepare_allocation()/parjob_prepare_allocation()'s global-phase branch (ctx->worker == NULL) now calls this new function into ctx->alloc_req.host immediately alongside the existing calc_wanted_global_resvals() call into .shared.\n\nSAFETY, traced precisely rather than assumed: plugin_prepare_allocation() (plugin.c:607-613) resets ONLY .shared at the top of a global-phase call and ONLY .host at the top of a per-host call -- never the other -- so .host is NOT reset when this new write happens. sched_fill_limit_requests() for obj->glimits runs later in that SAME plugin_prepare_allocation() invocation (plugin.c:644-647), reading and transferring the value into the limit's own permanent .shared record BEFORE any later per-host call ever resets .host. The value is therefore fully consumed and irrelevant by the time .host gets correctly wiped and repopulated with real per-host/per-bucket data for the actual capacity check -- two different consumers at two different times, not a conflict, confirmed by reading parjob_allocator.c's own per-bucket reset code directly (not inferred).\n\nTDD: two new regression tests calling seqjob_prepare_allocation()/parjob_prepare_allocation() directly with ctx.worker == NULL and a \"mem\"-named (non-\"slots\", non-global) rusage entry -- test_seqjob_prepare_allocation_global_phase_stores_host_resource_job_total (test_seqjob_allocator.c) and test_parjob_prepare_allocation_global_phase_stores_host_resource_job_total (test_parjob_combinatorial.c, maxTasks=3 specifically to prove the formula uses the job's WHOLE nslots=3, giving 2.0*3=6.0, not any bucket's share). Both assert .shared stays empty (mem correctly excluded, not globally flagged) and .host carries both the new mem entry and the pre-existing unconditional \"slots\" entry. Both PASS standalone with full Unity PASS lines confirmed (not just suite-level OK).\n\nFull `xce-build.sh all asan --incremental`: 92/94 Ok. The 2 failures are both pre-existing and unrelated, confirmed by cross-referencing this exact buglog: test_job_rusage_reconciles_mem_reservation_against_usage (bug-1762) and libxce test's 456-byte/29-allocation leak in test_xcealloc.c's make_alloc()/make_host_alloc() calls (bug-1733's fix note already confirms this exact leak signature -- 456 bytes, 29 allocations -- present byte-for-byte identically via an independent git-stash A/B rebuild in an earlier, unrelated session). LIVE-VERIFIED (2026-09-08, follow-up session). Added ug1_global_mem to limits.yaml -- a permanent, real GLOBAL limit on \"mem\" (a per-host resource, not \"slots\"), capping ug1's total cluster-wide mem usage at 5,000,000,000 -- exactly the previously-hypothetical scenario this bug was about, now real example config instead of untested. Fresh single-node deploy, 3 jobs submitted as user1 (ug1 member) each requesting rusage[mem=2000000000] (2GB) on queue normal: scheduler trace confirms ug1_global_mem matching (rc=1) and, for the first time, correctly computing a nonzero job-wide-total wanted value via calc_wanted_host_resval_job_totals() -- 'unsatisfy_ resource 36 total 5000000000.000000 inuse 4000000000.000000 uncomit 0.000000 wanted 2000000000.000000' / 'unsatisfy_ unmet resource 36 (limit=ug1_global_mem) shortfall 1000000000.000000' -- correctly detecting the 3rd job (4GB inuse + 2GB wanted = 6GB > 5GB cap) and PENDing it until an earlier job's mem freed up. Deploy config validated cleanly with zero new warnings. This closes the residual live-verification gap entirely -- bug-1790 is now proven correct for a resource other than \"slots\", not just unit-tested. REFACTORED (2026-09-08, same follow-up session, user-driven design review): the fix above worked and was live-verified, but put limit-plugin-specific logic (computing a job-total for a per-host resource, needed only when some GLOBAL limit references it) inside the ALLOCATOR plugins (seqjob_prepare_allocation()/parjob_prepare_allocation()) -- the wrong owner: those plugins should only ever compute what the job itself wants for capacity-satisfy purposes, and the duplication across two allocator plugins (a third would need it too) plus unconditionally computing totals for every per-host resource regardless of whether any limit needs one were both symptoms of that. Moved ownership to sched_fill_limit_requests() (resource.c) itself, called once centrally from plugin_prepare_allocation() regardless of which allocator ran: removed calc_wanted_host_resval_job_totals() and its calls from both allocator plugins entirely; added calc_wanted_job_total_resval() (resreq.c) -- a single-resource on-demand lookup (by name, no resmgr/key_find needed since the caller already has the resolved xce_resource_t), computing the SAME job-wide-total formula only for the one (scope, underlying-resource) combination that needs it (a GLOBAL limit on a per-host resource) -- the other three combinations are unchanged, still reading straight out of alloc_req. sched_fill_limit_requests() gained back rusages/nhosts/nslots parameters (undoing part of this session's earlier signature-trimming, but for a genuine need this time, not a re-added unused one); plugin.c's two call sites pass obj->u.primary->resreq->rusages (guarded on obj->tag == XCE_RESREQ_PRIMARY -- NULL for COMPOUND/ALTER, matching this codebase's existing scope convention for those tags) and ctx->nhosts/ctx->nslots (already correctly set by whichever allocator just ran, same invocation). All 23 testsuite link-satisfying stubs mechanically updated to the new 5-arg signature; the two allocator-level regression tests added by the first pass (test_seqjob_allocator.c/test_parjob_combinatorial.c) removed -- the behavior they proved no longer exists by design -- and test_resource_accounting.c's test_fill_limit_requests_global_limit_on_host_resource_reads_host rewritten to exercise the new on-demand path directly (job rusages instead of pre-staged .host, nslots=3 to prove the job-WIDE total is used, matching what the removed parjob-specific test proved). Full xce-build.sh all asan --incremental: 94/94 Ok, zero regressions. Live re-verified against the SAME ug1_global_mem deploy/scenario: byte-identical trace ('unsatisfy_ resource 36 total 5000000000.000000 inuse 4000000000.000000 uncomit 0.000000 wanted 2000000000.000000' / 'shortfall 1000000000.000000') to the pre-refactor result, confirming behavior-preserving.", "tags": [ "scheduler", "resource-accounting", diff --git a/.wolf/memory.md b/.wolf/memory.md index 71640eff..e92dd44e 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -4451,3 +4451,14 @@ | 02:01 | Edited src/lib/testsuite/test_xcealloc.c | 3→5 lines | ~41 | | 02:03 | Edited src/lib/misc.c | 7→8 lines | ~35 | | 02:10 | Edited src/scheduler/testsuite/test_core_events.c | expanded (+13 lines) | ~321 | +| 03:32 | Edited src/lib/resreq.c | added 2 condition(s) | ~1062 | +| 03:32 | Edited src/include/resreq.h | calc_wanted_host_resval_job_totals() → calc_wanted_job_total_resval() | ~74 | +| 03:32 | Edited src/scheduler/src/plugins/seqjob_allocator.c | reduced (-9 lines) | ~45 | +| 03:32 | Edited src/scheduler/src/plugins/parjob_allocator.c | reduced (-9 lines) | ~59 | +| 03:33 | Edited src/scheduler/src/resource.c | added 1 condition(s) | ~918 | +| 03:33 | Edited src/scheduler/include/scheduler.h | 1→2 lines | ~35 | +| 03:33 | Edited src/scheduler/src/plugin.c | modified if() | ~391 | +| 03:35 | Edited src/scheduler/testsuite/test_resource_accounting.c | modified combination() | ~1150 | +| 03:36 | Edited src/scheduler/testsuite/test_seqjob_allocator.c | removed 97 lines | ~64 | +| 03:36 | Edited src/scheduler/testsuite/test_parjob_combinatorial.c | removed 88 lines | ~8 | +| 03:36 | Edited src/scheduler/testsuite/test_parjob_combinatorial.c | 5→3 lines | ~27 |