From b63bb581d44aba7b5c326b07af467d5ccf058312 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Fri, 25 Sep 2026 13:15:43 +0000 Subject: [PATCH] chore(buglog): point fix references at the re-signed commit hashes The 16 commits since 534895c5 were re-signed (rebase --exec amend -S; tree unchanged), which changed their hashes. Update the buglog and the scheduler-allocation completion note to the new ones. Co-Authored-By: Claude Opus 5.5 (1M context) --- .kiro/specs/scheduler-allocation/spec.json | 2 +- .wolf/buglog.json | 24 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.kiro/specs/scheduler-allocation/spec.json b/.kiro/specs/scheduler-allocation/spec.json index ed96982b..dc5ee34d 100644 --- a/.kiro/specs/scheduler-allocation/spec.json +++ b/.kiro/specs/scheduler-allocation/spec.json @@ -35,5 +35,5 @@ "validation": 197 } }, - "completion_note": "Verified 2026-09-24/25 (kiro-verify-completion): scheduler suite 87/87 ASAN; live multi-host e2e PASS (33-task plain + array element split across 2 hosts, DONE) after bug-1857 fixture fix e216cb9a; HA soak sequential PASS + chaotic PASS (after bug-1858 393a07c2, bug-1859 444fd504, bug-1862 f0762962); bug-1770 (no parjob production path) fixed by resreq-maxtasks-wiring. Out-of-scope follow-ups remain open: bug-1766, bug-1768, bug-1769, bug-1771 (adjacent allocation defects), bug-1860." + "completion_note": "Verified 2026-09-24/25 (kiro-verify-completion): scheduler suite 87/87 ASAN; live multi-host e2e PASS (33-task plain + array element split across 2 hosts, DONE) after bug-1857 fixture fix 3cc2f6e3; HA soak sequential PASS + chaotic PASS (after bug-1858 9e9f2798, bug-1859 7c98c056, bug-1862 c29518b2); bug-1770 (no parjob production path) fixed by resreq-maxtasks-wiring. Out-of-scope follow-ups remain open: bug-1766, bug-1768, bug-1769, bug-1771 (adjacent allocation defects), bug-1860." } diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 11063033..428bfb10 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -15785,7 +15785,7 @@ }, { "id": "bug-1429", - "status_note": "574a5f84", + "status_note": "b5d59253", "status": "fixed", "status_updated": "2026-09-25", "timestamp": "2026-07-10", @@ -25239,11 +25239,11 @@ "id": "bug-1768", "status": "fixed", "status_updated": "2026-09-25", - "status_note": "34c52ccb", + "status_note": "b4e9a099", "timestamp": "2026-09-04T00:00:00Z", "error_message": "OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). seqjob_prepare_allocation() (src/scheduler/src/plugins/seqjob_allocator.c) and parjob_prepare_allocation() (src/scheduler/src/plugins/parjob_allocator.c) both silently `return 0` whenever `ctx->schedobj->tag != XCE_RESREQ_PRIMARY`. For a primary resource request nested inside a COMPOUND (or, transitively, an ALTER) job, `ctx->schedobj` remains the enclosing COMPOUND object for the entire per-primary allocation loop -- it is never swapped to a per-primary object -- so this guard fires for every nested primary and both prepare-allocation hooks no-op unconditionally.", "root_cause": "CONFIRMED via direct code read and `git log --follow -p`: the `ctx->schedobj->tag != XCE_RESREQ_PRIMARY` guard predates the entire scheduler-allocation feature (introduced in commit 12dcc2a6, months before this branch's base commit 5a670676); task 3.1's seqjob rewrap (b18c5f78) did not touch or introduce it. Effect: `ctx->alloc_req.host`/`ctx->alloc_req.shared` are never populated for a nested primary's resource want, so `sched_check_host_satisfy()` trivially passes against an empty want vector regardless of a candidate host's real free capacity. Only the SELECT expression (if the resreq specifies one) still gates candidate hosts for a nested primary -- slot count and rusage (memory, etc.) are unconditionally treated as satisfied. This means every COMPOUND job (and transitively every ALTER job, since ALTER delegates to COMPOUND dispatch) with a nested primary resource request has never had its real slots/rusage availability checked by the scheduler, since this guard was introduced.", - "fix": "NOT FIXED -- discovered and characterized during scheduler-allocation task 5.1's independent review (2026-09-04); deliberately left out of that task's scheduler.c-dispatch boundary. Fixing requires either (a) constructing/staging a transient per-primary schedobj view when dispatching a nested primary through prepare_allocation (mirroring task 5.1's own sched_primary_cache_obj_() stack-local-schedobj pattern used for cache-key synthesis), or (b) relaxing seqjob_prepare_allocation()/parjob_prepare_allocation()'s guard to operate directly on ctx->primary (already available on ctx regardless of the enclosing schedobj's tag) instead of gating on ctx->schedobj->tag at all. Needs its own dedicated bugfix task/spec given the severity and blast radius (all COMPOUND/ALTER jobs, not scoped to scheduler-allocation).\n\nUPDATE (2026-09-25): 2026-09-25 (bug-1769 fix 43974a21): with ALTER now allocating for real, this gap is what prevents a test where an ALTER alternative genuinely fails and the next one is used -- compound/alter-nested primaries never check slot/rusage availability, so every alternative 'fits'.", + "fix": "NOT FIXED -- discovered and characterized during scheduler-allocation task 5.1's independent review (2026-09-04); deliberately left out of that task's scheduler.c-dispatch boundary. Fixing requires either (a) constructing/staging a transient per-primary schedobj view when dispatching a nested primary through prepare_allocation (mirroring task 5.1's own sched_primary_cache_obj_() stack-local-schedobj pattern used for cache-key synthesis), or (b) relaxing seqjob_prepare_allocation()/parjob_prepare_allocation()'s guard to operate directly on ctx->primary (already available on ctx regardless of the enclosing schedobj's tag) instead of gating on ctx->schedobj->tag at all. Needs its own dedicated bugfix task/spec given the severity and blast radius (all COMPOUND/ALTER jobs, not scoped to scheduler-allocation).\n\nUPDATE (2026-09-25): 2026-09-25 (bug-1769 fix 1ecd348f): with ALTER now allocating for real, this gap is what prevents a test where an ALTER alternative genuinely fails and the next one is used -- compound/alter-nested primaries never check slot/rusage availability, so every alternative 'fits'.", "tags": [ "scheduler", "resource-accounting", @@ -25265,7 +25265,7 @@ "id": "bug-1769", "status": "fixed", "status_updated": "2026-09-25", - "status_note": "43974a21", + "status_note": "1ecd348f", "timestamp": "2026-09-04T00:00:00Z", "error_message": "OPEN, NOT A CRASH (but reachable) -- pre-existing, out-of-scope type confusion discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). core_build_alter_schedobj() (src/scheduler/src/core_events.c) stores raw `xce_compound_schedobj_t*` (sizeof == 24 bytes) into `obj->u.alters` (an xlist_t), but sched_make_alter_allocation() (src/scheduler/src/scheduler.c) reads each entry back cast as `xce_sched_obj_t*` (sizeof == 176 bytes, with `offsetof(xce_sched_obj_t, u) == 136`) and dereferences `obj->u.compound->resreq` through that mis-typed pointer.", "root_cause": "CONFIRMED via direct code read at both the construction site (core_events.c) and the read site (scheduler.c), plus a compiled offsetof/sizeof probe against the real headers during task 5.1's review: this is a genuine struct-layout mismatch, not a false alarm -- the two types do not share a compatible prefix layout. Any real ALTER-tagged job (the resreq grammar's `{A} || {B}` alternate syntax legitimately produces XCE_RESREQ_ALTER per xce_resreq.peg) dereferencing through this cast reads ~112 bytes past the actual 24-byte allocation, an out-of-bounds heap read. Separately confirmed during the same investigation that test_snapshot_load_allocator_regression.c's header comment claiming to already exercise this ALTER path is itself mistaken: its `\"type == any\"` fixture has no `{`/`}` braces, so per xce_resreq.peg's grammar it parses as XCE_RESREQ_PRIMARY, not ALTER -- it never actually drives sched_make_alter_allocation()'s u.alters read path. No test anywhere in the tree currently constructs a real multi-entry u.alters list and drives it through sched_make_alter_allocation(), so this defect is both real and currently completely untested.", @@ -25320,7 +25320,7 @@ "id": "bug-1771", "status": "fixed", "status_updated": "2026-09-25", - "status_note": "43974a21", + "status_note": "1ecd348f", "timestamp": "2026-09-04T00:00:00Z", "error_message": "ASAN SEGV (READ of address 0x0) in fcfs_del_job() (src/scheduler/src/plugins/fcfs.c:318, `job->queue->name`), reached via fcfs_commit() <- plugin_commit_decision() <- sched_commit_decision() <- commit_decision() <- sched_allocator(), the very first time a real DISPATCH decision was committed through the real, live sched_allocator() thread in this test directory. Discovered live during scheduler-allocation task 5.3's own end-to-end integration test construction (test_parjob_e2e_multislot.c) -- full ASAN backtrace captured.", "root_cause": "CONFIRMED via direct code read and the live ASAN backtrace. plugin_commit_decision() (src/scheduler/src/plugin.c:442-475) unconditionally fans a committed decision out to EVERY registered plugin's own `.hooks.commit_hook`, regardless of which plugin is the currently-ACTIVE job picker -- this includes the legacy 'xce_sched_fcfs' plugin (src/scheduler/src/plugins/fcfs.c), still registered alongside the modern default 'builtin_fcfs' by sched_register_plugins() (scheduler.c) even when it is never the resolved picker. fcfs_commit() (fcfs.c:326-337) calls fcfs_del_job(plugin, decision->u.alloc->jobid) for every XCE_DECISION_DISPATCH, and fcfs_del_job() (fcfs.c:303-324) unconditionally dereferences `job->queue->name` with no NULL check on `job->queue` -- sched_get_queue() (core.c:257-271) is get-ONLY (never auto-creates), so any job whose xce_sched_job_t.queue is NULL (i.e. its submitted queue name was never registered via a real QUEUE_UPDATE/CONFIG_PUSH event before the job reached DISPATCH) crashes this unrelated legacy plugin's own housekeeping, even though that plugin never elected or tracked the job at all. Every prior real-scheduler_srcs test in this directory that reaches a real sched_commit_decision() (test_scheduler_cache_dispatch.c, test_scheduler_cache_dispatch_compound.c, test_snapshot_load_allocator_regression.c) only ever produces PEND decisions (fcfs_commit()'s `default: break;` branch, never touching job->queue), so this NULL-deref had never been triggered before. Not reachable in real production as the system stands today (a real job's queue is always registered via controller-pushed config before any job can target it), but IS a genuine, real, unguarded NULL dereference in a code path that runs unconditionally for every real DISPATCH decision regardless of test/production context.", @@ -27829,7 +27829,7 @@ "id": "bug-1857", "status": "fixed", "status_updated": "2026-09-24", - "status_note": "e216cb9a", + "status_note": "3cc2f6e3", "timestamp": "2026-09-24T13:02:14Z", "error_message": "xce-test-multihost FAIL: job never reached RUN within 30s (last observed state: PEND); scheduler.log: 'unsatisfy_ unmet resource 33 (limit=limit1) shortfall 30.000000'", "file": ".claude/skills/xce-test-multihost/run-multihost-verify.sh,src/tools/deploy-ha.sh,src/configs/limits.yaml", @@ -27849,7 +27849,7 @@ "id": "bug-1858", "status": "fixed", "status_updated": "2026-09-24", - "status_note": "393a07c2", + "status_note": "9e9f2798", "timestamp": "2026-09-24T13:12:53Z", "error_message": "AddressSanitizer: attempting free on address which was not malloc()-ed in core_upsert_queue (core_events.c:383) <- core_apply_config_queues <- xce_core_push_events <- sched_sim_driver_run; address 56 bytes inside a 57-byte makeString instance. Seen as HA soak (sequential) FAIL: scheduler replay-parity SIMVERIFY errors=1 (simverify-3, rc=1)", "file": "src/scheduler/src/core_events.c,src/scheduler/src/schedobj.c", @@ -27870,12 +27870,12 @@ "id": "bug-1859", "status": "fixed", "status_updated": "2026-09-25", - "status_note": "444fd504", + "status_note": "7c98c056", "timestamp": "2026-09-24T13:45:31Z", "error_message": "xce-test-ha --chaotic FAIL: chaotic: 1/4 worker-resync job(s) never reconciled after the loop settled: 0-2-1790256202-546579(last=PEND). scheduler.log: 'JOBS_PUSH: refusing resync state 0x1 for jobid ... -- existing job is RUN with a real allocation on record (existing state 0x2 kept)'", "file": "src/scheduler/src/core_events.c,src/controller/job.c", "root_cause": "Not yet confirmed. Timeline (2026-09-24 chaotic soak, 6 rounds): job dispatched on primary ctrl-1 (term 6) at 13:23:23.74, commit_advance 80->81 at 13:23:23.79; worker started it and finished it at 13:23:43 (retained outcome exit_status=1); ctrl-1 killed in round 5, overlapping kills follow; after settle the cluster reports the job PEND while the scheduler keeps RUN+alloc and refuses the PEND resync (core_events.c:984), so neither side converges. Also: round 6 'scheduler restart did not confirm join within 20s (bug-1760 workaround)'. Candidates: dispatch/RUN not in the surviving committed prefix while the scheduler already applied it, or worker's retained finish never re-delivered to the new primary (cf. bug-1741 fire-and-forget finish notify). Same chaotic soak passed earlier the same day (21 jobs, 6 resync jobs reconciled), so intermittent.", - "fix": "UPDATE (2026-09-24): Frequency 2026-09-24: 1 failure in 3 chaotic 6-round soaks (pass 12:5x, FAIL 13:2x, pass 13:5x on 393a07c2). Failing run's logs kept at scratchpad ha-chaos-bug1859-run1 (session-local).\n\nUPDATE (2026-09-24): ROOT CAUSE (from preserved logs of the failing run): job 546579 dispatched+committed (LSN 81) on ctrl-1 term 6; worker finished it 13:23:43 and sent the fire-and-forget finish notify (xcecli_notify = single-shot send, rc ignored: 'FIXME: add error handling when msg send failure') into a dead connection during overlapping failovers; worker could not reach any primary for 84s; its retained finished outcome expired at finish_time + 60s (default window; xce.yaml sets no worker_resync_window_ms, controller pushes 0 -> default) DURING the outage; at rejoin (13:25:07) ctrl-2's resync query (op 43) got 'unrecognized' -> record_job_reconcile_pending_fact requeued it PEND while the scheduler was disconnected; scheduler rejoined 13:25:11 and its JOBS_PUSH bug-1812 guard refused the PEND (RUN+alloc on record) -> permanent divergence. FIX (uncommitted): worker retention window no longer counts outage time -- xcecd_on_reconnect() records the reconnect time and expiry is measured from max(finish_time, last_reconnect); nothing expires while checking_connection()'s hello probe reports unhealthy. Reported finish_time_ms unchanged. Tests: test_finished_job_retention.c test_reconnect_restarts_window_for_outcome_finished_before_outage (84s outage) + test_unhealthy_connection_suspends_expiry, both red first; worker suite 10/10. The scheduler-side divergence (controller requeue while scheduler disconnected is refused by bug-1812 guard) logged separately.", + "fix": "UPDATE (2026-09-24): Frequency 2026-09-24: 1 failure in 3 chaotic 6-round soaks (pass 12:5x, FAIL 13:2x, pass 13:5x on 9e9f2798). Failing run's logs kept at scratchpad ha-chaos-bug1859-run1 (session-local).\n\nUPDATE (2026-09-24): ROOT CAUSE (from preserved logs of the failing run): job 546579 dispatched+committed (LSN 81) on ctrl-1 term 6; worker finished it 13:23:43 and sent the fire-and-forget finish notify (xcecli_notify = single-shot send, rc ignored: 'FIXME: add error handling when msg send failure') into a dead connection during overlapping failovers; worker could not reach any primary for 84s; its retained finished outcome expired at finish_time + 60s (default window; xce.yaml sets no worker_resync_window_ms, controller pushes 0 -> default) DURING the outage; at rejoin (13:25:07) ctrl-2's resync query (op 43) got 'unrecognized' -> record_job_reconcile_pending_fact requeued it PEND while the scheduler was disconnected; scheduler rejoined 13:25:11 and its JOBS_PUSH bug-1812 guard refused the PEND (RUN+alloc on record) -> permanent divergence. FIX (uncommitted): worker retention window no longer counts outage time -- xcecd_on_reconnect() records the reconnect time and expiry is measured from max(finish_time, last_reconnect); nothing expires while checking_connection()'s hello probe reports unhealthy. Reported finish_time_ms unchanged. Tests: test_finished_job_retention.c test_reconnect_restarts_window_for_outcome_finished_before_outage (84s outage) + test_unhealthy_connection_suspends_expiry, both red first; worker suite 10/10. The scheduler-side divergence (controller requeue while scheduler disconnected is refused by bug-1812 guard) logged separately.", "tags": [ "ha", "chaotic-soak", @@ -27914,7 +27914,7 @@ "id": "bug-1861", "status": "fixed", "status_updated": "2026-09-25", - "status_note": "574a5f84", + "status_note": "b5d59253", "timestamp": "2026-09-24T15:05:56Z", "error_message": "config_repl.c do_upgrade_config_with_bytes: temp decode file /tmp/xce_upgrade_cfg..tmp -- the name is identical across processes (parallel controller_unit_test instances failed handle_upgrade_config with -EINVAL, 'a majority-acked upgrade must succeed') and is opened O_CREAT|O_TRUNC without O_EXCL in world-writable /tmp", "file": "src/controller/config_repl.c", @@ -27934,7 +27934,7 @@ "id": "bug-1862", "status": "fixed", "status_updated": "2026-09-25", - "status_note": "f0762962", + "status_note": "c29518b2", "timestamp": "2026-09-24T15:24:57Z", "error_message": "xcecd_virtual ASAN heap-use-after-free in find_worker_by_jobid (xcecd_worker_utils.c:702) <- xcecd_resync_classify_job (xcecd_resync.c:33) <- xcecd_resync_build_report <- handle_jobstatequery (controller_connector.c:1662) on a vtask thread; freed by remove_worker (xcecd_worker_utils.c:283) <- cleanup_worker_list <- main loop (main.c:652). Worker aborted mid chaotic HA soak (2026-09-24, run 1 of 3), so later rounds lost worker-resync coverage.", "file": "src/worker/src/xcecd_resync.c,src/worker/src/xcecd_worker_utils.c,src/worker/src/controller_connector.c,src/worker/src/main.c", @@ -27956,7 +27956,7 @@ "id": "bug-1863", "status": "fixed", "status_updated": "2026-09-25", - "status_note": "71287ef1", + "status_note": "aea34738", "timestamp": "2026-09-25T13:07:56Z", "error_message": "A COMPOUND resreq with 2+ primaries (e.g. '1*{A} + 2*{B}') keeps only the LAST primary's allocation: seqjob/parjob my_alloc assign ctx->alloc = make_job_alloc(...) per primary, overwriting (and leaking) the previous primary's job alloc, while every primary's hosts were already debited into ctx->uncomitted -- the dispatched allocation omits earlier primaries' hosts and accounting diverges from the decision.", "file": "src/scheduler/src/scheduler.c,src/scheduler/src/plugins/seqjob_allocator.c,src/scheduler/src/plugins/parjob_allocator.c",