From 1ba643b8be58cb3869ddb891e2de51754b119a09 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Fri, 25 Sep 2026 15:42:10 +0000 Subject: [PATCH] chore(buglog): bug-1714, bug-1840 cannot-reproduce on current builds; log bug-1866 Co-Authored-By: Claude Opus 5.5 (1M context) --- .wolf/buglog.json | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 86e26489..be0462e8 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -23648,9 +23648,10 @@ }, { "id": "bug-1714", - "status": "investigating", - "status_updated": "2026-09-24", - "status_note": "intermittent scheduler segfault under load; crash site not found, d5ec1eb3 mitigation only", + "repro_attempts": "0/2 runs, 2026-09-25: documented recipe (3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' = 2400 array elements each) on b63bb581 build.dbg and on d87b78dc-era ASAN build -- no crash, scheduler alive throughout, 0 redelivered array pushes (0 'sched_add_job failed' duplicates; the original repro saw up to 18x redelivery, the recipe's precondition)", + "status": "cannot-reproduce", + "status_updated": "2026-09-25", + "status_note": "recipe no longer reproduces on this machine: its trigger (array pushes redelivered to the scheduler) did not occur on either build, so it was already gone before 2026-09-25's fixes; ASAN run clean. Plausibly related fixes since: bug-1864 (per-endpoint message reordering in the vtask dispatcher), bug-1769 (ALTER heap overflow write). Reopen to investigating if a scheduler segfault recurs.", "timestamp": "2026-08-26T18:35:00Z", "error_message": "User-reported live deployment: xce_scheduler process died and restarted unexpectedly (PID changed from 3964918 to 4137745 in scheduler.log with a ~79s gap, no manual restart by the user) during a live 3-node cluster test session; xceadmin briefly reported 'Error: scheduler disconnected' (the already-correct bug-1693 disconnect-detection path in cluster.c, working as designed -- not itself a bug) then recovered on the next attempt once the scheduler rejoined. User confirmed via `sudo dmesg -T` a REAL kernel-level segfault: `xce_scheduler[2815660]: segfault at 30 ip 0000601569e1fb35 sp 00007db7db1f6e00 error 4 in xce_scheduler[23b35,601569e01000+de000] likely on CPU 5 (core 16, socket 0)` -- though this specific dmesg entry is dated Aug 25 19:21:39, a DIFFERENT, earlier occurrence than today's specific restart (Aug 26 ~18:00-18:02), not necessarily the direct cause of THIS restart. No core dump exists for xce_scheduler anywhere in /var/lib/systemd/coredump/ (only xce_controller and python entries present, despite the scheduler segfault being kernel-confirmed) -- a separate, notable gap in this environment's crash-capture coverage for the scheduler process specifically.", "file": "src/scheduler/ (exact crash site unknown -- no core dump, no backtrace available for either occurrence)", @@ -27402,11 +27403,12 @@ }, { "id": "bug-1840", + "repro_attempts": "0/1 ASAN run, 2026-09-25: 3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' (2400 elements) on the current ASAN build -- no AddressSanitizer report from the scheduler; plus ~10 ASAN HA soaks today (sequential/chaotic, all with scheduler replay-verify) with no scheduler heap error", "gitea_synced_status": "investigating", "gitea_issue": 74, - "status": "investigating", - "status_updated": "2026-09-24", - "status_note": "scheduler heap corruption (malloc_consolidate abort) during TLS handshake; awaiting ASAN repro", + "status": "cannot-reproduce", + "status_updated": "2026-09-25", + "status_note": "no ASAN repro found. Strongest code-level candidate fixed 2026-09-25: bug-1769 (sched_make_alter_allocation wrote obj1->alloc 136 bytes into a 24-byte heap chunk for any ALTER '{..}||{..}' job -- exactly the kind of out-of-bounds write that corrupts glibc fastbin metadata and surfaces later in an unrelated malloc_consolidate). Reopen to investigating if it recurs, ideally captured on an ASAN build.", "timestamp": "2026-09-22T02:35:00Z", "error_message": "Live 3-node HA cluster crash (same soak run as bug-1839): xce_scheduler SIGABRT, glibc `malloc_consolidate(): unaligned fastbin chunk detected`, surfacing during an OpenSSL TLS handshake's MAC-buffer growth: abort -> malloc_printerr -> malloc_consolidate -> _int_malloc -> __libc_malloc2 -> CRYPTO_malloc -> CRYPTO_clear_realloc -> BUF_MEM_grow_clean -> mem_write -> bwrite_conv -> BIO_write -> ssl3_finish_mac -> tls_get_message_body -> state_machine -> handle_ssl_handshake (netio.c:548) -> io_handler (netio.c:588) -> io_worker_dispatch_batch (netio.c:648) -> io_worker (netio.c:689).", "file": "unknown -- heap corruption detected far from its cause", @@ -28017,6 +28019,25 @@ "related_bugs": [], "occurrences": 1, "last_seen": "2026-09-25T14:43:27Z" + }, + { + "id": "bug-1866", + "status": "open", + "status_updated": "2026-09-25", + "status_note": "remove_worker never frees worker->launch_type", + "timestamp": "2026-09-25T15:42:02Z", + "error_message": "LeakSanitizer at xcecd_virtual exit (burst run 2026-09-25): 'Direct leak of 12 byte(s) in 12 object(s)' from strndup in add_worker (xcecd_worker_utils.c:140) <- xcecd_process_command <- handle_message_from_queue", + "file": "src/worker/src/xcecd_worker_utils.c", + "root_cause": "add_worker strndup's startinfo->launch_type into worker->launch_type, but remove_worker frees exechome/execcwd/execuser/job_cmd/pids/job_exdata and never launch_type -- one string leaked per finished job that carried a launch type.", + "fix": "", + "tags": [ + "worker", + "leak", + "asan" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-25T15:42:02Z" } ] }