From ca6da7eb6dc3a4b44a6ed300c75f1f54ad2ec42a Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Mon, 17 Aug 2026 05:04:02 +0000 Subject: [PATCH] chore(wolf): checkpoint buglog.json (925 bugs) --- .wolf/buglog.json | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/.wolf/buglog.json b/.wolf/buglog.json index fcba940a..7ff544d7 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -16500,6 +16500,61 @@ "related_bugs": [], "occurrences": 1, "last_seen": "2026-08-17T03:45:49.314Z" + }, + { + "id": "bug-923", + "timestamp": "2026-08-17T04:59:39.812Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-08-17T04:59:39.812Z" + }, + { + "id": "bug-924", + "timestamp": "2026-08-17T05:01:33.795Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-08-17T05:01:33.795Z" + }, + { + "id": "bug-1563", + "timestamp": "2026-08-17T04:59:00.000Z", + "error_message": "40-round chaotic /xce-test-ha soak: ctrl-2's own log shows 309,671 'persist: flusher fdatasync FAILED (errno=22 Invalid argument) -- durable_lsn held at 163 (tip=164 NOT made durable)' ERROR lines, 706 of them landing within a single 705ms window (04:52:54.000-04:52:54.705) on the freshly-promoted primary -- a sustained sync failure the flusher never recovered from or backed off on for the rest of the run. The soak's own verdict still reports PASS (committed_divergence=0, committed_job_loss=0) because the other two standbys' acks alone satisfy majority and the bug-924 fdatasync-error exclusion swallows the count -- but this specific pattern (a stuck LSN, EINVAL not the documented EIO, hundreds of retries per second) is a distinct, real defect from the known overlayfs sandbox quirk, not just noise.", + "file": "src/controller/persist.c (fsync_worker)", + "root_cause": "fsync_worker's loop (persist.c) only calls pthread_cond_timedwait (its ONLY throttle/backoff mechanism) when flusher_work_pending_locked(p) is false. But that predicate is purely 'durable_lsn < tip' -- it cannot distinguish 'genuine fresh work just landed' from 'still stuck from the previous failed attempt', because flusher_flush_once()'s fail-loud gate (correct, unchanged) deliberately never advances durable_lsn on a failed fdatasync. Once fdatasync starts failing with no new appends arriving (a standby/primary transition mid-chaotic-kill with a broken local fd, or any sustained real disk error in production), durable_lsn < tip stays true forever, so 'pending' is permanently true and the loop skips the wait on every single iteration -- flusher_flush_once (and thus fdatasync) gets called as fast as the CPU/lock overhead allows, pinning one core and flooding the log. Confirmed by direct code read of flusher_work_pending_locked (dur < tip, no failure-awareness) and fsync_worker's loop structure (wait only in the !pending branch), and reproduced deterministically in a unit test (test_force_fsync_fail seam) before any live soak was re-run.", + "fix": "FIXED (TDD RED->GREEN). Added test_flusher_flush_attempt_count (persist.h, XCE_TESTING-only atomic counter, incremented once per flusher_flush_once() call) and test_flusher_backs_off_on_sustained_fsync_failure (test_term_safety_adapter.c): forces every fdatasync to fail via the existing test_force_fsync_fail seam on a PRIMARY persist_t with wal_sync_ms=20, appends one entry (primary's append path defers all sync to the async flusher, unlike a standby's inline persist-before-ACK sync -- see test_on_persist_entry_failure_propagates_no_ack), waits 300ms, and asserts attempt count stays under 60 (~15 expected at a 20ms cadence). RED confirmed pre-fix (assertion failed -- unbounded spin). Fix: fsync_worker's loop (persist.c) now also waits (same pthread_cond_timedwait/wal_sync_ms cadence the idle path already uses, no new tunable) when the just-completed flush WAS pending but made no progress (after == before) -- i.e. specifically the failure case, never the legitimate 'nothing to sync' idle case (pending was false there, left untouched) nor a successful flush (after > before, still loops immediately to catch a fresh append with zero latency, unchanged fast path). GREEN: test_flusher_backs_off_on_sustained_fsync_failure passes with attempts=15 (was unbounded/thousands pre-fix in the same 300ms window). Full controller_unit_test: 1103 tests, 2 failures -- both pre-existing, unrelated overlayfs fdatasync EIO sandbox flakiness in test_config_repl.c (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), confirmed unchanged from before this fix. LIVE VALIDATION: re-ran the 40-round chaotic /xce-test-ha soak that originally surfaced this after rebuilding -- see follow-up note for the result.", + "tags": [ + "persist", + "fsync", + "livelock", + "busy-loop", + "backoff", + "durability", + "chaotic-soak", + "xce-test-ha" + ], + "related_bugs": [ + "bug-661" + ], + "occurrences": 1, + "last_seen": "2026-08-17T04:59:00.000Z" } ] }