From d26ff8870d7eaa8f4217ea984e7d14645998778d Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Mon, 28 Sep 2026 03:40:11 +0000 Subject: [PATCH] feat(buglog): one Markdown record per bug, checked against a record standard The buglog was one 27K-line JSON file whose fields covered only symptom, root cause and fix, as free text: 2% of entries named the code version a bug was found on, 12% the build profile, 36% a way to reproduce, and 58% of the file was OpenWolf hook auto-detections ("Missing guard clause", "Function not marked async" on .md files). - .claude/skills/buglog/bug-record.md: the record standard. Eight sections (Summary as the title, Description, Environment, Reproduction; Root cause, Fix, Verification, References once fixed), each with the content it must contain -- e.g. Environment needs the OS, the libc the failing binary loads (from its ELF interpreter: glibc 2.42 in the nix shell vs 2.40 on the host), toolchain, build profile and commit. - .wolf/bugs/.md: 500 records migrated from buglog.json; every field verified to round-trip. They carry legacy: true and an explicit "not recorded" where the old log had no such field; component derived from file paths, severity from keywords (else unknown), fix_commits from hashes in status_note/fix that git resolves (91 of 463 fixed). - .wolf/buglog-archive.json: the 710 hook auto-detections, verbatim. - .wolf/buglog.json is now an index generated from the records, keeping the fields OpenWolf's hooks and `openwolf bug search` read. - scripts/buglog.py rewritten: new (template + Environment collected from the machine and --binary), check, status (refuses a status the record doesn't support; fixed needs Root cause, Fix, Verification and --commit hashes git resolves), note, recur, set, show, list, triage, env, index. 29 unittest cases in scripts/tests/test_buglog.py. - scripts/git-hooks/pre-commit runs `check --staged`: incomplete staged records, a legacy record hand-changed to fixed without the fix-time sections, a new record claiming legacy: true, or a stale index refuse the commit. - scripts/gitea-sync.py reads the records: issue body = the record's sections plus linked fix commits (origin remote, GITEA_COMMIT_REPO to override), and a body hash so an issue's text is refreshed when the record changes. - bug-1866 rewritten to the full standard as the worked example. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/rules/buglog-status.md | 42 +- .claude/skills/buglog/SKILL.md | 72 +- .claude/skills/buglog/bug-record.md | 318 + .claude/skills/gitea-sync/SKILL.md | 136 +- .claude/skills/xce-audit-invariants/SKILL.md | 12 +- .wolf/buglog-archive.json | 14148 +++++++++++ .wolf/buglog.json | 22387 ++++------------- .wolf/bugs/bug-006.md | 38 + .wolf/bugs/bug-007.md | 38 + .wolf/bugs/bug-1013.md | 38 + .wolf/bugs/bug-1023.md | 38 + .wolf/bugs/bug-1028.md | 38 + .wolf/bugs/bug-1060.md | 38 + .wolf/bugs/bug-1067.md | 38 + .wolf/bugs/bug-1077.md | 38 + .wolf/bugs/bug-1095.md | 42 + .wolf/bugs/bug-1096.md | 38 + .wolf/bugs/bug-1112.md | 39 + .wolf/bugs/bug-1128.md | 38 + .wolf/bugs/bug-1147.md | 38 + .wolf/bugs/bug-1197.md | 39 + .wolf/bugs/bug-1198.md | 38 + .wolf/bugs/bug-1199.md | 38 + .wolf/bugs/bug-1210.md | 38 + .wolf/bugs/bug-1224.md | 38 + .wolf/bugs/bug-1225.md | 38 + .wolf/bugs/bug-1272.md | 39 + .wolf/bugs/bug-1281.md | 38 + .wolf/bugs/bug-1282.md | 39 + .wolf/bugs/bug-1286.md | 39 + .wolf/bugs/bug-1296.md | 39 + .wolf/bugs/bug-1298.md | 39 + .wolf/bugs/bug-1306.md | 38 + .wolf/bugs/bug-1307.md | 38 + .wolf/bugs/bug-1376.md | 38 + .wolf/bugs/bug-1381.md | 38 + .wolf/bugs/bug-1386.md | 38 + .wolf/bugs/bug-1389.md | 38 + .wolf/bugs/bug-1396.md | 39 + .wolf/bugs/bug-1400.md | 38 + .wolf/bugs/bug-1401.md | 38 + .wolf/bugs/bug-1406.md | 38 + .wolf/bugs/bug-1407.md | 39 + .wolf/bugs/bug-1408.md | 38 + .wolf/bugs/bug-1409.md | 38 + .wolf/bugs/bug-1410.md | 38 + .wolf/bugs/bug-1411.md | 38 + .wolf/bugs/bug-1412.md | 38 + .wolf/bugs/bug-1413.md | 38 + .wolf/bugs/bug-1414.md | 38 + .wolf/bugs/bug-1415.md | 38 + .wolf/bugs/bug-1416.md | 38 + .wolf/bugs/bug-1417.md | 38 + .wolf/bugs/bug-1418.md | 41 + .wolf/bugs/bug-1419.md | 47 + .wolf/bugs/bug-1420.md | 38 + .wolf/bugs/bug-1421.md | 38 + .wolf/bugs/bug-1422.md | 38 + .wolf/bugs/bug-1423.md | 38 + .wolf/bugs/bug-1424.md | 38 + .wolf/bugs/bug-1425.md | 60 + .wolf/bugs/bug-1426.md | 41 + .wolf/bugs/bug-1427.md | 41 + .wolf/bugs/bug-1428.md | 41 + .wolf/bugs/bug-1429.md | 54 + .wolf/bugs/bug-1430.md | 38 + .wolf/bugs/bug-1431.md | 38 + .wolf/bugs/bug-1432.md | 38 + .wolf/bugs/bug-1433.md | 39 + .wolf/bugs/bug-1434.md | 38 + .wolf/bugs/bug-1435.md | 38 + .wolf/bugs/bug-1436.md | 38 + .wolf/bugs/bug-1437.md | 38 + .wolf/bugs/bug-1438.md | 38 + .wolf/bugs/bug-1439.md | 41 + .wolf/bugs/bug-1440.md | 39 + .wolf/bugs/bug-1441.md | 38 + .wolf/bugs/bug-1442.md | 38 + .wolf/bugs/bug-1443.md | 39 + .wolf/bugs/bug-1444.md | 39 + .wolf/bugs/bug-1445.md | 38 + .wolf/bugs/bug-1446.md | 40 + .wolf/bugs/bug-1447.md | 47 + .wolf/bugs/bug-1448.md | 40 + .wolf/bugs/bug-1449.md | 40 + .wolf/bugs/bug-1450.md | 42 + .wolf/bugs/bug-1451.md | 42 + .wolf/bugs/bug-1452.md | 41 + .wolf/bugs/bug-1453.md | 40 + .wolf/bugs/bug-1454.md | 41 + .wolf/bugs/bug-1455.md | 41 + .wolf/bugs/bug-1456.md | 41 + .wolf/bugs/bug-1457.md | 40 + .wolf/bugs/bug-1458.md | 38 + .wolf/bugs/bug-1459.md | 38 + .wolf/bugs/bug-1460.md | 38 + .wolf/bugs/bug-1461.md | 38 + .wolf/bugs/bug-1462.md | 42 + .wolf/bugs/bug-1463.md | 38 + .wolf/bugs/bug-1464.md | 38 + .wolf/bugs/bug-1465.md | 38 + .wolf/bugs/bug-1466.md | 38 + .wolf/bugs/bug-1467.md | 38 + .wolf/bugs/bug-1468.md | 38 + .wolf/bugs/bug-1469.md | 45 + .wolf/bugs/bug-1470.md | 38 + .wolf/bugs/bug-1471.md | 41 + .wolf/bugs/bug-1472-remainder.md | 41 + .wolf/bugs/bug-1472.md | 41 + .wolf/bugs/bug-1473.md | 38 + .wolf/bugs/bug-1474.md | 53 + .wolf/bugs/bug-1475.md | 48 + .wolf/bugs/bug-1476.md | 44 + .wolf/bugs/bug-1477.md | 44 + .wolf/bugs/bug-1478.md | 50 + .wolf/bugs/bug-1479.md | 48 + .wolf/bugs/bug-1480.md | 39 + .wolf/bugs/bug-1481.md | 39 + .wolf/bugs/bug-1482.md | 38 + .wolf/bugs/bug-1483.md | 38 + .wolf/bugs/bug-1484.md | 38 + .wolf/bugs/bug-1485.md | 43 + .wolf/bugs/bug-1486.md | 38 + .wolf/bugs/bug-1487.md | 38 + .wolf/bugs/bug-1489.md | 38 + .wolf/bugs/bug-1490.md | 38 + .wolf/bugs/bug-1491.md | 38 + .wolf/bugs/bug-1492.md | 38 + .wolf/bugs/bug-1493.md | 38 + .wolf/bugs/bug-1494.md | 38 + .wolf/bugs/bug-1495.md | 38 + .wolf/bugs/bug-1496.md | 38 + .wolf/bugs/bug-1497.md | 38 + .wolf/bugs/bug-1498.md | 38 + .wolf/bugs/bug-1499.md | 38 + .wolf/bugs/bug-1500.md | 38 + .wolf/bugs/bug-1501.md | 38 + .wolf/bugs/bug-1502.md | 38 + .wolf/bugs/bug-1503.md | 38 + .wolf/bugs/bug-1504.md | 38 + .wolf/bugs/bug-1505.md | 38 + .wolf/bugs/bug-1506.md | 38 + .wolf/bugs/bug-1507.md | 38 + .wolf/bugs/bug-1508.md | 41 + .wolf/bugs/bug-1509.md | 38 + .wolf/bugs/bug-1514.md | 38 + .wolf/bugs/bug-1515.md | 38 + .wolf/bugs/bug-1519.md | 39 + .wolf/bugs/bug-1532.md | 38 + .wolf/bugs/bug-1542.md | 38 + .wolf/bugs/bug-1548.md | 38 + .wolf/bugs/bug-1549.md | 38 + .wolf/bugs/bug-1553.md | 38 + .wolf/bugs/bug-1554.md | 38 + .wolf/bugs/bug-1555.md | 38 + .wolf/bugs/bug-1556.md | 38 + .wolf/bugs/bug-1557.md | 38 + .wolf/bugs/bug-1558.md | 38 + .wolf/bugs/bug-1559.md | 38 + .wolf/bugs/bug-1560.md | 40 + .wolf/bugs/bug-1561.md | 65 + .wolf/bugs/bug-1562.md | 38 + .wolf/bugs/bug-1563.md | 38 + .wolf/bugs/bug-1564.md | 38 + .wolf/bugs/bug-1565.md | 38 + .wolf/bugs/bug-1566.md | 38 + .wolf/bugs/bug-1567.md | 38 + .wolf/bugs/bug-1568.md | 38 + .wolf/bugs/bug-1569.md | 38 + .wolf/bugs/bug-1570.md | 38 + .wolf/bugs/bug-1571.md | 38 + .wolf/bugs/bug-1572.md | 39 + .wolf/bugs/bug-1573.md | 38 + .wolf/bugs/bug-1574.md | 38 + .wolf/bugs/bug-1575.md | 38 + .wolf/bugs/bug-1576.md | 38 + .wolf/bugs/bug-1577.md | 38 + .wolf/bugs/bug-1578.md | 48 + .wolf/bugs/bug-1579.md | 40 + .wolf/bugs/bug-1580.md | 38 + .wolf/bugs/bug-1581.md | 38 + .wolf/bugs/bug-1582.md | 38 + .wolf/bugs/bug-1583.md | 39 + .wolf/bugs/bug-1584.md | 38 + .wolf/bugs/bug-1585.md | 42 + .wolf/bugs/bug-1586.md | 38 + .wolf/bugs/bug-1587.md | 39 + .wolf/bugs/bug-1588.md | 86 + .wolf/bugs/bug-1589.md | 38 + .wolf/bugs/bug-1590.md | 45 + .wolf/bugs/bug-1591.md | 38 + .wolf/bugs/bug-1592.md | 38 + .wolf/bugs/bug-1593.md | 38 + .wolf/bugs/bug-1594.md | 38 + .wolf/bugs/bug-1595.md | 38 + .wolf/bugs/bug-1596.md | 38 + .wolf/bugs/bug-1597.md | 39 + .wolf/bugs/bug-1598.md | 38 + .wolf/bugs/bug-1599.md | 38 + .wolf/bugs/bug-1600.md | 39 + .wolf/bugs/bug-1601.md | 38 + .wolf/bugs/bug-1602.md | 42 + .wolf/bugs/bug-1603.md | 39 + .wolf/bugs/bug-1604.md | 38 + .wolf/bugs/bug-1605.md | 39 + .wolf/bugs/bug-1606.md | 38 + .wolf/bugs/bug-1607.md | 38 + .wolf/bugs/bug-1608.md | 38 + .wolf/bugs/bug-1609.md | 38 + .wolf/bugs/bug-1610.md | 38 + .wolf/bugs/bug-1611.md | 47 + .wolf/bugs/bug-1612.md | 38 + .wolf/bugs/bug-1613.md | 59 + .wolf/bugs/bug-1614.md | 39 + .wolf/bugs/bug-1615.md | 38 + .wolf/bugs/bug-1616.md | 38 + .wolf/bugs/bug-1617.md | 38 + .wolf/bugs/bug-1618.md | 38 + .wolf/bugs/bug-1619.md | 38 + .wolf/bugs/bug-1620.md | 38 + .wolf/bugs/bug-1621.md | 38 + .wolf/bugs/bug-1622.md | 38 + .wolf/bugs/bug-1623.md | 38 + .wolf/bugs/bug-1624.md | 39 + .wolf/bugs/bug-1625.md | 39 + .wolf/bugs/bug-1626.md | 39 + .wolf/bugs/bug-1627.md | 38 + .wolf/bugs/bug-1628.md | 39 + .wolf/bugs/bug-1629.md | 38 + .wolf/bugs/bug-1630.md | 39 + .wolf/bugs/bug-1631.md | 43 + .wolf/bugs/bug-1632.md | 38 + .wolf/bugs/bug-1673.md | 41 + .wolf/bugs/bug-1674.md | 40 + .wolf/bugs/bug-1675.md | 40 + .wolf/bugs/bug-1676.md | 46 + .wolf/bugs/bug-1677.md | 38 + .wolf/bugs/bug-1678.md | 47 + .wolf/bugs/bug-1679.md | 44 + .wolf/bugs/bug-1680.md | 44 + .wolf/bugs/bug-1681.md | 46 + .wolf/bugs/bug-1682.md | 38 + .wolf/bugs/bug-1683.md | 40 + .wolf/bugs/bug-1684.md | 40 + .wolf/bugs/bug-1685.md | 43 + .wolf/bugs/bug-1686.md | 47 + .wolf/bugs/bug-1687.md | 44 + .wolf/bugs/bug-1688.md | 44 + .wolf/bugs/bug-1689.md | 42 + .wolf/bugs/bug-1690.md | 44 + .wolf/bugs/bug-1691.md | 46 + .wolf/bugs/bug-1692.md | 50 + .wolf/bugs/bug-1693.md | 38 + .wolf/bugs/bug-1694.md | 38 + .wolf/bugs/bug-1695.md | 53 + .wolf/bugs/bug-1696.md | 38 + .wolf/bugs/bug-1697.md | 38 + .wolf/bugs/bug-1698.md | 38 + .wolf/bugs/bug-1699.md | 38 + .wolf/bugs/bug-1700.md | 42 + .wolf/bugs/bug-1701.md | 38 + .wolf/bugs/bug-1702.md | 38 + .wolf/bugs/bug-1703.md | 38 + .wolf/bugs/bug-1704.md | 38 + .wolf/bugs/bug-1705.md | 38 + .wolf/bugs/bug-1706.md | 39 + .wolf/bugs/bug-1707.md | 38 + .wolf/bugs/bug-1708.md | 38 + .wolf/bugs/bug-1709.md | 38 + .wolf/bugs/bug-1710.md | 38 + .wolf/bugs/bug-1711.md | 39 + .wolf/bugs/bug-1712.md | 38 + .wolf/bugs/bug-1713.md | 38 + .wolf/bugs/bug-1714.md | 137 + .wolf/bugs/bug-1715.md | 38 + .wolf/bugs/bug-1716.md | 38 + .wolf/bugs/bug-1717.md | 38 + .wolf/bugs/bug-1718.md | 38 + .wolf/bugs/bug-1719.md | 38 + .wolf/bugs/bug-1720.md | 38 + .wolf/bugs/bug-1721.md | 38 + .wolf/bugs/bug-1722.md | 38 + .wolf/bugs/bug-1723.md | 38 + .wolf/bugs/bug-1724.md | 38 + .wolf/bugs/bug-1725.md | 38 + .wolf/bugs/bug-1726.md | 39 + .wolf/bugs/bug-1727.md | 44 + .wolf/bugs/bug-1728.md | 39 + .wolf/bugs/bug-1729.md | 38 + .wolf/bugs/bug-1730.md | 42 + .wolf/bugs/bug-1731.md | 73 + .wolf/bugs/bug-1732.md | 38 + .wolf/bugs/bug-1733.md | 46 + .wolf/bugs/bug-1734.md | 53 + .wolf/bugs/bug-1735.md | 53 + .wolf/bugs/bug-1736.md | 47 + .wolf/bugs/bug-1737.md | 38 + .wolf/bugs/bug-1738.md | 38 + .wolf/bugs/bug-1739.md | 38 + .wolf/bugs/bug-1740.md | 38 + .wolf/bugs/bug-1741.md | 39 + .wolf/bugs/bug-1742.md | 40 + .wolf/bugs/bug-1743.md | 43 + .wolf/bugs/bug-1744.md | 43 + .wolf/bugs/bug-1745.md | 41 + .wolf/bugs/bug-1746.md | 38 + .wolf/bugs/bug-1747.md | 39 + .wolf/bugs/bug-1748.md | 38 + .wolf/bugs/bug-1750.md | 38 + .wolf/bugs/bug-1751.md | 38 + .wolf/bugs/bug-1752.md | 38 + .wolf/bugs/bug-1753.md | 38 + .wolf/bugs/bug-1754.md | 42 + .wolf/bugs/bug-1755.md | 38 + .wolf/bugs/bug-1756.md | 38 + .wolf/bugs/bug-1757.md | 38 + .wolf/bugs/bug-1758.md | 40 + .wolf/bugs/bug-1759.md | 38 + .wolf/bugs/bug-1760.md | 67 + .wolf/bugs/bug-1761.md | 38 + .wolf/bugs/bug-1762.md | 38 + .wolf/bugs/bug-1763.md | 38 + .wolf/bugs/bug-1764.md | 38 + .wolf/bugs/bug-1765.md | 38 + .wolf/bugs/bug-1766.md | 39 + .wolf/bugs/bug-1767.md | 39 + .wolf/bugs/bug-1768.md | 41 + .wolf/bugs/bug-1769.md | 39 + .wolf/bugs/bug-1770.md | 39 + .wolf/bugs/bug-1771.md | 39 + .wolf/bugs/bug-1772.md | 41 + .wolf/bugs/bug-1773.md | 39 + .wolf/bugs/bug-1774.md | 38 + .wolf/bugs/bug-1775.md | 38 + .wolf/bugs/bug-1776.md | 38 + .wolf/bugs/bug-1777.md | 38 + .wolf/bugs/bug-1778.md | 38 + .wolf/bugs/bug-1779.md | 38 + .wolf/bugs/bug-1780.md | 38 + .wolf/bugs/bug-1781.md | 41 + .wolf/bugs/bug-1782.md | 41 + .wolf/bugs/bug-1784.md | 41 + .wolf/bugs/bug-1785.md | 41 + .wolf/bugs/bug-1786.md | 41 + .wolf/bugs/bug-1787.md | 41 + .wolf/bugs/bug-1788.md | 41 + .wolf/bugs/bug-1789.md | 41 + .wolf/bugs/bug-1790.md | 46 + .wolf/bugs/bug-1791.md | 38 + .wolf/bugs/bug-1792.md | 38 + .wolf/bugs/bug-1793.md | 38 + .wolf/bugs/bug-1794.md | 38 + .wolf/bugs/bug-1795.md | 38 + .wolf/bugs/bug-1796.md | 38 + .wolf/bugs/bug-1797.md | 38 + .wolf/bugs/bug-1798.md | 41 + .wolf/bugs/bug-1799.md | 42 + .wolf/bugs/bug-1800.md | 41 + .wolf/bugs/bug-1801.md | 41 + .wolf/bugs/bug-1802.md | 40 + .wolf/bugs/bug-1803.md | 41 + .wolf/bugs/bug-1804.md | 41 + .wolf/bugs/bug-1805.md | 38 + .wolf/bugs/bug-1806.md | 38 + .wolf/bugs/bug-1807.md | 38 + .wolf/bugs/bug-1808.md | 38 + .wolf/bugs/bug-1809.md | 38 + .wolf/bugs/bug-1810.md | 38 + .wolf/bugs/bug-1811.md | 38 + .wolf/bugs/bug-1812.md | 38 + .wolf/bugs/bug-1813.md | 38 + .wolf/bugs/bug-1814.md | 40 + .wolf/bugs/bug-1815.md | 38 + .wolf/bugs/bug-1816.md | 38 + .wolf/bugs/bug-1817.md | 38 + .wolf/bugs/bug-1818.md | 38 + .wolf/bugs/bug-1819.md | 38 + .wolf/bugs/bug-1820.md | 38 + .wolf/bugs/bug-1821.md | 38 + .wolf/bugs/bug-1822.md | 38 + .wolf/bugs/bug-1823.md | 38 + .wolf/bugs/bug-1824.md | 38 + .wolf/bugs/bug-1825.md | 48 + .wolf/bugs/bug-1826.md | 38 + .wolf/bugs/bug-1827.md | 38 + .wolf/bugs/bug-1828.md | 37 + .wolf/bugs/bug-1829.md | 38 + .wolf/bugs/bug-1830.md | 42 + .wolf/bugs/bug-1831.md | 41 + .wolf/bugs/bug-1832.md | 41 + .wolf/bugs/bug-1833.md | 41 + .wolf/bugs/bug-1834.md | 41 + .wolf/bugs/bug-1835.md | 41 + .wolf/bugs/bug-1836.md | 41 + .wolf/bugs/bug-1837.md | 41 + .wolf/bugs/bug-1838.md | 41 + .wolf/bugs/bug-1839.md | 41 + .wolf/bugs/bug-1840.md | 42 + .wolf/bugs/bug-1841.md | 41 + .wolf/bugs/bug-1842.md | 41 + .wolf/bugs/bug-1843.md | 41 + .wolf/bugs/bug-1844.md | 41 + .wolf/bugs/bug-1845.md | 41 + .wolf/bugs/bug-1846.md | 42 + .wolf/bugs/bug-1847.md | 38 + .wolf/bugs/bug-1848.md | 38 + .wolf/bugs/bug-1849.md | 38 + .wolf/bugs/bug-1850.md | 39 + .wolf/bugs/bug-1851.md | 38 + .wolf/bugs/bug-1852.md | 38 + .wolf/bugs/bug-1853.md | 38 + .wolf/bugs/bug-1854.md | 41 + .wolf/bugs/bug-1855.md | 43 + .wolf/bugs/bug-1856.md | 41 + .wolf/bugs/bug-1857.md | 41 + .wolf/bugs/bug-1858.md | 41 + .wolf/bugs/bug-1859.md | 43 + .wolf/bugs/bug-1860.md | 41 + .wolf/bugs/bug-1861.md | 41 + .wolf/bugs/bug-1862.md | 41 + .wolf/bugs/bug-1863.md | 41 + .wolf/bugs/bug-1864.md | 41 + .wolf/bugs/bug-1865.md | 41 + .wolf/bugs/bug-1866.md | 59 + .wolf/bugs/bug-1867.md | 42 + .wolf/bugs/bug-1868.md | 42 + .wolf/bugs/bug-1869.md | 42 + .wolf/bugs/bug-1870.md | 41 + .wolf/bugs/bug-1871.md | 41 + .wolf/bugs/bug-206.md | 38 + .wolf/bugs/bug-208.md | 38 + .wolf/bugs/bug-320.md | 38 + .wolf/bugs/bug-335.md | 38 + .wolf/bugs/bug-336.md | 38 + .wolf/bugs/bug-337.md | 38 + .wolf/bugs/bug-338.md | 38 + .wolf/bugs/bug-339.md | 38 + .wolf/bugs/bug-340.md | 38 + .wolf/bugs/bug-343.md | 38 + .wolf/bugs/bug-351.md | 38 + .wolf/bugs/bug-352.md | 38 + .wolf/bugs/bug-353.md | 38 + .wolf/bugs/bug-378.md | 38 + .wolf/bugs/bug-396.md | 38 + .wolf/bugs/bug-410.md | 38 + .wolf/bugs/bug-411.md | 39 + .wolf/bugs/bug-435.md | 38 + .wolf/bugs/bug-436.md | 38 + .wolf/bugs/bug-445.md | 38 + .wolf/bugs/bug-446.md | 38 + .wolf/bugs/bug-447.md | 38 + .wolf/bugs/bug-448.md | 39 + .wolf/bugs/bug-451.md | 41 + .wolf/bugs/bug-459.md | 38 + .wolf/bugs/bug-465.md | 38 + .wolf/bugs/bug-466.md | 38 + .wolf/bugs/bug-477.md | 38 + .wolf/bugs/bug-478.md | 38 + .wolf/bugs/bug-503.md | 38 + .wolf/bugs/bug-538.md | 38 + .wolf/bugs/bug-543.md | 38 + .wolf/bugs/bug-561.md | 38 + .wolf/bugs/bug-562.md | 38 + .wolf/bugs/bug-563.md | 38 + .wolf/bugs/bug-604.md | 38 + .wolf/bugs/bug-605.md | 38 + .wolf/bugs/bug-606.md | 38 + .wolf/bugs/bug-608.md | 38 + .wolf/bugs/bug-609.md | 38 + .wolf/bugs/bug-610.md | 38 + .wolf/bugs/bug-646.md | 38 + .wolf/bugs/bug-648.md | 38 + .wolf/bugs/bug-650.md | 38 + .wolf/bugs/bug-652.md | 38 + .wolf/bugs/bug-660.md | 38 + .wolf/bugs/bug-661.md | 39 + .wolf/bugs/bug-693.md | 38 + .wolf/bugs/bug-704.md | 38 + .wolf/bugs/bug-760.md | 38 + .wolf/bugs/bug-765.md | 39 + .wolf/bugs/bug-777.md | 39 + .wolf/bugs/bug-778.md | 39 + .wolf/bugs/bug-786.md | 39 + .wolf/bugs/bug-795.md | 42 + .wolf/bugs/bug-817.md | 41 + .wolf/bugs/bug-827.md | 39 + .wolf/bugs/bug-828-part2.md | 39 + .wolf/bugs/bug-828.md | 39 + .wolf/bugs/bug-845.md | 38 + .wolf/bugs/bug-846.md | 38 + .wolf/bugs/bug-847.md | 38 + .wolf/bugs/bug-848.md | 38 + .wolf/bugs/bug-849.md | 38 + .wolf/bugs/bug-850.md | 38 + .wolf/bugs/bug-851.md | 45 + .wolf/bugs/bug-854.md | 38 + .wolf/bugs/bug-855.md | 38 + .wolf/bugs/bug-857.md | 41 + .wolf/bugs/bug-884.md | 39 + .wolf/bugs/bug-918.md | 41 + .wolf/bugs/bug-924.md | 44 + .wolf/bugs/bug-970.md | 38 + .wolf/bugs/bug-976.md | 38 + .wolf/bugs/bug-981.md | 38 + .wolf/bugs/bug-990-fix.md | 38 + .wolf/bugs/bug-990.md | 41 + .wolf/bugs/bug-991.md | 38 + CLAUDE.md | 2 +- scripts/buglog.py | 865 +- scripts/git-hooks/pre-commit | 7 + scripts/gitea-sync.py | 255 +- scripts/tests/test_buglog.py | 340 + 512 files changed, 40724 insertions(+), 17843 deletions(-) create mode 100644 .claude/skills/buglog/bug-record.md create mode 100644 .wolf/buglog-archive.json create mode 100644 .wolf/bugs/bug-006.md create mode 100644 .wolf/bugs/bug-007.md create mode 100644 .wolf/bugs/bug-1013.md create mode 100644 .wolf/bugs/bug-1023.md create mode 100644 .wolf/bugs/bug-1028.md create mode 100644 .wolf/bugs/bug-1060.md create mode 100644 .wolf/bugs/bug-1067.md create mode 100644 .wolf/bugs/bug-1077.md create mode 100644 .wolf/bugs/bug-1095.md create mode 100644 .wolf/bugs/bug-1096.md create mode 100644 .wolf/bugs/bug-1112.md create mode 100644 .wolf/bugs/bug-1128.md create mode 100644 .wolf/bugs/bug-1147.md create mode 100644 .wolf/bugs/bug-1197.md create mode 100644 .wolf/bugs/bug-1198.md create mode 100644 .wolf/bugs/bug-1199.md create mode 100644 .wolf/bugs/bug-1210.md create mode 100644 .wolf/bugs/bug-1224.md create mode 100644 .wolf/bugs/bug-1225.md create mode 100644 .wolf/bugs/bug-1272.md create mode 100644 .wolf/bugs/bug-1281.md create mode 100644 .wolf/bugs/bug-1282.md create mode 100644 .wolf/bugs/bug-1286.md create mode 100644 .wolf/bugs/bug-1296.md create mode 100644 .wolf/bugs/bug-1298.md create mode 100644 .wolf/bugs/bug-1306.md create mode 100644 .wolf/bugs/bug-1307.md create mode 100644 .wolf/bugs/bug-1376.md create mode 100644 .wolf/bugs/bug-1381.md create mode 100644 .wolf/bugs/bug-1386.md create mode 100644 .wolf/bugs/bug-1389.md create mode 100644 .wolf/bugs/bug-1396.md create mode 100644 .wolf/bugs/bug-1400.md create mode 100644 .wolf/bugs/bug-1401.md create mode 100644 .wolf/bugs/bug-1406.md create mode 100644 .wolf/bugs/bug-1407.md create mode 100644 .wolf/bugs/bug-1408.md create mode 100644 .wolf/bugs/bug-1409.md create mode 100644 .wolf/bugs/bug-1410.md create mode 100644 .wolf/bugs/bug-1411.md create mode 100644 .wolf/bugs/bug-1412.md create mode 100644 .wolf/bugs/bug-1413.md create mode 100644 .wolf/bugs/bug-1414.md create mode 100644 .wolf/bugs/bug-1415.md create mode 100644 .wolf/bugs/bug-1416.md create mode 100644 .wolf/bugs/bug-1417.md create mode 100644 .wolf/bugs/bug-1418.md create mode 100644 .wolf/bugs/bug-1419.md create mode 100644 .wolf/bugs/bug-1420.md create mode 100644 .wolf/bugs/bug-1421.md create mode 100644 .wolf/bugs/bug-1422.md create mode 100644 .wolf/bugs/bug-1423.md create mode 100644 .wolf/bugs/bug-1424.md create mode 100644 .wolf/bugs/bug-1425.md create mode 100644 .wolf/bugs/bug-1426.md create mode 100644 .wolf/bugs/bug-1427.md create mode 100644 .wolf/bugs/bug-1428.md create mode 100644 .wolf/bugs/bug-1429.md create mode 100644 .wolf/bugs/bug-1430.md create mode 100644 .wolf/bugs/bug-1431.md create mode 100644 .wolf/bugs/bug-1432.md create mode 100644 .wolf/bugs/bug-1433.md create mode 100644 .wolf/bugs/bug-1434.md create mode 100644 .wolf/bugs/bug-1435.md create mode 100644 .wolf/bugs/bug-1436.md create mode 100644 .wolf/bugs/bug-1437.md create mode 100644 .wolf/bugs/bug-1438.md create mode 100644 .wolf/bugs/bug-1439.md create mode 100644 .wolf/bugs/bug-1440.md create mode 100644 .wolf/bugs/bug-1441.md create mode 100644 .wolf/bugs/bug-1442.md create mode 100644 .wolf/bugs/bug-1443.md create mode 100644 .wolf/bugs/bug-1444.md create mode 100644 .wolf/bugs/bug-1445.md create mode 100644 .wolf/bugs/bug-1446.md create mode 100644 .wolf/bugs/bug-1447.md create mode 100644 .wolf/bugs/bug-1448.md create mode 100644 .wolf/bugs/bug-1449.md create mode 100644 .wolf/bugs/bug-1450.md create mode 100644 .wolf/bugs/bug-1451.md create mode 100644 .wolf/bugs/bug-1452.md create mode 100644 .wolf/bugs/bug-1453.md create mode 100644 .wolf/bugs/bug-1454.md create mode 100644 .wolf/bugs/bug-1455.md create mode 100644 .wolf/bugs/bug-1456.md create mode 100644 .wolf/bugs/bug-1457.md create mode 100644 .wolf/bugs/bug-1458.md create mode 100644 .wolf/bugs/bug-1459.md create mode 100644 .wolf/bugs/bug-1460.md create mode 100644 .wolf/bugs/bug-1461.md create mode 100644 .wolf/bugs/bug-1462.md create mode 100644 .wolf/bugs/bug-1463.md create mode 100644 .wolf/bugs/bug-1464.md create mode 100644 .wolf/bugs/bug-1465.md create mode 100644 .wolf/bugs/bug-1466.md create mode 100644 .wolf/bugs/bug-1467.md create mode 100644 .wolf/bugs/bug-1468.md create mode 100644 .wolf/bugs/bug-1469.md create mode 100644 .wolf/bugs/bug-1470.md create mode 100644 .wolf/bugs/bug-1471.md create mode 100644 .wolf/bugs/bug-1472-remainder.md create mode 100644 .wolf/bugs/bug-1472.md create mode 100644 .wolf/bugs/bug-1473.md create mode 100644 .wolf/bugs/bug-1474.md create mode 100644 .wolf/bugs/bug-1475.md create mode 100644 .wolf/bugs/bug-1476.md create mode 100644 .wolf/bugs/bug-1477.md create mode 100644 .wolf/bugs/bug-1478.md create mode 100644 .wolf/bugs/bug-1479.md create mode 100644 .wolf/bugs/bug-1480.md create mode 100644 .wolf/bugs/bug-1481.md create mode 100644 .wolf/bugs/bug-1482.md create mode 100644 .wolf/bugs/bug-1483.md create mode 100644 .wolf/bugs/bug-1484.md create mode 100644 .wolf/bugs/bug-1485.md create mode 100644 .wolf/bugs/bug-1486.md create mode 100644 .wolf/bugs/bug-1487.md create mode 100644 .wolf/bugs/bug-1489.md create mode 100644 .wolf/bugs/bug-1490.md create mode 100644 .wolf/bugs/bug-1491.md create mode 100644 .wolf/bugs/bug-1492.md create mode 100644 .wolf/bugs/bug-1493.md create mode 100644 .wolf/bugs/bug-1494.md create mode 100644 .wolf/bugs/bug-1495.md create mode 100644 .wolf/bugs/bug-1496.md create mode 100644 .wolf/bugs/bug-1497.md create mode 100644 .wolf/bugs/bug-1498.md create mode 100644 .wolf/bugs/bug-1499.md create mode 100644 .wolf/bugs/bug-1500.md create mode 100644 .wolf/bugs/bug-1501.md create mode 100644 .wolf/bugs/bug-1502.md create mode 100644 .wolf/bugs/bug-1503.md create mode 100644 .wolf/bugs/bug-1504.md create mode 100644 .wolf/bugs/bug-1505.md create mode 100644 .wolf/bugs/bug-1506.md create mode 100644 .wolf/bugs/bug-1507.md create mode 100644 .wolf/bugs/bug-1508.md create mode 100644 .wolf/bugs/bug-1509.md create mode 100644 .wolf/bugs/bug-1514.md create mode 100644 .wolf/bugs/bug-1515.md create mode 100644 .wolf/bugs/bug-1519.md create mode 100644 .wolf/bugs/bug-1532.md create mode 100644 .wolf/bugs/bug-1542.md create mode 100644 .wolf/bugs/bug-1548.md create mode 100644 .wolf/bugs/bug-1549.md create mode 100644 .wolf/bugs/bug-1553.md create mode 100644 .wolf/bugs/bug-1554.md create mode 100644 .wolf/bugs/bug-1555.md create mode 100644 .wolf/bugs/bug-1556.md create mode 100644 .wolf/bugs/bug-1557.md create mode 100644 .wolf/bugs/bug-1558.md create mode 100644 .wolf/bugs/bug-1559.md create mode 100644 .wolf/bugs/bug-1560.md create mode 100644 .wolf/bugs/bug-1561.md create mode 100644 .wolf/bugs/bug-1562.md create mode 100644 .wolf/bugs/bug-1563.md create mode 100644 .wolf/bugs/bug-1564.md create mode 100644 .wolf/bugs/bug-1565.md create mode 100644 .wolf/bugs/bug-1566.md create mode 100644 .wolf/bugs/bug-1567.md create mode 100644 .wolf/bugs/bug-1568.md create mode 100644 .wolf/bugs/bug-1569.md create mode 100644 .wolf/bugs/bug-1570.md create mode 100644 .wolf/bugs/bug-1571.md create mode 100644 .wolf/bugs/bug-1572.md create mode 100644 .wolf/bugs/bug-1573.md create mode 100644 .wolf/bugs/bug-1574.md create mode 100644 .wolf/bugs/bug-1575.md create mode 100644 .wolf/bugs/bug-1576.md create mode 100644 .wolf/bugs/bug-1577.md create mode 100644 .wolf/bugs/bug-1578.md create mode 100644 .wolf/bugs/bug-1579.md create mode 100644 .wolf/bugs/bug-1580.md create mode 100644 .wolf/bugs/bug-1581.md create mode 100644 .wolf/bugs/bug-1582.md create mode 100644 .wolf/bugs/bug-1583.md create mode 100644 .wolf/bugs/bug-1584.md create mode 100644 .wolf/bugs/bug-1585.md create mode 100644 .wolf/bugs/bug-1586.md create mode 100644 .wolf/bugs/bug-1587.md create mode 100644 .wolf/bugs/bug-1588.md create mode 100644 .wolf/bugs/bug-1589.md create mode 100644 .wolf/bugs/bug-1590.md create mode 100644 .wolf/bugs/bug-1591.md create mode 100644 .wolf/bugs/bug-1592.md create mode 100644 .wolf/bugs/bug-1593.md create mode 100644 .wolf/bugs/bug-1594.md create mode 100644 .wolf/bugs/bug-1595.md create mode 100644 .wolf/bugs/bug-1596.md create mode 100644 .wolf/bugs/bug-1597.md create mode 100644 .wolf/bugs/bug-1598.md create mode 100644 .wolf/bugs/bug-1599.md create mode 100644 .wolf/bugs/bug-1600.md create mode 100644 .wolf/bugs/bug-1601.md create mode 100644 .wolf/bugs/bug-1602.md create mode 100644 .wolf/bugs/bug-1603.md create mode 100644 .wolf/bugs/bug-1604.md create mode 100644 .wolf/bugs/bug-1605.md create mode 100644 .wolf/bugs/bug-1606.md create mode 100644 .wolf/bugs/bug-1607.md create mode 100644 .wolf/bugs/bug-1608.md create mode 100644 .wolf/bugs/bug-1609.md create mode 100644 .wolf/bugs/bug-1610.md create mode 100644 .wolf/bugs/bug-1611.md create mode 100644 .wolf/bugs/bug-1612.md create mode 100644 .wolf/bugs/bug-1613.md create mode 100644 .wolf/bugs/bug-1614.md create mode 100644 .wolf/bugs/bug-1615.md create mode 100644 .wolf/bugs/bug-1616.md create mode 100644 .wolf/bugs/bug-1617.md create mode 100644 .wolf/bugs/bug-1618.md create mode 100644 .wolf/bugs/bug-1619.md create mode 100644 .wolf/bugs/bug-1620.md create mode 100644 .wolf/bugs/bug-1621.md create mode 100644 .wolf/bugs/bug-1622.md create mode 100644 .wolf/bugs/bug-1623.md create mode 100644 .wolf/bugs/bug-1624.md create mode 100644 .wolf/bugs/bug-1625.md create mode 100644 .wolf/bugs/bug-1626.md create mode 100644 .wolf/bugs/bug-1627.md create mode 100644 .wolf/bugs/bug-1628.md create mode 100644 .wolf/bugs/bug-1629.md create mode 100644 .wolf/bugs/bug-1630.md create mode 100644 .wolf/bugs/bug-1631.md create mode 100644 .wolf/bugs/bug-1632.md create mode 100644 .wolf/bugs/bug-1673.md create mode 100644 .wolf/bugs/bug-1674.md create mode 100644 .wolf/bugs/bug-1675.md create mode 100644 .wolf/bugs/bug-1676.md create mode 100644 .wolf/bugs/bug-1677.md create mode 100644 .wolf/bugs/bug-1678.md create mode 100644 .wolf/bugs/bug-1679.md create mode 100644 .wolf/bugs/bug-1680.md create mode 100644 .wolf/bugs/bug-1681.md create mode 100644 .wolf/bugs/bug-1682.md create mode 100644 .wolf/bugs/bug-1683.md create mode 100644 .wolf/bugs/bug-1684.md create mode 100644 .wolf/bugs/bug-1685.md create mode 100644 .wolf/bugs/bug-1686.md create mode 100644 .wolf/bugs/bug-1687.md create mode 100644 .wolf/bugs/bug-1688.md create mode 100644 .wolf/bugs/bug-1689.md create mode 100644 .wolf/bugs/bug-1690.md create mode 100644 .wolf/bugs/bug-1691.md create mode 100644 .wolf/bugs/bug-1692.md create mode 100644 .wolf/bugs/bug-1693.md create mode 100644 .wolf/bugs/bug-1694.md create mode 100644 .wolf/bugs/bug-1695.md create mode 100644 .wolf/bugs/bug-1696.md create mode 100644 .wolf/bugs/bug-1697.md create mode 100644 .wolf/bugs/bug-1698.md create mode 100644 .wolf/bugs/bug-1699.md create mode 100644 .wolf/bugs/bug-1700.md create mode 100644 .wolf/bugs/bug-1701.md create mode 100644 .wolf/bugs/bug-1702.md create mode 100644 .wolf/bugs/bug-1703.md create mode 100644 .wolf/bugs/bug-1704.md create mode 100644 .wolf/bugs/bug-1705.md create mode 100644 .wolf/bugs/bug-1706.md create mode 100644 .wolf/bugs/bug-1707.md create mode 100644 .wolf/bugs/bug-1708.md create mode 100644 .wolf/bugs/bug-1709.md create mode 100644 .wolf/bugs/bug-1710.md create mode 100644 .wolf/bugs/bug-1711.md create mode 100644 .wolf/bugs/bug-1712.md create mode 100644 .wolf/bugs/bug-1713.md create mode 100644 .wolf/bugs/bug-1714.md create mode 100644 .wolf/bugs/bug-1715.md create mode 100644 .wolf/bugs/bug-1716.md create mode 100644 .wolf/bugs/bug-1717.md create mode 100644 .wolf/bugs/bug-1718.md create mode 100644 .wolf/bugs/bug-1719.md create mode 100644 .wolf/bugs/bug-1720.md create mode 100644 .wolf/bugs/bug-1721.md create mode 100644 .wolf/bugs/bug-1722.md create mode 100644 .wolf/bugs/bug-1723.md create mode 100644 .wolf/bugs/bug-1724.md create mode 100644 .wolf/bugs/bug-1725.md create mode 100644 .wolf/bugs/bug-1726.md create mode 100644 .wolf/bugs/bug-1727.md create mode 100644 .wolf/bugs/bug-1728.md create mode 100644 .wolf/bugs/bug-1729.md create mode 100644 .wolf/bugs/bug-1730.md create mode 100644 .wolf/bugs/bug-1731.md create mode 100644 .wolf/bugs/bug-1732.md create mode 100644 .wolf/bugs/bug-1733.md create mode 100644 .wolf/bugs/bug-1734.md create mode 100644 .wolf/bugs/bug-1735.md create mode 100644 .wolf/bugs/bug-1736.md create mode 100644 .wolf/bugs/bug-1737.md create mode 100644 .wolf/bugs/bug-1738.md create mode 100644 .wolf/bugs/bug-1739.md create mode 100644 .wolf/bugs/bug-1740.md create mode 100644 .wolf/bugs/bug-1741.md create mode 100644 .wolf/bugs/bug-1742.md create mode 100644 .wolf/bugs/bug-1743.md create mode 100644 .wolf/bugs/bug-1744.md create mode 100644 .wolf/bugs/bug-1745.md create mode 100644 .wolf/bugs/bug-1746.md create mode 100644 .wolf/bugs/bug-1747.md create mode 100644 .wolf/bugs/bug-1748.md create mode 100644 .wolf/bugs/bug-1750.md create mode 100644 .wolf/bugs/bug-1751.md create mode 100644 .wolf/bugs/bug-1752.md create mode 100644 .wolf/bugs/bug-1753.md create mode 100644 .wolf/bugs/bug-1754.md create mode 100644 .wolf/bugs/bug-1755.md create mode 100644 .wolf/bugs/bug-1756.md create mode 100644 .wolf/bugs/bug-1757.md create mode 100644 .wolf/bugs/bug-1758.md create mode 100644 .wolf/bugs/bug-1759.md create mode 100644 .wolf/bugs/bug-1760.md create mode 100644 .wolf/bugs/bug-1761.md create mode 100644 .wolf/bugs/bug-1762.md create mode 100644 .wolf/bugs/bug-1763.md create mode 100644 .wolf/bugs/bug-1764.md create mode 100644 .wolf/bugs/bug-1765.md create mode 100644 .wolf/bugs/bug-1766.md create mode 100644 .wolf/bugs/bug-1767.md create mode 100644 .wolf/bugs/bug-1768.md create mode 100644 .wolf/bugs/bug-1769.md create mode 100644 .wolf/bugs/bug-1770.md create mode 100644 .wolf/bugs/bug-1771.md create mode 100644 .wolf/bugs/bug-1772.md create mode 100644 .wolf/bugs/bug-1773.md create mode 100644 .wolf/bugs/bug-1774.md create mode 100644 .wolf/bugs/bug-1775.md create mode 100644 .wolf/bugs/bug-1776.md create mode 100644 .wolf/bugs/bug-1777.md create mode 100644 .wolf/bugs/bug-1778.md create mode 100644 .wolf/bugs/bug-1779.md create mode 100644 .wolf/bugs/bug-1780.md create mode 100644 .wolf/bugs/bug-1781.md create mode 100644 .wolf/bugs/bug-1782.md create mode 100644 .wolf/bugs/bug-1784.md create mode 100644 .wolf/bugs/bug-1785.md create mode 100644 .wolf/bugs/bug-1786.md create mode 100644 .wolf/bugs/bug-1787.md create mode 100644 .wolf/bugs/bug-1788.md create mode 100644 .wolf/bugs/bug-1789.md create mode 100644 .wolf/bugs/bug-1790.md create mode 100644 .wolf/bugs/bug-1791.md create mode 100644 .wolf/bugs/bug-1792.md create mode 100644 .wolf/bugs/bug-1793.md create mode 100644 .wolf/bugs/bug-1794.md create mode 100644 .wolf/bugs/bug-1795.md create mode 100644 .wolf/bugs/bug-1796.md create mode 100644 .wolf/bugs/bug-1797.md create mode 100644 .wolf/bugs/bug-1798.md create mode 100644 .wolf/bugs/bug-1799.md create mode 100644 .wolf/bugs/bug-1800.md create mode 100644 .wolf/bugs/bug-1801.md create mode 100644 .wolf/bugs/bug-1802.md create mode 100644 .wolf/bugs/bug-1803.md create mode 100644 .wolf/bugs/bug-1804.md create mode 100644 .wolf/bugs/bug-1805.md create mode 100644 .wolf/bugs/bug-1806.md create mode 100644 .wolf/bugs/bug-1807.md create mode 100644 .wolf/bugs/bug-1808.md create mode 100644 .wolf/bugs/bug-1809.md create mode 100644 .wolf/bugs/bug-1810.md create mode 100644 .wolf/bugs/bug-1811.md create mode 100644 .wolf/bugs/bug-1812.md create mode 100644 .wolf/bugs/bug-1813.md create mode 100644 .wolf/bugs/bug-1814.md create mode 100644 .wolf/bugs/bug-1815.md create mode 100644 .wolf/bugs/bug-1816.md create mode 100644 .wolf/bugs/bug-1817.md create mode 100644 .wolf/bugs/bug-1818.md create mode 100644 .wolf/bugs/bug-1819.md create mode 100644 .wolf/bugs/bug-1820.md create mode 100644 .wolf/bugs/bug-1821.md create mode 100644 .wolf/bugs/bug-1822.md create mode 100644 .wolf/bugs/bug-1823.md create mode 100644 .wolf/bugs/bug-1824.md create mode 100644 .wolf/bugs/bug-1825.md create mode 100644 .wolf/bugs/bug-1826.md create mode 100644 .wolf/bugs/bug-1827.md create mode 100644 .wolf/bugs/bug-1828.md create mode 100644 .wolf/bugs/bug-1829.md create mode 100644 .wolf/bugs/bug-1830.md create mode 100644 .wolf/bugs/bug-1831.md create mode 100644 .wolf/bugs/bug-1832.md create mode 100644 .wolf/bugs/bug-1833.md create mode 100644 .wolf/bugs/bug-1834.md create mode 100644 .wolf/bugs/bug-1835.md create mode 100644 .wolf/bugs/bug-1836.md create mode 100644 .wolf/bugs/bug-1837.md create mode 100644 .wolf/bugs/bug-1838.md create mode 100644 .wolf/bugs/bug-1839.md create mode 100644 .wolf/bugs/bug-1840.md create mode 100644 .wolf/bugs/bug-1841.md create mode 100644 .wolf/bugs/bug-1842.md create mode 100644 .wolf/bugs/bug-1843.md create mode 100644 .wolf/bugs/bug-1844.md create mode 100644 .wolf/bugs/bug-1845.md create mode 100644 .wolf/bugs/bug-1846.md create mode 100644 .wolf/bugs/bug-1847.md create mode 100644 .wolf/bugs/bug-1848.md create mode 100644 .wolf/bugs/bug-1849.md create mode 100644 .wolf/bugs/bug-1850.md create mode 100644 .wolf/bugs/bug-1851.md create mode 100644 .wolf/bugs/bug-1852.md create mode 100644 .wolf/bugs/bug-1853.md create mode 100644 .wolf/bugs/bug-1854.md create mode 100644 .wolf/bugs/bug-1855.md create mode 100644 .wolf/bugs/bug-1856.md create mode 100644 .wolf/bugs/bug-1857.md create mode 100644 .wolf/bugs/bug-1858.md create mode 100644 .wolf/bugs/bug-1859.md create mode 100644 .wolf/bugs/bug-1860.md create mode 100644 .wolf/bugs/bug-1861.md create mode 100644 .wolf/bugs/bug-1862.md create mode 100644 .wolf/bugs/bug-1863.md create mode 100644 .wolf/bugs/bug-1864.md create mode 100644 .wolf/bugs/bug-1865.md create mode 100644 .wolf/bugs/bug-1866.md create mode 100644 .wolf/bugs/bug-1867.md create mode 100644 .wolf/bugs/bug-1868.md create mode 100644 .wolf/bugs/bug-1869.md create mode 100644 .wolf/bugs/bug-1870.md create mode 100644 .wolf/bugs/bug-1871.md create mode 100644 .wolf/bugs/bug-206.md create mode 100644 .wolf/bugs/bug-208.md create mode 100644 .wolf/bugs/bug-320.md create mode 100644 .wolf/bugs/bug-335.md create mode 100644 .wolf/bugs/bug-336.md create mode 100644 .wolf/bugs/bug-337.md create mode 100644 .wolf/bugs/bug-338.md create mode 100644 .wolf/bugs/bug-339.md create mode 100644 .wolf/bugs/bug-340.md create mode 100644 .wolf/bugs/bug-343.md create mode 100644 .wolf/bugs/bug-351.md create mode 100644 .wolf/bugs/bug-352.md create mode 100644 .wolf/bugs/bug-353.md create mode 100644 .wolf/bugs/bug-378.md create mode 100644 .wolf/bugs/bug-396.md create mode 100644 .wolf/bugs/bug-410.md create mode 100644 .wolf/bugs/bug-411.md create mode 100644 .wolf/bugs/bug-435.md create mode 100644 .wolf/bugs/bug-436.md create mode 100644 .wolf/bugs/bug-445.md create mode 100644 .wolf/bugs/bug-446.md create mode 100644 .wolf/bugs/bug-447.md create mode 100644 .wolf/bugs/bug-448.md create mode 100644 .wolf/bugs/bug-451.md create mode 100644 .wolf/bugs/bug-459.md create mode 100644 .wolf/bugs/bug-465.md create mode 100644 .wolf/bugs/bug-466.md create mode 100644 .wolf/bugs/bug-477.md create mode 100644 .wolf/bugs/bug-478.md create mode 100644 .wolf/bugs/bug-503.md create mode 100644 .wolf/bugs/bug-538.md create mode 100644 .wolf/bugs/bug-543.md create mode 100644 .wolf/bugs/bug-561.md create mode 100644 .wolf/bugs/bug-562.md create mode 100644 .wolf/bugs/bug-563.md create mode 100644 .wolf/bugs/bug-604.md create mode 100644 .wolf/bugs/bug-605.md create mode 100644 .wolf/bugs/bug-606.md create mode 100644 .wolf/bugs/bug-608.md create mode 100644 .wolf/bugs/bug-609.md create mode 100644 .wolf/bugs/bug-610.md create mode 100644 .wolf/bugs/bug-646.md create mode 100644 .wolf/bugs/bug-648.md create mode 100644 .wolf/bugs/bug-650.md create mode 100644 .wolf/bugs/bug-652.md create mode 100644 .wolf/bugs/bug-660.md create mode 100644 .wolf/bugs/bug-661.md create mode 100644 .wolf/bugs/bug-693.md create mode 100644 .wolf/bugs/bug-704.md create mode 100644 .wolf/bugs/bug-760.md create mode 100644 .wolf/bugs/bug-765.md create mode 100644 .wolf/bugs/bug-777.md create mode 100644 .wolf/bugs/bug-778.md create mode 100644 .wolf/bugs/bug-786.md create mode 100644 .wolf/bugs/bug-795.md create mode 100644 .wolf/bugs/bug-817.md create mode 100644 .wolf/bugs/bug-827.md create mode 100644 .wolf/bugs/bug-828-part2.md create mode 100644 .wolf/bugs/bug-828.md create mode 100644 .wolf/bugs/bug-845.md create mode 100644 .wolf/bugs/bug-846.md create mode 100644 .wolf/bugs/bug-847.md create mode 100644 .wolf/bugs/bug-848.md create mode 100644 .wolf/bugs/bug-849.md create mode 100644 .wolf/bugs/bug-850.md create mode 100644 .wolf/bugs/bug-851.md create mode 100644 .wolf/bugs/bug-854.md create mode 100644 .wolf/bugs/bug-855.md create mode 100644 .wolf/bugs/bug-857.md create mode 100644 .wolf/bugs/bug-884.md create mode 100644 .wolf/bugs/bug-918.md create mode 100644 .wolf/bugs/bug-924.md create mode 100644 .wolf/bugs/bug-970.md create mode 100644 .wolf/bugs/bug-976.md create mode 100644 .wolf/bugs/bug-981.md create mode 100644 .wolf/bugs/bug-990-fix.md create mode 100644 .wolf/bugs/bug-990.md create mode 100644 .wolf/bugs/bug-991.md create mode 100755 scripts/git-hooks/pre-commit create mode 100644 scripts/tests/test_buglog.py diff --git a/.claude/rules/buglog-status.md b/.claude/rules/buglog-status.md index fbf2bcee..02896093 100644 --- a/.claude/rules/buglog-status.md +++ b/.claude/rules/buglog-status.md @@ -1,34 +1,34 @@ -# Buglog status field +# Bug records and their status Kept separate from `openwolf.md` because `openwolf update` overwrites that file. -OpenWolf itself ignores this field (search, pre-write hints and the hook-created -entries don't read or set it), so the convention is enforced only by this rule. -- Every `.wolf/buglog.json` entry carries a `status`, and it is the only source - of truth for whether a bug is open. Never infer it from `fix`: many entries - have a populated `fix` that says "NOT FIXED", "PENDING" or "PARTIAL". +- Each bug is a Markdown record, `.wolf/bugs/.md`, written to the standard + in `.claude/skills/buglog/bug-record.md` (what Description, Environment, + Reproduction, Root cause, Fix and Verification must contain). Create and + change records with the `buglog` skill (`scripts/buglog.py`), which checks + them against that standard. +- `.wolf/buglog.json` is an index generated from the records; never edit it. + `.wolf/buglog-archive.json` holds retired OpenWolf hook auto-detections. +- The front-matter `status` is the only source of truth for whether a bug is + open. Never infer it from the Fix text or tags: migrated records often + quote "NOT FIXED" or "PENDING" from earlier stages. - Values: - `open`: defect confirmed and still in the code. Intermittent test failures that do still recur count as `open`, not `cannot-reproduce`. - `partial`: part fixed; `status_note` says what remains. - `investigating`: real failure observed, root cause not found yet or waiting for a reproduction. - - `fixed`: fix committed; cite the commit in `fix` or `status_note`. + - `fixed`: fix committed; `fix_commits` lists the commit(s) and the + Verification section says how the fix was proven. - `cannot-reproduce`: reproduction was attempted and failed; record what was tried in `repro_attempts` (e.g. "0/40 runs, 2026-08-01"). - `wontfix`: not a bug, accepted limitation, or deferred with no plan. - - `duplicate`: same defect as another entry; set `duplicate_of`. -- Whenever you log a bug or change its state, set `status` and - `status_updated` (YYYY-MM-DD), plus `status_note` unless the status is - `fixed` with the commit already in `fix`. Put them right after `"id"`. -- A `cannot-reproduce` bug that recurs: bump `occurrences`/`last_seen` and set - `status` back to `investigating`. -- Use the `buglog` skill (`scripts/buglog.py`) to add entries and change - status: `add`, `status`, `note`, `recur`, `list`, `triage`. It picks the - next free id and edits only the touched entry's lines. Don't re-serialize - the whole file with `json.dumps`/`jq`; that rewrites ~1,200 unrelated lines. -- Open-bug list: + - `duplicate`: same defect as another record; set `duplicate_of`. +- Change status only with `python3 scripts/buglog.py status ` + (`--note` for every status except `fixed`; `--commit ` for `fixed`). + It sets `status_updated` and refuses a change the record doesn't support. +- A `cannot-reproduce` bug that recurs: `buglog.py recur ` bumps + `occurrences`/`last_seen` and sets `status` back to `investigating`. +- Open-bug list: `python3 scripts/buglog.py list --open`, or from the index: `jq -r '.bugs[] | select(.status | IN("open","partial","investigating")) | "\(.id) [\(.status)] \(.status_note // .error_message[0:90])"' .wolf/buglog.json` -- Entries with no `status` (hand-written without it, or created by an OpenWolf - hook) need triage: - `jq -r '.bugs[] | select(.status == null) | .id' .wolf/buglog.json` +- Records with no valid status need triage: `python3 scripts/buglog.py triage`. diff --git a/.claude/skills/buglog/SKILL.md b/.claude/skills/buglog/SKILL.md index 99ae855d..d5761283 100644 --- a/.claude/skills/buglog/SKILL.md +++ b/.claude/skills/buglog/SKILL.md @@ -1,7 +1,7 @@ --- name: buglog -description: Log a new bug, change a bug's status (fixed / partial / cannot-reproduce / ...), record a recurrence, or list open bugs in the OpenWolf buglog (.wolf/buglog.json). Use whenever you log or fix a bug, or are asked for the open-bug list. -argument-hint: add | status | note | recur | list [--open] | show | triage +description: Log a new bug, change a bug's status (fixed / partial / cannot-reproduce / ...), record a recurrence, or list open bugs. Bugs are Markdown records in .wolf/bugs/.md checked against a record standard. Use whenever you log or fix a bug, or are asked for the open-bug list. +argument-hint: new | status | note | recur | list [--open] | show | check triggers: - /buglog - log a bug @@ -11,12 +11,23 @@ triggers: # Buglog Skill -Every operation goes through `scripts/buglog.py` (run from the repo root). It -edits only the lines of the entry it touches, re-parses the file before -saving, and writes atomically. **Never** rewrite `.wolf/buglog.json` with -`json.dumps`/`jq`/the Write tool: entries mix raw UTF-8 and `\uXXXX` escapes, -so a re-serialize rewrites ~1,200 unrelated lines. Never read the file whole -either (~27K lines); use `list`/`show`. +Each bug is one Markdown file, `.wolf/bugs/.md`: short machine fields in +front matter, `# : `, then the sections of the record standard. +**Read `bug-record.md` (next to this file) before writing a record**: it says +what every section must contain (e.g. Environment needs the OS, the libc the +failing binary loads, toolchain, build profile, commit and deployment). + +- `.wolf/buglog.json` is an **index generated** from the records (for + OpenWolf's hooks, `openwolf bug search` and `jq`). Never edit it; every + `buglog.py` write regenerates it, and `python3 scripts/buglog.py index` does + it after a hand edit of a record. +- `.wolf/buglog-archive.json` holds the retired OpenWolf hook auto-detections + (710 entries, template messages like "Missing guard clause"). They are + not bug reports; leave them there. +- Records migrated from the old JSON log carry `legacy: true` and + `_Not recorded (migrated from buglog.json)._` where the old log had no such + field. They are exempt from the reporting sections, but changing one to + `fixed`/`partial` still needs the fix-time sections. Status values and when to use each: `.claude/rules/buglog-status.md`. @@ -29,36 +40,35 @@ or `python3 scripts/buglog.py list --grep ''`. | Task | Command | |------|---------| -| Log a new bug | `python3 scripts/buglog.py add --error "" --root-cause "" --file [,] --tags scheduler,asan --status open --note ""` → prints the new id | -| Log a bug you already fixed | same, with `--fix ", commit " --status fixed` | -| Mark fixed | `python3 scripts/buglog.py status fixed --note ""` | -| Mark partly fixed | `python3 scripts/buglog.py status partial --note ""` | +| Log a new bug | `python3 scripts/buglog.py new --summary "" --component --severity [--binary ] [--files a,b] [--tags x,y] [--related bug-N]` → prints the file path. Environment is filled in from this machine (pass `--binary` so libc and build profile come from that binary). Then edit the file: replace every `TODO` in Description, Environment (`Deployment`) and Reproduction, and run `check `. | +| Environment block only | `python3 scripts/buglog.py env []` | +| Check records | `python3 scripts/buglog.py check …` / `check --all` | +| Mark fixed | Fill in Root cause, Fix and Verification (`- Test:` RED before/GREEN after, `- Suites:`, `- Live:`), then `python3 scripts/buglog.py status fixed --commit [--commit ]`. Refused, and nothing written, until the record meets the standard; each `--commit` must be a real commit. | +| Mark partly fixed | `python3 scripts/buglog.py status partial --note ""` (Root cause + Fix required) | | Couldn't reproduce | `python3 scripts/buglog.py status cannot-reproduce --repro "0/40 runs, 2026-09-24" --note ""` | | Real failure, cause unknown | `python3 scripts/buglog.py status investigating --note ""` | | Not a bug / accepted | `python3 scripts/buglog.py status wontfix --note ""` | | Duplicate | `python3 scripts/buglog.py status duplicate --duplicate-of --note ""` | -| Add findings to an entry | `python3 scripts/buglog.py note ""` (appends an `UPDATE (date):` paragraph to `fix`) | +| Add findings | `python3 scripts/buglog.py note "" [--section "Root cause"]` (appends `UPDATE (date): …`; default section Description) | | Seen again | `python3 scripts/buglog.py recur [--note ""]` (bumps occurrences/last_seen; reopens a `fixed`/`wontfix`/`cannot-reproduce` bug) | | Open-bug list | `python3 scripts/buglog.py list --open` (add `--tag scheduler`, `--grep`, `--since YYYY-MM-DD`) | -| One entry | `python3 scripts/buglog.py show ` | -| Entries missing a status | `python3 scripts/buglog.py triage` | -| Any other single-line field | `python3 scripts/buglog.py set key= ...` | +| One record | `python3 scripts/buglog.py show ` | +| Front-matter field | `python3 scripts/buglog.py set key= ...` | -`status` requires `--note` for every status except `fixed`, and always -rewrites the note, so a stale note never outlives its status. +Components: controller, scheduler, worker, lib, harness, apiserver, xcecli, +xceadmin, autoscaler, build, config, tooling. Severities: data-loss, crash, +hang, wrong-result, leak, security, noise. ## Rules -- **Ids**: let `add` pick the id (one past the highest existing). Don't number - entries by hand or by entry count. OpenWolf's post-write hook numbers by - entry count, which collides with existing ids; its auto-detection is turned - off in `.wolf/config.json` for that reason (`openwolf.buglog.auto_detect`). -- **Write the symptom first**: `--error` holds the observed error text or - behavior, not a status banner ("OPEN, NOT FIXED --" goes in `status`). -- **Fixing a bug in code**: after the fix commit, run `status fixed - --note `. If only part is fixed, use `partial` with what remains. -- **Arrays** (`tags`, `related_bugs`, `files`) can only be set at `add` time; - to change them later, edit that entry's lines by hand and re-check with - `python3 -c "import json; json.load(open('.wolf/buglog.json'))"`. -- **Gitea**: status changes reach Gitea on the next `/gitea-sync`, which opens, - closes or reopens issues from `status`. +- **Ids**: let `new` pick the id (one past the highest in records + archive). +- **Editing sections**: edit the Markdown file directly (Edit tool), keeping + the `##
` headings; then `check ` and `index`. Status changes + go through `status`, never a hand edit of `status:`. +- **Enforcement**: the pre-commit hook (`scripts/git-hooks/pre-commit`; install + once per clone with `ln -sf ../../scripts/git-hooks/pre-commit + .git/hooks/pre-commit`) runs `check --staged`: it refuses staged records + that miss the standard, a legacy record hand-changed to fixed/partial without + the fix-time sections, and a stale `buglog.json`. +- **Gitea**: `/gitea-sync` mirrors status, and refreshes an issue's title and + body whenever the record's text changes; fix commits are linked. diff --git a/.claude/skills/buglog/bug-record.md b/.claude/skills/buglog/bug-record.md new file mode 100644 index 00000000..734b8c57 --- /dev/null +++ b/.claude/skills/buglog/bug-record.md @@ -0,0 +1,318 @@ +# Bug Record Standard + +Each bug is one file, `.wolf/bugs/.md`. `scripts/buglog.py` creates +records, changes their status and checks them against this standard; the +pre-commit hook refuses a commit whose staged records do not meet it. See +[Storage and enforcement](#storage-and-enforcement). + +Every bug record is read by two audiences: a person deciding what to work on, +and an agent that has to reproduce, fix or verify the bug with no other +context. So a record uses **few fields**, and each field is plain Markdown +text that must satisfy the rules in its section below. The rules say what the +text must contain, how to collect it, and what does not count. + +Write every field so that someone who has never seen the bug, on a different +day and a different machine, could act on it. + +## Fields at a glance + +| # | Field | When required | One-line rule | +|---|-------|---------------|---------------| +| 1 | **Summary** | always | The `# : ` title: one line, ≤ 100 characters, symptom + component. | +| 2 | **Description** | always | What happened vs. what should happen, with the exact error text and the impact. | +| 3 | **Environment** | always | OS, libc, toolchain, build profile, deployment, and the exact code version. | +| 4 | **Reproduction** | always | Steps someone else can run, how often it fails, and where the evidence is kept. | +| 5 | **Root cause** | status `fixed` or `partial` | The mechanism, in code terms, that produces the symptom. | +| 6 | **Fix** | status `fixed` or `partial` | What was changed and why this change removes the cause. | +| 7 | **Verification** | status `fixed` | Proof the fix works, and that the same test failed before the fix. | +| 8 | **References** | status `fixed` | Fix commits, related bugs, specs or docs. | + +Machine fields stay short and live in the front matter, outside these +sections: `id`, `status`, `status_updated`, `status_note`, `component`, +`severity`, `timestamp`, `fix_commits`, and a few more. Their values are defined in `.claude/rules/buglog-status.md` +and in [Machine fields](#machine-fields). + +--- + +## 1. Summary + +**Must include** +- The observable symptom, not the suspected cause. +- The component or binary where it shows (`xce_controller`, `xcecd_virtual`, + `xce_scheduler --sim`, `gitea-sync`, …). + +**Rules** +- Written as the record's title, `# : `; the summary itself is + one line, ≤ 100 characters, no trailing period. It becomes the Gitea issue title. +- No status words (`OPEN`, `FIXED`, `NOT FIXED`, `PRIORITY: LOW`). Status lives in `status`. + +**Not acceptable**: `Missing guard clause` · `Incorrect value in code` · +`crash` · `bug in ha.c`. + +**Good**: `xcecd_virtual leaks worker strings on every finished job (LeakSanitizer at exit)` + +## 2. Description + +**Must include** +- **Observed**: what happened, with the exact error text, log line, assertion + or sanitizer headline in a code block, copied, not paraphrased. +- **Expected**: what should have happened instead. +- **Impact**: who or what is affected and how badly: crash, hang, data loss, + wrong result, leak (and its growth rate), noise only. +- **How it was found**: user report, HA soak, ASAN run, audit, code review, + unit test, or a spec task (name it). + +**Include when relevant** +- The first bad state you can see, when the visible symptom comes much later. +- Which other bugs showed up in the same run. + +**Rules** +- Keep it to the facts known at the time you log the bug. Later findings go + in Root cause, or in a dated `UPDATE (YYYY-MM-DD):` line at the end of this + section. Never rewrite history silently. +- No guesses stated as facts. Label a suspicion as `Suspected:`. + +## 3. Environment + +The goal: another person can tell whether their machine and build are +affected, and can recreate the same conditions. + +**Must include** + +- **OS**: distribution and version, and the kernel. + - Kernel: `uname -srm`. + - Distribution: `/etc/os-release` `PRETTY_NAME`. If that file is missing, + say so and give what identifies the system instead. For example, this dev + host has no `/etc/os-release`. It is a Docker container (`/.dockerenv` + exists) on a NixOS kernel (`/proc/version` shows `#1-NixOS`). + - Container or VM, when it applies. +- **libc**: name and version of the C library **the failing binary actually + loads**, not the host's. + - Get it from the binary: + `readelf -l | grep 'program interpreter'` → `/nix/store/…-glibc-2.42-47/lib/ld-linux-x86-64.so.2`. + - Why it matters: on this host `ldd --version` outside the Nix shell reports + glibc **2.40**, but every XCE binary built in the dev shell runs on glibc + **2.42**. The host value is the wrong answer. +- **Toolchain**: compiler and version (`gcc --version`) plus Meson, and Go or + Python for those components, taken from the same shell that built the binary. +- **Build**: profile and directory: `asan` (`xce.build`), `debug` + (`build.dbg`) or `release` (`build.release`); and the sanitizer options in + force when the failure was seen (e.g. `ASAN_OPTIONS=detect_leaks=0`). +- **Code version**: the commit the failing binary was built from, and the + branch (`git rev-parse --short HEAD` at build time; add `-dirty` if the tree + had uncommitted changes). For a bug seen on a deployed cluster, the version + of **each** component involved (controller, scheduler, worker, apiserver, + xceadmin) when they differ. +- **Deployment**: topology (single controller, 3-node HA, multihost), worker + kind (`xcecd_virtual` or real root `xcecd`), and host names when more than + one machine is involved. + +**Include when relevant** +- Config set (`src/configs` or a fixture directory) and any non-default values. +- CPU count and memory, for races, timeouts and load-dependent bugs. +- systemd and cgroup version, for worker process or accounting bugs. +- Anything else that ran on the machine at the same time (parallel builds, + other soaks), for timing-sensitive failures. + +**Not acceptable**: `Linux` · `local` · `latest code` · `ASAN build` with no +commit · the host's `ldd --version` for a Nix-built binary. + +**Collect it with** (Nix-built binaries): + +```bash +uname -srm; cat /etc/os-release 2>/dev/null | grep PRETTY_NAME || echo "no /etc/os-release" +[ -f /.dockerenv ] && echo "docker container" +readelf -l | grep -o '/nix/store/[^]]*ld-linux[^]]*' # libc actually loaded +nix develop ./src --command bash -c 'gcc --version | head -1; meson --version' +git rev-parse --short HEAD; git diff --quiet || echo dirty +``` + +## 4. Reproduction + +**Must include** +- **Steps**: numbered commands someone else can paste, starting from a clean + checkout at the commit in Environment. Name the test binary, soak mode, + replay file or script and its arguments. +- **Frequency**: `deterministic`, or measured as `N/M runs` with the date + (e.g. `2/4 force-snapshot runs, 2026-09-26`). Never `sometimes`. +- **Evidence**: where the logs, cores, replay files or captures are kept. Use + a path that survives the session: copy them out of `/tmp` and out of + per-session scratch directories first. If they could not be kept, say so. + +**Include when relevant** +- A smaller reproduction (unit test, single command) once you have one. Keep + the original steps too. +- Conditions that stop it reproducing (e.g. passes with `--jobs 1`). + +**When it does not reproduce**: record what was tried, how many runs, on +which commit and date; this is what `status cannot-reproduce --repro` stores. + +## 5. Root cause *(fixed or partial)* + +**Must include** +- The mechanism in code terms: the function(s) and file(s), the state or + ordering that goes wrong, and why that produces the symptom in Description. +- Since when: the commit or change that introduced it when known + (`git log -S`, `git bisect`), otherwise `pre-existing, origin unknown`. +- Scope: other call sites or components with the same defect, checked or + not checked. + +**Rules** +- Describe the cause, not the fix. `Added a NULL check` is a fix, not a cause. +- If part of the explanation is still unproven, mark that part `Unconfirmed:`. + +## 6. Fix *(fixed or partial)* + +**Must include** +- What changed, per file or function, in a few lines. +- Why this change removes the cause in Root cause. +- For `partial`: what is left and where it is tracked. + +**Include when relevant** +- Alternatives that were rejected, and why. +- Behaviour changes, compatibility or migration notes (wire format, WAL, + snapshot, config). + +**Rules** +- A proposal that is not implemented yet is labelled `Proposed:` and needs + status `open` or `investigating`, never `fixed`. + +## 7. Verification *(fixed)* + +**Must include** +- **Test added or changed**: the test name and file, and a statement that it + **failed before the fix** (RED) and passes after (GREEN). If no automated + test is possible, say why and describe the manual check. +- **Suites run**: which suites, on which build profile, with results + (e.g. `worker suite 12/12, asan`). +- **Live check** for HA, persistence, scheduler or worker-process bugs: the + soak or deploy run used, with its mode, rounds and result. + +**Not acceptable**: `tests pass` · `verified` · `should be fixed now`. + +## 8. References *(fixed)* + +**Must include** +- Fix commit(s), short SHA, on a pushed branch. They live in the front-matter + field `fix_commits`, set by `buglog.py status fixed --commit `, + which refuses a hash that is not a commit in this repository. + +**Include when relevant** +- Related bugs, with the relationship: `regression of`, `split from`, + `split into`, `same root cause as`, `duplicate of`. +- Spec, steering or design documents that changed or explain the design. +- The Gitea issue number (filled in by `gitea-sync`). + +--- + +## Machine fields + +Front matter: one `key: ` per line (valid YAML), between `---` lines. + +| Field | Values | +|-------|--------| +| `id` | `bug-`; must match the file name. `buglog.py new` assigns it. | +| `status` | see `.claude/rules/buglog-status.md`; change it with `buglog.py status` | +| `status_updated`, `status_note` | set by `buglog.py status`; a note is required for every status except `fixed` | +| `component` | `controller` · `scheduler` · `worker` · `lib` · `harness` · `apiserver` · `xcecli` · `xceadmin` · `autoscaler` · `build` · `config` · `tooling` | +| `severity` | `data-loss` · `crash` · `hang` · `wrong-result` · `leak` · `security` · `noise` | +| `timestamp` | when the bug was first seen, ISO-8601 UTC, real time (no placeholder `T00:00:00Z`) | +| `last_seen`, `occurrences` | updated by `buglog.py recur` | +| `files`, `tags`, `related` | JSON lists; old status words are not tags (`fixed`, `partially-fixed`, `closed-by-…`) | +| `fix_commits` | short SHAs, required for `fixed` | +| `duplicate_of`, `repro_attempts` | required for `duplicate` / `cannot-reproduce` | +| `legacy` | `true` on records migrated from the old JSON log | +| `gitea_issue`, `gitea_synced_status`, `gitea_synced_hash` | written by `gitea-sync.py` only | + +A value of `TODO…` counts as empty: `buglog.py new` writes the template with +`TODO` markers, and `check` lists each one still unfilled. + +## Template + +`buglog.py new` writes this, with Environment already filled in: + +````markdown +--- +id: "bug-1872" +status: "open" +status_updated: "2026-09-27" +status_note: "newly logged" +component: "scheduler" +severity: "hang" +timestamp: "2026-09-27T20:05:00Z" +files: [] +tags: [] +related: [] +--- + +# bug-1872: + +## Description +**Observed:** +``` + +``` +**Expected:** <…> +**Impact:** <…> +**Found by:** + +## Environment +- OS: ; kernel ; +- libc: +- Toolchain: gcc <…>, meson <…> +- Build: (), +- Code: @ [-dirty] +- Deployment: , , + +## Reproduction +1. +2. +- Frequency: +- Evidence: + +## Root cause + + +## Fix + + +## Verification +- Test: in : RED before, GREEN after +- Suites: +- Live: + +## References +- Related: , … +- Docs: , … +```` + +## Example + +`.wolf/bugs/bug-1866.md` is a complete record written to this standard +(a worker leak: exact sanitizer text, Environment from `buglog.py env`, a +reproduction on the discovery commit, RED/GREEN verification, fix commit). + +## Storage and enforcement + +- **Records**: `.wolf/bugs/.md`, one per bug. People and agents read and + edit them directly; keep the `##
` headings. +- **Index**: `.wolf/buglog.json` is generated from the records (for OpenWolf's + hooks, `openwolf bug search` and `jq`). Never edit it; `buglog.py` rewrites + it on every write, and `buglog.py index` after a hand edit. +- **Archive**: `.wolf/buglog-archive.json` keeps the 710 retired OpenWolf hook + auto-detections (template messages such as "Missing guard clause", ids + between bug-002 and bug-1672) verbatim. They are not bug reports and are + not in the index. +- **Migrated records** (`legacy: true`, 2026-09-27) show + `_Not recorded (migrated from buglog.json)._` where the old log had no such + field. They are exempt from the reporting sections; changing one to + `fixed` or `partial` still needs the fix-time sections. +- **Enforcement**: + - `buglog.py status` refuses a status whose requirements the record does + not meet, and writes nothing. + - `buglog.py check |--all` lists every unmet rule. + - The pre-commit hook (`scripts/git-hooks/pre-commit`, installed per clone + with `ln -sf ../../scripts/git-hooks/pre-commit .git/hooks/pre-commit`) + runs `check --staged`: it refuses staged records that miss the standard, + a legacy record hand-changed to fixed/partial without the fix-time + sections, and a stale index. diff --git a/.claude/skills/gitea-sync/SKILL.md b/.claude/skills/gitea-sync/SKILL.md index c74e5d32..e8defb51 100644 --- a/.claude/skills/gitea-sync/SKILL.md +++ b/.claude/skills/gitea-sync/SKILL.md @@ -1,6 +1,6 @@ --- name: gitea-sync -description: Sync cc-sdd specs and OpenWolf buglog to Gitea milestones/issues in nexa/xce +description: Sync cc-sdd specs and bug records (.wolf/bugs/*.md) to Gitea milestones/issues in nexa/xce argument-hint: [--dry-run] [--spec ] [--bugs] [--since ] [--ids ] triggers: - /gitea-sync @@ -11,7 +11,7 @@ triggers: # Gitea Sync Skill -Sync cc-sdd specs (`.kiro/specs/`) and OpenWolf buglog (`.wolf/buglog.json`) to +Sync cc-sdd specs (`.kiro/specs/`) and bug records (`.wolf/bugs/*.md`) to Gitea issues in `nexa/xce`. Files are ground truth — Gitea is a read-only mirror. Sync state (issue numbers) is stored in source files for idempotency. @@ -54,7 +54,7 @@ Keep the script and this procedure in step when changing either. with `--since`). Needed because bulk triage has bumped `last_seen` / `status_updated` on most of the buglog, and for a never-synced bug `--since` only checks `last_seen` — a recently fixed older bug is otherwise - unreachable. Always `--dry-run` first: `--since` alone can match ~1100 bugs. + unreachable. Always `--dry-run` first: `--since` alone can match hundreds of bugs. ### 1. Ensure labels exist @@ -149,9 +149,12 @@ For each spec directory in `.kiro/specs/` (or just `--spec `): ### 3. Sync Bugs -Read `.wolf/buglog.json`. Each entry's `status` field is the source of truth -for open/closed — **never infer it from `fix`**: many entries have a populated -`fix` that says "NOT FIXED", "PENDING" or "PARTIALLY FIXED". +Read the bug records, `.wolf/bugs/.md` (format and rules: +`.claude/skills/buglog/bug-record.md`). Each record's front-matter `status` is +the source of truth for open/closed — **never infer it from the Fix text or +tags**: migrated records often quote "NOT FIXED" or "PENDING" from earlier +stages. `.wolf/buglog-archive.json` (retired hook auto-detections) is never +synced. **Status → Gitea state / label** (`status` values are defined in `.claude/rules/buglog-status.md`): @@ -165,114 +168,93 @@ for open/closed — **never infer it from `fix`**: many entries have a populated | `cannot-reproduce` | closed | `bug:cannot-reproduce` | | `wontfix` | closed | `bug:wontfix` | | `duplicate` | closed | `bug:duplicate` | -| missing (hook-created entry) | open | — ; report it in the summary as "needs status" | +| missing / unknown | open | — ; report it in the summary as "needs status" | An issue carries exactly one status label. When changing it, remove only the other `bug:` labels and add the new one; never replace the issue's whole label set, which would wipe labels a person added. -**Shared issues**: several entries can point at one `gitea_issue` (e.g. a +**Shared issues**: several records can point at one `gitea_issue` (e.g. a user-reported issue whose cause was split into several bugs, like #46). Such -an issue follows its "most open" entry, in this order: `open`, +an issue follows its "most open" record, in this order: `open`, `investigating`, `partial`, (no status), `cannot-reproduce`, `duplicate`, `wontfix`, `fixed`. -For each entry (after the `--since` filter): +**Title**: `[] ` — the record's `#` title (max 120 chars). -1. **Skip** if it has `gitea_issue` AND `gitea_synced_status == status` - (already mirrored, nothing changed). +**Body**: +``` +**Status:** — +**Component:** · **Severity:** +**Fix commits:** [``](//commit/), … (if any) +**Duplicate of:** / **Repro attempts:** (if set) -2. **Create issue** for entries without `gitea_issue`: - Use `mcp__gitea__issue_write` method `create`. +## - **Title**: `[] ` (max 120 chars), - after dropping any leading status banner such as "OPEN, NOT FIXED --" or - "PRIORITY: LOW --" (status lives in the state and labels). +## Affected Files +- `` for each of `files` - **Body**: - ``` - **Status:** (, if present) - <"Duplicate of " / "Repro attempts: ", if present> +## Tags +`tag1` `tag2` ... - ## Error - +--- +*Synced from `.wolf/bugs/.md`* +``` +Commit links point at `GITEA_COMMIT_REPO`, default the `origin` remote's +owner/repo (fixes land on a fork branch before they reach `nexa/xce`). - ## Root Cause - +**Labels**: `["bug"]` + the status label, `bug:crash`/`bug:data-loss` from +`severity`, and known tags: `asan`/`ubsan` → `bug:asan`, +`SEGV`/`double-free`/`UAF` → `bug:crash`, `race`/`toctou` → `bug:race`, +`ha`/`raft`/`election`/`failover` → `bug:ha`, `data-loss` → `bug:data-loss`. - ## Fix - (if `fix` is an array, join the elements with blank lines; - "*Not yet fixed*" if empty) +For each record (after the `--since`/`--ids` filter): - ## Affected Files - - (split on commas), or each element of `files` +1. **Skip** if it has `gitea_issue`, `gitea_synced_status == status` and + `gitea_synced_hash` equals the hash of the title + body it would write now. - ## Tags - `tag1` `tag2` ... +2. **Create issue** for records without `gitea_issue`, then close it if the + state is closed (`state: "closed"` may be ignored on create). No milestone. - ## Related Bugs - - +3. **Update changed issues**: + - text changed (hash differs): `update` the issue's title and body, so an + issue created while the bug was open gets its Root cause, Fix, + Verification and fix commits once it is fixed. + - status changed: `update` the state and swap the status label (aggregate + status for shared issues), and post a one-line comment with the new + status, `status_note` and fix commits. This also **reopens** bugs whose + status went back to `open`/`investigating`. - --- - *Synced from `.wolf/buglog.json`* - ``` - - **Labels**: `["bug"]` + the status label from the table above. Map known - tags to labels: `asan`/`ubsan` → `bug:asan`, `SEGV`/`double-free`/`UAF` → - `bug:crash`, `race`/`toctou` → `bug:race`, `ha`/`raft`/`election`/`failover` - → `bug:ha`, `data-loss` → `bug:data-loss`. - - **State**: from the table above. `state: "closed"` may be ignored on - `create` — create as open, then `update` to closed if needed. - - **Milestone**: None (bugs span specs). - -3. **Update changed issues**: for entries WITH `gitea_issue` whose `status` - differs from `gitea_synced_status` (or that have no `gitea_synced_status` - yet), `update` the issue's state and swap its status label per the table - (using the aggregate status for shared issues). - This both closes newly fixed bugs and **reopens** bugs whose status went - back to `open`/`investigating` (e.g. a `cannot-reproduce` bug that recurred). - Post a one-line comment with the new status and `status_note`. - - **One-time repair**: issues synced before the `status` field existed were - closed whenever `fix` was non-empty, so entries with `gitea_issue` and no - `gitea_synced_status` whose status is `open`/`partial`/`investigating` are - probably wrongly closed — step 3 reopens them. - -4. **Write sync state back** to the buglog entry immediately after each - issue is created/updated (not once at the end — a failure part-way through - must not lose an issue number, or the next run creates a duplicate issue), with the surgical helper — - never re-serialize the whole file (`json.dumps`/`jq` rewrite ~1,200 - unrelated lines because entries mix raw UTF-8 and `\uXXXX` escapes): +4. **Write sync state back** to the record's front matter immediately after + each issue is created/updated (a failure part-way through must not lose an + issue number, or the next run creates a duplicate issue): ```bash - python3 scripts/buglog.py set \ - gitea_issue= 'gitea_synced_status=""' + python3 scripts/buglog.py set gitea_issue= \ + 'gitea_synced_status=""' 'gitea_synced_hash=""' ``` - Run it from the repo root. It edits only that entry's lines and refuses to - write anything that no longer parses. For entries without a `status`, - record `gitea_synced_status="unknown"`. + For records without a valid `status`, record `gitea_synced_status="unknown"`. ### 4. Report -Print summary: "Specs: N synced, M skipped | Bugs: X created, Y closed, R reopened, Z skipped" -and list any entries with no `status` (hook-created entries) so they can be triaged. +Print summary: "Specs: N synced, M skipped | Bugs: X created, Y closed, R reopened, F refreshed, Z skipped" +and list any records with no valid `status` so they can be triaged. ## Important Rules - **Files are ground truth**. Never read Gitea state to update local files. The gitea_issue/milestone_id fields in local files only store sync mappings. - **Idempotent**. Items with existing `gitea_issue` or `gitea` state are skipped - unless their phase changed (specs) or `status != gitea_synced_status` (bugs). + unless their phase changed (specs) or their status or text changed (bugs). - **Labels**: Check existing labels before creating. Use label IDs (not names) in issue create calls — fetch with `mcp__gitea__label_read` first. - **Rate limiting**: Batch issue creation — create issues sequentially (a few per turn) to stay within API limits. -- **Buglog is large** (~1,200 entries, ~27K lines). Always use `--since` filter - for practical runs (e.g., `--since 2026-07-18`). Without `--since`, sync ALL - bugs. Filter with `jq`, don't read the file whole. +- **~400 records have never been synced** (older bugs). A run without + `--since`/`--ids` creates an issue for every one of them; use the filters + unless that is the intent, and always `--dry-run` first. - **Private repo**: `nexa/xce` is private. MCP auth must have access. ## Gitea MCP Tool Reference diff --git a/.claude/skills/xce-audit-invariants/SKILL.md b/.claude/skills/xce-audit-invariants/SKILL.md index f16b510e..9e7d5c27 100644 --- a/.claude/skills/xce-audit-invariants/SKILL.md +++ b/.claude/skills/xce-audit-invariants/SKILL.md @@ -269,11 +269,13 @@ several minutes per mode. ## Step 5 — Record and report - Log every genuinely new root-caused bug with the `buglog` skill: - `python3 scripts/buglog.py add --error ... --root-cause ... --file ... - --tags ,... --fix "" --status fixed` (or `open` / - `partial` with `--note` for what remains). It assigns the id; don't pick one - by hand. For a violation that an existing entry already covers, update that - entry instead: `scripts/buglog.py status ...` / `note ...`. + `python3 scripts/buglog.py new --summary "" + --component --severity --binary `, fill in + the record per `.claude/skills/buglog/bug-record.md`, and once the fix is + committed `scripts/buglog.py status fixed --commit ` (or + `partial --note ""`). It assigns the id; don't pick one by + hand. For a violation that an existing record already covers, update that + record instead: `scripts/buglog.py status ...` / `note ...`. - Append durable, non-obvious learnings to `.wolf/cerebrum.md`'s Key Learnings section (anchor the edit near an existing dated entry to avoid the "## Key Learnings" heading collision — the file has one per dated diff --git a/.wolf/buglog-archive.json b/.wolf/buglog-archive.json new file mode 100644 index 00000000..41cdaea6 --- /dev/null +++ b/.wolf/buglog-archive.json @@ -0,0 +1,14148 @@ +{ + "archived": "2026-09-27", + "reason": "OpenWolf post-write hook auto-detections (template messages such as 'Missing guard clause', 'Function not marked async'): not real bug reports. Kept verbatim; not part of .wolf/bugs/ or the index.", + "bugs": [ + { + "id": "bug-002", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-21T07:46:54.198Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/kiro-spec-init/SKILL.md", + "root_cause": "Had `.kiro/settings/templates/specs/init.json`", + "fix": "Changed to `templates/specs/init.json`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 12, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-004", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-21T07:58:54.673Z", + "error_message": "Wrong reference: g_tmpdir should be g_instdir", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Used \"g_tmpdir\" instead of \"g_instdir\"", + "fix": "Changed g_tmpdir → g_instdir", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-005", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-21T08:24:10.738Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: rc == 0", + "fix": "Added guard clause: if (rc == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-008", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-21T13:20:28.953Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: getaddrinfo(host, port_str, &hints, &res", + "fix": "Added guard clause: if (getaddrinfo(host, port_str, &hints, &res)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-020", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-25T11:31:55.826Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-023", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T07:17:31.980Z", + "error_message": "Missing guard clause", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "No early return/throw for edge case: req->term < my_term", + "fix": "Added guard clause: if (req->term < my_term)", + "tags": [ + "auto-detected", + "guard-clause", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-024", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T07:18:36.066Z", + "error_message": "Missing guard clause", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "No early return/throw for edge case: n_peers == 2", + "fix": "Added guard clause: if (n_peers == 2)", + "tags": [ + "auto-detected", + "guard-clause", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-026", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T07:26:48.363Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Was returning: 1;", + "fix": "Now returns: 1; // single-node: always primary", + "tags": [ + "auto-detected", + "return-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-027", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T07:27:34.925Z", + "error_message": "Wrong condition in logic", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Condition was: if (n_peers == 2)", + "fix": "Changed to: if (n_peers == 1)", + "tags": [ + "auto-detected", + "logic-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-029", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T07:29:11.741Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Had `main.c`", + "fix": "Changed to `-EINVAL`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-032", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T09:07:23.015Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Had `-EINVAL`", + "fix": "Changed to `--vcpus N`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-034", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T09:23:02.818Z", + "error_message": "Missing guard clause", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "No early return/throw for edge case: rc == 0", + "fix": "Added guard clause: if (rc == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-035", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T09:46:12.902Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Had `sysconf(_SC_NPROCESSORS_ONLN)`", + "fix": "Changed to ` when not specified. Values `", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-038", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T10:09:32.424Z", + "error_message": "Missing guard clause", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "No early return/throw for edge case: c->myconf->config_version_tag != 0", + "fix": "Added guard clause: if (c->myconf->config_version_tag != 0)", + "tags": [ + "auto-detected", + "guard-clause", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-039", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T10:09:57.711Z", + "error_message": "Wrong return value", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Was returning: 0 (standby); fresher peer runs its own election", + "fix": "Now returns: `-ENODATA`. Not a normal runtime path.", + "tags": [ + "auto-detected", + "return-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-041", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T10:10:40.149Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Had `test_fetch_config_from_best_peer`", + "fix": "Changed to `test_handle_config_request_standbys_serve_config`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-042", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T11:35:44.097Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-046", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T13:51:32.274Z", + "error_message": "Missing error handling in unknown", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - [ ] `test_heartbeat_carries_committed_lsn`: heartbeat payl; `committed_lsn`; standby updates `ha.last_known_committed_ls", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-047", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-26T13:52:49.044Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: // Catch-up gate (Clarification 6): behind nodes must not in; if (c->ha.in_catch_up) {", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-048", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-27T03:02:34.912Z", + "error_message": "Missing error handling in unknown", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: name, key, applied_lsn, and config_tag immediately after TCP; - [ ] `test_ctrl_hello_registers_standby_in_repl`: handle_ct", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-059", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-27T07:24:56.724Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Had `-ENOSYS`", + "fix": "Changed to `bootstrap_config_if_needed()`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-060", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-27T09:24:44.651Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/spec.json", + "root_cause": "Had \"2026-05-27T01:24:00-07:00\"", + "fix": "Changed to \"2026-05-27T01:30:00-07:00\"", + "tags": [ + "auto-detected", + "wrong-value", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-067", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T03:42:53.810Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'worker.c'", + "fix": "Changed to 'startup_validation.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-068", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T04:01:56.031Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/main.c", + "root_cause": "Condition was: if (arguments->vcpus < 1)", + "fix": "Changed to: if (validate_vcpus(arguments->vcpus)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-069", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T04:02:06.925Z", + "error_message": "Wrong reference: vcpus should be nres", + "file": "src/include/clusterconfig.h", + "root_cause": "Used \"vcpus\" instead of \"nres\"", + "fix": "Changed vcpus → nres", + "tags": [ + "auto-detected", + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-071", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T04:02:48.429Z", + "error_message": "Incorrect value in code", + "file": "src/schema/controller.capnp", + "root_cause": "Had \"vcpus\"", + "fix": "Changed to \"resources\"", + "tags": [ + "auto-detected", + "wrong-value", + "capnp", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-074", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T05:59:28.525Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (opcode == OPCode_requestVote || opcode == OPCode_p)", + "fix": "Changed to: if (opcode == OPCode_requestVote ||\n opcod)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-075", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T07:04:33.895Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_election.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-078", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T07:22:56.418Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: peer_idx < 0 || peer_idx >= ctx->n_peers", + "fix": "Added guard clause: if (peer_idx < 0 || peer_idx >= ctx->n_peers)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-081", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T07:53:10.883Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: role != CTRL_STANDBY", + "fix": "Added guard clause: if (role != CTRL_STANDBY)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-082", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T08:23:40.040Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ctx->done_posted", + "fix": "Added guard clause: if (ctx->done_posted)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-083", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T08:45:37.921Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !host || !tag_out", + "fix": "Added guard clause: if (!host || !tag_out)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-084", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T08:48:58.609Z", + "error_message": "Wrong reference: capn_str_null should be str", + "file": "src/controller/config_repl.c", + "root_cause": "Used \"capn_str_null\" instead of \"str\"", + "fix": "Changed capn_str_null → str", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-086", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T09:13:17.676Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_bootstrap_support.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-087", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T10:13:52.421Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !workdir", + "fix": "Added guard clause: if (!workdir)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-091", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T13:53:09.924Z", + "error_message": "Wrong reference: in_catch_up should be acked_config_lsn", + "file": "src/controller/repl.c", + "root_cause": "Used \"in_catch_up\" instead of \"acked_config_lsn\"", + "fix": "Changed in_catch_up → acked_config_lsn", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-092", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T14:11:04.583Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn", + "fix": "Added guard clause: if (p->current_lsn == 0 || cutoff_lsn + 1 >=)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-094", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T14:47:44.445Z", + "error_message": "Wrong reference: ha_state_t should be _Atomic", + "file": "src/controller/ha.h", + "root_cause": "Used \"ha_state_t\" instead of \"_Atomic\"", + "fix": "Changed ha_state_t → _Atomic", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-095", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T15:04:44.613Z", + "error_message": "Wrong reference: socket should be epoll", + "file": "src/controller/ha.c", + "root_cause": "Used \"socket\" instead of \"epoll\"", + "fix": "Changed socket → epoll", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-096", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-28T15:05:23.129Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ha_tcp_connect_hook", + "fix": "Added guard clause: if (ha_tcp_connect_hook)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-097", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T02:48:17.371Z", + "error_message": "Missing guard clause", + "file": "src/lib/request.c", + "root_cause": "No early return/throw for edge case: (ep == NULL", + "fix": "Added guard clause: if ((ep == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-098", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T02:50:46.444Z", + "error_message": "Wrong reference: XCE_ERR_STANDBY should be E_XCE_STANDBY", + "file": "src/lib/request.c", + "root_cause": "Used \"XCE_ERR_STANDBY\" instead of \"E_XCE_STANDBY\"", + "fix": "Changed XCE_ERR_STANDBY → E_XCE_STANDBY", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-099", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T02:51:03.529Z", + "error_message": "Wrong reference: XCE_ERR_NOT_READY should be E_XCE_NOT_READY", + "file": "src/lib/testsuite/test_libxce_retry.c", + "root_cause": "Used \"XCE_ERR_NOT_READY\" instead of \"E_XCE_NOT_READY\"", + "fix": "Changed XCE_ERR_NOT_READY → E_XCE_NOT_READY", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-102", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T03:37:52.117Z", + "error_message": "Wrong reference: data should be ctrl_hello_reply_t", + "file": "src/include/typemap.h", + "root_cause": "Used \"data\" instead of \"ctrl_hello_reply_t\"", + "fix": "Changed data → ctrl_hello_reply_t", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-104", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T03:54:23.205Z", + "error_message": "Missing error handling in struct", + "file": "src/include/typemap.h", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: wal_catchup_entry_t* walcatchupentrymsg;; snapshot_transfer_t* snapshottransfermsg; | Also: /* Forward declarations for types defined later in this file; typedef struct snapshot_transfer_t snapshot_transfer_fw", + "tags": [ + "auto-detected", + "error-handling", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-109", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T06:59:14.045Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha", + "fix": "Added guard clause: if (!ha)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-111", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T07:02:31.287Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (c->myconf && c->myconf->cluster_config)", + "fix": "Changed to: if (c->myconf && c->myconf->peers_cfg.candidates_count)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-117", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T13:41:46.841Z", + "error_message": "Incorrect value in code", + "file": "src/controller/myapp.c", + "root_cause": "Had \"Pre-election config loaded: tag=%016\"", + "fix": "Changed to \"Pre-election config loaded: tag=%016llx\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-118", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T13:51:42.299Z", + "error_message": "Wrong reference: nres should be vcpus", + "file": "src/include/clusterconfig.h", + "root_cause": "Used \"nres\" instead of \"vcpus\"", + "fix": "Changed nres → vcpus", + "tags": [ + "auto-detected", + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-119", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-29T16:32:10.285Z", + "error_message": "Wrong reference: data should be request_vote_reply_t", + "file": "src/include/typemap.h", + "root_cause": "Used \"data\" instead of \"request_vote_reply_t\"", + "fix": "Changed data → request_vote_reply_t", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-122", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T00:05:02.052Z", + "error_message": "Wrong reference: cluster_init_phase2 should be cluster_register_listener", + "file": "src/controller/include/cluster.h", + "root_cause": "Used \"cluster_init_phase2\" instead of \"cluster_register_listener\"", + "fix": "Changed cluster_init_phase2 → cluster_register_listener", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-123", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T00:20:27.244Z", + "error_message": "Missing guard clause", + "file": "src/controller/cluster.c", + "root_cause": "No early return/throw for edge case: c->ep.flags == 0", + "fix": "Added guard clause: if (c->ep.flags == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-127", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T01:12:38.572Z", + "error_message": "Wrong reference: CTRL_PERF_NHASH should be c", + "file": "src/controller/cluster.c", + "root_cause": "Used \"CTRL_PERF_NHASH\" instead of \"c\"", + "fix": "Changed CTRL_PERF_NHASH → c", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-129", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T01:21:20.140Z", + "error_message": "Wrong reference: true should be false", + "file": ".claude/skills/xce-build/SKILL.md", + "root_cause": "Used \"true\" instead of \"false\"", + "fix": "Changed true → false", + "tags": [ + "auto-detected", + "wrong-reference", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-130", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T01:28:22.079Z", + "error_message": "Wrong reference: 120 should be 300", + "file": "src/controller/meson.build", + "root_cause": "Used \"120\" instead of \"300\"", + "fix": "Changed 120 → 300", + "tags": [ + "auto-detected", + "wrong-reference", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-135", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T06:19:47.181Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/election-iomgr-refactor/design.md", + "root_cause": "Had `cluster_init_phase2`", + "fix": "Changed to `cluster_config == NULL`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-136", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T06:38:09.325Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/election-iomgr-refactor/design.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* After election determines role: */; int is_first_boot = !c->running; /* running=1 only after cl | Also: #### `ha_dispatch_gate` — CTRL_PRIMARY_COMMITTING case; ```c | Also: ### 3-node startup — ctrl-a wins (with COMMITTING phase); participant PeerC as ctrl-c endpoint | Also: - **`ha_start_election_async()` is fully non-blocking**: Arm; - *Initial startup* (`ha->phase2_done == 0`): calls `cluster", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-139", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T06:58:05.115Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/election-iomgr-refactor/design.md", + "root_cause": "Had `delay_task_cancel()`", + "fix": "Changed to `clock_gettime(CLOCK_MONOTONIC)`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-142", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T07:57:14.052Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: xce_vote_request_hook != NULL", + "fix": "Added guard clause: if (xce_vote_request_hook != NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-143", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T08:16:03.222Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !pkt || !pkt->data || pkt->sz == 0", + "fix": "Added guard clause: if (!pkt || !pkt->data || pkt->sz == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-144", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T08:49:04.058Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-145", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T08:49:59.817Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: atomic_load(&t->cancelled", + "fix": "Added guard clause: if (atomic_load(&t->cancelled)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-147", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T09:13:19.377Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-149", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T10:31:53.836Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_vote_gate.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-153", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T11:43:01.989Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_repl_push.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-154", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T11:43:44.829Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Push config to standbys with lower config_tag (Req 7.1) *; #ifdef XCE_TESTING", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-157", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T12:02:39.009Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-159", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T12:22:57.013Z", + "error_message": "Wrong return value", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Was returning: in < 10ms.", + "fix": "Now returns: in < 5ms (design spec).", + "tags": [ + "auto-detected", + "return-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-160", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T12:23:18.560Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Had \"myapp_run path: ha_start_election_async must retu", + "fix": "Changed to \"myapp_run path: ha_start_election_async must retu", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-161", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T12:29:45.867Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-162", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T12:30:10.570Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-163", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T12:31:02.203Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_failover.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-164", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T12:45:22.182Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-166", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T12:50:35.870Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_bootstrap_support.c", + "root_cause": "No early return/throw for edge case: c->myconf->config_version_tag != 0", + "fix": "Added guard clause: if (c->myconf->config_version_tag != 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-167", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T13:06:57.152Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: cancelled_val", + "fix": "Added guard clause: if (cancelled_val)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-168", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T13:24:06.024Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_repl_ack.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-169", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T13:42:28.077Z", + "error_message": "Wrong reference: third should be second", + "file": "src/controller/testsuite/test_ha_repl_ack.c", + "root_cause": "Used \"third\" instead of \"second\"", + "fix": "Changed third → second", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-170", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T14:35:07.679Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-172", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-30T23:42:12.644Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-176", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T02:09:39.630Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (c->myconf && c->myconf->peers_cfg.candidates_count)", + "fix": "Changed to: if (ha->my_priority > 0)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-183", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T03:02:33.311Z", + "error_message": "Missing guard clause", + "file": "src/lib/request.c", + "root_cause": "No early return/throw for edge case: req == NULL", + "fix": "Added guard clause: if (req == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-186", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T04:11:45.534Z", + "error_message": "Missing error handling in struct", + "file": ".kiro/specs/raft-library/design.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: Caller->>RaftSM: raft_on_vote_request(state, from, term); alt term < current_term", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-188", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T04:19:54.524Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-library/design.md", + "root_cause": "Had `raft.c`", + "fix": "Changed to `raft.h`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-190", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T05:07:37.759Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: cfg->n_peers < 1", + "fix": "Added guard clause: if (cfg->n_peers < 1)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-191", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T05:23:19.394Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->role == RAFT_CANDIDATE || s->role == RAFT_LEADER", + "fix": "Added guard clause: if (s->role == RAFT_CANDIDATE || s->role == )", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-197", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T07:08:23.387Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-207", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T08:47:08.484Z", + "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_even_peers_returns_einval", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_even_peers_returns_einval\"", + "fix": "Changed test_raft_init_zero_peers_returns_einval → test_raft_init_even_peers_returns_einval", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-209", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T10:19:47.014Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (ha_rdhup_election_hook != NULL)", + "fix": "Changed to: if (c->myconf && c->myconf->peers_cfg.candidates_count)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-211", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T11:29:03.301Z", + "error_message": "Wrong reference: 2000 should be 5000", + "file": "src/controller/ha.h", + "root_cause": "Used \"2000\" instead of \"5000\"", + "fix": "Changed 2000 → 5000", + "tags": [ + "auto-detected", + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-215", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T12:43:45.333Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !pkt || !pkt->data || pkt->sz == 0", + "fix": "Added guard clause: if (!pkt || !pkt->data || pkt->sz == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-216", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T12:49:40.554Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-peer-connection-reuse/tasks.md", + "root_cause": "Had 's vote endpoints to standby control channels\n\n- [", + "fix": "Changed to 's vote endpoints to standby control channels\n\n- [", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-218", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T12:58:44.435Z", + "error_message": "Wrong reference: Request_body_nodata should be Request_body_nomsg", + "file": "src/controller/ha.c", + "root_cause": "Used \"Request_body_nodata\" instead of \"Request_body_nomsg\"", + "fix": "Changed Request_body_nodata → Request_body_nomsg", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-220", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-05-31T14:20:41.161Z", + "error_message": "Wrong reference: vote_reply_msg_handler should be vote_reply_msg_handler_for_test", + "file": "src/controller/ha.c", + "root_cause": "Used \"vote_reply_msg_handler\" instead of \"vote_reply_msg_handler_for_test\"", + "fix": "Changed vote_reply_msg_handler → vote_reply_msg_handler_for_test", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-224", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T02:07:55.176Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c->delaymgr.running", + "fix": "Added guard clause: if (!c->delaymgr.running)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-229", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T02:49:09.141Z", + "error_message": "Wrong operator: && should be !=", + "file": "src/controller/ha.c", + "root_cause": "Used \"&&\" instead of \"!=\"", + "fix": "Changed operator && → !=", + "tags": [ + "auto-detected", + "operator-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-233", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T03:07:51.264Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_connect.c'", + "fix": "Changed to 'testsuite/test_ha_ctrlhello.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-235", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T03:13:14.521Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_connection_reuse.c'", + "fix": "Changed to 'testsuite/test_ha_connect.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-236", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T03:15:29.129Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: rc < 0", + "fix": "Added guard clause: if (rc < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-238", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T03:23:32.483Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_connect.c'", + "fix": "Changed to 'testsuite/test_ha_connection_reuse.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-239", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T03:28:46.956Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ha_tcp_connect_hook != NULL", + "fix": "Added guard clause: if (ha_tcp_connect_hook != NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-240", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T04:08:07.950Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_connect.c'", + "fix": "Changed to 'testsuite/test_ha_connection_reuse.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-241", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T04:13:03.001Z", + "error_message": "Wrong reference: test_ha_connect should be test_ha_ctrlhello", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_ha_connect\" instead of \"test_ha_ctrlhello\"", + "fix": "Changed test_ha_connect → test_ha_ctrlhello", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-242", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T05:19:12.427Z", + "error_message": "Wrong reference: vcpu should be vcpus", + "file": "src/tools/deploy-ha.sh", + "root_cause": "Used \"vcpu\" instead of \"vcpus\"", + "fix": "Changed vcpu → vcpus", + "tags": [ + "auto-detected", + "wrong-reference", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-250", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T07:46:05.128Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ep || !ep->data", + "fix": "Added guard clause: if (!ep || !ep->data)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-252", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T08:35:29.969Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_rdhup_integration.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-255", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T12:02:55.635Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (rc != 0)", + "fix": "Changed to: if (validate_heartbeat_config(ha_cfg.heartbeat_timeout)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-256", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T12:36:28.381Z", + "error_message": "Wrong reference: startup_validation should be testsuite_test_ha_liveness", + "file": "src/xce.build/build.ninja", + "root_cause": "Used \"startup_validation\" instead of \"testsuite_test_ha_liveness\"", + "fix": "Changed startup_validation → testsuite_test_ha_liveness", + "tags": [ + "wrong-reference", + "ninja", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-260", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T14:42:41.325Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: interval_s <= 0", + "fix": "Added guard clause: if (interval_s <= 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-263", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-01T15:32:59.304Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: !f", + "fix": "Added guard clause: if (!f)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-265", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-03T08:47:25.835Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-266", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-03T08:47:54.376Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-269", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-03T08:50:58.837Z", + "error_message": "Wrong reference: test_am_i_primary should be test_ha_cluster", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_am_i_primary\" instead of \"test_ha_cluster\"", + "fix": "Changed test_am_i_primary → test_ha_cluster", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-279", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T11:54:56.114Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-repl-hardening/design.md", + "root_cause": "Had `(checkpoint_lsn, last_received_chunk)`", + "fix": "Changed to `snap.part`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-284", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T14:33:03.464Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ── Task 2.1: OPCode_walCatchupEntry handler ─────────────; #ifdef XCE_TESTING | Also: /* Task 2.1: OPCode_walCatchupEntry handler */; RUN_TEST(test_wal_catchup_entry_advances_applied_lsn);", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-285", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T14:41:35.073Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/cluster.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: case OPCode_walCatchupEntry:; /* Task 2.1: standby applies a single WAL catch-up entry fro", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-288", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T14:46:44.652Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: #include \"persist.h\" /* persist_t, apply_wal_entry */; #include \"typemap.h\" /* wal_catchup_entry_t */", + "tags": [ + "auto-detected", + "error-handling", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-292", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T16:15:43.743Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES", + "fix": "Added guard clause: if (peer_idx < 0 || peer_idx >= MAX_HA_CANDI)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-294", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T16:45:25.724Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: standby_tag < committed_tag", + "fix": "Added guard clause: if (standby_tag < committed_tag)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-295", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T16:57:59.213Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_role.c", + "root_cause": "Had \"test_evaluate_repl_action_stream_wal_from_zero: P", + "fix": "Changed to \"test_evaluate_repl_action_send_snapshot_lsn_zero:", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-296", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T16:58:16.627Z", + "error_message": "Wrong reference: test_evaluate_repl_action_stream_wal_from_zero should be test_evaluate_repl_action_send_snapshot_lsn_zero", + "file": "src/controller/testsuite/test_ha_role.c", + "root_cause": "Used \"test_evaluate_repl_action_stream_wal_from_zero\" instead of \"test_evaluate_repl_action_send_snapshot_lsn_zero\"", + "fix": "Changed test_evaluate_repl_action_stream_wal_from_zero → test_evaluate_repl_action_send_snapshot_lsn_zero", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-297", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T17:08:18.571Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES", + "fix": "Added guard clause: if (!ha || peer_idx < 0 || peer_idx >= MAX_H)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-301", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T23:09:43.701Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: g_snaptest_chunk_count >= 16", + "fix": "Added guard clause: if (g_snaptest_chunk_count >= 16)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-302", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-04T23:10:44.426Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !snap_dir", + "fix": "Added guard clause: if (!snap_dir)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-308", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-05T01:19:20.356Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !d", + "fix": "Added guard clause: if (!d)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-309", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-05T01:33:20.146Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !p || !snap_dir || !chunk", + "fix": "Added guard clause: if (!ha || !p || !snap_dir || !chunk)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-310", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-05T01:34:06.660Z", + "error_message": "Missing guard clause", + "file": "src/controller/cluster.c", + "root_cause": "No early return/throw for edge case: !chunk", + "fix": "Added guard clause: if (!chunk)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-311", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-05T02:21:15.033Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ua > ub", + "fix": "Added guard clause: if (ua > ub)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-312", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-05T02:47:32.237Z", + "error_message": "Wrong reference: uintptr_t should be void", + "file": "src/controller/testsuite/test_ha_evaluate_committing.c", + "root_cause": "Used \"uintptr_t\" instead of \"void\"", + "fix": "Changed uintptr_t → void", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-314", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-05T04:01:58.721Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha", + "fix": "Added guard clause: if (!ha)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-315", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-05T04:52:32.206Z", + "error_message": "Wrong reference: 10000 should be 60000", + "file": "src/controller/ha.h", + "root_cause": "Used \"10000\" instead of \"60000\"", + "fix": "Changed 10000 → 60000", + "tags": [ + "auto-detected", + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-317", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-05T04:53:55.034Z", + "error_message": "Wrong reference: 10000 should be HA_DEFAULT_COMMITTING_TIMEOUT_MS", + "file": "src/controller/testsuite/test_ha_evaluate_committing.c", + "root_cause": "Used \"10000\" instead of \"HA_DEFAULT_COMMITTING_TIMEOUT_MS\"", + "fix": "Changed 10000 → HA_DEFAULT_COMMITTING_TIMEOUT_MS", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-326", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-06T11:06:46.509Z", + "error_message": "Wrong reference: DEBUG2 should be INFO", + "file": "src/controller/ha.c", + "root_cause": "Used \"DEBUG2\" instead of \"INFO\"", + "fix": "Changed DEBUG2 → INFO", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-327", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-06T12:11:21.422Z", + "error_message": "Incorrect value in code", + "file": "src/controller/ha.c", + "root_cause": "Had \"HA: heartbeat tick (role=%d, n_standbys=%d)\"", + "fix": "Changed to \"HA: primary heartbeat armed\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-328", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-06T13:02:49.052Z", + "error_message": "Missing guard clause", + "file": "src/lib/delay.c", + "root_cause": "No early return/throw for edge case: t1_is_ms && t2_is_ms", + "fix": "Added guard clause: if (t1_is_ms && t2_is_ms)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-329", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-08T13:12:54.436Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-library/research.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-334", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-09T12:56:11.095Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (my_role == CTRL_STANDBY && ha_heartbeat_is_fresh(h)", + "fix": "Changed to: if (my_role == CTRL_STANDBY && ha->primary_ctrl_fd >= )", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-341", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T00:23:52.563Z", + "error_message": "Wrong reference: last_heartbeat should be voted_for_term", + "file": "src/controller/ha.h", + "root_cause": "Used \"last_heartbeat\" instead of \"voted_for_term\"", + "fix": "Changed last_heartbeat → voted_for_term", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-344", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T00:32:34.054Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_bootstrap_progress.md", + "root_cause": "Had `ha->last_heartbeat`", + "fix": "Changed to `ha->last_activity`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-349", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T01:39:19.368Z", + "error_message": "Missing guard clause", + "file": "src/lib/request.c", + "root_cause": "No early return/throw for edge case: l.p.type == CAPN_NULL", + "fix": "Added guard clause: if (l.p.type == CAPN_NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-350", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T01:39:31.683Z", + "error_message": "Missing error handling in unknown", + "file": "src/lib/request.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: if (req->body_which == Request_body_walcatchupentrymsg; && req->body.walcatchupentrymsg != NULL) { | Also: /* wal_entry_to_wire: fill a wire WalCatchupEntry from a C e; * fields and raw payload bytes. Sub-entry nesting is one le", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-358", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T03:55:44.698Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/config_repl.c", + "root_cause": "Condition was: if (!rcpr->raw_bytes || rcpr->raw_bytes_len == 0 || !r)", + "fix": "Changed to: if ((ctrl_role_t)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-359", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T04:16:25.052Z", + "error_message": "Missing guard clause", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "No early return/throw for edge case: r == NULL", + "fix": "Added guard clause: if (r == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-361", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T04:47:09.048Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (opcode == OPCode_requestVote || opcode == OPCode_p)", + "fix": "Changed to: if (opcode == OPCode_requestVote ||\n opcode)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-364", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T06:26:24.340Z", + "error_message": "Incorrect value in code", + "file": "src/controller/CLAUDE.md", + "root_cause": "Had `CTRL_ELECTING`", + "fix": "Changed to `CTRL_INIT`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-367", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T07:56:21.347Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/testsuite/test_repl_unit.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* =========================================================; * Task 1.1: repl_broadcast send path tests", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-368", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T07:57:40.792Z", + "error_message": "Wrong reference: WAL_JOB should be WAL_JOB_SUBMIT", + "file": "src/controller/testsuite/test_repl_unit.c", + "root_cause": "Used \"WAL_JOB\" instead of \"WAL_JOB_SUBMIT\"", + "fix": "Changed WAL_JOB → WAL_JOB_SUBMIT", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-371", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T07:59:24.268Z", + "error_message": "Wrong reference: DEBUG3 should be TRACE", + "file": "src/controller/repl.c", + "root_cause": "Used \"DEBUG3\" instead of \"TRACE\"", + "fix": "Changed DEBUG3 → TRACE", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-373", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T08:45:30.400Z", + "error_message": "Wrong reference: request should be ep", + "file": "src/controller/cluster.c", + "root_cause": "Used \"request\" instead of \"ep\"", + "fix": "Changed request → ep", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-374", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T08:47:48.296Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_ha_role.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 3.3 (ha-wal-replication): dispatch gate standby-role; RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_ro", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-376", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T09:17:35.218Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (opcode == OPCode_nop)", + "fix": "Changed to: if (rep_s.data_which == RequestReply_data_walackreplym)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-380", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T10:37:21.924Z", + "error_message": "Wrong reference: test_catchup_full_lifecycle_integration should be test_single_node_committed_lsn_unchanged", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Used \"test_catchup_full_lifecycle_integration\" instead of \"test_single_node_committed_lsn_unchanged\"", + "fix": "Changed test_catchup_full_lifecycle_integration → test_single_node_committed_lsn_unchanged", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-382", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T11:54:19.749Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-wal-replication/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-384", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T14:13:01.581Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/tech.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-386", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T14:43:53.161Z", + "error_message": "Function not marked async", + "file": "CLAUDE.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-388", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-10T14:54:18.503Z", + "error_message": "Function not marked async", + "file": "src/controller/CLAUDE.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-389", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T02:46:08.749Z", + "error_message": "Wrong reference: signal should be stdio", + "file": "src/lib/delay.c", + "root_cause": "Used \"signal\" instead of \"stdio\"", + "fix": "Changed signal → stdio", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-391", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T03:41:14.912Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha", + "fix": "Added guard clause: if (!ha)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-392", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T03:46:02.642Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: sync_rc_out", + "fix": "Added guard clause: if (sync_rc_out)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-394", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T05:09:37.037Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had `persist_truncate_wal_above`", + "fix": "Changed to `restore_job`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-398", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T06:35:26.288Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had `restore_job`", + "fix": "Changed to \"irrevocable\"", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-399", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T06:43:14.709Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had 't span syscalls and degrade to serial mode), C ec", + "fix": "Changed to 't span syscalls), C ecosystem dead (libitm unmain", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-400", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T07:23:53.901Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had \"return quickly\"", + "fix": "Changed to `--vcpus`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-401", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T07:47:34.756Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had `mark_job_*`", + "fix": "Changed to \"pending facts\"", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-403", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T08:34:47.868Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-consistency-invariants/design.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-405", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T09:47:16.750Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-single-leader-role/design.md", + "root_cause": "Had `writable_lsn`", + "fix": "Changed to 's \"", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-406", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T09:49:04.296Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-single-leader-role/tasks.md", + "root_cause": "Had \" --exclude-dir=.git --exclude-dir=.kiro --exclude", + "fix": "Changed to \" --exclude-dir=.git --exclude-dir=specs --exclude", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-407", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T10:01:15.181Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-barrier-rollout/design.md", + "root_cause": "Had `repl.h`", + "fix": "Changed to `entry_lsn`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-408", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T10:01:39.424Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-convergence-recovery/tasks.md", + "root_cause": "Had `-XCE_ERR_NOT_READY`", + "fix": "Changed to `OPCode_notify`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-413", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T15:28:35.990Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_engine.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ═════════════════════════════════════════════════════════; * Task 3.1 — membership re-evaluation (commit_on_membership_ | Also: /* Task 3.1 — membership re-evaluation */; RUN_TEST(test_catchup_standby_ack_does_not_count_toward_pend", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-420", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T16:19:51.195Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_engine.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 4.2 — wired gate: registry truth at ha_dispatch_gate; RUN_TEST(test_gate_wired_registry_catchup_standby_rejects_th", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-421", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T16:23:46.118Z", + "error_message": "Missing guard clause", + "file": "src/controller/repl.c", + "root_cause": "No early return/throw for edge case: !r", + "fix": "Added guard clause: if (!r)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-423", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T22:39:24.704Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_engine.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ═════════════════════════════════════════════════════════; * Task 5.1 — integration: ACK feed and the job-submit sync-a | Also: #include \"typemap.h\" /* wal_catchup_entry_t (task 5; #include \"wal_types.h\" /* WAL_JOB_SUBMIT (task 5.1 sy", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-424", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T23:08:49.909Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-engine/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-425", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T23:15:25.014Z", + "error_message": "Wrong return value", + "file": "src/controller/repl.c", + "root_cause": "Was returning: idx;", + "fix": "Now returns: * above mutates nothing, so it deliberately skips ", + "tags": [ + "auto-detected", + "return-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-428", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T23:43:45.216Z", + "error_message": "Incorrect value in code", + "file": "src/include/xce/cluster_def.h", + "root_cause": "Had \"ha.h\"", + "fix": "Changed to \"commit.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-429", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T23:44:16.357Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (ha_is_primary(&c->ha)", + "fix": "Changed to: if (c->commit.p == NULL)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-431", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T23:45:16.518Z", + "error_message": "Incorrect value in code", + "file": "src/controller/persist.c", + "root_cause": "Had \"xce/types.h\"", + "fix": "Changed to \"xce/errnox.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-432", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T23:45:42.031Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-433", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-11T23:45:51.649Z", + "error_message": "Wrong reference: test_single_node_committed_lsn_unchanged should be test_single_node_committed_lsn_advances", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Used \"test_single_node_committed_lsn_unchanged\" instead of \"test_single_node_committed_lsn_advances\"", + "fix": "Changed test_single_node_committed_lsn_unchanged → test_single_node_committed_lsn_advances", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-434", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-12T00:23:08.815Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-437", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-12T00:51:52.673Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-engine/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-438", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-12T03:52:57.142Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'testsuite/test_commit_barrier_rollout.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-440", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-12T04:18:19.158Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: detail == HOOK_DETAIL_WORKER_JOIN ||\n detail == H", + "fix": "Added guard clause: if (detail == HOOK_DETAIL_WORKER_JOIN ||\n )", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-442", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-12T04:50:32.154Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_commit_barrier_rollout.c", + "root_cause": "Had \"xce/types.h\"", + "fix": "Changed to \"xce/errnox.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-443", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-12T05:14:36.392Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: p == NULL", + "fix": "Added guard clause: if (p == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-452", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-12T15:03:38.962Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_barrier_reply.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-457", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T00:18:17.080Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c->ha.standby_endpoints", + "fix": "Added guard clause: if (!c->ha.standby_endpoints)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-461", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T01:11:49.673Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/config_repl.c", + "root_cause": "Condition was: if (n == 0)", + "fix": "Changed to: if (n_out)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-467", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T01:38:08.308Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (rep_s.data_which == RequestReply_data_replconfigpu)", + "fix": "Changed to: if (0 /* REVIEWER RED CHECK */ && rep_s.data_which == )", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-468", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T03:56:09.185Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_barrier_reply.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-469", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T06:33:41.449Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_cluster.c'", + "fix": "Changed to 'testsuite/test_ha_role_lifecycle.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-470", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T06:34:01.929Z", + "error_message": "Wrong reference: test_ha_cluster should be test_ha_role_lifecycle", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_ha_cluster\" instead of \"test_ha_role_lifecycle\"", + "fix": "Changed test_ha_cluster → test_ha_role_lifecycle", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-471", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T06:41:01.532Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "No early return/throw for edge case: pid < 0", + "fix": "Added guard clause: if (pid < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-473", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T07:32:43.287Z", + "error_message": "Wrong return value", + "file": "src/controller/persist.c", + "root_cause": "Was returning: WAL_MODE_LOCAL;", + "fix": "Now returns: WAL_MODE_RETAIN;", + "tags": [ + "auto-detected", + "return-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-476", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T08:38:00.471Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: c->ha.hooks_registered", + "fix": "Added guard clause: if (c->ha.hooks_registered)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-480", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T08:57:05.687Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-482", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T09:36:17.806Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_start_election_async.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-483", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T09:45:01.531Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-484", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T10:16:06.331Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-single-leader-role/tasks.md", + "root_cause": "Had `lifecycle_cluster_setup`", + "fix": "Changed to `ha_promote_to_primary`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-493", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T12:08:04.621Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (role == CTRL_PRIMARY || role == CTRL_PRIMARY_COMMI)", + "fix": "Changed to: if (role == CTRL_PRIMARY)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-494", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T12:10:08.938Z", + "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", + "file": "src/controller/ha.c", + "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", + "fix": "Changed CTRL_PRIMARY_COMMITTING → CTRL_PRIMARY", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-496", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T13:07:42.791Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "Had \"winning election must land DIRECTLY in CTRL_PRIMA", + "fix": "Changed to \"winning election must land DIRECTLY in CTRL_PRIMA", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-497", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T13:10:05.452Z", + "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", + "file": "src/controller/testsuite/test_ha_connection_reuse.c", + "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", + "fix": "Changed CTRL_PRIMARY_COMMITTING → CTRL_PRIMARY", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-498", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T13:10:16.164Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_connection_reuse.c", + "root_cause": "Had \"err_handler: ep must be removed from standby_endp", + "fix": "Changed to \"err_handler: ep must be removed from standby_endp", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-500", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T13:14:11.496Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Had \"test_dispatch_gate_committing_allows_repl_ops: PA", + "fix": "Changed to \"test_dispatch_gate_primary_no_quorum_allows_repl_", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-501", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T13:15:33.777Z", + "error_message": "Wrong reference: test_nonblocking_single_node_election_enters_committing should be test_nonblocking_single_node_election_enters_primary_directly", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Used \"test_nonblocking_single_node_election_enters_committing\" instead of \"test_nonblocking_single_node_election_enters_primary_directly\"", + "fix": "Changed test_nonblocking_single_node_election_enters_committing → test_nonblocking_single_node_election_enters_primary_directly", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-505", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T14:01:32.553Z", + "error_message": "Wrong reference: uint32_t should be ha_config_t", + "file": "src/controller/ha.h", + "root_cause": "Used \"uint32_t\" instead of \"ha_config_t\"", + "fix": "Changed uint32_t → ha_config_t", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-507", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T14:43:54.194Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/ha-async-io.md", + "root_cause": "Had `CTRL_PRIMARY_COMMITTING → CTRL_PRIMARY`", + "fix": "Changed to 's own-term commit rule (§5.4.2): an election winn", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-508", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T14:54:21.781Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-deploy-ha/SKILL.md", + "root_cause": "Had `→ CTRL_PRIMARY_COMMITTING`", + "fix": "Changed to `HA FSM (raft): → CTRL_PRIMARY (term=...)`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-509", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-13T15:31:21.740Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_consistency_invariants.md", + "root_cause": "Had \"XCE HA Raft+CP umbrella — sub-specs 1&2 of 4 IMPL", + "fix": "Changed to \"XCE HA Raft+CP umbrella — sub-specs 1,2,3 of 4 IM", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-510", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T00:30:31.077Z", + "error_message": "Wrong reference: stdint should be stdbool", + "file": "src/controller/include/cluster.h", + "root_cause": "Used \"stdint\" instead of \"stdbool\"", + "fix": "Changed stdint → stdbool", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-512", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T00:34:38.745Z", + "error_message": "Wrong reference: xlist_destroy should be xlist_fini", + "file": "src/controller/testsuite/test_egress_filter.c", + "root_cause": "Used \"xlist_destroy\" instead of \"xlist_fini\"", + "fix": "Changed xlist_destroy → xlist_fini", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-513", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T00:35:13.711Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'testsuite/test_egress_filter.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-514", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T00:37:31.926Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_egress_filter.c", + "root_cause": "No early return/throw for edge case: iter_make(&it, c->jobmgr", + "fix": "Added guard clause: if (iter_make(&it, c->jobmgr)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-516", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T01:46:09.600Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-518", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T01:50:24.429Z", + "error_message": "Missing guard clause", + "file": "src/controller/sched.c", + "root_cause": "No early return/throw for edge case: l->count > 0", + "fix": "Added guard clause: if (l->count > 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-519", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T01:52:56.833Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_egress_filter.c", + "root_cause": "Had \"\\n>> commit-gated scheduler egress tests (RED, ta", + "fix": "Changed to \"\\n>> commit-gated scheduler egress tests (GREEN, ", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-521", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T03:55:55.380Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_demotion_convergence.c", + "root_cause": "Had \"commit.h\"", + "fix": "Changed to \"cluster.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-523", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T04:21:18.071Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha_convergence.c", + "root_cause": "No early return/throw for edge case: ctx == NULL || ctx->c == NULL", + "fix": "Added guard clause: if (ctx == NULL || ctx->c == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-526", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T04:52:58.813Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-convergence-recovery/tasks.md", + "root_cause": "Had \"No space left on device\"", + "fix": "Changed to `/tmp/xce_*`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-527", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T04:57:22.178Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /*; * ha_walack_continuation_for_test: test-only entry point for", + "tags": [ + "auto-detected", + "error-handling", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-532", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T06:12:36.513Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha_convergence.c", + "root_cause": "No early return/throw for edge case: ha == NULL", + "fix": "Added guard clause: if (ha == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-533", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T06:37:35.904Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_convergence_verification.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: printf(\"\\n>> Regression pins: fan-out / snapshot fallback / ; \"re-inclusion (task 6.1 — PASS against current code)\\n\\n\"); | Also: /* ── Fan-out counting send-hook (task 6.1 (a)/(d), umbrella; * repl_broadcast routes each enqueue through repl_broadcast_", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-534", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T06:38:38.792Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_convergence_verification.c", + "root_cause": "Had \"delay.h\"", + "fix": "Changed to \"controller.capnp.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-536", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T07:56:01.819Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_rdhup_integration.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-537", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T08:02:40.832Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-542", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T09:29:31.490Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'testsuite/test_coldstart_lsn0.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-550", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T11:38:09.614Z", + "error_message": "Wrong reference: test_pin_dispatch_gate_degraded_allowlist should be test_empty_standby_fresh_cluster_streams_not_snapshot", + "file": "src/controller/testsuite/test_convergence_verification.c", + "root_cause": "Used \"test_pin_dispatch_gate_degraded_allowlist\" instead of \"test_empty_standby_fresh_cluster_streams_not_snapshot\"", + "fix": "Changed test_pin_dispatch_gate_degraded_allowlist → test_empty_standby_fresh_cluster_streams_not_snapshot", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-551", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T11:45:02.411Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (standby_lsn < oldest_wal_lsn)", + "fix": "Changed to: if (standby_lsn + 1 < oldest_wal_lsn)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-555", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T12:23:52.351Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_repl_apply.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-556", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T12:25:37.723Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-557", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T12:26:04.137Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-560", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-14T19:54:55.428Z", + "error_message": "Wrong reference: myconf_t should be uint32_t", + "file": "src/include/myconf.h", + "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", + "fix": "Changed myconf_t → uint32_t", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-570", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-15T11:10:10.553Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/product.md", + "root_cause": "Had `raft.c`", + "fix": "Changed to `src/lib/raft/`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-572", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-15T11:10:26.312Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/tech.md", + "root_cause": "Had `raft.c`", + "fix": "Changed to `src/lib/raft/`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-575", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T00:10:49.295Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-prevote/research.md", + "root_cause": "Had 't understand PreVote (older binary) — does the ca", + "fix": "Changed to 's \"", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-576", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T00:18:28.793Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-prevote/research.md", + "root_cause": "Had 's \"", + "fix": "Changed to `ha_heartbeat_is_fresh()`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-578", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T03:00:48.818Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-prevote/design.md", + "root_cause": "Had `ha_elect_ctx_t`", + "fix": "Changed to `raft_on_vote_reply`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 9, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-579", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T03:01:07.659Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/ha-prevote/design.md", + "root_cause": "Was returning: codes (e.g. `RAFT_PRE_VOTE_WON`, `RAFT_PRE_VOTE_LO", + "fix": "Now returns: codes (e.g. `RAFT_PRE_VOTE_WON`, `RAFT_PRE_VOTE_LO", + "tags": [ + "auto-detected", + "return-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-584", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T04:27:50.039Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->cbs->on_trace", + "fix": "Added guard clause: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-585", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T04:36:28.796Z", + "error_message": "Missing guard clause", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "No early return/throw for edge case: rc != 0", + "fix": "Added guard clause: if (rc != 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-586", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T04:50:38.855Z", + "error_message": "Missing error handling in struct", + "file": "src/controller/testsuite/test_ha_vote_gate.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 3.1 — voter-side PreVote dispatch gate (stickiness +; * no mutation, reply term unchanged) */", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-591", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T05:27:09.380Z", + "error_message": "Wrong reference: test_commit_engine should be test_ha_prevote", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_prevote\"", + "fix": "Changed test_commit_engine → test_ha_prevote", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-592", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T05:28:01.236Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c || !c->myconf", + "fix": "Added guard clause: if (!c || !c->myconf)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-593", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T05:44:57.430Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-prevote/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-594", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T05:50:25.729Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_commit_engine.c'", + "fix": "Changed to 'testsuite/test_ha_failover_prevote.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-595", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T05:50:35.123Z", + "error_message": "Wrong reference: test_commit_engine should be test_ha_failover_prevote", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_failover_prevote\"", + "fix": "Changed test_commit_engine → test_ha_failover_prevote", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-597", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T05:59:43.939Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-598", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T06:33:05.644Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: role == CTRL_STANDBY && ha->raft.pre_vote_in_progress", + "fix": "Added guard clause: if (role == CTRL_STANDBY && ha->raft.pre_vot)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-603", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T08:18:48.567Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-613", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T13:15:11.048Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-614", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T13:18:29.113Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Had `xce_ctx_call`", + "fix": "Changed to `xcecli_retry_next`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-617", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T23:38:24.000Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Had `redirect_host/port`", + "fix": "Changed to `host/port`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-619", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-16T23:40:22.003Z", + "error_message": "Wrong reference: address should be hint", + "file": ".kiro/specs/libxce-failover/tasks.md", + "root_cause": "Used \"address\" instead of \"hint\"", + "fix": "Changed address → hint", + "tags": [ + "auto-detected", + "wrong-reference", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-623", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T00:04:51.775Z", + "error_message": "Wrong reference: XCE_MAXHOST_LEN should be XCE_MAXHOSTNAME_LEN", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Used \"XCE_MAXHOST_LEN\" instead of \"XCE_MAXHOSTNAME_LEN\"", + "fix": "Changed XCE_MAXHOST_LEN → XCE_MAXHOSTNAME_LEN", + "tags": [ + "wrong-reference", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-624", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T02:35:29.695Z", + "error_message": "Incorrect value in code", + "file": "src/lib/meson.build", + "root_cause": "Had 'dataq.c'", + "fix": "Changed to 'client_failover.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-626", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T02:48:27.547Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: ac == NULL || host_out == NULL || host_sz == 0 || port_out =", + "fix": "Added guard clause: if (ac == NULL || host_out == NULL || host_s)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-629", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T03:18:22.365Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: base >= (double", + "fix": "Added guard clause: if (base >= (double)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-631", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T03:31:29.120Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: ac == NULL", + "fix": "Added guard clause: if (ac == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-633", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T03:40:10.853Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: ac == NULL", + "fix": "Added guard clause: if (ac == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-635", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T03:51:08.767Z", + "error_message": "Wrong return value", + "file": "src/lib/client.c", + "root_cause": "Was returning: xce_reslist(&(ctx->ctx), res);", + "fix": "Now returns: XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,", + "tags": [ + "auto-detected", + "return-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 8, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-637", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T03:57:33.232Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Had `nop`", + "fix": "Changed to `jobinfo`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-638", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T04:02:05.572Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: ac == NULL", + "fix": "Added guard clause: if (ac == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-639", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T04:11:59.759Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_p1_integration.c'", + "fix": "Changed to 'testsuite/test_join_redirect.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-640", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T04:12:01.203Z", + "error_message": "Wrong reference: test_p1_integration should be test_join_redirect", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_p1_integration\" instead of \"test_join_redirect\"", + "fix": "Changed test_p1_integration → test_join_redirect", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-641", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T04:19:58.697Z", + "error_message": "Wrong reference: test_recover_resets_err_state_to_run should be test_decide_jitter_staggers_many_clients", + "file": "src/lib/testsuite/test_libxce_retry.c", + "root_cause": "Used \"test_recover_resets_err_state_to_run\" instead of \"test_decide_jitter_staggers_many_clients\"", + "fix": "Changed test_recover_resets_err_state_to_run → test_decide_jitter_staggers_many_clients", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-644", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T08:15:04.978Z", + "error_message": "Incorrect value in code", + "file": "src/lib/meson.build", + "root_cause": "Had 'testsuite/test_raft.c'", + "fix": "Changed to 'testsuite/test_netio_arm_failure.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-647", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T08:21:56.735Z", + "error_message": "Wrong reference: 611 should be 646", + "file": "src/lib/netio.c", + "root_cause": "Used \"611\" instead of \"646\"", + "fix": "Changed 611 → 646", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-651", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T10:44:37.350Z", + "error_message": "Wrong return value", + "file": "src/lib/request.c", + "root_cause": "Was returning: 0;", + "fix": "Now returns: (enq_rc < 0) ? enq_rc : 0;", + "tags": [ + "auto-detected", + "return-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-654", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T11:21:51.571Z", + "error_message": "Wrong return value", + "file": "src/lib/testsuite/test_netio_arm_failure.c", + "root_cause": "Was returning: a negative error and (2) invoke the endpoint's err", + "fix": "Now returns: a negative, RETRIABLE error so the caller propagat", + "tags": [ + "auto-detected", + "return-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-656", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T11:22:17.916Z", + "error_message": "Wrong reference: test_enq_arm_failure_is_loud_and_fires_errhandler should be test_enq_arm_failure_is_loud_no_errhandler", + "file": "src/lib/testsuite/test_netio_arm_failure.c", + "root_cause": "Used \"test_enq_arm_failure_is_loud_and_fires_errhandler\" instead of \"test_enq_arm_failure_is_loud_no_errhandler\"", + "fix": "Changed test_enq_arm_failure_is_loud_and_fires_errhandler → test_enq_arm_failure_is_loud_no_errhandler", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-658", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T14:46:54.135Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)", + "fix": "Changed to: if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-659", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T14:47:25.446Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (role == CTRL_STANDBY &&\n )", + "fix": "Changed to: if (role == CTRL_STANDBY &&\n )", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-662", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T23:34:32.115Z", + "error_message": "Function not marked async", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-664", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-17T23:59:44.577Z", + "error_message": "Incorrect value in code", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Had \"${2:-6}\"", + "fix": "Changed to \"${2:-8}\"", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-666", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T02:32:07.670Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-durability/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-667", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T02:35:08.157Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-668", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T03:12:26.259Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had `persist.c:515`", + "fix": "Changed to `fsync_worker`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-671", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T03:15:13.287Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `fsync_worker`", + "fix": "Changed to `/kiro-validate-design`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-673", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T03:50:26.761Z", + "error_message": "Function not marked async", + "file": "src/controller/CLAUDE.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-676", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T04:00:06.872Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had `src/include/persist.h`", + "fix": "Changed to `persist_wal_sync_append`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-677", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T04:39:24.460Z", + "error_message": "Wrong reference: myconf_t should be uint32_t", + "file": "src/include/myconf.h", + "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", + "fix": "Changed myconf_t → uint32_t", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-678", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T04:50:36.884Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'testsuite/test_commit_durability.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-679", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T04:51:30.430Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-682", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T05:02:07.556Z", + "error_message": "Incorrect value in code", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Had \"$(mktemp /tmp/xce-durability-acked.XXXXXX)\"", + "fix": "Changed to \"$(mktemp \"", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-684", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T05:28:05.352Z", + "error_message": "Function not marked async", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-686", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T05:43:02.761Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-688", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T05:54:46.908Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-690", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T05:55:33.474Z", + "error_message": "Wrong reference: else should be RUN_TEST", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Used \"else\" instead of \"RUN_TEST\"", + "fix": "Changed else → RUN_TEST", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-692", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T05:59:17.869Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-694", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:09:08.298Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", + "root_cause": "Had `/kiro-spec-init ha-commit-durability`", + "fix": "Changed to 't fence the stale node because the entries were d", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-695", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:12:59.924Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-696", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:13:32.891Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-697", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:13:40.659Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-699", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:14:16.158Z", + "error_message": "Function not marked async", + "file": "src/controller/commit.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-701", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:15:45.248Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Had \"config.h\"", + "fix": "Changed to \"commit.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-705", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:33:12.523Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-707", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:34:06.875Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: p", + "fix": "Added guard clause: if (p)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-708", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:34:36.356Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-711", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T06:50:49.987Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: written_lsn > durable", + "fix": "Added guard clause: if (written_lsn > durable)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-714", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T07:00:36.008Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: written_lsn > durable", + "fix": "Added guard clause: if (written_lsn > durable)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-720", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T07:52:20.595Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* task 3.2 — Boot Policy Decision Table skeletons (IGNORED,; * 3.3/4.3). Registered so each row is reported as IGNORED, n", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-721", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T08:04:18.857Z", + "error_message": "Missing error handling in enum", + "file": "src/include/persist.h", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Bounded catch-up window (ha-commit-durability task 3.3, R; * The FINITE deadline a recovery-truncated / behind member i", + "tags": [ + "auto-detected", + "error-handling", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-726", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T08:19:47.036Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-728", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T08:39:54.889Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had `durable_floor ≤ applied_lsn`", + "fix": "Changed to `applied_lsn`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-732", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T08:51:34.635Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_vote_gate.c", + "root_cause": "No early return/throw for edge case: !mkdtemp(tmp", + "fix": "Added guard clause: if (!mkdtemp(tmp)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-733", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T08:52:12.840Z", + "error_message": "Wrong return value", + "file": "src/controller/ha.c", + "root_cause": "Was returning: persist_durable_floor(p);", + "fix": "Now returns: floor > 8ULL ? floor : 8ULL; /* BUG: pretends lear", + "tags": [ + "auto-detected", + "return-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-735", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T09:08:09.215Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ── ha-commit-durability task 4.3: pre-promotion eligibili; * Refuse promotion BEFORE the node becomes authoritative CTR", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-736", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T09:09:28.348Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fr; * member is not behind anything and must be promotable (no f", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-738", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T09:45:40.163Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-739", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T09:46:18.019Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-740", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T09:46:46.814Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-741", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T09:49:58.840Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Test-injectable send hook (defined in repl.c under XCE_TE; * repl_broadcast from dereferencing the dummy endpoints as r", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-743", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T10:25:02.673Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ha_cancel_promotion_fence_timer: cancel a pending bounded; * (idempotent; no-op when none is armed). Called when the m", + "tags": [ + "auto-detected", + "error-handling", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-745", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T10:26:34.227Z", + "error_message": "Wrong operator: != should be ==", + "file": "src/controller/ha.c", + "root_cause": "Used \"!=\" instead of \"==\"", + "fix": "Changed operator != → ==", + "tags": [ + "auto-detected", + "operator-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-746", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T10:30:18.073Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* task 4.4 — bounded-window catch-up before FATAL (no synch; * deferred-timer FATAL only after a real bounded failure; fe", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-755", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T10:55:31.898Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had \"bounded window\"", + "fix": "Changed to `DEFAULT_RECOVERY_CATCHUP_WINDOW_MS`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-759", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T11:21:44.844Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", + "root_cause": "Had `--wal-sync-ms`", + "fix": "Changed to `:FAIL:`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-761", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T13:13:49.968Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_barrier_rollout.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-762", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T13:18:43.638Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-763", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-18T14:54:25.478Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-766", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-19T02:55:49.212Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", + "root_cause": "Was returning: `-ENOENT` if job not found", + "fix": "Now returns: `-ENOENT` if job not found. _Remediation (bug-765,", + "tags": [ + "auto-detected", + "return-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-768", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-19T03:30:28.056Z", + "error_message": "Wrong condition in logic", + "file": "src/scheduler/src/schedobj.c", + "root_cause": "Condition was: if (job->rusages != NULL && job->rusages != rusages)", + "fix": "Changed to: if (job->rusages != rusages)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-772", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-19T03:49:23.624Z", + "error_message": "Wrong reference: xce_sched_op_name should be sched_op_name", + "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", + "root_cause": "Used \"xce_sched_op_name\" instead of \"sched_op_name\"", + "fix": "Changed xce_sched_op_name → sched_op_name", + "tags": [ + "wrong-reference", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-773", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-19T03:49:39.373Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", + "root_cause": "Had `xce_sched_op_name()`", + "fix": "Changed to `sched_op_name()`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-774", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-19T03:50:12.460Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", + "root_cause": "Had `xce_sched_op_name()`", + "fix": "Changed to `sched_op_name()`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-781", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-19T14:02:49.235Z", + "error_message": "Wrong reference: hdr should be payload_len", + "file": "src/controller/persist.c", + "root_cause": "Used \"hdr\" instead of \"payload_len\"", + "fix": "Changed hdr → payload_len", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-785", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-19T14:08:12.866Z", + "error_message": "Wrong reference: NULL should be type", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "Used \"NULL\" instead of \"type\"", + "fix": "Changed NULL → type", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-788", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-19T15:34:41.623Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/testsuite/test.c", + "root_cause": "Condition was: if (active_ctx->parent != NULL)", + "fix": "Changed to: if (i > 1)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-792", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-19T16:13:02.715Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test.c", + "root_cause": "Had \"shutdown\"", + "fix": "Changed to \"expect-jobs\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-796", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T00:32:43.144Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/spec.json", + "root_cause": "Had \"2026-06-18T04:00:00Z\"", + "fix": "Changed to \"2026-06-19T00:00:00Z\"", + "tags": [ + "auto-detected", + "wrong-value", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-799", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T01:05:25.091Z", + "error_message": "Incorrect value in code", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Had \"$label: DATA LOSS — a committed job vanished (bug", + "fix": "Changed to \"$label: DATA LOSS — a committed job vanished from", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-804", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T03:29:22.678Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-lsid/requirements.md", + "root_cause": "Had \"none\"", + "fix": "Changed to \"unknown\"", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-805", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T03:29:29.570Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/controller-lsid/requirements.md", + "root_cause": "Was returning: a well-defined \"unknown\" or \"none\" value for that ", + "fix": "Now returns: a well-defined \"unknown\" value for that field rath", + "tags": [ + "auto-detected", + "return-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-808", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T04:08:49.684Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-status/design.md", + "root_cause": "Had `committed_config_tag`", + "fix": "Changed to `committedConfigVersion`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-809", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T04:40:36.648Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'ctrlstatus.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-810", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T04:45:48.985Z", + "error_message": "Wrong reference: Data should be Text", + "file": "src/schema/controller.capnp", + "root_cause": "Used \"Data\" instead of \"Text\"", + "fix": "Changed Data → Text", + "tags": [ + "auto-detected", + "wrong-reference", + "capnp", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-811", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T04:45:49.790Z", + "error_message": "Wrong reference: xce_data_t should be char", + "file": "src/include/typemap.h", + "root_cause": "Used \"xce_data_t\" instead of \"char\"", + "fix": "Changed xce_data_t → char", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-812", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T05:08:41.030Z", + "error_message": "Incorrect value in code", + "file": "src/lib/meson.build", + "root_cause": "Had 'raft/raft_sim.c'", + "fix": "Changed to 'testsuite/test_ctrlstatus.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-813", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T05:42:31.707Z", + "error_message": "Wrong operator: >= should be ==", + "file": "src/controller/testsuite/test.c", + "root_cause": "Used \">=\" instead of \"==\"", + "fix": "Changed operator >= → ==", + "tags": [ + "auto-detected", + "operator-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-818", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T06:11:49.084Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/testsuite/test.c", + "root_cause": "Condition was: if (!g_continue_on_error)", + "fix": "Changed to: if (!g_continue_on_error && !g_interactive)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-820", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T06:16:34.685Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/xce-test-e2e-primitives/tasks.md", + "root_cause": "Had `g_interactive = isatty(STDIN_FILENO)`", + "fix": "Changed to `-e`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-821", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T08:14:43.577Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-status/requirements.md", + "root_cause": "Had 's identity and role, the configured controller to", + "fix": "Changed to 's identity and role, the configured controller to", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-824", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T09:30:23.040Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/controller-status/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-825", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T09:31:24.076Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/controller-status/spec.json", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-829", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T11:44:25.496Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'ctrlstatus.c'", + "fix": "Changed to 'testsuite/test_ha_heartbeat_committed.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-830", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T11:45:26.943Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-831", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T11:54:30.060Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_heartbeat_committed.c", + "root_cause": "Had \"xce/cluster_def.h\"", + "fix": "Changed to \"typemap.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-832", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T11:56:28.240Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c || !hb", + "fix": "Added guard clause: if (!c || !hb)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-834", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T12:22:39.011Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", + "root_cause": "Had `heartbeatmsg`", + "fix": "Changed to `xce-build.sh controller`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-835", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T12:26:15.999Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'ctrlstatus.c'", + "fix": "Changed to 'testsuite/test_ha_vote_floor_heartbeat.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-836", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T12:31:51.359Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-837", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T12:32:25.549Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-heartbeat-committed-propagation/spec.json", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-840", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T14:05:49.912Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test.c", + "root_cause": "Had \"xce/cluster.h\"", + "fix": "Changed to \"waldump.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-841", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T14:06:11.393Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/testsuite/test.c", + "root_cause": "Condition was: if (active_ctx->parent != NULL)", + "fix": "Changed to: if (argc < 2)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-843", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T14:15:35.599Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `!`", + "fix": "Changed to `xce_test`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-853", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-20T15:27:41.698Z", + "error_message": "Wrong reference: test_wal_batch_entry_without_payload should be test_wal_batch_roundtrip_large_payloads", + "file": "src/lib/testsuite/test_wal_wire.c", + "root_cause": "Used \"test_wal_batch_entry_without_payload\" instead of \"test_wal_batch_roundtrip_large_payloads\"", + "fix": "Changed test_wal_batch_entry_without_payload → test_wal_batch_roundtrip_large_payloads", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-861", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T00:20:55.974Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_persist_recovery.c", + "root_cause": "Had \"persist.h\"", + "fix": "Changed to \"ha.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-864", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T03:39:07.483Z", + "error_message": "Missing guard clause", + "file": "src/controller/repl.c", + "root_cause": "No early return/throw for edge case: r->standbys[i].ep == ep", + "fix": "Added guard clause: if (r->standbys[i].ep == ep)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-871", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T04:01:07.089Z", + "error_message": "Wrong reference: test_repl_write_rejects_checkpoint should be test_repl_write_rejects_forward_gap", + "file": "src/controller/testsuite/test_repl_apply.c", + "root_cause": "Used \"test_repl_write_rejects_checkpoint\" instead of \"test_repl_write_rejects_forward_gap\"", + "fix": "Changed test_repl_write_rejects_checkpoint → test_repl_write_rejects_forward_gap", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-875", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T10:04:37.876Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/requirements.md", + "root_cause": "Had `OPCode_replConfigPush`", + "fix": "Changed to `snapshot_transfer_s`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-878", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T10:34:30.593Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-879", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T10:59:29.212Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/TLA-gap-analysis.md", + "root_cause": "Had `raft.c`", + "fix": "Changed to 's split architecture, the chosen direction is to ", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-880", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T12:56:07.740Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/requirements.md", + "root_cause": "Had `snapshot_transfer_s`", + "fix": "Changed to `(term, lsn)`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-881", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T13:00:46.210Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/design.md", + "root_cause": "Had `xce_test`", + "fix": "Changed to `OPCode_replConfigPush`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-883", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T13:05:44.284Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/research.md", + "root_cause": "Had `config_repl.c:1857-1944`", + "fix": "Changed to `xzarray_freeze`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-885", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T15:21:59.689Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: index < 1 || index > RAFT_SIM_MAX_LOG", + "fix": "Added guard clause: if (index < 1 || index > RAFT_SIM_MAX_LOG)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-886", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T15:41:42.176Z", + "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_oversize_peers_returns_einval", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_oversize_peers_returns_einval\"", + "fix": "Changed test_raft_init_zero_peers_returns_einval → test_raft_init_oversize_peers_returns_einval", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-887", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T15:45:29.785Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->role != RAFT_LEADER", + "fix": "Added guard clause: if (s->role != RAFT_LEADER)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-888", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T15:52:45.272Z", + "error_message": "Missing guard clause", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "No early return/throw for edge case: index >= 1 && index <= RAFT_SIM_MAX_LOG", + "fix": "Added guard clause: if (index >= 1 && index <= RAFT_SIM_MAX_LOG)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-889", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T15:53:14.276Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->cbs->on_trace", + "fix": "Added guard clause: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-890", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T15:59:59.295Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: rc != 0", + "fix": "Added guard clause: if (rc != 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-891", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T16:06:37.923Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->cbs->on_trace", + "fix": "Added guard clause: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-893", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T16:16:47.908Z", + "error_message": "Wrong reference: g_te_spy should be g_pe_spy", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"g_te_spy\" instead of \"g_pe_spy\"", + "fix": "Changed g_te_spy → g_pe_spy", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-894", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T16:20:39.535Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->cbs->on_trace", + "fix": "Added guard clause: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-895", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T16:29:35.778Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: c->nodes[src].partition || c->nodes[dst].partition", + "fix": "Added guard clause: if (c->nodes[src].partition || c->nodes[dst])", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-899", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T16:42:30.241Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: node_idx >= 0 && node_idx < c->n", + "fix": "Added guard clause: if (node_idx >= 0 && node_idx < c->n)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-900", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T16:51:25.960Z", + "error_message": "Missing error handling in unknown", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: RUN_TEST(test_sim_9_1_replication_and_partition_catch_up);; RUN_TEST(test_sim_9_2_divergent_suffix_truncation);", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-903", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T17:04:17.830Z", + "error_message": "Wrong operator: <= should be >=", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"<=\" instead of \">=\"", + "fix": "Changed operator <= → >=", + "tags": [ + "auto-detected", + "operator-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-904", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T21:44:26.804Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: node_idx < 0 || node_idx >= c->n", + "fix": "Added guard clause: if (node_idx < 0 || node_idx >= c->n)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-905", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T21:49:10.796Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-log-replication/tasks.md", + "root_cause": "Had `initial_log_len`", + "fix": "Changed to `raft_sim_restart_node`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-907", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T23:14:37.837Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->role != RAFT_LEADER", + "fix": "Added guard clause: if (s->role != RAFT_LEADER)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-908", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T23:14:55.547Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: node_idx >= 0 && node_idx < c->n", + "fix": "Added guard clause: if (node_idx >= 0 && node_idx < c->n)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-909", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T23:30:04.636Z", + "error_message": "Missing guard clause", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "No early return/throw for edge case: g_mock.get_entry_term_log_len != 0 &&\n index == g_moc", + "fix": "Added guard clause: if (g_mock.get_entry_term_log_len != 0 &&\n )", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-911", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-21T23:32:38.225Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft.c", + "root_cause": "Condition was: if (s->cbs->on_vote_eligible != NULL)", + "fix": "Changed to: if (!raft_log_ok(s, cand_last_log_term, cand_last_log_)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-913", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T00:19:56.768Z", + "error_message": "Wrong reference: TEST_ASSERT_NOT_EQUAL_INT should be TEST_ASSERT_NOT_EQUAL", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"TEST_ASSERT_NOT_EQUAL_INT\" instead of \"TEST_ASSERT_NOT_EQUAL\"", + "fix": "Changed TEST_ASSERT_NOT_EQUAL_INT → TEST_ASSERT_NOT_EQUAL", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-914", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T01:24:34.541Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/spec.json", + "root_cause": "Had \"design-generated\"", + "fix": "Changed to \"tasks-generated\"", + "tags": [ + "auto-detected", + "wrong-value", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-920", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T03:58:08.538Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_term_safety_adapter.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-921", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T03:58:19.392Z", + "error_message": "Wrong reference: test_waldump should be test_term_safety_adapter", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_waldump\" instead of \"test_term_safety_adapter\"", + "fix": "Changed test_waldump → test_term_safety_adapter", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-922", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T03:59:43.099Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || lsn == 0", + "fix": "Added guard clause: if (!p || lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-923", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T04:01:37.467Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/persist.c", + "root_cause": "Condition was: if (hdr.lsn <= from_lsn)", + "fix": "Changed to: if (hdr.type < WAL_CHECKPOINT)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-925", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T04:42:28.897Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-926", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T04:45:07.018Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", + "fix": "Added guard clause: if (!ha || !ha->raft_c)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-928", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T04:55:58.789Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-929", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T04:57:03.605Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", + "fix": "Added guard clause: if (!ha || !ha->raft_c)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-930", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T04:57:25.141Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-931", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T05:45:09.373Z", + "error_message": "Function not marked async", + "file": "src/include/repl.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-933", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T05:47:50.037Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !d", + "fix": "Added guard clause: if (!d)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-937", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T06:39:51.351Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", + "fix": "Added guard clause: if (!ha || !ha->raft_c)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-941", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T07:53:06.457Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-942", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T07:54:52.956Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-943", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T07:55:30.714Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-944", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T07:56:08.623Z", + "error_message": "Wrong reference: ha_arm_own_term_gate should be ha_arm_own_term_gate_for_test", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Used \"ha_arm_own_term_gate\" instead of \"ha_arm_own_term_gate_for_test\"", + "fix": "Changed ha_arm_own_term_gate → ha_arm_own_term_gate_for_test", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-945", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T08:09:41.508Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-948", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T08:21:57.062Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat ; * library as a 0-entry AppendEntries so the LIBRARY performs", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-953", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T08:48:45.792Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: rc == 0 && out_lsn", + "fix": "Added guard clause: if (rc == 0 && out_lsn)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-957", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T09:08:41.177Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ep_is_vote", + "fix": "Added guard clause: if (ep_is_vote)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-960", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T09:20:00.000Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "ha-term-safety", + "raft", + "peer-index", + "match_index", + "oob", + "false-quorum", + "task-3.3", + "remediation", + "c", + "auto-detected", + "guard-clause", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [ + "bug-851" + ], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-959", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T09:31:11.227Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !sha256 || !buf || !out_len", + "fix": "Added guard clause: if (!sha256 || !buf || !out_len)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-961", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T09:45:31.779Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c || !sha256", + "fix": "Added guard clause: if (!c || !sha256)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-962", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T10:04:06.960Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: committed == 0", + "fix": "Added guard clause: if (committed == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-964", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T10:05:22.007Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: acked_out", + "fix": "Added guard clause: if (acked_out)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-966", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T10:10:11.668Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-967", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T10:10:17.811Z", + "error_message": "Wrong reference: persist_roll_segment should be persist_test_force_new_segment", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Used \"persist_roll_segment\" instead of \"persist_test_force_new_segment\"", + "fix": "Changed persist_roll_segment → persist_test_force_new_segment", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-971", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T10:39:44.072Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: !mc", + "fix": "Added guard clause: if (!mc)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-972", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T10:40:31.131Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-975", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T11:00:42.661Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha", + "fix": "Added guard clause: if (!ha)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-977", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T11:21:09.742Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-980", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T11:32:35.966Z", + "error_message": "Wrong reference: ClusterConfig should be cluster_config_t", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Used \"ClusterConfig\" instead of \"cluster_config_t\"", + "fix": "Changed ClusterConfig → cluster_config_t", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-983", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T11:53:36.582Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: commit_index", + "fix": "Added guard clause: if (commit_index)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-984", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T12:03:15.865Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-985", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T12:06:43.530Z", + "error_message": "Wrong return value", + "file": "src/controller/testsuite/waldump.c", + "root_cause": "Was returning: -EINVAL;", + "fix": "Now returns: waldump_path_visit(path, out, stats, NULL, NULL);", + "tags": [ + "auto-detected", + "return-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-986", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T12:08:51.529Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test.c", + "root_cause": "Had \"waldump.h\"", + "fix": "Changed to \"walcompare.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-987", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T12:09:28.306Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test.c", + "root_cause": "No early return/throw for edge case: st.n_gaps > 0 || st.crc_broken", + "fix": "Added guard clause: if (st.n_gaps > 0 || st.crc_broken)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-993", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T15:10:39.717Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || lsn == 0", + "fix": "Added guard clause: if (!p || lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-998", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T15:27:10.703Z", + "error_message": "Wrong reference: test_repl_ack_currentterm_quorum_advances_commit should be test_live_broadcast_receive_persists_leader_term", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Used \"test_repl_ack_currentterm_quorum_advances_commit\" instead of \"test_live_broadcast_receive_persists_leader_term\"", + "fix": "Changed test_repl_ack_currentterm_quorum_advances_commit → test_live_broadcast_receive_persists_leader_term", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1000", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T15:44:38.466Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1001", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T15:44:54.113Z", + "error_message": "Missing error handling in errno", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - _Blocked (live portion only — build+unit gate CONFIRMED GR", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1002", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T21:10:31.345Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Had 's term and standbys persist it, proven at the int", + "fix": "Changed to 's REAL term (2) for the committed prefix (pre-3.4", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1003", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T22:09:23.064Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft.c", + "root_cause": "Condition was: if (agreed == 0)", + "fix": "Changed to: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1005", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T22:31:03.315Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1006", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T22:32:53.722Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Advance applied_lsn to last_good_lsn after the full batch; * last contiguously-held LSN, NOT last_good_lsn + 1 — see th", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1009", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-22T22:44:50.365Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1025", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T04:09:25.647Z", + "error_message": "Wrong reference: $MASTERS should be $MASTERS_LIVE", + "file": "src/controller/testsuite/clean_failover_preservation.sh", + "root_cause": "Used \"$MASTERS\" instead of \"$MASTERS_LIVE\"", + "fix": "Changed $MASTERS → $MASTERS_LIVE", + "tags": [ + "wrong-reference", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1033", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T08:05:43.916Z", + "error_message": "Incorrect value in code", + "file": "repro854.sh", + "root_cause": "Had 'set masters %s\\nwait-primary 6000\\nctrlstatus\\nqu", + "fix": "Changed to 'set masters %s\\nset myname failover-test\\nclient\\", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1035", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T09:53:37.812Z", + "error_message": "Incorrect value in code", + "file": "src/lib/xcectx.c", + "root_cause": "Had \"xcectx.h\"", + "fix": "Changed to \"controller.capnp.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1036", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T10:44:48.074Z", + "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", + "file": "src/lib/netio.c", + "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", + "fix": "Changed handle_ssl_handshake → endpoint_ssl_handshake", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1037", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T10:48:18.493Z", + "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", + "file": "src/controller/cluster.c", + "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", + "fix": "Changed handle_ssl_handshake → endpoint_ssl_handshake", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1040", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T10:54:14.342Z", + "error_message": "Wrong reference: endpoint_ssl_handshake should be handle_ssl_handshake", + "file": "src/lib/netio.c", + "root_cause": "Used \"endpoint_ssl_handshake\" instead of \"handle_ssl_handshake\"", + "fix": "Changed endpoint_ssl_handshake → handle_ssl_handshake", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1045", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T14:08:05.466Z", + "error_message": "Missing error handling in unknown", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: # ── Steps 7–10 looped: kill primary, verify, submit, restar; #", + "tags": [ + "auto-detected", + "error-handling", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1049", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T14:39:01.881Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (0 && recovered_tip > ha->raft.log_len)", + "fix": "Changed to: if (recovered_tip > ha->raft.log_len)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1051", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T14:48:18.998Z", + "error_message": "Wrong operator: >= should be ==", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Used \">=\" instead of \"==\"", + "fix": "Changed operator >= → ==", + "tags": [ + "auto-detected", + "operator-fix", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1054", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-23T16:02:42.489Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/SKILL.md", + "root_cause": "Had \"End-to-end HA failover soak test for an XCE contr", + "fix": "Changed to \"End-to-end HA failover soak test for an XCE contr", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1066", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T04:36:50.602Z", + "error_message": "Incorrect value in code", + "file": "src/controller/CLAUDE.md", + "root_cause": "Had `persist_truncate_wal_above(safe_committed_lsn)`", + "fix": "Changed to 's own durable log**: the cutoff is `", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1070", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T08:18:27.510Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/config_repl.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: if (c && c->persist.wal_path[0]) {; /* bug-1067: pass the leader's CURRENT term so historical ca", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1075", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T08:23:04.980Z", + "error_message": "Wrong reference: standby_lsn should be leader_term", + "file": "src/controller/testsuite/test_coldstart_lsn0.c", + "root_cause": "Used \"standby_lsn\" instead of \"leader_term\"", + "fix": "Changed standby_lsn → leader_term", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1078", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T09:18:52.605Z", + "error_message": "Wrong reference: handle_reply should be xce_handle_reply", + "file": "src/lib/xcectx.c", + "root_cause": "Used \"handle_reply\" instead of \"xce_handle_reply\"", + "fix": "Changed handle_reply → xce_handle_reply", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1079", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T09:27:59.772Z", + "error_message": "Wrong reference: libxce_failover_client_hang should be project_libxce_failover_hang", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1077_reply_union_type_confusion.md", + "root_cause": "Used \"libxce_failover_client_hang\" instead of \"project_libxce_failover_hang\"", + "fix": "Changed libxce_failover_client_hang → project_libxce_failover_hang", + "tags": [ + "wrong-reference", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1080", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T10:24:07.439Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-sim-conformance/design.md", + "root_cause": "Had \"on_get_entry_term -> node.log[idx].term | 0\"", + "fix": "Changed to \"on_get_entry_term -> node.log[idx].term | 0\"", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1081", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T10:30:45.585Z", + "error_message": "Wrong reference: apply should be advanced", + "file": ".kiro/specs/raft-sim-conformance/design.md", + "root_cause": "Used \"apply\" instead of \"advanced\"", + "fix": "Changed apply → advanced", + "tags": [ + "wrong-reference", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1082", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T10:30:56.237Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-sim-conformance/design.md", + "root_cause": "Had \"vote/AE requests + responses;
Drop/Duplicate/", + "fix": "Changed to \"vote/AE requests + responses,
Drop/Duplicate/", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1083", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T11:45:52.473Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: n > sizeof(v", + "fix": "Added guard clause: if (n > sizeof(v)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1085", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T12:00:01.924Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: i != last", + "fix": "Added guard clause: if (i != last)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1086", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T12:13:27.532Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: i < 0 || i >= c->bag_count", + "fix": "Added guard clause: if (i < 0 || i >= c->bag_count)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1087", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T12:22:22.094Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: sa.role != RAFT_LEADER", + "fix": "Added guard clause: if (sa.role != RAFT_LEADER)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1088", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T12:30:35.436Z", + "error_message": "Wrong reference: test_sim_check_election_safety_fail_on_two_leaders should be test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_sim_check_election_safety_fail_on_two_leaders\" instead of \"test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix\"", + "fix": "Changed test_sim_check_election_safety_fail_on_two_leaders → test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1089", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T12:38:40.360Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: sl.role != RAFT_LEADER", + "fix": "Added guard clause: if (sl.role != RAFT_LEADER)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1090", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T12:47:03.843Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: shared > RAFT_SIM_MAX_LOG", + "fix": "Added guard clause: if (shared > RAFT_SIM_MAX_LOG)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1091", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T13:04:27.057Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: c->trace_len >= RAFT_SIM_MAX_STEPS", + "fix": "Added guard clause: if (c->trace_len >= RAFT_SIM_MAX_STEPS)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1094", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T13:22:05.858Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: c->bag[i].kind != RAFT_SIM_MSG_VOTE_RSP", + "fix": "Added guard clause: if (c->bag[i].kind != RAFT_SIM_MSG_VOTE_RSP)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1099", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T14:08:44.250Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: failing != RAFT_SIM_INV_NONE", + "fix": "Added guard clause: if (failing != RAFT_SIM_INV_NONE)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1104", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T14:28:33.169Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "Condition was: if (c->n >= 2 && c->nodes[0].commit_index >= 1 && c->n)", + "fix": "Changed to: if (c->n >= 2)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1108", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T14:38:06.432Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: chk < 0", + "fix": "Added guard clause: if (chk < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1110", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T14:51:30.855Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: i == leader", + "fix": "Added guard clause: if (i == leader)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1111", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T15:13:15.333Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: p == leader", + "fix": "Added guard clause: if (p == leader)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1113", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T15:16:52.683Z", + "error_message": "Missing error handling in unknown", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /*; * sim_scen_catch_up_commit: drive bounded AppendEntries-to-a | Also: * on a CONNECTED quorum, yet raft_advance_commit must NOT co; * count alone — the current-term Figure-8 rule (raft.c G4) f", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1115", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T15:18:59.248Z", + "error_message": "Wrong operator: != should be ==", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "Used \"!=\" instead of \"==\"", + "fix": "Changed operator != → ==", + "tags": [ + "auto-detected", + "operator-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1116", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T15:25:17.886Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: i == leader", + "fix": "Added guard clause: if (i == leader)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1117", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T15:32:34.085Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft.c", + "root_cause": "Condition was: if (s->cbs->on_get_entry_term(s->ctx, agreed)", + "fix": "Changed to: if (0 && s->cbs->on_get_entry_term(s->ctx, agreed)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1120", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T16:09:04.749Z", + "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_1_fault_switch_mechanism", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_1_fault_switch_mechanism\"", + "fix": "Changed test_sim_6_1_seed_sweep → test_sim_7_1_fault_switch_mechanism", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1123", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T16:22:58.427Z", + "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_2_negative_both_directions", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_2_negative_both_directions\"", + "fix": "Changed test_sim_6_1_seed_sweep → test_sim_7_2_negative_both_directions", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1125", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T16:42:32.261Z", + "error_message": "Wrong reference: test_sim_4_2_step_liveness_and_coverage should be test_sim_8_1_export_trace_named_and_gated", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_sim_4_2_step_liveness_and_coverage\" instead of \"test_sim_8_1_export_trace_named_and_gated\"", + "fix": "Changed test_sim_4_2_step_liveness_and_coverage → test_sim_8_1_export_trace_named_and_gated", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1126", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-24T16:43:39.427Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "Condition was: if (node_idx < 0 || node_idx >= c->n)", + "fix": "Changed to: if (getenv(\"RAFT_SIM_EXPORT\")", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1127", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T03:17:45.530Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_demotion_convergence.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1133", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T07:12:01.309Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-log-replication/spec.json", + "root_cause": "Had \"2026-06-21T11:00:00Z\"", + "fix": "Changed to \"2026-06-25T00:00:00Z\"", + "tags": [ + "auto-detected", + "wrong-value", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1134", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T08:18:43.915Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1135", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T09:26:49.441Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1136", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T09:27:00.312Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-apply-decoupling/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1138", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T11:36:52.942Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1139", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T14:02:51.225Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/research.md", + "root_cause": "Had `VTASK_IO`", + "fix": "Changed to `endpoint_block_tasks()`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1141", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T15:12:16.193Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/research.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1142", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T15:13:00.445Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `.kiro/specs/raft-drive-async/brief.md`", + "fix": "Changed to `delay_task_ms`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1143", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T23:37:38.579Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - Commit-advance under pool load (R4.5): with the vtask pool; - Snapshot/checkpoint concurrency (R1.1/1.3/1.5): concurrent", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1144", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T23:37:49.697Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Had `persist_checkpoint`", + "fix": "Changed to `wait_all_with_timeout`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1145", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-25T23:38:08.746Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1148", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T03:46:51.038Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Had 's boundary or any standby'", + "fix": "Changed to `persist.c:1374`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1149", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T03:48:48.222Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `/kiro-spec-design raft-drive-async`", + "fix": "Changed to `approved: false`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1151", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:31:53.879Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Had `commit_submit`", + "fix": "Changed to `commit.c`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1152", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:32:20.960Z", + "error_message": "Wrong reference: PeerCtrlDispatch should be LivenessAdmission", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Used \"PeerCtrlDispatch\" instead of \"LivenessAdmission\"", + "fix": "Changed PeerCtrlDispatch → LivenessAdmission", + "tags": [ + "wrong-reference", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1153", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:33:10.286Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `COMMIT_WAIT_RESERVED=2`", + "fix": "Changed to `commit.c:16`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1154", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:37:18.052Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `VTASK_LIVENESS_RESERVED`", + "fix": "Changed to `R = clamp(n_peers − 1, 1, n_vcpus − 1)`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1155", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:47:49.937Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_drive_async_config.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1156", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:48:34.246Z", + "error_message": "Function not marked async", + "file": "src/include/config.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1157", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:48:50.005Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1158", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:49:04.568Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1159", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:49:44.821Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1160", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T04:57:57.877Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1161", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T05:02:36.720Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_slowlane.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1162", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T05:06:30.148Z", + "error_message": "Function not marked async", + "file": "src/include/xce/cluster_def.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1163", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T05:06:42.680Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (ha_is_primary(&c->ha)", + "fix": "Changed to: if (c->slowlane.workers == NULL)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1164", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T05:06:53.129Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1165", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T06:51:00.808Z", + "error_message": "Wrong reference: test_request_send_propagation should be test_netio_bounded_connect", + "file": "src/lib/testsuite/test.c", + "root_cause": "Used \"test_request_send_propagation\" instead of \"test_netio_bounded_connect\"", + "fix": "Changed test_request_send_propagation → test_netio_bounded_connect", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1166", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T06:51:57.649Z", + "error_message": "Function not marked async", + "file": "src/lib/netio.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1168", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T06:52:21.608Z", + "error_message": "Function not marked async", + "file": "src/include/netio.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1169", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:16:36.377Z", + "error_message": "Function not marked async", + "file": "src/include/vtaskmgr.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1171", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:18:02.307Z", + "error_message": "Function not marked async", + "file": "src/lib/vtaskmgr.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1172", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:19:14.857Z", + "error_message": "Function not marked async", + "file": "src/controller/include/commit.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1173", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:19:45.059Z", + "error_message": "Function not marked async", + "file": "src/controller/commit.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1174", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:20:07.233Z", + "error_message": "Wrong operator: >= should be !=", + "file": "src/controller/commit.c", + "root_cause": "Used \">=\" instead of \"!=\"", + "fix": "Changed operator >= → !=", + "tags": [ + "auto-detected", + "operator-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1175", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:20:41.448Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1177", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:47:19.731Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1178", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:48:00.209Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1179", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:51:01.491Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: deadline.tv_nsec >= 1000000000L", + "fix": "Added guard clause: if (deadline.tv_nsec >= 1000000000L)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1180", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:51:10.764Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1183", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T07:57:35.484Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1184", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T08:12:52.624Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1185", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T08:13:19.511Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1186", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T08:15:44.315Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx)", + "fix": "Changed to: if (0 && cur_sctx && cur_sctx->peer_idx == sctx->peer_)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1188", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T10:20:33.560Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1189", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T10:21:12.319Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1190", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T10:24:21.473Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1191", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T10:27:00.557Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: atomic_load(&g_pc_gate_on", + "fix": "Added guard clause: if (atomic_load(&g_pc_gate_on)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1192", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T10:27:14.358Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1193", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T10:56:43.461Z", + "error_message": "Function not marked async", + "file": "src/include/vtaskmgr.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1194", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T10:57:18.280Z", + "error_message": "Function not marked async", + "file": "src/lib/vtaskmgr.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1195", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T10:57:44.750Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1196", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T11:00:15.213Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1202", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T11:33:28.328Z", + "error_message": "Missing error handling in task", + "file": "src/controller/testsuite/test_e2e_checkpoint.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Helper: check if a file exists in a directory ; static int file_exists_in(const char* dir, const char* name)", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1204", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T11:35:37.597Z", + "error_message": "Wrong reference: file_exists_for_trim_test should be seg_file_exists", + "file": "src/controller/testsuite/test_e2e_checkpoint.c", + "root_cause": "Used \"file_exists_for_trim_test\" instead of \"seg_file_exists\"", + "fix": "Changed file_exists_for_trim_test → seg_file_exists", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1205", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T11:39:53.357Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1206", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T15:16:40.996Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1207", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T15:28:19.885Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ha_repl_send_snapshot_hook != NULL", + "fix": "Added guard clause: if (ha_repl_send_snapshot_hook != NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1211", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T15:49:46.434Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1212", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T15:50:27.084Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1213", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T15:51:29.747Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ha_configpush_probe != NULL", + "fix": "Added guard clause: if (ha_configpush_probe != NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1214", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T15:54:05.463Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1216", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T16:54:57.384Z", + "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", + "file": "src/include/netio.h", + "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", + "fix": "Changed ssl_handshake_done → tearing_down", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1217", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T16:55:15.332Z", + "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", + "file": "src/lib/netio.c", + "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", + "fix": "Changed ssl_handshake_done → tearing_down", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1218", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T16:55:48.690Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ectx", + "fix": "Added guard clause: if (ectx)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1222", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-26T17:44:25.670Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_ha_dispatch_integration.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1226", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T02:53:39.273Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Had \"$REPO/src/xce.build/controller/xce_controller\"", + "fix": "Changed to \"${XCE_HA_BIN:-$REPO/src/xce.build/controller/xce_", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1227", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T03:04:05.891Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1228", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T04:13:35.943Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/design.md", + "root_cause": "Had `; no silent retention of a stale-term entry.\n- In", + "fix": "Changed to `; no silent retention of a stale-term entry.\n- In", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1233", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T04:59:57.805Z", + "error_message": "Missing error handling in unknown", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: # ── Chaotic-mode primitives (count outcomes into the verdic; # Run `walcompare --report` across all three members and ADD | Also: # ── Chaotic mode: overlapping-kill rounds → isolated verdic; # Opt-in (--chaotic / XCE_HA_MODE=chaotic). Kills land insid", + "tags": [ + "auto-detected", + "error-handling", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1234", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T05:00:06.726Z", + "error_message": "Function not marked async", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1236", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T09:27:45.080Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/design.md", + "root_cause": "Had `raft_on_append_entries_request`", + "fix": "Changed to `research.md`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1237", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T09:28:07.944Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/raft-ha-core-safety/design.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - **Technical debt addressed:** the silent empty-payload shi | Also: - `.claude/skills/xce-test-ha/run-ha-soak.sh` (and `SKILL.md; - `src/controller/` log init (only if the client-probe appro | Also: | Catch-up Payload Integrity | Adapter | Stream a non-empty ; | Persist Conflict Guard | Persist | `(term,lsn,payload-crc) | Also: - The live-repro gate (task 1.3) has **already run and refut; - A green synthetic unit test over a still-red soak is not a", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1243", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T10:02:56.820Z", + "error_message": "Wrong operator: == should be <<", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Used \"==\" instead of \"<<\"", + "fix": "Changed operator == → <<", + "tags": [ + "auto-detected", + "operator-fix", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1244", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T10:04:31.979Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Had \"no primary elected within 45s (check $TOP/logs)\"", + "fix": "Changed to \"no primary elected within ~90s via client probe (", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1246", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T10:07:37.692Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/SKILL.md", + "root_cause": "Had `no primary elected within 45s`", + "fix": "Changed to `no primary elected within ~90s via client probe`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1247", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T10:25:38.366Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || lsn == 0", + "fix": "Added guard clause: if (!p || lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1249", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T10:45:00.686Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/persist.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* (a) non-empty incoming whose payload differs at an existi; * — a genuine payload conflict. Branch on tip vs below-tip.", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1250", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T10:45:34.063Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1252", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T10:47:06.628Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/persist.c", + "root_cause": "Condition was: if (lsn != tip_lsn)", + "fix": "Changed to: if (trc0 != 0)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1254", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T11:14:27.133Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_ha_core_safety.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: * TASK 2.3 — cross-term catch-up streamer payload integrity ; * | Also: /* Task 2.3 cross-term catch-up streamer payload integrity:; * CORE RED — the streamer must never ship a non-empty cro", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1255", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-27T11:28:45.351Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !pbuf", + "fix": "Added guard clause: if (!pbuf)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1264", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-29T04:13:02.187Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_ha_partition.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1266", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-29T04:25:39.111Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_partition.c", + "root_cause": "No early return/throw for edge case: peer_idx == g_partition_peer_idx && g_reconnect_gate", + "fix": "Added guard clause: if (peer_idx == g_partition_peer_idx && g_re)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1269", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T01:47:07.635Z", + "error_message": "Missing await", + "file": "src/controller/ha.c", + "root_cause": "Async call without await — returned Promise instead of value", + "fix": "Added await to async call", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1271", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T02:25:32.055Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - _Depends: 1.5, 6.2, 5.2, 5.3_; - _Blocked: first soak run (HEAD 01419e1, Phase-2+5 fixes) i", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1273", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T02:41:41.017Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_ha_core_safety.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 5.3 lagging-but-safe survivor electability under ove; * CORE RED — a catching-up survivor with NO live primary m", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1275", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T02:55:23.808Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1276", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T02:55:37.693Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `research.md`", + "fix": "Changed to 's persist flusher jams on `", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1279", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T03:33:06.273Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had 's persist flusher jams on `", + "fix": "Changed to `xce_controller`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1280", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T03:57:32.784Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had \"Task 7.2 on ~/tmp\"", + "fix": "Changed to \"Task 7.2/5.4 — UNIFIED root cause\"", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1283", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T04:17:58.579Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `xce_controller`", + "fix": "Changed to `src/lib`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1285", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T08:24:48.860Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `vtask_eligible`", + "fix": "Changed to `endpoint_is_task_blocked`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1290", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T09:15:12.861Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had 't crash) — STILL blocks the gate; likely an endpo", + "fix": "Changed to 't crash) — is now **ROOT-CAUSED** (via the new `", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1292", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T09:33:08.918Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `1+2*njobs`", + "fix": "Changed to `c = ep->owner`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1295", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T13:19:42.706Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: jmax > 0", + "fix": "Added guard clause: if (jmax > 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1297", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T13:28:01.170Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: jmax > 0", + "fix": "Added guard clause: if (jmax > 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1299", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T13:39:09.194Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1304", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T14:24:36.874Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/research.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1308", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T21:22:32.991Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `research.md`", + "fix": "Changed to 't-preempt-in-flight-election; reduced term churn)", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1310", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-06-30T18:40:00Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: p == NULL || jlist == NULL", + "fix": "Added guard clause: if (p == NULL || jlist == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "election-liveness", + "rdhup", + "term-escalation", + "debounce", + "5.10b", + "raft", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [ + "bug-1296", + "bug-1306" + ], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1312", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T00:29:56.872Z", + "error_message": "Wrong reference: test_job_hook_start_writes_wal_entry should be test_job_hook_new_is_notify_only", + "file": "src/controller/testsuite/test_persist_hooks.c", + "root_cause": "Used \"test_job_hook_start_writes_wal_entry\" instead of \"test_job_hook_new_is_notify_only\"", + "fix": "Changed test_job_hook_start_writes_wal_entry → test_job_hook_new_is_notify_only", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1313", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T00:42:13.591Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1315", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T01:26:38.341Z", + "error_message": "Wrong reference: unistd should be time", + "file": "src/controller/testsuite/test_ha_rdhup_integration.c", + "root_cause": "Used \"unistd\" instead of \"time\"", + "fix": "Changed unistd → time", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1319", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T04:04:13.051Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-entity-snapshots/design.md", + "root_cause": "Had \"collect+encode jobs\\n(RTTI_LOCK_READ on jobmgr)\"", + "fix": "Changed to \"collect+encode jobs\\n(table READ lock + per-job R", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1320", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T04:47:19.662Z", + "error_message": "Wrong reference: test_ha_partition should be test_persist_snapshot_entities", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_ha_partition\" instead of \"test_persist_snapshot_entities\"", + "fix": "Changed test_ha_partition → test_persist_snapshot_entities", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1326", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T06:17:52.049Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_persist_snapshot_entities.c", + "root_cause": "No early return/throw for edge case: fd < 0", + "fix": "Added guard clause: if (fd < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1327", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T06:18:39.877Z", + "error_message": "Wrong reference: pthread should be fcntl", + "file": "src/controller/testsuite/test_persist_snapshot_entities.c", + "root_cause": "Used \"pthread\" instead of \"fcntl\"", + "fix": "Changed pthread → fcntl", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1328", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T06:33:25.117Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_persist_snapshot_entities.c", + "root_cause": "No early return/throw for edge case: fd < 0", + "fix": "Added guard clause: if (fd < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1330", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T07:24:22.541Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: cleared == 0", + "fix": "Added guard clause: if (cleared == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1331", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T07:46:42.046Z", + "error_message": "Wrong return value", + "file": "src/controller/persist.c", + "root_cause": "Was returning: load_snapshot(p, path, lsn);", + "fix": "Now returns: rc;", + "tags": [ + "auto-detected", + "return-value", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1340", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T09:55:17.967Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || checkpoint_lsn == 0", + "fix": "Added guard clause: if (!p || checkpoint_lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1343", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T13:27:42.519Z", + "error_message": "Missing error handling in every", + "file": "src/controller/testsuite/test_ha_core_safety.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 6.1 integration: leader->standby replication carries; * payload) faithfully end-to-end across BOTH live in-term re", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1348", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T14:10:48.987Z", + "error_message": "Wrong reference: test_pre_vote_catch_up_denies should be test_pre_vote_catch_up_grants_when_primary_unreachable", + "file": "src/controller/testsuite/test_ha_vote_gate.c", + "root_cause": "Used \"test_pre_vote_catch_up_denies\" instead of \"test_pre_vote_catch_up_grants_when_primary_unreachable\"", + "fix": "Changed test_pre_vote_catch_up_denies → test_pre_vote_catch_up_grants_when_primary_unreachable", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1349", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-01T14:24:21.310Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (atomic_load_explicit(&ha->in_catch_up, memory_orde)", + "fix": "Changed to: if (0 && atomic_load_explicit(&ha->in_catch_up, memory)", + "tags": [ + "auto-detected", + "logic-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1355", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T03:03:21.985Z", + "error_message": "Wrong reference: myconf_t should be uint32_t", + "file": "src/include/myconf.h", + "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", + "fix": "Changed myconf_t → uint32_t", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1357", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T03:03:51.842Z", + "error_message": "Function not marked async", + "file": "src/controller/main.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1362", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T03:50:14.650Z", + "error_message": "Wrong reference: persist_config_t should be uint32_t", + "file": "src/include/persist.h", + "root_cause": "Used \"persist_config_t\" instead of \"uint32_t\"", + "fix": "Changed persist_config_t → uint32_t", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1363", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T03:50:42.583Z", + "error_message": "Wrong reference: WAL_SEGMENT_SIZE_BYTES should be p", + "file": "src/controller/persist.c", + "root_cause": "Used \"WAL_SEGMENT_SIZE_BYTES\" instead of \"p\"", + "fix": "Changed WAL_SEGMENT_SIZE_BYTES → p", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1364", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T03:52:09.933Z", + "error_message": "Wrong reference: myconf_t should be uint32_t", + "file": "src/include/myconf.h", + "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", + "fix": "Changed myconf_t → uint32_t", + "tags": [ + "wrong-reference", + "h", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1369", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T03:57:52.302Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Had \"MODE=force-snapshot — forcing one full-checkpoint", + "fix": "Changed to \"MODE=force-snapshot — forcing one full-checkpoint", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1370", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T04:10:10.430Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1372", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T04:39:29.527Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: fd < 0", + "fix": "Added guard clause: if (fd < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1380", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T07:16:35.420Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `, differential-verified: this specific soak scena", + "fix": "Changed to `, differential-verified). (4) Re-ran `", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1382", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T07:23:50.910Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `snapshot_lane_fn`", + "fix": "Changed to `configpush_offload`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1383", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T07:31:28.737Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: waited >= timeout_ms", + "fix": "Added guard clause: if (waited >= timeout_ms)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1384", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T07:37:33.427Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had ` (test_ha_liveness.c). Blocked on **bug-1381** (o", + "fix": "Changed to ` (test_ha_liveness.c). **bug-1381 CLOSED 2026-07-", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1388", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T08:19:04.015Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: g_snaptest_chunk_count >= 16", + "fix": "Added guard clause: if (g_snaptest_chunk_count >= 16)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1393", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T12:05:51.415Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `~/tmp/xce-ha-deploy/controller/ctrl-{1,2}/data/sn", + "fix": "Changed to `ha_send_snapshot_to_standby`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1395", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T13:25:19.081Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `[~]`", + "fix": "Changed to `[x]`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1397", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T13:57:56.610Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1398", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T13:58:18.885Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "Had `/kiro-validate-impl`", + "fix": "Changed to `src/tools/test-commit-durability.sh ~/tmp/xce-dur", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1399", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T14:03:20.814Z", + "error_message": "Incorrect value in code", + "file": "src/controller/CLAUDE.md", + "root_cause": "Had `xce-test-e2e-primitives`", + "fix": "Changed to `ha-commit-durability`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1402", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-02T14:21:35.080Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-status/tasks.md", + "root_cause": "Had `/xce-deploy-ha`", + "fix": "Changed to `~/tmp`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1405", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-07-03T00:08:25.320Z", + "error_message": "Wrong reference: test_xcecli_ctrlstatus_null_ctx_typed_failure should be test_xce_free_ctrlstatus_frees_the_struct_itself", + "file": "src/lib/testsuite/test_ctrlstatus.c", + "root_cause": "Used \"test_xcecli_ctrlstatus_null_ctx_typed_failure\" instead of \"test_xce_free_ctrlstatus_frees_the_struct_itself\"", + "fix": "Changed test_xcecli_ctrlstatus_null_ctx_typed_failure → test_xce_free_ctrlstatus_frees_the_struct_itself", + "tags": [ + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1510", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-10T13:07:36.020Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/design.md", + "root_cause": "Had `hardware_monitor`", + "fix": "Changed to `worker_process_update()`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1511", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-10T13:09:14.910Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/component-failover-adoption/design.md", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 3→4 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1512", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-10T13:10:14.229Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/component-failover-adoption/requirements.md", + "root_cause": "8 lines replaced/restructured", + "fix": "Rewrote 21→7 lines (8 removed)", + "tags": [ + "auto-detected", + "refactor", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1513", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-10T13:47:21.938Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 6→6 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1516", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T11:35:00.157Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/testsuite/meson.build", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 5→12 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "build", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1517", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T11:35:50.895Z", + "error_message": "Wrong reference: true should be false", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "Used \"true\" instead of \"false\"", + "fix": "Changed true → false", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1518", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T12:22:54.348Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 7→11 lines (2 removed) | Also: ERROR(\"%s: xcecli_join() failed across %d candidat; __func__, sched->cluster_cfg.candidates_count, rc)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1520", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T13:05:01.694Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 12→10 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1521", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T13:14:08.399Z", + "error_message": "Wrong reference: test_sched_init_no_peers_default_also_fails_cleanly should be test_all_candidates_unreachable_log_names_every_candidate_host_port", + "file": "src/scheduler/testsuite/test_scheduler_connect.c", + "root_cause": "Used \"test_sched_init_no_peers_default_also_fails_cleanly\" instead of \"test_all_candidates_unreachable_log_names_every_candidate_host_port\"", + "fix": "Changed test_sched_init_no_peers_default_also_fails_cleanly → test_all_candidates_unreachable_log_names_every_candidate_host_port", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1522", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T13:14:51.964Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/testsuite/test_scheduler_connect.c", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 11→21 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1523", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T14:34:18.839Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/testsuite/test_scheduler_decision_redelivery.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 6→7 lines (2 removed) | Also: * Fixtures call xce_sched_core_init()/xce_sched_co; * the full sched_init()) -- the same minimal-fixtu", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1524", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T14:37:43.571Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 26→27 lines (5 removed) | Also: /* TEMPORARY RED-PHASE PROBE #2 (task 3.4 evidence; * immediately after use) -- core.stop checked only", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1525", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T23:39:27.643Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "15 lines replaced/restructured", + "fix": "Rewrote 27→11 lines (15 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1526", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-11T23:49:56.443Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1527", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T02:41:37.052Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/clib.go", + "root_cause": "6 lines replaced/restructured", + "fix": "Rewrote 21→26 lines (6 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1528", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T02:41:56.690Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/config.go", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 5→8 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1529", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T03:09:17.720Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/candidates.go", + "root_cause": "13 lines replaced/restructured", + "fix": "Rewrote 40→74 lines (13 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1530", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T04:33:27.866Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had `scheduler.c`", + "fix": "Changed to `xce_parse_peers()`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1531", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T07:05:08.345Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/clib.go", + "root_cause": "40 lines replaced/restructured", + "fix": "Rewrote 83→229 lines (40 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1533", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T07:46:01.677Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had `scheduler.c`", + "fix": "Changed to 's two connect mechanisms into one — the deleted `", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1534", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T08:45:46.229Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/reconnect_test.go", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 9→14 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1535", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T09:15:53.632Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had `scheduler.c`", + "fix": "Changed to `go_backend_rejoin_handler`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1536", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T09:25:02.926Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/heartbeat.go", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 17→62 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1537", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T09:25:33.041Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/clib.go", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 7→14 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1538", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T09:26:08.038Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/heartbeat_test.go", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 48→214 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1539", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T09:58:02.265Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had `scheduler.c`", + "fix": "Changed to `runHeartbeat()`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1540", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T10:16:08.309Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/clib.go", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 23→95 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1541", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T10:17:17.099Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/reconnect.go", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 16→26 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1543", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T10:20:03.350Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/reconnect_test.go", + "root_cause": "8 lines replaced/restructured", + "fix": "Rewrote 17→30 lines (8 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1544", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T10:51:41.466Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had `sched_init()`", + "fix": "Changed to `callBackend()`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1545", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T12:21:30.458Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had `sched_init()`", + "fix": "Changed to \"zero new code is a valid outcome when genuinely t", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1546", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T12:42:35.605Z", + "error_message": "Significant refactor of ", + "file": "../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-scheduler-framework/93f96538-d154-48c8-b92b-c8f9342a87cf/scratchpad/phase2_failover.sh", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 11→13 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1547", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T12:48:36.145Z", + "error_message": "Incorrect value in code", + "file": "../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-scheduler-framework/93f96538-d154-48c8-b92b-c8f9342a87cf/scratchpad/phase2_failover.sh", + "root_cause": "Had \"$TOP/logs/apiserver.stdout.log\"", + "fix": "Changed to \"$APISERVER_LOG\"", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1550", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-12T13:22:40.852Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had `sched_init()`", + "fix": "Changed to `xcecd_virtual`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1551", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T02:38:03.922Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had `sched_init()`", + "fix": "Changed to 's bug-1548 finding (heartbeat-only, zero write tra", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1552", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T03:02:32.434Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had `sched_init()`", + "fix": "Changed to `client_failover.c`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-872", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T07:20:20.800Z", + "error_message": "Significant refactor of ", + "file": "src/xceadmin/src/xcecommon/mapping.py", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 8→14 lines (7 removed)", + "tags": [ + "auto-detected", + "refactor", + "py", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-873", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T07:20:42.831Z", + "error_message": "Significant refactor of ", + "file": "src/xceadmin/tests/test_mapping.py", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 3→5 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "py", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-874", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T07:25:51.662Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_worker_join.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1633", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T07:27:23.041Z", + "error_message": "Significant refactor of ", + "file": "src/controller/worker.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 49→67 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1634", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T08:06:37.286Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 30→41 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1635", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T08:16:40.941Z", + "error_message": "Significant refactor of ", + "file": "src/schema/controller.capnp", + "root_cause": "11 lines replaced/restructured", + "fix": "Rewrote 14→3 lines (11 removed) | Also: # Display-unit hint copied from the resource's own; # at reply-build time -- empty/unset means \"raw nu", + "tags": [ + "auto-detected", + "refactor", + "capnp", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-882", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T08:18:53.065Z", + "error_message": "Significant refactor of ", + "file": "src/controller/opget.c", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 20→9 lines (7 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1636", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T08:19:26.708Z", + "error_message": "Incorrect value in code", + "file": "src/configs/schema/resources.schema.json", + "root_cause": "Had \"unit\"", + "fix": "Changed to \"resourcemap\"", + "tags": [ + "auto-detected", + "wrong-value", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1637", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T08:20:21.520Z", + "error_message": "Significant refactor of ", + "file": "src/xceadmin/src/xcecommon/mapping.py", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 5→3 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "py", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1638", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T08:22:12.974Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test.c", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 8→37 lines (5 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1639", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T09:22:19.958Z", + "error_message": "Significant refactor of ", + "file": "src/worker/src/xcecd.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 3→7 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1640", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T09:22:43.708Z", + "error_message": "Incorrect value in code", + "file": "src/configs/resources.yaml", + "root_cause": "Had \"available memory in MB (built-in dynamic load ind", + "fix": "Changed to \"available memory in bytes (built-in dynamic load ", + "tags": [ + "auto-detected", + "wrong-value", + "yaml", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1641", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T09:23:07.016Z", + "error_message": "Significant refactor of ", + "file": "src/configs/resources.yaml", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 7→7 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "yaml", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1642", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T09:23:47.981Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test.c", + "root_cause": "6 lines replaced/restructured", + "fix": "Rewrote 12→16 lines (6 removed) | Also: /* MaxMem is stored/wired as raw bytes (xce_resour; * for the maxmem resource definition) -- humanize ", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1643", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T09:25:25.166Z", + "error_message": "Significant refactor of ", + "file": "src/configs/schema/hosts.schema.json", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 10→17 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1644", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T09:25:49.927Z", + "error_message": "Significant refactor of ", + "file": "src/configs/schema/queues.schema.json", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 10→17 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-892", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T09:26:55.689Z", + "error_message": "Significant refactor of ", + "file": "src/xceadmin/src/xcecommon/mapping.py", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 16→58 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "py", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1645", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T10:45:55.441Z", + "error_message": "Significant refactor of ", + "file": "src/tools/deploy-ha.sh", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 21→34 lines (2 removed) | Also: # NOT retroactively update an already-built snapsh; # priority order (as this used to be) silently pic", + "tags": [ + "auto-detected", + "refactor", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1646", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T10:52:01.122Z", + "error_message": "Incorrect value in code", + "file": "src/tools/deploy-ha.sh", + "root_cause": "Had \"Warning: no xceadmin found in $BINARY'", + "fix": "Changed to \"Warning: no xceadmin found alongside $BINARY (che", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-896", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T13:08:50.126Z", + "error_message": "Significant refactor of ", + "file": "src/xceadmin/src/xcecommon/mapping.py", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 3→5 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "py", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-897", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-13T13:11:27.503Z", + "error_message": "Wrong reference: write_output should be dump_cluster_config", + "file": "src/xceadmin/src/xceadmin/cli.py", + "root_cause": "Used \"write_output\" instead of \"dump_cluster_config\"", + "fix": "Changed write_output → dump_cluster_config", + "tags": [ + "auto-detected", + "wrong-reference", + "py", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1647", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-15T13:02:22.188Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/apiserver/backend/clib.go", + "root_cause": "6 lines replaced/restructured", + "fix": "Rewrote 14→30 lines (6 removed)", + "tags": [ + "auto-detected", + "refactor", + "go", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-901", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-15T13:05:02.816Z", + "error_message": "Significant refactor of ", + "file": "src/frontend/Makefile", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 3→6 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "unknown", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-902", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-15T13:06:10.183Z", + "error_message": "Wrong reference: 039 should be 1548", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Used \"039\" instead of \"1548\"", + "fix": "Changed 039 → 1548", + "tags": [ + "auto-detected", + "wrong-reference", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1648", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-15T13:07:00.267Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Had 's own scope to fix (task 5.x is cross-component v", + "fix": "Changed to `/kiro-validate-impl`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1649", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-15T13:07:21.491Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/component-failover-adoption/spec.json", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 20→30 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1650", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-15T14:20:32.090Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/component-failover-adoption/tasks.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-906", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-15T14:20:51.010Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/component-failover-adoption/spec.json", + "root_cause": "Had \"2026-08-15T13:10:00Z\"", + "fix": "Changed to \"2026-08-15T14:20:00Z\"", + "tags": [ + "auto-detected", + "wrong-value", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1651", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-15T14:21:07.376Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/component-failover-adoption/spec.json", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 11→17 lines (5 removed)", + "tags": [ + "auto-detected", + "refactor", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1652", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T03:42:14.435Z", + "error_message": "Missing error handling in unknown", + "file": "reports/aug08-aug15-2026.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-910", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T03:57:07.719Z", + "error_message": "Incorrect value in code", + "file": "reports/aug08-aug15-2026.md", + "root_cause": "Had `build.py`", + "fix": "Changed to `3afe3d5`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1653", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T04:29:20.063Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/config-upgrade-snapshot/spec.json", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 21→29 lines (3 removed) | Also: \"updated_at\": \"2026-07-29T09:20:00Z\",; \"phase\": \"tasks-generated\",", + "tags": [ + "auto-detected", + "refactor", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1654", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T04:51:52.141Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_commit_barrier_reply.c", + "root_cause": "11 lines replaced/restructured", + "fix": "Rewrote 11→16 lines (11 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1655", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T04:54:26.304Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: j == NULL", + "fix": "Added guard clause: if (j == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-912", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T05:04:02.838Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 37→86 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1656", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T05:22:23.386Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_e2e_checkpoint.c", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1657", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T05:25:35.899Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 6→17 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-915", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T06:54:02.070Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ha->standby_endpoints", + "fix": "Added guard clause: if (ha->standby_endpoints)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-916", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T07:03:34.559Z", + "error_message": "Significant refactor of ", + "file": "src/controller/worker.c", + "root_cause": "9 lines replaced/restructured", + "fix": "Rewrote 12→23 lines (9 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-917", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T09:54:06.127Z", + "error_message": "Significant refactor of ", + "file": "src/controller/worker.c", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 15→8 lines (7 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1658", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T11:12:53.822Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/admin-config-tool/tasks.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-919", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T12:12:34.395Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/spec.json", + "root_cause": "Had \"All tasks 1.x-11.x done (verified 2026-06-09 gap ", + "fix": "Changed to \"CLOSED 2026-08-16. All 3 known deviations from th", + "tags": [ + "auto-detected", + "wrong-value", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1659", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-16T12:12:57.617Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/resval-id-decoupling/spec.json", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 21→22 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "json", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1660", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T03:45:49.314Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 6→8 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1661", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T04:59:39.812Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1662", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T05:01:33.795Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1663", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T06:41:48.625Z", + "error_message": "Significant refactor of ", + "file": "src/tools/deploy-ha.sh", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 40→93 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-927", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T06:44:51.034Z", + "error_message": "Significant refactor of ", + "file": ".claude/skills/xce-deploy-ha/SKILL.md", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 65→132 lines (7 removed) | Also: # 1. Stop running controllers; bash \"$TOP/scripts/cleanup.sh\"", + "tags": [ + "auto-detected", + "refactor", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1664", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T06:51:13.785Z", + "error_message": "Significant refactor of ", + "file": ".claude/skills/xce-build/SKILL.md", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 3→26 lines (2 removed) | Also: does not need to run from inside this repo or the ; **not** bundle its dependencies (`click`, `pycapnp | Also: ## Installing the xceadmin wheel elsewhere; `/xce-build xceadmin` always leaves a fresh, teste", + "tags": [ + "auto-detected", + "refactor", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1665", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T06:52:21.795Z", + "error_message": "Significant refactor of ", + "file": "src/xceadmin/pyproject.toml", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 6→9 lines (5 removed)", + "tags": [ + "auto-detected", + "refactor", + "toml", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1666", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T07:00:05.291Z", + "error_message": "Significant refactor of ", + "file": "src/tools/deploy-ha.sh", + "root_cause": "13 lines replaced/restructured", + "fix": "Rewrote 22→44 lines (13 removed)", + "tags": [ + "auto-detected", + "refactor", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1667", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T07:00:21.077Z", + "error_message": "Incorrect value in code", + "file": "src/tools/deploy-ha.sh", + "root_cause": "Had \"xceadmin installed at $TOP/xceadmin-venv -- use i", + "fix": "Changed to \"xceadmin installed at $TOP/xceadmin-site -- use i", + "tags": [ + "auto-detected", + "wrong-value", + "sh", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-932", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T07:15:26.246Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1668", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T07:15:58.937Z", + "error_message": "Wrong reference: fire_count should be g_durable_advance_fire_count", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Used \"fire_count\" instead of \"g_durable_advance_fire_count\"", + "fix": "Changed fire_count → g_durable_advance_fire_count", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-936", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T08:02:07.121Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/persistence.md", + "root_cause": "Had `on_durable_advance(ctx)`", + "fix": "Changed to `fdatasync`", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1669", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T08:03:27.006Z", + "error_message": "Significant refactor of ", + "file": ".kiro/steering/build-deployment.md", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 32→48 lines (7 removed)", + "tags": [ + "auto-detected", + "refactor", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-938", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T08:18:40.345Z", + "error_message": "Significant refactor of ", + "file": ".claude/skills/xce-build/SKILL.md", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 11→12 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-939", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T08:19:13.328Z", + "error_message": "Wrong operator: && should be <<", + "file": ".claude/skills/xce-build/SKILL.md", + "root_cause": "Used \"&&\" instead of \"<<\"", + "fix": "Changed operator && → <<", + "tags": [ + "auto-detected", + "operator-fix", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-940", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T08:19:30.812Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/build-deployment.md", + "root_cause": "Had 're gone until something explicitly rebuilds them ", + "fix": "Changed to 're gone until something explicitly rebuilds them ", + "tags": [ + "auto-detected", + "wrong-value", + "md", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1670", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T08:24:49.130Z", + "error_message": "Incorrect value in code", + "file": "src/xceadmin/tests/test_cli.py", + "root_cause": "Had \", event_count=0, started_at=None)\n\n monkeypatc", + "fix": "Changed to \", event_count=0, started_at=0)\n\n monkeypatch.s", + "tags": [ + "auto-detected", + "wrong-value", + "py", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1671", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T09:14:31.784Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/src/sim_loop.c", + "root_cause": "47 lines replaced/restructured", + "fix": "Rewrote 52→12 lines (47 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-946", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T09:14:35.731Z", + "error_message": "Wrong reference: sim_register_default_picker should be sim_bootstrap_picker", + "file": "src/scheduler/src/sim_loop.c", + "root_cause": "Used \"sim_register_default_picker\" instead of \"sim_bootstrap_picker\"", + "fix": "Changed sim_register_default_picker → sim_bootstrap_picker", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-947", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T10:14:50.237Z", + "error_message": "Wrong reference: test_job_rusage_applies_state_when_hook_is_null should be test_workers_push_twice_updates_existing_worker_in_place", + "file": "src/scheduler/testsuite/test_core_events.c", + "root_cause": "Used \"test_job_rusage_applies_state_when_hook_is_null\" instead of \"test_workers_push_twice_updates_existing_worker_in_place\"", + "fix": "Changed test_job_rusage_applies_state_when_hook_is_null → test_workers_push_twice_updates_existing_worker_in_place", + "tags": [ + "auto-detected", + "wrong-reference", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-1672", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T10:16:21.240Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/src/core_events.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 74→151 lines (3 removed) | Also: xce_sched_worker_t* worker; = core_build_worker(core, ev->u.workers_push.worke | Also: * WORKERS_PUSH core_build_worker() + sched_; * No picker hook (Req 1 AC 2:", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-09-19T09:00:00Z" + }, + { + "id": "bug-949", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-08-17T10:31:46.356Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/src/repl_info.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 4→9 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c", + "fixed", + "closed-by-scheduler-sim-live-allocation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-19T09:00:00Z" + } + ] +} diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 087f3b8f..d4fffc5f 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -1,73 +1,20 @@ { "version": 1, + "generated": "by scripts/buglog.py from .wolf/bugs/*.md -- do not edit", "bugs": [ - { - "id": "bug-002", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-21T07:46:54.198Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/kiro-spec-init/SKILL.md", - "root_cause": "Had `.kiro/settings/templates/specs/init.json`", - "fix": "Changed to `templates/specs/init.json`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 12, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-004", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-21T07:58:54.673Z", - "error_message": "Wrong reference: g_tmpdir should be g_instdir", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Used \"g_tmpdir\" instead of \"g_instdir\"", - "fix": "Changed g_tmpdir \u2192 g_instdir", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-005", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-21T08:24:10.738Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: rc == 0", - "fix": "Added guard clause: if (rc == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, { "id": "bug-006", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-05-21T08:30:20.719Z", - "error_message": "AddressSanitizer: attempting double-free on sctx in ha_connect_to_peer", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "AddressSanitizer: attempting double-free on sctx in ha_connect_to_peer\n\nAddressSanitizer: attempting double-free on sctx in ha_connect_to_peer", "file": "src/controller/ha.c", - "root_cause": "endpoint_fini calls FREEUP(ep->data) which frees sctx, then ha_connect_to_peer also calls free(sctx) \u2014 double-free at lines 3594-3595 and 3600-3601", + "root_cause": "endpoint_fini calls FREEUP(ep->data) which frees sctx, then ha_connect_to_peer also calls free(sctx) — double-free at lines 3594-3595 and 3600-3601", "fix": "Set ep->data = NULL before endpoint_fini so FREEUP becomes a no-op; then free(sctx) explicitly in the error paths", "tags": [ "md", @@ -75,20 +22,23 @@ "asan", "ha_connect_to_peer", "endpoint_fini", - "integration-test", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "integration-test" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-006.md" }, { "id": "bug-007", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-05-21T12:53:13.789Z", - "error_message": "AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 \u2014 reading ctx->cancelled after ha_arm_heartbeat_timeout_vtask freed the old ctx", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 — reading ctx->cancelled…\n\nAddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 — reading ctx->cancelled after ha_arm_heartbeat_timeout_vtask freed the old ctx", "file": "src/controller/ha.c", "root_cause": "ha_arm_heartbeat_timeout_vtask freed the old liveness_poll_ctx_t immediately after setting cancelled=1, but the delaymgr task was already in flight reading ctx->cancelled=0 and then continued to read ctx->ha after free", "fix": "ha_arm_heartbeat_timeout_vtask no longer frees old ctx; liveness_poll_fn now frees ctx when it fires with cancelled=1 (guarded by #ifndef XCE_TESTING to avoid bad-free on stack-allocated test ctxs)", @@ -100,1671 +50,23 @@ "liveness_poll_fn", "ha_arm_heartbeat_timeout_vtask", "race-condition", - "integration-test", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "integration-test" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-008", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-21T13:20:28.953Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: getaddrinfo(host, port_str, &hints, &res", - "fix": "Added guard clause: if (getaddrinfo(host, port_str, &hints, &res)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-020", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-25T11:31:55.826Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-023", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T07:17:31.980Z", - "error_message": "Missing guard clause", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "No early return/throw for edge case: req->term < my_term", - "fix": "Added guard clause: if (req->term < my_term)", - "tags": [ - "auto-detected", - "guard-clause", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-024", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T07:18:36.066Z", - "error_message": "Missing guard clause", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "No early return/throw for edge case: n_peers == 2", - "fix": "Added guard clause: if (n_peers == 2)", - "tags": [ - "auto-detected", - "guard-clause", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-026", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T07:26:48.363Z", - "error_message": "Wrong return value", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Was returning: 1;", - "fix": "Now returns: 1; // single-node: always primary", - "tags": [ - "auto-detected", - "return-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-027", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T07:27:34.925Z", - "error_message": "Wrong condition in logic", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Condition was: if (n_peers == 2)", - "fix": "Changed to: if (n_peers == 1)", - "tags": [ - "auto-detected", - "logic-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-029", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T07:29:11.741Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Had `main.c`", - "fix": "Changed to `-EINVAL`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-032", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T09:07:23.015Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Had `-EINVAL`", - "fix": "Changed to `--vcpus N`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-034", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T09:23:02.818Z", - "error_message": "Missing guard clause", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "No early return/throw for edge case: rc == 0", - "fix": "Added guard clause: if (rc == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-035", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T09:46:12.902Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Had `sysconf(_SC_NPROCESSORS_ONLN)`", - "fix": "Changed to ` when not specified. Values `", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-038", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T10:09:32.424Z", - "error_message": "Missing guard clause", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "No early return/throw for edge case: c->myconf->config_version_tag != 0", - "fix": "Added guard clause: if (c->myconf->config_version_tag != 0)", - "tags": [ - "auto-detected", - "guard-clause", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-039", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T10:09:57.711Z", - "error_message": "Wrong return value", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Was returning: 0 (standby); fresher peer runs its own election", - "fix": "Now returns: `-ENODATA`. Not a normal runtime path.", - "tags": [ - "auto-detected", - "return-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-041", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T10:10:40.149Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Had `test_fetch_config_from_best_peer`", - "fix": "Changed to `test_handle_config_request_standbys_serve_config`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-042", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T11:35:44.097Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-046", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T13:51:32.274Z", - "error_message": "Missing error handling in unknown", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - [ ] `test_heartbeat_carries_committed_lsn`: heartbeat payl; `committed_lsn`; standby updates `ha.last_known_committed_ls", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-047", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-26T13:52:49.044Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: // Catch-up gate (Clarification 6): behind nodes must not in; if (c->ha.in_catch_up) {", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-048", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-27T03:02:34.912Z", - "error_message": "Missing error handling in unknown", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: name, key, applied_lsn, and config_tag immediately after TCP; - [ ] `test_ctrl_hello_registers_standby_in_repl`: handle_ct", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-059", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-27T07:24:56.724Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Had `-ENOSYS`", - "fix": "Changed to `bootstrap_config_if_needed()`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-060", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-27T09:24:44.651Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/spec.json", - "root_cause": "Had \"2026-05-27T01:24:00-07:00\"", - "fix": "Changed to \"2026-05-27T01:30:00-07:00\"", - "tags": [ - "auto-detected", - "wrong-value", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-067", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T03:42:53.810Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'worker.c'", - "fix": "Changed to 'startup_validation.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-068", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T04:01:56.031Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/main.c", - "root_cause": "Condition was: if (arguments->vcpus < 1)", - "fix": "Changed to: if (validate_vcpus(arguments->vcpus)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-069", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T04:02:06.925Z", - "error_message": "Wrong reference: vcpus should be nres", - "file": "src/include/clusterconfig.h", - "root_cause": "Used \"vcpus\" instead of \"nres\"", - "fix": "Changed vcpus \u2192 nres", - "tags": [ - "auto-detected", - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-071", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T04:02:48.429Z", - "error_message": "Incorrect value in code", - "file": "src/schema/controller.capnp", - "root_cause": "Had \"vcpus\"", - "fix": "Changed to \"resources\"", - "tags": [ - "auto-detected", - "wrong-value", - "capnp", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-074", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T05:59:28.525Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (opcode == OPCode_requestVote || opcode == OPCode_p)", - "fix": "Changed to: if (opcode == OPCode_requestVote ||\n opcod)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-075", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T07:04:33.895Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_election.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-078", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T07:22:56.418Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: peer_idx < 0 || peer_idx >= ctx->n_peers", - "fix": "Added guard clause: if (peer_idx < 0 || peer_idx >= ctx->n_peers)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-081", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T07:53:10.883Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: role != CTRL_STANDBY", - "fix": "Added guard clause: if (role != CTRL_STANDBY)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-082", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T08:23:40.040Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ctx->done_posted", - "fix": "Added guard clause: if (ctx->done_posted)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-083", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T08:45:37.921Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !host || !tag_out", - "fix": "Added guard clause: if (!host || !tag_out)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-084", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T08:48:58.609Z", - "error_message": "Wrong reference: capn_str_null should be str", - "file": "src/controller/config_repl.c", - "root_cause": "Used \"capn_str_null\" instead of \"str\"", - "fix": "Changed capn_str_null \u2192 str", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-086", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T09:13:17.676Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_bootstrap_support.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-087", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T10:13:52.421Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !workdir", - "fix": "Added guard clause: if (!workdir)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-091", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T13:53:09.924Z", - "error_message": "Wrong reference: in_catch_up should be acked_config_lsn", - "file": "src/controller/repl.c", - "root_cause": "Used \"in_catch_up\" instead of \"acked_config_lsn\"", - "fix": "Changed in_catch_up \u2192 acked_config_lsn", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-092", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T14:11:04.583Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn", - "fix": "Added guard clause: if (p->current_lsn == 0 || cutoff_lsn + 1 >=)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-094", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T14:47:44.445Z", - "error_message": "Wrong reference: ha_state_t should be _Atomic", - "file": "src/controller/ha.h", - "root_cause": "Used \"ha_state_t\" instead of \"_Atomic\"", - "fix": "Changed ha_state_t \u2192 _Atomic", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-095", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T15:04:44.613Z", - "error_message": "Wrong reference: socket should be epoll", - "file": "src/controller/ha.c", - "root_cause": "Used \"socket\" instead of \"epoll\"", - "fix": "Changed socket \u2192 epoll", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-096", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-28T15:05:23.129Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ha_tcp_connect_hook", - "fix": "Added guard clause: if (ha_tcp_connect_hook)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-097", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T02:48:17.371Z", - "error_message": "Missing guard clause", - "file": "src/lib/request.c", - "root_cause": "No early return/throw for edge case: (ep == NULL", - "fix": "Added guard clause: if ((ep == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-098", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T02:50:46.444Z", - "error_message": "Wrong reference: XCE_ERR_STANDBY should be E_XCE_STANDBY", - "file": "src/lib/request.c", - "root_cause": "Used \"XCE_ERR_STANDBY\" instead of \"E_XCE_STANDBY\"", - "fix": "Changed XCE_ERR_STANDBY \u2192 E_XCE_STANDBY", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-099", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T02:51:03.529Z", - "error_message": "Wrong reference: XCE_ERR_NOT_READY should be E_XCE_NOT_READY", - "file": "src/lib/testsuite/test_libxce_retry.c", - "root_cause": "Used \"XCE_ERR_NOT_READY\" instead of \"E_XCE_NOT_READY\"", - "fix": "Changed XCE_ERR_NOT_READY \u2192 E_XCE_NOT_READY", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-102", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T03:37:52.117Z", - "error_message": "Wrong reference: data should be ctrl_hello_reply_t", - "file": "src/include/typemap.h", - "root_cause": "Used \"data\" instead of \"ctrl_hello_reply_t\"", - "fix": "Changed data \u2192 ctrl_hello_reply_t", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-104", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T03:54:23.205Z", - "error_message": "Missing error handling in struct", - "file": "src/include/typemap.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: wal_catchup_entry_t* walcatchupentrymsg;; snapshot_transfer_t* snapshottransfermsg; | Also: /* Forward declarations for types defined later in this file; typedef struct snapshot_transfer_t snapshot_transfer_fw", - "tags": [ - "auto-detected", - "error-handling", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-109", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T06:59:14.045Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha", - "fix": "Added guard clause: if (!ha)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-111", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T07:02:31.287Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (c->myconf && c->myconf->cluster_config)", - "fix": "Changed to: if (c->myconf && c->myconf->peers_cfg.candidates_count)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-117", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T13:41:46.841Z", - "error_message": "Incorrect value in code", - "file": "src/controller/myapp.c", - "root_cause": "Had \"Pre-election config loaded: tag=%016\"", - "fix": "Changed to \"Pre-election config loaded: tag=%016llx\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-118", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T13:51:42.299Z", - "error_message": "Wrong reference: nres should be vcpus", - "file": "src/include/clusterconfig.h", - "root_cause": "Used \"nres\" instead of \"vcpus\"", - "fix": "Changed nres \u2192 vcpus", - "tags": [ - "auto-detected", - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-119", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-29T16:32:10.285Z", - "error_message": "Wrong reference: data should be request_vote_reply_t", - "file": "src/include/typemap.h", - "root_cause": "Used \"data\" instead of \"request_vote_reply_t\"", - "fix": "Changed data \u2192 request_vote_reply_t", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-122", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T00:05:02.052Z", - "error_message": "Wrong reference: cluster_init_phase2 should be cluster_register_listener", - "file": "src/controller/include/cluster.h", - "root_cause": "Used \"cluster_init_phase2\" instead of \"cluster_register_listener\"", - "fix": "Changed cluster_init_phase2 \u2192 cluster_register_listener", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-123", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T00:20:27.244Z", - "error_message": "Missing guard clause", - "file": "src/controller/cluster.c", - "root_cause": "No early return/throw for edge case: c->ep.flags == 0", - "fix": "Added guard clause: if (c->ep.flags == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-127", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T01:12:38.572Z", - "error_message": "Wrong reference: CTRL_PERF_NHASH should be c", - "file": "src/controller/cluster.c", - "root_cause": "Used \"CTRL_PERF_NHASH\" instead of \"c\"", - "fix": "Changed CTRL_PERF_NHASH \u2192 c", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-129", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T01:21:20.140Z", - "error_message": "Wrong reference: true should be false", - "file": ".claude/skills/xce-build/SKILL.md", - "root_cause": "Used \"true\" instead of \"false\"", - "fix": "Changed true \u2192 false", - "tags": [ - "auto-detected", - "wrong-reference", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-130", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T01:28:22.079Z", - "error_message": "Wrong reference: 120 should be 300", - "file": "src/controller/meson.build", - "root_cause": "Used \"120\" instead of \"300\"", - "fix": "Changed 120 \u2192 300", - "tags": [ - "auto-detected", - "wrong-reference", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-135", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T06:19:47.181Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "Had `cluster_init_phase2`", - "fix": "Changed to `cluster_config == NULL`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-136", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T06:38:09.325Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* After election determines role: */; int is_first_boot = !c->running; /* running=1 only after cl | Also: #### `ha_dispatch_gate` \u2014 CTRL_PRIMARY_COMMITTING case; ```c | Also: ### 3-node startup \u2014 ctrl-a wins (with COMMITTING phase); participant PeerC as ctrl-c endpoint | Also: - **`ha_start_election_async()` is fully non-blocking**: Arm; - *Initial startup* (`ha->phase2_done == 0`): calls `cluster", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-139", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T06:58:05.115Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "Had `delay_task_cancel()`", - "fix": "Changed to `clock_gettime(CLOCK_MONOTONIC)`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-142", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T07:57:14.052Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: xce_vote_request_hook != NULL", - "fix": "Added guard clause: if (xce_vote_request_hook != NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-143", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T08:16:03.222Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !pkt || !pkt->data || pkt->sz == 0", - "fix": "Added guard clause: if (!pkt || !pkt->data || pkt->sz == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-144", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T08:49:04.058Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-145", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T08:49:59.817Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: atomic_load(&t->cancelled", - "fix": "Added guard clause: if (atomic_load(&t->cancelled)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-147", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T09:13:19.377Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-149", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T10:31:53.836Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_vote_gate.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-153", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T11:43:01.989Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_repl_push.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-154", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T11:43:44.829Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Push config to standbys with lower config_tag (Req 7.1) *; #ifdef XCE_TESTING", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-157", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T12:02:39.009Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-159", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T12:22:57.013Z", - "error_message": "Wrong return value", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Was returning: in < 10ms.", - "fix": "Now returns: in < 5ms (design spec).", - "tags": [ - "auto-detected", - "return-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-160", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T12:23:18.560Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Had \"myapp_run path: ha_start_election_async must retu", - "fix": "Changed to \"myapp_run path: ha_start_election_async must retu", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-161", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T12:29:45.867Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-162", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T12:30:10.570Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-163", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T12:31:02.203Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_failover.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-164", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T12:45:22.182Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-166", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T12:50:35.870Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_bootstrap_support.c", - "root_cause": "No early return/throw for edge case: c->myconf->config_version_tag != 0", - "fix": "Added guard clause: if (c->myconf->config_version_tag != 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-167", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T13:06:57.152Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: cancelled_val", - "fix": "Added guard clause: if (cancelled_val)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-168", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T13:24:06.024Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_repl_ack.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-169", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T13:42:28.077Z", - "error_message": "Wrong reference: third should be second", - "file": "src/controller/testsuite/test_ha_repl_ack.c", - "root_cause": "Used \"third\" instead of \"second\"", - "fix": "Changed third \u2192 second", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-170", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T14:35:07.679Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-172", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-30T23:42:12.644Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-176", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T02:09:39.630Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (c->myconf && c->myconf->peers_cfg.candidates_count)", - "fix": "Changed to: if (ha->my_priority > 0)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-183", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T03:02:33.311Z", - "error_message": "Missing guard clause", - "file": "src/lib/request.c", - "root_cause": "No early return/throw for edge case: req == NULL", - "fix": "Added guard clause: if (req == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-186", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T04:11:45.534Z", - "error_message": "Missing error handling in struct", - "file": ".kiro/specs/raft-library/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: Caller->>RaftSM: raft_on_vote_request(state, from, term); alt term < current_term", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-188", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T04:19:54.524Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-library/design.md", - "root_cause": "Had `raft.c`", - "fix": "Changed to `raft.h`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-190", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T05:07:37.759Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: cfg->n_peers < 1", - "fix": "Added guard clause: if (cfg->n_peers < 1)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-191", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T05:23:19.394Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->role == RAFT_CANDIDATE || s->role == RAFT_LEADER", - "fix": "Added guard clause: if (s->role == RAFT_CANDIDATE || s->role == )", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-197", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T07:08:23.387Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-007.md" }, { "id": "bug-206", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-05-31T08:40:00.000Z", - "error_message": "test_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 \u2014 ha->term must be atomically updated to peer term when peer term > ours", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 — ha->term must be atomically updated…\n\ntest_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 — ha->term must be atomically updated to peer term when peer term > ours", "file": "src/controller/ha.c", "root_cause": "vote_reply_msg_handler delegated term fencing entirely to raft_cb_become_follower (via raft_on_vote_reply). In XCE_TESTING builds, vote_reply_dispatch calls xce_vote_reply_hook and returns early, bypassing raft_on_vote_reply and thus never updating ha->term.", "fix": "Added explicit CAS loop in vote_reply_msg_handler before vote_reply_dispatch to advance ha->term when vr_term > ha->term. This applies on both the production path and the XCE_TESTING hook path.", @@ -1774,39 +76,23 @@ "ha", "xce-testing", "test-seam", - "vote-reply", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "vote-reply" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-207", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T08:47:08.484Z", - "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_even_peers_returns_einval", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_even_peers_returns_einval\"", - "fix": "Changed test_raft_init_zero_peers_returns_einval \u2192 test_raft_init_even_peers_returns_einval", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-206.md" }, { "id": "bug-208", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "leak", "timestamp": "2026-05-31", - "error_message": "ASAN LeakSanitizer: detected memory leaks in scheduler testsuite \u2014 build exits with error code", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ASAN LeakSanitizer: detected memory leaks in scheduler testsuite — build exits with error code\n\nASAN LeakSanitizer: detected memory leaks in scheduler testsuite — build exits with error code", "file": "src/scheduler/testsuite/test.c", "root_cause": "sched_meet_host_selection() lazily allocates w->host_eval_ctx via build_host_eval_ctx(). Four test functions (test_meet_host_selection_always_true_passes, test_meet_host_selection_always_false_fails, test_meet_host_selection_numeric_match, test_meet_host_selection_string_match) called zarray_fini() without first calling sched_worker_release_host_ctx() to free the lazily-allocated host_eval_ctx.", "fix": "Added sched_worker_release_host_ctx(&w) calls before zarray_fini() in all four affected test functions.", @@ -1815,1014 +101,48 @@ "leak", "scheduler", "test", - "host_eval_ctx", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "host_eval_ctx" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-209", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T10:19:47.014Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (ha_rdhup_election_hook != NULL)", - "fix": "Changed to: if (c->myconf && c->myconf->peers_cfg.candidates_count)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-211", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T11:29:03.301Z", - "error_message": "Wrong reference: 2000 should be 5000", - "file": "src/controller/ha.h", - "root_cause": "Used \"2000\" instead of \"5000\"", - "fix": "Changed 2000 \u2192 5000", - "tags": [ - "auto-detected", - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-215", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T12:43:45.333Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !pkt || !pkt->data || pkt->sz == 0", - "fix": "Added guard clause: if (!pkt || !pkt->data || pkt->sz == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-216", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T12:49:40.554Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-peer-connection-reuse/tasks.md", - "root_cause": "Had 's vote endpoints to standby control channels\n\n- [", - "fix": "Changed to 's vote endpoints to standby control channels\n\n- [", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-218", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T12:58:44.435Z", - "error_message": "Wrong reference: Request_body_nodata should be Request_body_nomsg", - "file": "src/controller/ha.c", - "root_cause": "Used \"Request_body_nodata\" instead of \"Request_body_nomsg\"", - "fix": "Changed Request_body_nodata \u2192 Request_body_nomsg", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-220", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-05-31T14:20:41.161Z", - "error_message": "Wrong reference: vote_reply_msg_handler should be vote_reply_msg_handler_for_test", - "file": "src/controller/ha.c", - "root_cause": "Used \"vote_reply_msg_handler\" instead of \"vote_reply_msg_handler_for_test\"", - "fix": "Changed vote_reply_msg_handler \u2192 vote_reply_msg_handler_for_test", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-224", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T02:07:55.176Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c->delaymgr.running", - "fix": "Added guard clause: if (!c->delaymgr.running)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-229", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T02:49:09.141Z", - "error_message": "Wrong operator: && should be !=", - "file": "src/controller/ha.c", - "root_cause": "Used \"&&\" instead of \"!=\"", - "fix": "Changed operator && \u2192 !=", - "tags": [ - "auto-detected", - "operator-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-233", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T03:07:51.264Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_connect.c'", - "fix": "Changed to 'testsuite/test_ha_ctrlhello.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-235", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T03:13:14.521Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_connection_reuse.c'", - "fix": "Changed to 'testsuite/test_ha_connect.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-236", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T03:15:29.129Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: rc < 0", - "fix": "Added guard clause: if (rc < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-238", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T03:23:32.483Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_connect.c'", - "fix": "Changed to 'testsuite/test_ha_connection_reuse.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-239", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T03:28:46.956Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ha_tcp_connect_hook != NULL", - "fix": "Added guard clause: if (ha_tcp_connect_hook != NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-240", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T04:08:07.950Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_connect.c'", - "fix": "Changed to 'testsuite/test_ha_connection_reuse.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-241", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T04:13:03.001Z", - "error_message": "Wrong reference: test_ha_connect should be test_ha_ctrlhello", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_ha_connect\" instead of \"test_ha_ctrlhello\"", - "fix": "Changed test_ha_connect \u2192 test_ha_ctrlhello", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-242", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T05:19:12.427Z", - "error_message": "Wrong reference: vcpu should be vcpus", - "file": "src/tools/deploy-ha.sh", - "root_cause": "Used \"vcpu\" instead of \"vcpus\"", - "fix": "Changed vcpu \u2192 vcpus", - "tags": [ - "auto-detected", - "wrong-reference", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-250", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T07:46:05.128Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ep || !ep->data", - "fix": "Added guard clause: if (!ep || !ep->data)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-252", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T08:35:29.969Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_rdhup_integration.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-255", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T12:02:55.635Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/cluster.c", - "root_cause": "Condition was: if (rc != 0)", - "fix": "Changed to: if (validate_heartbeat_config(ha_cfg.heartbeat_timeout)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-256", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T12:36:28.381Z", - "error_message": "Wrong reference: startup_validation should be testsuite_test_ha_liveness", - "file": "src/xce.build/build.ninja", - "root_cause": "Used \"startup_validation\" instead of \"testsuite_test_ha_liveness\"", - "fix": "Changed startup_validation \u2192 testsuite_test_ha_liveness", - "tags": [ - "wrong-reference", - "ninja", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-260", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T14:42:41.325Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: interval_s <= 0", - "fix": "Added guard clause: if (interval_s <= 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-263", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-01T15:32:59.304Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "No early return/throw for edge case: !f", - "fix": "Added guard clause: if (!f)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-265", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-03T08:47:25.835Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-266", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-03T08:47:54.376Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-269", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-03T08:50:58.837Z", - "error_message": "Wrong reference: test_am_i_primary should be test_ha_cluster", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_am_i_primary\" instead of \"test_ha_cluster\"", - "fix": "Changed test_am_i_primary \u2192 test_ha_cluster", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-279", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T11:54:56.114Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-repl-hardening/design.md", - "root_cause": "Had `(checkpoint_lsn, last_received_chunk)`", - "fix": "Changed to `snap.part`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-284", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T14:33:03.464Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* \u2500\u2500 Task 2.1: OPCode_walCatchupEntry handler \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500; #ifdef XCE_TESTING | Also: /* Task 2.1: OPCode_walCatchupEntry handler */; RUN_TEST(test_wal_catchup_entry_advances_applied_lsn);", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-285", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T14:41:35.073Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/cluster.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: case OPCode_walCatchupEntry:; /* Task 2.1: standby applies a single WAL catch-up entry fro", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-288", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T14:46:44.652Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: #include \"persist.h\" /* persist_t, apply_wal_entry */; #include \"typemap.h\" /* wal_catchup_entry_t */", - "tags": [ - "auto-detected", - "error-handling", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-292", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T16:15:43.743Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES", - "fix": "Added guard clause: if (peer_idx < 0 || peer_idx >= MAX_HA_CANDI)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-294", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T16:45:25.724Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: standby_tag < committed_tag", - "fix": "Added guard clause: if (standby_tag < committed_tag)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-295", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T16:57:59.213Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "Had \"test_evaluate_repl_action_stream_wal_from_zero: P", - "fix": "Changed to \"test_evaluate_repl_action_send_snapshot_lsn_zero:", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-296", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T16:58:16.627Z", - "error_message": "Wrong reference: test_evaluate_repl_action_stream_wal_from_zero should be test_evaluate_repl_action_send_snapshot_lsn_zero", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "Used \"test_evaluate_repl_action_stream_wal_from_zero\" instead of \"test_evaluate_repl_action_send_snapshot_lsn_zero\"", - "fix": "Changed test_evaluate_repl_action_stream_wal_from_zero \u2192 test_evaluate_repl_action_send_snapshot_lsn_zero", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-297", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T17:08:18.571Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES", - "fix": "Added guard clause: if (!ha || peer_idx < 0 || peer_idx >= MAX_H)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-301", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T23:09:43.701Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: g_snaptest_chunk_count >= 16", - "fix": "Added guard clause: if (g_snaptest_chunk_count >= 16)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-302", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-04T23:10:44.426Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !snap_dir", - "fix": "Added guard clause: if (!snap_dir)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-308", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-05T01:19:20.356Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !d", - "fix": "Added guard clause: if (!d)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-309", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-05T01:33:20.146Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || !p || !snap_dir || !chunk", - "fix": "Added guard clause: if (!ha || !p || !snap_dir || !chunk)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-310", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-05T01:34:06.660Z", - "error_message": "Missing guard clause", - "file": "src/controller/cluster.c", - "root_cause": "No early return/throw for edge case: !chunk", - "fix": "Added guard clause: if (!chunk)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-311", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-05T02:21:15.033Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ua > ub", - "fix": "Added guard clause: if (ua > ub)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-312", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-05T02:47:32.237Z", - "error_message": "Wrong reference: uintptr_t should be void", - "file": "src/controller/testsuite/test_ha_evaluate_committing.c", - "root_cause": "Used \"uintptr_t\" instead of \"void\"", - "fix": "Changed uintptr_t \u2192 void", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-314", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-05T04:01:58.721Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha", - "fix": "Added guard clause: if (!ha)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-315", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-05T04:52:32.206Z", - "error_message": "Wrong reference: 10000 should be 60000", - "file": "src/controller/ha.h", - "root_cause": "Used \"10000\" instead of \"60000\"", - "fix": "Changed 10000 \u2192 60000", - "tags": [ - "auto-detected", - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-317", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-05T04:53:55.034Z", - "error_message": "Wrong reference: 10000 should be HA_DEFAULT_COMMITTING_TIMEOUT_MS", - "file": "src/controller/testsuite/test_ha_evaluate_committing.c", - "root_cause": "Used \"10000\" instead of \"HA_DEFAULT_COMMITTING_TIMEOUT_MS\"", - "fix": "Changed 10000 \u2192 HA_DEFAULT_COMMITTING_TIMEOUT_MS", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-208.md" }, { "id": "bug-320", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-05T06:50:00Z", - "error_message": "3-node startup: majority convergence never fires; cluster falls through to committing_timeout degraded-mode WARN", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "3-node startup: majority convergence never fires; cluster falls through to committing_timeout…\n\n3-node startup: majority convergence never fires; cluster falls through to committing_timeout degraded-mode WARN", "file": "src/controller/ha.c", - "root_cause": "ha->n_peers was set to candidates_count (3, total including self) in ha_start_election_async. ha_check_majority_convergence loops i=0..n_peers-1 checking standby_converged[], but standby_converged[] is indexed by votes_idx (relative, self excluded: 0..n_other-1). With n_peers=3, it checked standby_converged[2] which is never set, so n_converged (2) < n_peers (3) always \u2014 majority convergence path could never succeed.", + "root_cause": "ha->n_peers was set to candidates_count (3, total including self) in ha_start_election_async. ha_check_majority_convergence loops i=0..n_peers-1 checking standby_converged[], but standby_converged[] is indexed by votes_idx (relative, self excluded: 0..n_other-1). With n_peers=3, it checked standby_converged[2] which is never set, so n_converged (2) < n_peers (3) always — majority convergence path could never succeed.", "fix": "Changed ha->n_peers = n_peers to ha->n_peers = n_other (= n_peers-1 = other-node count). Moved the assignment after n_other is computed. ha_repl_ack_received quorum formula (ha->n_peers+2)/2 still yields correct majority for both old and new value.", "tags": [ "ha", "majority-convergence", "n_peers", "off-by-one", - "integration-test", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "integration-test" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-326", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-06T11:06:46.509Z", - "error_message": "Wrong reference: DEBUG2 should be INFO", - "file": "src/controller/ha.c", - "root_cause": "Used \"DEBUG2\" instead of \"INFO\"", - "fix": "Changed DEBUG2 \u2192 INFO", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-327", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-06T12:11:21.422Z", - "error_message": "Incorrect value in code", - "file": "src/controller/ha.c", - "root_cause": "Had \"HA: heartbeat tick (role=%d, n_standbys=%d)\"", - "fix": "Changed to \"HA: primary heartbeat armed\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-328", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-06T13:02:49.052Z", - "error_message": "Missing guard clause", - "file": "src/lib/delay.c", - "root_cause": "No early return/throw for edge case: t1_is_ms && t2_is_ms", - "fix": "Added guard clause: if (t1_is_ms && t2_is_ms)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-329", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-08T13:12:54.436Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-library/research.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-334", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-09T12:56:11.095Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (my_role == CTRL_STANDBY && ha_heartbeat_is_fresh(h)", - "fix": "Changed to: if (my_role == CTRL_STANDBY && ha->primary_ctrl_fd >= )", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-320.md" }, { "id": "bug-335", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-06-09T00:00:00Z", - "error_message": "UPDATE_RESVALS dequeue writes worker resval at index 0 unconditionally instead of resolving resource idx by name", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "UPDATE_RESVALS dequeue writes worker resval at index 0 unconditionally instead of resolving…\n\nUPDATE_RESVALS dequeue writes worker resval at index 0 unconditionally instead of resolving resource idx by name", "file": "src/scheduler/src/scheduler.c", "root_cause": "zarray_put(&w->resvals, 0, rv->val) hardcodes index 0; worker resvals are indexed by resource idx (see schedobj.c:291)", "fix": "FIXED 2026-06-10: UPDATE_RESVALS now resolves rv->name via key_find on sched->resmgr and writes to r->idx (matching UPDATE_RESVALS_FULL pattern) instead of hardcoding index 0.", @@ -2831,20 +151,23 @@ "resvals", "latent-bug", "gap-analysis", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "c" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-335.md" }, { "id": "bug-336", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-06-09T00:00:00Z", - "error_message": "XCE_SCHED_OP_UPDATE_RESOURCES cannot update existing resources; makeResource returns NULL/EEXIST and return value is silently ignored", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "XCE_SCHED_OP_UPDATE_RESOURCES cannot update existing resources; makeResource returns NULL/EEXIST…\n\nXCE_SCHED_OP_UPDATE_RESOURCES cannot update existing resources; makeResource returns NULL/EEXIST and return value is silently ignored", "file": "src/scheduler/src/scheduler.c", "root_cause": "makeResource_ in src/lib/resobj.c rejects duplicates with EEXIST; dequeue case ignores NULL return, so 'add or update' (EARS-UB-1.1) is only 'add'", "fix": "FIXED 2026-06-10: UPDATE_RESOURCES now checks makeResource return; on NULL (EEXIST) looks up existing resource by name and updates valtype/ordtype/flags/interval fields in place (add-or-update semantics).", @@ -2854,20 +177,23 @@ "latent-bug", "gap-analysis", "silent-error", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "c" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-336.md" }, { "id": "bug-337", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-09T00:02:00Z", - "error_message": "Vote-deny gate inert: ha_heartbeat_is_fresh reads ha->last_heartbeat which has no production writer after the liveness refactor", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Vote-deny gate inert: ha_heartbeat_is_fresh reads ha->last_heartbeat which has no production…\n\nVote-deny gate inert: ha_heartbeat_is_fresh reads ha->last_heartbeat which has no production writer after the liveness refactor", "file": "src/controller/ha.c", "root_cause": "ha-heartbeat-liveness refactor moved activity tracking to ha->last_activity but ha_heartbeat_is_fresh (ha.c:1762) still reads the orphaned last_heartbeat timespec; only tests write it, so freshness is always stale after ~heartbeat_timeout_ms of uptime and standbys grant votes despite a live primary (gates at ha.c:214 and ha.c:1841)", "fix": "FIXED 2026-06-09: ha_heartbeat_is_fresh now reads ha->last_activity (returns 0 when last_activity==0); dead last_heartbeat field removed from ha_state_t; test_ha_vote_gate.c helper rewritten as set_last_activity_age and fresh/stale tests now also set primary_ctrl_fd. All tests green.", @@ -2877,20 +203,23 @@ "liveness", "vote-gate", "dead-field", - "gap-analysis", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "gap-analysis" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-337.md" }, { "id": "bug-338", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-09T00:00:00Z", - "error_message": "controller_unit_test: test_vote_granted_to_same_candidate_same_term FAIL (Expected 1 Was 0); test_standby_fresh_heartbeat_denies_vote_request FAIL (Expected 0 Was 1, Req 6.2)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller_unit_test: test_vote_granted_to_same_candidate_same_term FAIL (Expected 1 Was 0)…\n\ncontroller_unit_test: test_vote_granted_to_same_candidate_same_term FAIL (Expected 1 Was 0); test_standby_fresh_heartbeat_denies_vote_request FAIL (Expected 0 Was 1, Req 6.2)", "file": "src/controller/ha.c", "root_cause": "Uncommitted handle_request_vote changes: (1) duplicate-vote deny made candidate-agnostic (denies same-candidate idempotent re-grant, diverging from raft.c which keeps re-grant); (2) STANDBY vote deny now requires primary_ctrl_fd >= 0, tests use ha_init default fd=-1", "fix": "FIXED 2026-06-09: handle_request_vote Req 2.6 check restored to same-candidate idempotent re-grant (deny only when voted_for_name differs from candidate_id), matching raft.c; test_standby_fresh_heartbeat_denies_vote_request updated to set primary_ctrl_fd=999. Also fixed myapp.c leader_known check to treat a vote for myconf->name as a self-vote (production records real name, not the 'self' test placeholder). All tests green.", @@ -2901,20 +230,23 @@ "raft", "unit-test", "uncommitted-drift", - "gap-analysis", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "gap-analysis" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-338.md" }, { "id": "bug-339", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-09T17:30:00Z", - "error_message": "After killing the primary in a 3-node controller HA cluster, the two surviving controllers enter an infinite election loop: both only ever raft_cb_become_follower (never become_leader), terms climb in lockstep (4..23), RDHUP -> CTRL_ELECTING -> CTRL_STANDBY repeats forever.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "After killing the primary in a 3-node controller HA cluster, the two surviving controllers enter…\n\nAfter killing the primary in a 3-node controller HA cluster, the two surviving controllers enter an infinite election loop: both only ever raft_cb_become_follower (never become_leader), terms climb in lockstep (4..23), RDHUP -> CTRL_ELECTING -> CTRL_STANDBY repeats forever.", "file": "src/controller/ha.c, src/controller/myapp.c, src/controller/testsuite/test_ha_rdhup_integration.c", "root_cause": "LSN-source asymmetry between candidate-advertised LSN and voter freshness floor, specific to the RDHUP failover election path. ha_on_primary_ctrl_fd_rdhup advertised c->persist.committed_lsn as the RequestVote last_log_lsn, but a STANDBY never advances persist.committed_lsn (only the primary's commit engine writes it at persist.c:1192). Standbys advance applied_lsn (ha_apply_wal_catchup_entry) and last_known_committed_lsn (heartbeat replies). The voter-side freshness floor (cluster.c OPCode_requestVote) is max(applied_lsn, last_known_committed_lsn) via ha_vote_self_freshness. So once a standby applied any WAL beyond its last snapshot, every surviving standby advertised a stale-low LSN (e.g. 0) while each peer computed a HIGH floor (e.g. 10); the freshness gate cand_lsn < my_lsn tripped on BOTH sides, each denied the other's vote, neither reached quorum (2 of 3), each election timed out -> become_follower -> 10s liveness timeout -> RDHUP -> re-elect at term+1. The boot path (myapp.c:217) already advertised applied_lsn, so cold-boot election worked; only the RDHUP failover path was inconsistent.", "fix": "In ha_on_primary_ctrl_fd_rdhup, compute candidate freshness ONCE via ha_vote_self_freshness(ha, config_version_tag, applied_lsn, &cand_tag, &cand_lsn) right after ha->raft_c=c, and use cand_tag/cand_lsn for BOTH the elect_ctx (votes[0].my_tag/my_lsn + self_tag/self_lsn) and the XCE_TESTING ha_rdhup_election_hook args. This makes candidate-advertised LSN identical to the voter-side floor, so for any two nodes A,B: A grants B iff floor_B >= floor_A -> the freshest node is always granted by every peer; equal-floor ties are broken by raft priority delay. Regression test added: test_rdhup_candidate_advertises_freshness_floor_not_stale_committed (applied/learned=10, persist.committed=0 -> asserts advertised my_lsn==10). RED (Was 0) -> GREEN verified. Follow-up: boot election path (myapp.c) also routed through ha_vote_self_freshness (was raw applied_lsn) so both election paths are symmetric with the voter floor.", @@ -2934,25 +266,28 @@ "committed_lsn", "applied_lsn", "split-vote", - "infinite-loop", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "infinite-loop" ], "related_bugs": [ "bug-338" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-339.md" }, { "id": "bug-340", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-10T00:23:21.918Z", - "error_message": "After kill+RESTART of a controller (ctrl-1), the restarted node loops forever: ha_on_primary_ctrl_fd_rdhup \u2192 CTRL_ELECTING \u2192 raft_cb_become_follower \u2192 CTRL_STANDBY every ~heartbeat_timeout, term climbing unbounded (473+), while ctrl-2 stays PRIMARY at term 3. On ctrl-2: every heartbeat tick logs `xce_encode op 0 (NOP)` then `WARN endpoint_update_epoll rc1=-1 < 0` (netio.c:575) \u2014 a wedged outbound standby endpoint.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "After kill+RESTART of a controller (ctrl-1), the restarted node loops forever…\n\nAfter kill+RESTART of a controller (ctrl-1), the restarted node loops forever: ha_on_primary_ctrl_fd_rdhup → CTRL_ELECTING → raft_cb_become_follower → CTRL_STANDBY every ~heartbeat_timeout, term climbing unbounded (473+), while ctrl-2 stays PRIMARY at term 3. On ctrl-2: every heartbeat tick logs `xce_encode op 0 (NOP)` then `WARN endpoint_update_epoll rc1=-1 < 0` (netio.c:575) — a wedged outbound standby endpoint.", "file": "src/controller/ha.c, src/controller/testsuite/test_ha_liveness.c", - "root_cause": "primary_heartbeat_fn never evicted a dead/wedged outbound standby endpoint from ha->standby_endpoints. When a peer is killed/restarted, its outbound ep on the primary wedges: handle_error (netio.c:420) runs iomgr_unmonitor (EPOLL_CTL_DEL, does NOT close the fd) and the err handler may leave the ep in standby_endpoints; epoll_ctl(EPOLL_CTL_MOD) on the DEL'd fd then returns ENOENT (rc1=-1) on every heartbeat enqueue. xce_nop_req still returns >=0 (enqueue ok) so the heartbeat layer never detected the wedge. Critically, primary_heartbeat_fn step 2 (reconnect) treats a peer as 'found' (connected) if ANY ep for it is in standby_endpoints \u2014 so the wedged ep SUPPRESSED reconnection. ctrl-2 therefore never opened a fresh connection to the restarted ctrl-1, never sent a ctrlhello, so ctrl-1 never set primary_ctrl_fd / refreshed last_activity and its liveness watchdog re-elected every heartbeat_timeout forever. (The earlier suspected 'become_follower doesn't reset last_activity' was NOT a bug: ha_arm_heartbeat_timeout_vtask already stamps last_activity=now, so the ~11s period is exactly one full timeout \u2014 confirming the watchdog is correct and the sole cause is the broken channel.)", - "fix": "In primary_heartbeat_fn (ha.c) step 1, added liveness eviction symmetric with the standby's own liveness poll: for each standby ep, if ep->ts_last_in > 0 (was responsive at least once) AND (now_wall - ep->ts_last_in) > heartbeat_timeout_s, remove it from ha->standby_endpoints (zarray_del, under ha->lock) and collect it; after releasing ha->lock, repl_remove_standby() each evicted ep. Removal is the load-bearing action \u2014 the reconnect scan (step 2) then re-establishes a fresh ctrlhello channel to the restarted peer on this/next tick, and ctrl-1 settles via ha_discover_primary (no term gate, ha.c:3807) \u2192 become_follower \u2192 STANDBY. ts_last_in/ts_last_out are existing endpoint_t fields (wall clock, set in handle_input/handle_output), so no libnetio change. The evicted ep object itself is reclaimed by its err handler when the dead connection finally surfaces an epoll event (bounded leak, consistent with codebase). Eviction keys on ts_last_in>0 so a just-connected joiner (ts_last_in==0) gets grace. Regression tests: test_primary_hb_evicts_silent_standby (RED\u2192GREEN), test_primary_hb_keeps_live_standby, test_primary_hb_keeps_unconfirmed_standby in test_ha_liveness.c.", + "root_cause": "primary_heartbeat_fn never evicted a dead/wedged outbound standby endpoint from ha->standby_endpoints. When a peer is killed/restarted, its outbound ep on the primary wedges: handle_error (netio.c:420) runs iomgr_unmonitor (EPOLL_CTL_DEL, does NOT close the fd) and the err handler may leave the ep in standby_endpoints; epoll_ctl(EPOLL_CTL_MOD) on the DEL'd fd then returns ENOENT (rc1=-1) on every heartbeat enqueue. xce_nop_req still returns >=0 (enqueue ok) so the heartbeat layer never detected the wedge. Critically, primary_heartbeat_fn step 2 (reconnect) treats a peer as 'found' (connected) if ANY ep for it is in standby_endpoints — so the wedged ep SUPPRESSED reconnection. ctrl-2 therefore never opened a fresh connection to the restarted ctrl-1, never sent a ctrlhello, so ctrl-1 never set primary_ctrl_fd / refreshed last_activity and its liveness watchdog re-elected every heartbeat_timeout forever. (The earlier suspected 'become_follower doesn't reset last_activity' was NOT a bug: ha_arm_heartbeat_timeout_vtask already stamps last_activity=now, so the ~11s period is exactly one full timeout — confirming the watchdog is correct and the sole cause is the broken channel.)", + "fix": "In primary_heartbeat_fn (ha.c) step 1, added liveness eviction symmetric with the standby's own liveness poll: for each standby ep, if ep->ts_last_in > 0 (was responsive at least once) AND (now_wall - ep->ts_last_in) > heartbeat_timeout_s, remove it from ha->standby_endpoints (zarray_del, under ha->lock) and collect it; after releasing ha->lock, repl_remove_standby() each evicted ep. Removal is the load-bearing action — the reconnect scan (step 2) then re-establishes a fresh ctrlhello channel to the restarted peer on this/next tick, and ctrl-1 settles via ha_discover_primary (no term gate, ha.c:3807) → become_follower → STANDBY. ts_last_in/ts_last_out are existing endpoint_t fields (wall clock, set in handle_input/handle_output), so no libnetio change. The evicted ep object itself is reclaimed by its err handler when the dead connection finally surfaces an epoll event (bounded leak, consistent with codebase). Eviction keys on ts_last_in>0 so a just-connected joiner (ts_last_in==0) gets grace. Regression tests: test_primary_hb_evicts_silent_standby (RED→GREEN), test_primary_hb_keeps_live_standby, test_primary_hb_keeps_unconfirmed_standby in test_ha_liveness.c.", "tags": [ "logic-fix", "c", @@ -2964,99 +299,51 @@ "reconnect", "epoll", "liveness", - "election-loop", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "election-loop" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-340.md" }, { - "id": "bug-341", + "id": "bug-343", "status": "fixed", "status_updated": "2026-09-24", - "timestamp": "2026-06-10T00:23:52.563Z", - "error_message": "Wrong reference: last_heartbeat should be voted_for_term", - "file": "src/controller/ha.h", - "root_cause": "Used \"last_heartbeat\" instead of \"voted_for_term\"", - "fix": "Changed last_heartbeat \u2192 voted_for_term", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], + "status_note": "", + "component": "controller", + "severity": "crash", + "timestamp": "2026-06-14T04:34:32+00:00", + "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-344", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T00:32:34.054Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_bootstrap_progress.md", - "root_cause": "Had `ha->last_heartbeat`", - "fix": "Changed to `ha->last_activity`", + "error_message": "AddressSanitizer: heap-use-after-free in ha_rejoin_reconcile_fn (ha_convergence.c:236) during…\n\nAddressSanitizer: heap-use-after-free in ha_rejoin_reconcile_fn (ha_convergence.c:236) during test_rejoin_second_call_no_second_restart", + "file": "src/controller/testsuite/test_demotion_convergence.c", + "root_cause": "dc_run_pending_task peeked tasks.head and ran entry(args) WITHOUT dequeuing. The once-guard correctly makes the 2nd ha_rejoin_reconcile schedule no new task, so head still pointed at the first (already-run) task whose body had freed its heap ctx (args). The 2nd dc_run_pending_task re-ran that task on the freed ctx. The convergence task bodies free their own args (production delaymgr killer frees only the delay_task_t node, never args; production execv never re-runs a one-shot).", + "fix": "Made dc_run_pending_task DEQUEUE via ordq_get(&mgr->tasks, NULL), run the body, then free(t)/free(it) — mirroring the real delaymgr worker (ordq_get -> runner_fn -> FREEUP for runcount==1). A 2nd call now returns DC_NO_TASK as production would. Production ha_rejoin_reconcile/ha_rejoin_reconcile_fn unchanged (the body-frees-ctx pattern is correct).", "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "asan", + "use-after-free", + "ha", + "rejoin", + "delaymgr", + "test-harness", + "ha-convergence-recovery", + "task-3.2" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-349", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T01:39:19.368Z", - "error_message": "Missing guard clause", - "file": "src/lib/request.c", - "root_cause": "No early return/throw for edge case: l.p.type == CAPN_NULL", - "fix": "Added guard clause: if (l.p.type == CAPN_NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-350", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T01:39:31.683Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/request.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: if (req->body_which == Request_body_walcatchupentrymsg; && req->body.walcatchupentrymsg != NULL) { | Also: /* wal_entry_to_wire: fill a wire WalCatchupEntry from a C e; * fields and raw payload bytes. Sub-entry nesting is one le", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-343.md" }, { "id": "bug-351", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-09T00:00:00Z", - "error_message": "request.c: passing argument 'from' of xce_wal_batch_encode_raw / xce_snapshot_chunk_encode_raw from incompatible pointer type (xce_peerinfo_t** vs xce_peerinfo_t*)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "request.c: passing argument 'from' of xce_wal_batch_encode_raw / xce_snapshot_chunk_encode_raw…\n\nrequest.c: passing argument 'from' of xce_wal_batch_encode_raw / xce_snapshot_chunk_encode_raw from incompatible pointer type (xce_peerinfo_t** vs xce_peerinfo_t*)", "file": "src/lib/request.c", "root_cause": "ep->peer is already xce_peerinfo_t* (netio.h endpoint struct); call sites passed &ep->peer producing xce_peerinfo_t**", "fix": "Dropped the & at both call sites (xce_wal_batch_req_raw line 547, xce_snapshot_chunk_req_raw line 633) to pass ep->peer directly", @@ -3065,20 +352,23 @@ "pointer-type", "c", "request.c", - "ha-replication", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha-replication" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-351.md" }, { "id": "bug-352", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-06-10T01:46:28.589Z", - "error_message": "ASAN heap-use-after-free in free_packet (netio.c:333) via endpoint_fini <- raft_cb_become_follower (loser path) <- raft_on_vote_reply; election loser crashes on vote-reply processing", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "ASAN heap-use-after-free in free_packet (netio.c:333) via endpoint_fini <- raft_cb_become_follower…\n\nASAN heap-use-after-free in free_packet (netio.c:333) via endpoint_fini <- raft_cb_become_follower (loser path) <- raft_on_vote_reply; election loser crashes on vote-reply processing", "file": "src/controller/ha.c", "root_cause": "vote_reply_decode_and_dispatch took pkt = ep->fsm_in.pkt and freed it WITHOUT clearing ep->fsm_in.pkt; vote_reply_dispatch can synchronously reach raft_cb_become_follower -> endpoint_fini(ep), which frees the stale fsm_in.pkt again. repl_ack_decode_and_dispatch already nulls the pointer (correct pattern); the vote-reply path missed it.", "fix": "FIXED 2026-06-10: ep->fsm_in.pkt = NULL immediately after taking ownership in vote_reply_decode_and_dispatch (ha.c:2413), matching repl_ack_decode_and_dispatch. Found by task 7.7 E2E (3-node deploy, election loser crash).", @@ -3090,20 +380,23 @@ "use-after-free", "packet-ownership", "netio", - "e2e", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "e2e" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-352.md" }, { "id": "bug-353", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-10T01:46:45.213Z", - "error_message": "Config push silently rejected when standby is in CTRL_ELECTING: ha_dispatch_gate only allows requestVote/probeConfigTag, returning -E_XCE_ERR_NOT_READY for all other opcodes. If liveness poll fires during election window, any incoming config push is discarded.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Config push silently rejected when standby is in CTRL_ELECTING: ha_dispatch_gate only allows…\n\nConfig push silently rejected when standby is in CTRL_ELECTING: ha_dispatch_gate only allows requestVote/probeConfigTag, returning -E_XCE_ERR_NOT_READY for all other opcodes. If liveness poll fires during election window, any incoming config push is discarded.", "file": "src/controller/ha.c", "root_cause": "CTRL_ELECTING gate in ha_dispatch_gate (ha.c:1464) allowed only vote/probe opcodes. Config push from primary arriving during the ELECTING window (after liveness-poll-triggered failover, before election result) was rejected with -E_XCE_ERR_NOT_READY (10013). Standby never saved config, leaving conf dir empty.", "fix": "FIXED 2026-06-10: Added OPCode_hello, OPCode_nop, OPCode_replConfigPush, OPCode_walCatchupEntry, OPCode_snapshotTransfer to CTRL_ELECTING allowlist. A campaigning node must accept replication from the current primary to stay current for both win and loss paths. Controller suite green.", @@ -3114,222 +407,26 @@ "dispatch-gate", "election", "config-replication", - "silent-failure", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "silent-failure" ], "related_bugs": [ "bug-335", "bug-336" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-358", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T03:55:44.698Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/config_repl.c", - "root_cause": "Condition was: if (!rcpr->raw_bytes || rcpr->raw_bytes_len == 0 || !r)", - "fix": "Changed to: if ((ctrl_role_t)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-359", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T04:16:25.052Z", - "error_message": "Missing guard clause", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "No early return/throw for edge case: r == NULL", - "fix": "Added guard clause: if (r == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-361", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T04:47:09.048Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (opcode == OPCode_requestVote || opcode == OPCode_p)", - "fix": "Changed to: if (opcode == OPCode_requestVote ||\n opcode)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-364", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T06:26:24.340Z", - "error_message": "Incorrect value in code", - "file": "src/controller/CLAUDE.md", - "root_cause": "Had `CTRL_ELECTING`", - "fix": "Changed to `CTRL_INIT`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-367", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T07:56:21.347Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/testsuite/test_repl_unit.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* =========================================================; * Task 1.1: repl_broadcast send path tests", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-368", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T07:57:40.792Z", - "error_message": "Wrong reference: WAL_JOB should be WAL_JOB_SUBMIT", - "file": "src/controller/testsuite/test_repl_unit.c", - "root_cause": "Used \"WAL_JOB\" instead of \"WAL_JOB_SUBMIT\"", - "fix": "Changed WAL_JOB \u2192 WAL_JOB_SUBMIT", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-371", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T07:59:24.268Z", - "error_message": "Wrong reference: DEBUG3 should be TRACE", - "file": "src/controller/repl.c", - "root_cause": "Used \"DEBUG3\" instead of \"TRACE\"", - "fix": "Changed DEBUG3 \u2192 TRACE", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-373", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T08:45:30.400Z", - "error_message": "Wrong reference: request should be ep", - "file": "src/controller/cluster.c", - "root_cause": "Used \"request\" instead of \"ep\"", - "fix": "Changed request \u2192 ep", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-374", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T08:47:48.296Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 3.3 (ha-wal-replication): dispatch gate standby-role; RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_ro", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-376", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T09:17:35.218Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (opcode == OPCode_nop)", - "fix": "Changed to: if (rep_s.data_which == RequestReply_data_walackreplym)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-353.md" }, { "id": "bug-378", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-10T10:02:00.000Z", - "error_message": "AddressSanitizer: global-buffer-overflow in dataq_enq via xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: global-buffer-overflow in dataq_enq via xce_wal_batch_req_raw <- repl_broadcast…\n\nAddressSanitizer: global-buffer-overflow in dataq_enq via xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async", "file": "src/controller/testsuite/test_p2_integration.c", "root_cause": "Tests 8 and 9 declared ep_a/ep_b/g_ep9_a/g_ep9_b as 'static int' (4 bytes) and passed them to repl_add_standby without (void*) cast. repl_broadcast then called xce_wal_batch_req_raw(s->ep, ...) using the stored int* as a real network endpoint, causing an 8-byte read past the 4-byte int. Neither test installed repl_broadcast_send_hook to intercept the call.", "fix": "Added #include typemap.h; added extern repl_broadcast_send_hook declaration guarded by XCE_TESTING; added p2_drop_send_hook no-op; installed hook around persist_wal_sync_append in tests 8 and 9; fixed missing (void*) casts on all four ep variables", @@ -3338,199 +435,23 @@ "global-buffer-overflow", "repl_broadcast", "unit-test", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "c" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-380", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T10:37:21.924Z", - "error_message": "Wrong reference: test_catchup_full_lifecycle_integration should be test_single_node_committed_lsn_unchanged", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Used \"test_catchup_full_lifecycle_integration\" instead of \"test_single_node_committed_lsn_unchanged\"", - "fix": "Changed test_catchup_full_lifecycle_integration \u2192 test_single_node_committed_lsn_unchanged", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-382", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T11:54:19.749Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-wal-replication/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-384", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T14:13:01.581Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/tech.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-386", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T14:43:53.161Z", - "error_message": "Function not marked async", - "file": "CLAUDE.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-388", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-10T14:54:18.503Z", - "error_message": "Function not marked async", - "file": "src/controller/CLAUDE.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-389", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T02:46:08.749Z", - "error_message": "Wrong reference: signal should be stdio", - "file": "src/lib/delay.c", - "root_cause": "Used \"signal\" instead of \"stdio\"", - "fix": "Changed signal \u2192 stdio", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-391", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T03:41:14.912Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha", - "fix": "Added guard clause: if (!ha)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-392", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T03:46:02.642Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: sync_rc_out", - "fix": "Added guard clause: if (sync_rc_out)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-394", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T05:09:37.037Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had `persist_truncate_wal_above`", - "fix": "Changed to `restore_job`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-378.md" }, { "id": "bug-396", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "worker", + "severity": "unknown", "timestamp": "2026-06-10T22:45:00-07:00", - "error_message": "Worker job state notifications (OPCode_notify) are permanently lost when the controller dispatch gate rejects them (quorum loss / COMMITTING window): worker never reads replies, never retries, no rejoin resync", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Worker job state notifications (OPCode_notify) are permanently lost when the controller dispatch…\n\nWorker job state notifications (OPCode_notify) are permanently lost when the controller dispatch gate rejects them (quorum loss / COMMITTING window): worker never reads replies, never retries, no rejoin resync", "file": "src/worker/src/controller_connector.c, src/lib/xceapi.c, src/controller/cluster.c", "root_cause": "xce_notify is enqueue-and-forget (no cbuf_get); error replies dropped via cbuf_tryput -EAGAIN (xcectx.c:304); xcecd_job_finished return ignored (FIXME at xcecd_worker_utils.c:298); xce_join_req carries no running-job state so rejoin cannot resync", "fix": "FIXED 2026-06-12 (ha-commit-barrier-rollout tasks 1.2+4.2+4.3): dispatch gate admits fact opcodes (notify/update/setpreason/clrpreason) without write quorum (ha.c CTRL_PRIMARY allowlist); facts append via RETAIN (log-is-the-queue, no timeout) and commit when quorum returns. End-to-end retention proof: test_fact_retention_across_quorum_loss. Worker-side redelivery gap remains (worker-rejoin-resync backlog covers disconnect windows).", @@ -3540,484 +461,75 @@ "silent-data-loss", "worker", "quorum", - "ha-consistency-invariants", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha-consistency-invariants" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-398", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T06:35:26.288Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had `restore_job`", - "fix": "Changed to \"irrevocable\"", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-399", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T06:43:14.709Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had 't span syscalls and degrade to serial mode), C ec", - "fix": "Changed to 't span syscalls), C ecosystem dead (libitm unmain", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-400", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T07:23:53.901Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had \"return quickly\"", - "fix": "Changed to `--vcpus`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-401", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T07:47:34.756Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had `mark_job_*`", - "fix": "Changed to \"pending facts\"", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-403", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T08:34:47.868Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-consistency-invariants/design.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-405", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T09:47:16.750Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-single-leader-role/design.md", - "root_cause": "Had `writable_lsn`", - "fix": "Changed to 's \"", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-406", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T09:49:04.296Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-single-leader-role/tasks.md", - "root_cause": "Had \" --exclude-dir=.git --exclude-dir=.kiro --exclude", - "fix": "Changed to \" --exclude-dir=.git --exclude-dir=specs --exclude", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-407", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T10:01:15.181Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-barrier-rollout/design.md", - "root_cause": "Had `repl.h`", - "fix": "Changed to `entry_lsn`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-408", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T10:01:39.424Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-convergence-recovery/tasks.md", - "root_cause": "Had `-XCE_ERR_NOT_READY`", - "fix": "Changed to `OPCode_notify`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-396.md" }, { "id": "bug-410", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-11T13:30:00Z", - "error_message": "test_commit_engine fixture: Expected 0 Was 1 from repl_add_standby", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_commit_engine fixture: Expected 0 Was 1 from repl_add_standby\n\ntest_commit_engine fixture: Expected 0 Was 1 from repl_add_standby", "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "repl_add_standby returns the slot index (>=0) on success, not 0 \u2014 second add legitimately returns 1", + "root_cause": "repl_add_standby returns the slot index (>=0) on success, not 0 — second add legitimately returns 1", "fix": "fixture asserts TEST_ASSERT_EQUAL_INT(i, repl_add_standby(...)) per standby slot", "tags": [ "controller", "testsuite", "repl", - "return-convention", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "return-convention" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-410.md" }, { "id": "bug-411", "status": "fixed", "status_updated": "2026-09-24", "status_note": "95a10ced", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-11T13:35:00Z", - "error_message": "3 pre-existing controller testsuite failures masked by exit code 0: test_committed_lsn_does_not_advance_on_timeout (-110), test_repl_ack_after_promote_gate_allows_jobsub (-10013), test_rdhup_primary_dispatch_gate_opens (-10013)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "3 pre-existing controller testsuite failures masked by exit code 0…\n\n3 pre-existing controller testsuite failures masked by exit code 0: test_committed_lsn_does_not_advance_on_timeout (-110), test_repl_ack_after_promote_gate_allows_jobsub (-10013), test_rdhup_primary_dispatch_gate_opens (-10013)", "file": "src/controller/testsuite/controller_test.c", - "root_cause": "each test group calls UNITY_BEGIN/UNITY_END internally, resetting Unity counters \u2014 main()'s final UNITY_END returns 0 unless the LAST group fails, so meson reported the suite green despite 3 real failures; Unity also misattributes failure FILE names to whichever file's UNITY_BEGIN ran (e.g. reports test_repl_unit.c for a test living in test_p2_integration.c)", - "fix": "not fixed (out of task 1.1 boundary) \u2014 documented; failures exist identically on baseline without ha-commit-engine changes (verified by stash/rebuild). persist_wal_sync_append returns -ETIMEDOUT while the test expects rc==0 with sync_rc==-ETIMEDOUT; the two gate tests fail ha_has_write_quorum (standby_endpoints empty with n_peers=2)", + "root_cause": "each test group calls UNITY_BEGIN/UNITY_END internally, resetting Unity counters — main()'s final UNITY_END returns 0 unless the LAST group fails, so meson reported the suite green despite 3 real failures; Unity also misattributes failure FILE names to whichever file's UNITY_BEGIN ran (e.g. reports test_repl_unit.c for a test living in test_p2_integration.c)", + "fix": "not fixed (out of task 1.1 boundary) — documented; failures exist identically on baseline without ha-commit-engine changes (verified by stash/rebuild). persist_wal_sync_append returns -ETIMEDOUT while the test expects rc==0 with sync_rc==-ETIMEDOUT; the two gate tests fail ha_has_write_quorum (standby_endpoints empty with n_peers=2)", "tags": [ "controller", "testsuite", "unity", "exit-code", "pre-existing", - "ha", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-413", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T15:28:35.990Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550; * Task 3.1 \u2014 membership re-evaluation (commit_on_membership_ | Also: /* Task 3.1 \u2014 membership re-evaluation */; RUN_TEST(test_catchup_standby_ack_does_not_count_toward_pend", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "95a10ced" ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-420", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T16:19:51.195Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 4.2 \u2014 wired gate: registry truth at ha_dispatch_gate; RUN_TEST(test_gate_wired_registry_catchup_standby_rejects_th", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-421", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T16:23:46.118Z", - "error_message": "Missing guard clause", - "file": "src/controller/repl.c", - "root_cause": "No early return/throw for edge case: !r", - "fix": "Added guard clause: if (!r)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-423", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T22:39:24.704Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550; * Task 5.1 \u2014 integration: ACK feed and the job-submit sync-a | Also: #include \"typemap.h\" /* wal_catchup_entry_t (task 5; #include \"wal_types.h\" /* WAL_JOB_SUBMIT (task 5.1 sy", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-424", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T23:08:49.909Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-engine/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-425", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T23:15:25.014Z", - "error_message": "Wrong return value", - "file": "src/controller/repl.c", - "root_cause": "Was returning: idx;", - "fix": "Now returns: * above mutates nothing, so it deliberately skips ", - "tags": [ - "auto-detected", - "return-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-428", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T23:43:45.216Z", - "error_message": "Incorrect value in code", - "file": "src/include/xce/cluster_def.h", - "root_cause": "Had \"ha.h\"", - "fix": "Changed to \"commit.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-429", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T23:44:16.357Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/cluster.c", - "root_cause": "Condition was: if (ha_is_primary(&c->ha)", - "fix": "Changed to: if (c->commit.p == NULL)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-431", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T23:45:16.518Z", - "error_message": "Incorrect value in code", - "file": "src/controller/persist.c", - "root_cause": "Had \"xce/types.h\"", - "fix": "Changed to \"xce/errnox.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-432", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T23:45:42.031Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-433", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-11T23:45:51.649Z", - "error_message": "Wrong reference: test_single_node_committed_lsn_unchanged should be test_single_node_committed_lsn_advances", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Used \"test_single_node_committed_lsn_unchanged\" instead of \"test_single_node_committed_lsn_advances\"", - "fix": "Changed test_single_node_committed_lsn_unchanged \u2192 test_single_node_committed_lsn_advances", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-434", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-12T00:23:08.815Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-411.md" }, { "id": "bug-435", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-11T00:00:00Z", - "error_message": "test_committed_lsn_does_not_advance_on_timeout: Expected 0 Was -110 (baseline failure, masked by bug-411)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_committed_lsn_does_not_advance_on_timeout: Expected 0 Was -110 (baseline failure, masked by…\n\ntest_committed_lsn_does_not_advance_on_timeout: Expected 0 Was -110 (baseline failure, masked by bug-411)", "file": "src/controller/testsuite/test_p2_integration.c", "root_cause": "Stale test expectation: pinned the old accept-on-timeout contract (rc==0 + sync_rc=-ETIMEDOUT advisory). ha-commit-engine made persist_wal_sync_append reject-on-timeout (returns -ETIMEDOUT, Req 2.1/4.4). Fixture was also engine-less (deprecated ce==NULL legacy path).", "fix": "Updated expectation to rc==-ETIMEDOUT and wired a commit engine (n_peers=3) with a priming async append so the sync entry lands at LSN 1 on the engine path; kept the committed_lsn-unmoved assertion.", @@ -4025,22 +537,25 @@ "ha-commit-engine", "task-6.1", "stale-expectation", - "unity-misattribution", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "unity-misattribution" ], "related_bugs": [ "bug-411" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-435.md" }, { "id": "bug-436", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-11T00:00:00Z", - "error_message": "test_repl_ack_after_promote_gate_allows_jobsub / test_rdhup_primary_dispatch_gate_opens: Expected 0 Was -10013 (NOT_READY; baseline failures, masked by bug-411)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_repl_ack_after_promote_gate_allows_jobsub / test_rdhup_primary_dispatch_gate_opens: Expected…\n\ntest_repl_ack_after_promote_gate_allows_jobsub / test_rdhup_primary_dispatch_gate_opens: Expected 0 Was -10013 (NOT_READY; baseline failures, masked by bug-411)", "file": "src/controller/testsuite/test_ha_repl_ack.c, src/controller/testsuite/test_ha_rdhup_integration.c", "root_cause": "CTRL_PRIMARY dispatch gate now requires write quorum (>= n_peers/2 connected standbys). Fixtures promote via a standby ACK but never register the connected standby; legacy fallback counts ha->standby_endpoints (NULL -> 0). In the rdhup test ha_on_primary_ctrl_fd_rdhup also overwrites ha->n_peers=1 with candidates_count=2 (ha.c:3484).", "fix": "Populated standby_endpoints with one endpoint (the ACKing standby) in both fixtures so quorum legitimately holds on a healthy primary; gate unchanged. Freed the zarray before ha_fini (ha_fini does not own it).", @@ -4048,122 +563,25 @@ "ha-commit-engine", "task-6.1", "quorum-gate", - "fixture-fix", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fixture-fix" ], "related_bugs": [ "bug-411" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-437", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-12T00:51:52.673Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-engine/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-438", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-12T03:52:57.142Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'testsuite/test_commit_barrier_rollout.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-440", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-12T04:18:19.158Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: detail == HOOK_DETAIL_WORKER_JOIN ||\n detail == H", - "fix": "Added guard clause: if (detail == HOOK_DETAIL_WORKER_JOIN ||\n )", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-442", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-12T04:50:32.154Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_commit_barrier_rollout.c", - "root_cause": "Had \"xce/types.h\"", - "fix": "Changed to \"xce/errnox.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-443", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-12T05:14:36.392Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: p == NULL", - "fix": "Added guard clause: if (p == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-436.md" }, { "id": "bug-445", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-06-12T09:30:00Z", - "error_message": "AddressSanitizer SEGV in hash_string via makeResource when running controller_unit_test -t test_commit_barrier_reply standalone", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer SEGV in hash_string via makeResource when running controller_unit_test -t…\n\nAddressSanitizer SEGV in hash_string via makeResource when running controller_unit_test -t test_commit_barrier_reply standalone", "file": "src/controller/testsuite/test_commit_barrier_reply.c", "root_cause": "Global xstring table uninitialised in standalone runs: makeString() returns NULL, resobj name becomes NULL, key_add hashes NULL. In full-suite runs test_worker_join.c:520 initialises the table unguarded, so the bug only appears with -t selection.", "fix": "Probe makeString(\"cbrp-probe\") in the group entry; call xstring_table_init(139) only when the probe returns NULL (avoids double init in full-suite order).", @@ -4172,20 +590,23 @@ "xstring", "fixture", "segv", - "ha-commit-barrier-rollout", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha-commit-barrier-rollout" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-445.md" }, { "id": "bug-446", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-12T09:30:00Z", - "error_message": "makeJob() returns NULL (Expected Non-NULL) in commit-barrier reply fixture", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "makeJob() returns NULL (Expected Non-NULL) in commit-barrier reply fixture\n\nmakeJob() returns NULL (Expected Non-NULL) in commit-barrier reply fixture", "file": "src/controller/testsuite/test_commit_barrier_reply.c", "root_cause": "job_init (lib/jobj.c) returns -EINVAL when xce_job_t.submit == NULL, and makeJobsub requires job_obj_init() module tables which no controller test group initialises.", "fix": "Fixture calls job_obj_init(64, 64) once per process (static guard) and provides a zeroed static job_submit_data_t as jdef.submit.", @@ -4193,20 +614,23 @@ "testsuite", "jobj", "fixture", - "ha-commit-barrier-rollout", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha-commit-barrier-rollout" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-446.md" }, { "id": "bug-447", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "leak", "timestamp": "2026-06-12T09:30:00Z", - "error_message": "queue_ctrl/worker_ctrl append NO WAL entry and reply status 1 on success; job_ctrl also returns 1 (key_find found-rc leak)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "queue_ctrl/worker_ctrl append NO WAL entry and reply status 1 on success; job_ctrl also returns 1…\n\nqueue_ctrl/worker_ctrl append NO WAL entry and reply status 1 on success; job_ctrl also returns 1 (key_find found-rc leak)", "file": "src/controller/queue.c", "root_cause": "key_find (xkvmgr_find, lib/kvmgr.c:130-151) returns 1 when the key is found. queue_ctrl (queue.c:60), worker_ctrl (worker.c:643) and job_ctrl (job.c:314) store it in rc and never reset it: the 'if (rc == 0)' AFTER_EVENT hook gate is skipped (no WAL_QUEUE_CTRL / WAL_WORKER_CTRL append on the found path) and the requester reply carries status 1 instead of 0. Production gap discovered by task-3.1 RED tests; queue/host control state changes are currently never persisted or replicated.", "fix": "FIXED (task 3.2, 2026-06-12): normalized the key_find found-rc to 0 in queue_ctrl (queue.c), worker_ctrl (worker.c) and job_ctrl (job.c) after the miss/-ENOENT checks, so the AFTER_EVENT hook gate fires (WAL append + replication) and the handlers' commit-barrier pattern sees 0. Handlers do_jobcontrol_op/do_queuecontrol_op/do_hostcontrol_op now run persist_commit_barrier(&c->persist, 0) on rc==0 before replying; handle_worker_join barriers between AFTER_EVENT and the success reply. All 12 test_commit_barrier_reply tests PASS; full controller suite 0 FAIL.", @@ -4218,21 +642,23 @@ "key_find", "wal", "ha-commit-barrier-rollout", - "red-finding", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "red-finding" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-447.md" }, { "id": "bug-448", "status": "fixed", "status_updated": "2026-09-27", "status_note": "0c1bac44", + "component": "controller", + "severity": "leak", "timestamp": "2026-06-12T10:05:12.464Z", - "error_message": "job_ctrl reference leak: reference(j) taken at job.c:317 but normal exit path (job.c:444-446) never calls dereference(j); only the workermgr-miss error path releases it", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "job_ctrl reference leak: reference(j) taken at job.c:317 but normal exit path (job.c:444-446)…\n\njob_ctrl reference leak: reference(j) taken at job.c:317 but normal exit path (job.c:444-446) never calls dereference(j); only the workermgr-miss error path releases it", "file": "src/controller/job.c", "root_cause": "asymmetric reference/dereference across exit paths, predates ha-commit-barrier-rollout", "fix": "NOT YET FIXED - discovered by task 3.2 adversarial review; add dereference(j) on the normal exit path", @@ -4243,24 +669,29 @@ "job", "refcount", "leak", - "pre-existing", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pre-existing" ], "related_bugs": [ "bug-447" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "0c1bac44" + ], + "record": ".wolf/bugs/bug-448.md" }, { "id": "bug-451", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-12T11:05:00-07:00", - "error_message": "va_list type confusion: WAL_JOB_RUSAGE hook payload reads job_t fields from an xce_job_rusages_t*", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "va_list type confusion: WAL_JOB_RUSAGE hook payload reads job_t fields from an xce_job_rusages_t*\n\nva_list type confusion: WAL_JOB_RUSAGE hook payload reads job_t fields from an xce_job_rusages_t*", "file": "src/controller/job.c (caller) / src/controller/persist.c (consumer)", - "root_cause": "update_job_rusages fires AFTER_EVENT HOOK_DETAIL_JOB_RUSAGE_UPDATE with (cluster_t*, xce_job_rusages_t*) at job.c:1015-1016, but persist_job_hook_fun's default arm (persist.c:1354-1359) reads the second vararg as job_t* and dereferences j->job.state at offset 56 == sizeof(xce_job_rusages_t) \u2014 an out-of-bounds read on an exactly-sized allocation; the WAL_JOB_RUSAGE payload state field is garbage (jobid at offset 0 lines up by accident)", + "root_cause": "update_job_rusages fires AFTER_EVENT HOOK_DETAIL_JOB_RUSAGE_UPDATE with (cluster_t*, xce_job_rusages_t*) at job.c:1015-1016, but persist_job_hook_fun's default arm (persist.c:1354-1359) reads the second vararg as job_t* and dereferences j->job.state at offset 56 == sizeof(xce_job_rusages_t) — an out-of-bounds read on an exactly-sized allocation; the WAL_JOB_RUSAGE payload state field is garbage (jobid at offset 0 lines up by accident)", "fix": "FIXED (TDD RED->GREEN). Added a dedicated HOOK_DETAIL_JOB_RUSAGE_UPDATE arm to persist_job_hook_fun (persist.c) that reads the real xce_job_rusages_t* payload and looks the job up by jobid via the cluster_t* the hook already carries (c->jobmgr), instead of falling through to the default arm that misread it as a job_t*. Fix candidate (1) from the original investigation was used -- a dedicated RUSAGE arm -- not candidate (2) (firing the hook with (c, j) like other job hooks), because sched_job_rusage_update_fun/apiserver_job_rusage_update_fun (this hook's other two listeners, sched.c/apiserver.c) genuinely need the full rusages payload (pids/pgids/usages), so changing what job.c fires would have broken them. Regression test (test_commit_barrier_reply.c's test_update_op_rusage_receipt_reply_facts_retained, extended): drives the real do_update_op/update_job_rusages call path and scans the resulting WAL_JOB_RUSAGE entry, asserting state == JOB_STAT_PEND (the job's real state) -- RED under the old code (read back 0, the zeroed padding past xce_job_rusages_t's end), GREEN under the fix. Also fixed test_persist_hooks.c's test_job_hook_rusage_writes_wal_entry, which previously masked the bug by passing a job_t* stub instead of the real xce_job_rusages_t* shape -- now builds a minimal real cluster+jobmgr+job and passes the correct payload type; added a sibling test for the job-not-found graceful-skip path (returns 0, appends nothing, no crash). Full controller_unit_test suite: 1096 tests, 2 pre-existing unrelated failures (test_config_repl.c, filed as bug-1561 -- overlayfs fdatasync artifact, zero diff in that file this session), no regressions.\n\n[Original entry]\nNOT FIXED (discovered while writing task 3.3 receipt-ACK verdict pins; production out of boundary). Test test_update_op_rusage_receipt_reply_facts_retained uses a padded union so the pin stays in-bounds under ASAN without masking the bug; fix candidates: dedicated RUSAGE arm in persist_job_hook_fun reading xce_job_rusages_t*, or fire the hook with (c, j) like the other job hooks", "tags": [ "pre-existing", @@ -4270,62 +701,26 @@ "rusage", "asan", "oob-read", - "ha-commit-barrier-rollout", - "fixed" + "ha-commit-barrier-rollout" ], "related_bugs": [ "bug-447", "bug-448" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-452", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-12T15:03:38.962Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-457", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T00:18:17.080Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !c->ha.standby_endpoints", - "fix": "Added guard clause: if (!c->ha.standby_endpoints)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-451.md" }, { "id": "bug-459", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-13T00:30:00Z", - "error_message": "configAck quorum fabricated success: counted acked_config_lsn > 0 (any previous upgrade's ack), denominator n_active with no self-count, and n_active==0 short-circuited to success; upgradeConfig also ACKed the admin tool unconditionally and requested restart even when the majority wait timed out (logged-only result)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "configAck quorum fabricated success: counted acked_config_lsn > 0 (any previous upgrade's ack)…\n\nconfigAck quorum fabricated success: counted acked_config_lsn > 0 (any previous upgrade's ack), denominator n_active with no self-count, and n_active==0 short-circuited to success; upgradeConfig also ACKed the admin tool unconditionally and requested restart even when the majority wait timed out (logged-only result)", "file": "src/controller/repl.c, src/controller/config_repl.c", "root_cause": "legacy connectivity-based counting on the configAck track predated the commit engine's per-push arithmetic; push_config_to_standbys returned the transport push count and only logged the barrier outcome, so do_upgrade_config_with_bytes could not observe failure", "fix": "task 5.2: configAck branch counts acked_config_lsn >= push_tag (entry_lsn carries the tag), majority 1+count > n_peers/2 over engine n_peers, n_peers<=1 immediate 0, ce==NULL -EINVAL fail-loud; push_config_to_standbys returns the barrier result (push count via acked_out); do_upgrade_config_with_bytes sends send_upgrade_error and skips success ACK + restart on non-zero; committed_config_tag advance stays success-branch-only", @@ -4335,42 +730,25 @@ "quorum", "configAck", "commit-barrier", - "upgradeConfig", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "upgradeConfig" ], "related_bugs": [ "bug-396" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-461", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T01:11:49.673Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/config_repl.c", - "root_cause": "Condition was: if (n == 0)", - "fix": "Changed to: if (n_out)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-459.md" }, { "id": "bug-465", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-13T01:30:00Z", - "error_message": "Config-ACK signal never produced in production: repl_on_config_ack had zero production callers; standby replconfigpushreplymsg silently dropped in repl_ack_decode_and_dispatch (ha.c) and myreq_handler default case (cluster.c) -> every multi-member config upgrade barrier timed out (upgrades bricked)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Config-ACK signal never produced in production: repl_on_config_ack had zero production callers…\n\nConfig-ACK signal never produced in production: repl_on_config_ack had zero production callers; standby replconfigpushreplymsg silently dropped in repl_ack_decode_and_dispatch (ha.c) and myreq_handler default case (cluster.c) -> every multi-member config upgrade barrier timed out (upgrades bricked)", "file": "src/controller/ha.c", "root_cause": "ha-config-bootstrap task 8.1's production ingestion half was never landed; the 5.2 barrier relied on a signal nothing produced (tests injected repl_on_config_ack directly)", "fix": "Added replconfigpushreplymsg decode branch in repl_ack_decode_and_dispatch (ha.c) and an OPCode_reply ingestion case in cluster.c myreq_handler, both calling repl_on_config_ack(&c->persist.repl, ep, versionTag); wire-path integration test added (test_config_ack_wire_path_completes_barrier)", @@ -4380,20 +758,23 @@ "barrier", "capnp", "wire-path", - "task-5.2", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-5.2" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-465.md" }, { "id": "bug-466", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-06-13T01:30:00Z", - "error_message": "Unconditional 500ms config barrier wait in push_config_to_standbys reachable from iomgr epoll thread (ha.c raft_cb_become_leader, ha_dispatch_repl_action_for_standby, legacy election path; join.c controller join) -> blocks all controller I/O; the awaited ACK arrives on the blocked thread (structural deadlock)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Unconditional 500ms config barrier wait in push_config_to_standbys reachable from iomgr epoll…\n\nUnconditional 500ms config barrier wait in push_config_to_standbys reachable from iomgr epoll thread (ha.c raft_cb_become_leader, ha_dispatch_repl_action_for_standby, legacy election path; join.c controller join) -> blocks all controller I/O; the awaited ACK arrives on the blocked thread (structural deadlock)", "file": "src/controller/config_repl.c", "root_cause": "task 5.2 made the barrier unconditional inside push_config_to_standbys while best-effort catch-up/election pushes on iomgr-reachable paths still called the same entry", "fix": "Split the API: exported transport-only fan-out push_config_transport_only (no barrier, iomgr-safe) and switched all 5 ha.c sites + join.c:187 to it; push_config_to_standbys (transport+barrier) kept for the vtask-context upgrade path only; doc comments updated in config_repl.h", @@ -4403,158 +784,26 @@ "deadlock", "config-repl", "api-split", - "task-5.2", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-5.2" ], "related_bugs": [ "bug-465" ], + "fix_commits": [], + "record": ".wolf/bugs/bug-466.md" + }, + { + "id": "bug-477", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", + "timestamp": "2026-06-13", + "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-467", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T01:38:08.308Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (rep_s.data_which == RequestReply_data_replconfigpu)", - "fix": "Changed to: if (0 /* REVIEWER RED CHECK */ && rep_s.data_which == )", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-468", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T03:56:09.185Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-469", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T06:33:41.449Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_cluster.c'", - "fix": "Changed to 'testsuite/test_ha_role_lifecycle.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-470", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T06:34:01.929Z", - "error_message": "Wrong reference: test_ha_cluster should be test_ha_role_lifecycle", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_ha_cluster\" instead of \"test_ha_role_lifecycle\"", - "fix": "Changed test_ha_cluster \u2192 test_ha_role_lifecycle", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-471", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T06:41:01.532Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "No early return/throw for edge case: pid < 0", - "fix": "Added guard clause: if (pid < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-473", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T07:32:43.287Z", - "error_message": "Wrong return value", - "file": "src/controller/persist.c", - "root_cause": "Was returning: WAL_MODE_LOCAL;", - "fix": "Now returns: WAL_MODE_RETAIN;", - "tags": [ - "auto-detected", - "return-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-476", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T08:38:00.471Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: c->ha.hooks_registered", - "fix": "Added guard clause: if (c->ha.hooks_registered)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "error_message": "AddressSanitizer: global-buffer-overflow in xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async <- ha_arm_own_term_gate <- ha_promote_to_primary, during test_promotion_registers_persist_hooks (test_ha_role_lifecycle.c case 3)", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect \u2014 OUT OF task 2.2 boundary)", + "error_message": "AddressSanitizer: global-buffer-overflow in xce_wal_batch_req_raw <- repl_broadcast <…\n\nAddressSanitizer: global-buffer-overflow in xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async <- ha_arm_own_term_gate <- ha_promote_to_primary, during test_promotion_registers_persist_hooks (test_ha_role_lifecycle.c case 3)", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect — OUT OF task 2.2 boundary)", "root_cause": "task-2.2's ha_promote_to_primary now appends a WAL_NOOP via persist_wal_async, which broadcasts to registered standbys. The lifecycle fixture (lifecycle_cluster_setup) registers fake `int g_lc_eps[]` endpoints via repl_add_standby and NEVER installs repl_broadcast_send_hook, so repl_broadcast calls xce_wal_batch_req_raw(s->ep) and reads ep->peer (offset 36) past the 4-byte int. IDENTICAL to the prior test_p2_integration tests 8/9 bug already in buglog.", "fix": "Test-side fix (parent must apply, mirrors the documented p2_integration fix): in test_ha_role_lifecycle.c add `extern int (*repl_broadcast_send_hook)(void*, wal_catchup_entry_t**, int);` and a no-op drop hook, install it in lifecycle_cluster_setup before promotion (reset to NULL in teardown), and cast endpoints to (void*). Production ha.c is correct per design (persist_wal_async appends+broadcasts by design).", "tags": [ @@ -4564,264 +813,49 @@ "asan", "repl_broadcast", "global-buffer-overflow", - "out-of-boundary", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "out-of-boundary" ], - "id": "bug-477", + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-477.md" + }, + { + "id": "bug-478", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-13", "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, - "related_bugs": [] - }, - { - "error_message": "task-2.1 lifecycle fixture off-by-one: case 5/6/7 assert writable_lsn==noop_lsn and current_lsn==noop_lsn+1, but lifecycle_cluster_setup primes current_lsn = appended_lsn + 1, so a real persist_wal_async appends the noop at appended_lsn+1 (current_lsn->appended_lsn+2), giving noop_lsn=appended_lsn+1 != asserted appended_lsn.", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect \u2014 OUT OF task 2.2 boundary)", + "error_message": "task-2.1 lifecycle fixture off-by-one: case 5/6/7 assert writable_lsn==noop_lsn and…\n\ntask-2.1 lifecycle fixture off-by-one: case 5/6/7 assert writable_lsn==noop_lsn and current_lsn==noop_lsn+1, but lifecycle_cluster_setup primes current_lsn = appended_lsn + 1, so a real persist_wal_async appends the noop at appended_lsn+1 (current_lsn->appended_lsn+2), giving noop_lsn=appended_lsn+1 != asserted appended_lsn.", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect — OUT OF task 2.2 boundary)", "root_cause": "persist_wal_async assigns hdr.lsn = current_lsn++ (entry takes the CURRENT value). The fixture comment assumes the next append takes current_lsn-1, which is off by one. To make the noop land at `appended_lsn`, the prime must be `current_lsn = appended_lsn`, not `appended_lsn + 1`.", - "fix": "Test-side fix (parent): set g_lc.persist.current_lsn = appended_lsn in lifecycle_cluster_setup. Production noop_lsn = current_lsn-1 (post-append) is correct per design \u00a7Promotion Rewrite / brief.", + "fix": "Test-side fix (parent): set g_lc.persist.current_lsn = appended_lsn in lifecycle_cluster_setup. Production noop_lsn = current_lsn-1 (post-append) is correct per design §Promotion Rewrite / brief.", "tags": [ "ha-single-leader-role", "task-2.2", "test-fixture", "off-by-one", "lsn", - "out-of-boundary", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "id": "bug-478", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13", - "last_seen": "2026-09-19T09:00:00Z", - "occurrences": 1, - "related_bugs": [] - }, - { - "id": "bug-480", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T08:57:05.687Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "out-of-boundary" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-482", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T09:36:17.806Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_start_election_async.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-483", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T09:45:01.531Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-484", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T10:16:06.331Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-single-leader-role/tasks.md", - "root_cause": "Had `lifecycle_cluster_setup`", - "fix": "Changed to `ha_promote_to_primary`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-493", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T12:08:04.621Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (role == CTRL_PRIMARY || role == CTRL_PRIMARY_COMMI)", - "fix": "Changed to: if (role == CTRL_PRIMARY)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-494", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T12:10:08.938Z", - "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", - "file": "src/controller/ha.c", - "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", - "fix": "Changed CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-496", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T13:07:42.791Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "Had \"winning election must land DIRECTLY in CTRL_PRIMA", - "fix": "Changed to \"winning election must land DIRECTLY in CTRL_PRIMA", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-497", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T13:10:05.452Z", - "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", - "file": "src/controller/testsuite/test_ha_connection_reuse.c", - "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", - "fix": "Changed CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-498", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T13:10:16.164Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_connection_reuse.c", - "root_cause": "Had \"err_handler: ep must be removed from standby_endp", - "fix": "Changed to \"err_handler: ep must be removed from standby_endp", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-500", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T13:14:11.496Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Had \"test_dispatch_gate_committing_allows_repl_ops: PA", - "fix": "Changed to \"test_dispatch_gate_primary_no_quorum_allows_repl_", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-501", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T13:15:33.777Z", - "error_message": "Wrong reference: test_nonblocking_single_node_election_enters_committing should be test_nonblocking_single_node_election_enters_primary_directly", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Used \"test_nonblocking_single_node_election_enters_committing\" instead of \"test_nonblocking_single_node_election_enters_primary_directly\"", - "fix": "Changed test_nonblocking_single_node_election_enters_committing \u2192 test_nonblocking_single_node_election_enters_primary_directly", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-478.md" }, { "id": "bug-503", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-13T13:30:00.000Z", - "error_message": "test_dispatch_gate_primary_no_quorum_denies_job_ops: Expected -10013 Was 0 (jobsub allowed, expected -XCE_ERR_NOT_READY)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_dispatch_gate_primary_no_quorum_denies_job_ops: Expected -10013 Was 0 (jobsub allowed…\n\ntest_dispatch_gate_primary_no_quorum_denies_job_ops: Expected -10013 Was 0 (jobsub allowed, expected -XCE_ERR_NOT_READY)", "file": "src/controller/testsuite/test_ha_startup.c", "root_cause": "Retargeting the dispatch-gate test from CTRL_PRIMARY_COMMITTING to CTRL_PRIMARY (ha-single-leader-role task 5.1) changed gate semantics: the deleted COMMITTING branch denied job ops UNCONDITIONALLY, but the CTRL_PRIMARY branch only denies job ops when ha_has_write_quorum() is false. ha_init leaves n_peers=0, so ha_has_write_quorum returns true (single-node = self-majority, ha.c:1437) and jobsub was ALLOWED (rc=0). A blind role swap does not reproduce the original deny.", "fix": "Set ha.n_peers = 2 in both retargeted dispatch-gate tests so a multi-node primary with zero connected standbys genuinely lacks write quorum (connected 0 < n_peers/2=1), making the universal commit barrier deny job ops with -XCE_ERR_NOT_READY while still permitting replication ops.", @@ -4831,442 +865,23 @@ "write-quorum", "test-retarget", "commit-barrier", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "c" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-505", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T14:01:32.553Z", - "error_message": "Wrong reference: uint32_t should be ha_config_t", - "file": "src/controller/ha.h", - "root_cause": "Used \"uint32_t\" instead of \"ha_config_t\"", - "fix": "Changed uint32_t \u2192 ha_config_t", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-507", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T14:43:54.194Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/ha-async-io.md", - "root_cause": "Had `CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY`", - "fix": "Changed to 's own-term commit rule (\u00a75.4.2): an election winn", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-508", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T14:54:21.781Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-deploy-ha/SKILL.md", - "root_cause": "Had `\u2192 CTRL_PRIMARY_COMMITTING`", - "fix": "Changed to `HA FSM (raft): \u2192 CTRL_PRIMARY (term=...)`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-509", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-13T15:31:21.740Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_consistency_invariants.md", - "root_cause": "Had \"XCE HA Raft+CP umbrella \u2014 sub-specs 1&2 of 4 IMPL", - "fix": "Changed to \"XCE HA Raft+CP umbrella \u2014 sub-specs 1,2,3 of 4 IM", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-510", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T00:30:31.077Z", - "error_message": "Wrong reference: stdint should be stdbool", - "file": "src/controller/include/cluster.h", - "root_cause": "Used \"stdint\" instead of \"stdbool\"", - "fix": "Changed stdint \u2192 stdbool", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-512", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T00:34:38.745Z", - "error_message": "Wrong reference: xlist_destroy should be xlist_fini", - "file": "src/controller/testsuite/test_egress_filter.c", - "root_cause": "Used \"xlist_destroy\" instead of \"xlist_fini\"", - "fix": "Changed xlist_destroy \u2192 xlist_fini", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-513", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T00:35:13.711Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'testsuite/test_egress_filter.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-514", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T00:37:31.926Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_egress_filter.c", - "root_cause": "No early return/throw for edge case: iter_make(&it, c->jobmgr", - "fix": "Added guard clause: if (iter_make(&it, c->jobmgr)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-516", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T01:46:09.600Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-518", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T01:50:24.429Z", - "error_message": "Missing guard clause", - "file": "src/controller/sched.c", - "root_cause": "No early return/throw for edge case: l->count > 0", - "fix": "Added guard clause: if (l->count > 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-519", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T01:52:56.833Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_egress_filter.c", - "root_cause": "Had \"\\n>> commit-gated scheduler egress tests (RED, ta", - "fix": "Changed to \"\\n>> commit-gated scheduler egress tests (GREEN, ", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-521", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T03:55:55.380Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "Had \"commit.h\"", - "fix": "Changed to \"cluster.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-523", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T04:21:18.071Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha_convergence.c", - "root_cause": "No early return/throw for edge case: ctx == NULL || ctx->c == NULL", - "fix": "Added guard clause: if (ctx == NULL || ctx->c == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-343", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T04:34:32+00:00", - "error_message": "AddressSanitizer: heap-use-after-free in ha_rejoin_reconcile_fn (ha_convergence.c:236) during test_rejoin_second_call_no_second_restart", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "dc_run_pending_task peeked tasks.head and ran entry(args) WITHOUT dequeuing. The once-guard correctly makes the 2nd ha_rejoin_reconcile schedule no new task, so head still pointed at the first (already-run) task whose body had freed its heap ctx (args). The 2nd dc_run_pending_task re-ran that task on the freed ctx. The convergence task bodies free their own args (production delaymgr killer frees only the delay_task_t node, never args; production execv never re-runs a one-shot).", - "fix": "Made dc_run_pending_task DEQUEUE via ordq_get(&mgr->tasks, NULL), run the body, then free(t)/free(it) \u2014 mirroring the real delaymgr worker (ordq_get -> runner_fn -> FREEUP for runcount==1). A 2nd call now returns DC_NO_TASK as production would. Production ha_rejoin_reconcile/ha_rejoin_reconcile_fn unchanged (the body-frees-ctx pattern is correct).", - "tags": [ - "asan", - "use-after-free", - "ha", - "rejoin", - "delaymgr", - "test-harness", - "ha-convergence-recovery", - "task-3.2", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-526", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T04:52:58.813Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-convergence-recovery/tasks.md", - "root_cause": "Had \"No space left on device\"", - "fix": "Changed to `/tmp/xce_*`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-527", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T04:57:22.178Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /*; * ha_walack_continuation_for_test: test-only entry point for", - "tags": [ - "auto-detected", - "error-handling", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-532", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T06:12:36.513Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha_convergence.c", - "root_cause": "No early return/throw for edge case: ha == NULL", - "fix": "Added guard clause: if (ha == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-533", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T06:37:35.904Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: printf(\"\\n>> Regression pins: fan-out / snapshot fallback / ; \"re-inclusion (task 6.1 \u2014 PASS against current code)\\n\\n\"); | Also: /* \u2500\u2500 Fan-out counting send-hook (task 6.1 (a)/(d), umbrella; * repl_broadcast routes each enqueue through repl_broadcast_", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-534", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T06:38:38.792Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "Had \"delay.h\"", - "fix": "Changed to \"controller.capnp.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-536", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T07:56:01.819Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_rdhup_integration.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-537", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T08:02:40.832Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-503.md" }, { "id": "bug-538", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-14", - "error_message": "config-less controller (my_config_tag==0) became PRIMARY after RDHUP and pushed empty config, breaking replication on a live 3-node cluster", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "config-less controller (my_config_tag==0) became PRIMARY after RDHUP and pushed empty config…\n\nconfig-less controller (my_config_tag==0) became PRIMARY after RDHUP and pushed empty config, breaking replication on a live 3-node cluster", "file": "src/controller/ha.c", "root_cause": "ha_on_primary_ctrl_fd_rdhup did not apply the no-config voter-only guard that ha_start_election_async has; after primary RDHUP it set CTRL_ELECTING, allocated elect_ctx and called raft_on_leader_lost even for tag=0 nodes. Two tag=0 nodes granted each other (freshness gate only denies LOWER tag; tag0==tag0 granted), electing a config-less primary.", "fix": "Added an election-safety guard in ha_on_primary_ctrl_fd_rdhup after the WAL-truncate block and before elect_ctx alloc/hook/raft_on_leader_lost: if my_config_tag==0 && candidates_count>1, log + set role=CTRL_STANDBY and return (stays passive voter). Mirrors ha_start_election_async. Single-node (n_peers==1) unaffected.", @@ -5277,43 +892,26 @@ "config-bootstrap", "Req2.8", "Req10.5", - "voter-only-guard", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "voter-only-guard" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-542", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T09:29:31.490Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'testsuite/test_coldstart_lsn0.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-538.md" }, { "id": "bug-543", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-14T09:36:36.967406Z", - "error_message": "WAL HOLE on cold-start catch-up: a fresh EMPTY standby (standby_lsn=0) catching up to a primary holding [0,1] receives ONLY lsn 1, missing lsn 0", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "WAL HOLE on cold-start catch-up: a fresh EMPTY standby (standby_lsn=0) catching up to a primary…\n\nWAL HOLE on cold-start catch-up: a fresh EMPTY standby (standby_lsn=0) catching up to a primary holding [0,1] receives ONLY lsn 1, missing lsn 0", "file": "src/controller/config_repl.c", "root_cause": "ha_stream_wal_to_standby filter `if (hdr.lsn > standby_lsn)` excludes lsn 0 when standby_lsn==0 (0>0 is false). standby_lsn=0 is ambiguous: it means BOTH 'empty standby (applied nothing)' AND 'standby has applied lsn 0'. The streamer cannot distinguish, computes start_lsn=standby_lsn+1=1, and drops the genuine lsn-0 entry. Only non-cosmetic because lsn-0 entries can be REAL data (first-ever WAL entry of a fresh cluster is lsn 0 from a real write OR the promotion NOOP).", - "fix": "FIXED via reserve-lsn-0: persist_init() now sets current_lsn=1 on a fresh store (persist.c), so lsn 0 is reserved as the none/empty sentinel and the first-ever WAL entry (promotion NOOP) is lsn 1. The streamer filter hdr.lsn>standby_lsn now yields 1>0=true for an empty standby (standby_lsn=0), streaming the first entry \u2014 no hole. No streamer change needed; the reserved base makes the existing filter correct. Also fixes the uncommittable cold-start NOOP (now lsn 1, committable 0->1) and the own-term writable_lsn gate (now 1, effective). Test: test_coldstart_lsn0.c H1-H4 all green incl H4 no-hole. Base-shift fallout fixed in test_commit_engine, test_commit_barrier_rollout, test_commit_barrier_reply, test_p2_integration, test_egress_filter (removed now-redundant priming appends / wired engine).", + "fix": "FIXED via reserve-lsn-0: persist_init() now sets current_lsn=1 on a fresh store (persist.c), so lsn 0 is reserved as the none/empty sentinel and the first-ever WAL entry (promotion NOOP) is lsn 1. The streamer filter hdr.lsn>standby_lsn now yields 1>0=true for an empty standby (standby_lsn=0), streaming the first entry — no hole. No streamer change needed; the reserved base makes the existing filter correct. Also fixes the uncommittable cold-start NOOP (now lsn 1, committable 0->1) and the own-term writable_lsn gate (now 1, effective). Test: test_coldstart_lsn0.c H1-H4 all green incl H4 no-hole. Base-shift fallout fixed in test_commit_engine, test_commit_barrier_rollout, test_commit_barrier_reply, test_p2_integration, test_egress_filter (removed now-redundant priming appends / wired engine).", "tags": [ "ha", "replication", @@ -5322,138 +920,23 @@ "lsn-0", "catch-up", "correctness", - "off-by-one", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "off-by-one" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-550", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T11:38:09.614Z", - "error_message": "Wrong reference: test_pin_dispatch_gate_degraded_allowlist should be test_empty_standby_fresh_cluster_streams_not_snapshot", - "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "Used \"test_pin_dispatch_gate_degraded_allowlist\" instead of \"test_empty_standby_fresh_cluster_streams_not_snapshot\"", - "fix": "Changed test_pin_dispatch_gate_degraded_allowlist \u2192 test_empty_standby_fresh_cluster_streams_not_snapshot", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-551", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T11:45:02.411Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (standby_lsn < oldest_wal_lsn)", - "fix": "Changed to: if (standby_lsn + 1 < oldest_wal_lsn)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-555", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T12:23:52.351Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_repl_apply.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-556", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T12:25:37.723Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-557", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T12:26:04.137Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-560", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-14T19:54:55.428Z", - "error_message": "Wrong reference: myconf_t should be uint32_t", - "file": "src/include/myconf.h", - "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t \u2192 uint32_t", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-543.md" }, { "id": "bug-561", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-14T20:05:48.755409", - "error_message": "jobsub intermittent -ETIMEDOUT \"Connection timed out\": commit: WAIT deadline expired without commit (timeout_ms=50)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "jobsub intermittent -ETIMEDOUT \"Connection timed out\": commit: WAIT deadline expired without…\n\njobsub intermittent -ETIMEDOUT \"Connection timed out\": commit: WAIT deadline expired without commit (timeout_ms=50)", "file": "src/include/config.h", "root_cause": "DEFAULT_SYNC_TIMEOUT_MS=50 too tight for durable sync majority replication (each standby fdatasyncs before ACK); not configurable (controller always used persist_config_defaults).", "fix": "Raised DEFAULT_SYNC_TIMEOUT_MS 50->500; added --sync-timeout-ms CLI flag plumbed arguments->myconf.sync_timeout_ms->pcfg.sync_timeout_ms (0=use default) in main.c/myconf.h/cluster.c.", @@ -5463,20 +946,23 @@ "timeout", "replication", "jobsub", - "config", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "config" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-561.md" }, { "id": "bug-562", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-06-14T23:55:00Z", - "error_message": "AddressSanitizer: heap-use-after-free in encode_RequestReply / strlen (lib/controller.capnp.c:13168) on ctrl-2 startup, via xce_reply->xce_encode from OPCode_nop handler (cluster.c:307). Also: job submission succeeds but joblist shows no jobs.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: heap-use-after-free in encode_RequestReply / strlen…\n\nAddressSanitizer: heap-use-after-free in encode_RequestReply / strlen (lib/controller.capnp.c:13168) on ctrl-2 startup, via xce_reply->xce_encode from OPCode_nop handler (cluster.c:307). Also: job submission succeeds but joblist shows no jobs.", "file": "src/lib/request.c", "root_cause": "init_reply_by_req() only set seq+opcode, leaving the rest of the stack-declared xce_reply_t uninitialized. The Cap'n-C generated encode_RequestReply reads the common trailer field redirect_host UNCONDITIONALLY (gated only on != NULL), so a garbage stack pointer in reply.redirect_host was strlen'd. The garbage aliased a 4KB capnp segment freed moments earlier by the WAL-ack reply encode on the same thread -> UAF. Handlers that build a reply via init_reply_by_req and only set status/data_which/data (e.g. the NOP/state-report path) never cleared redirect_host. Empty joblist is downstream: ctrl-2 crash-looping on this UAF destabilizes replication/quorum, so a get request that fails over to the not-caught-up standby reads an empty jobmgr (standby serves get from its own committed state).", "fix": "memset(reply,0,sizeof(*reply)) at the top of init_reply_by_req() (src/lib/request.c) before setting seq/opcode. All 30+ reply sites follow declare->init->populate, so zeroing in the shared initializer fixes the whole class (redirect_host/redirect_port/status/data garbage) at the source. Added regression test test_init_reply_by_req_zeroes_reply in src/lib/testsuite/test_request.c (poisons reply with 0xFF, asserts redirect_host==NULL etc).", @@ -5489,23 +975,26 @@ "init_reply_by_req", "ha", "joblist", - "uninitialized-stack", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "uninitialized-stack" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-562.md" }, { "id": "bug-563", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-15T09:16:53.679025Z", - "error_message": "jobsub returns ok (status 0, jobid) but joblist AND jobinfo show the job absent on a stable single primary (ctrl-1); reslist works.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "jobsub returns ok (status 0, jobid) but joblist AND jobinfo show the job absent on a stable single…\n\njobsub returns ok (status 0, jobid) but joblist AND jobinfo show the job absent on a stable single primary (ctrl-1); reslist works.", "file": "src/lib/request.c", - "root_cause": "SOLVED (reproduced locally via /xce-build + 3-node deploy-ha, ASAN). cluster_start() \u2014 the ONLY place that sets c->running=1 \u2014 is ORPHANED (zero callers; grep confirms). The HA startup path (myapp_run) starts iomgr/vtaskmgr/delaymgr/persist_start itself but never calls cluster_start, so c->running stays 0 (calloc default) on the primary. check_controller() (the streaming gate passed to xce_reply_get for joblist/workerlist) returns -EINVAL when !c->running; xce_reply_get then sets nc=0 (chunk count), shipping an EMPTY list to the client even though collect_jobs/key_find found the job. reslist passes checkf=NULL so it was immune. DIAG proof: 'reply_get send: table=2 pos=0 nc=0 nrows=1 checkstatus=-22' (pre-fix) -> 'nc=1 ... checkstatus=1' (post-fix). Single-node/legacy cluster_init path called cluster_start, which is why non-replicated worked.", - "fix": "Set c->running=1 at the TOP of cluster_run() (src/controller/cluster.c), symmetric with the existing c->running=0 on shutdown \u2014 the orphaned cluster_start() used to set it. c->running is a liveness/shutdown flag: check_controller() is polled PER CHUNK inside xce_reply_get()'s streaming loop so a long multi-chunk fetch (e.g. thousands of jobs) ABORTS mid-stream (nc=0) if a shutdown request arrives during it. The bug: running started 0 and was never set under HA, so the gate fired on every reply, not just during shutdown. Fix restores running=1 for the run-loop lifetime; the shutdown abort path (running=0 when cluster_run's cbuf_get loop breaks on the stop signal) is unchanged. Verified E2E on a local 3-node deploy-ha (ASAN): jobs now appear in joblist; DIAG showed checkstatus/nc flip -22/0 -> 1/1. Final fix is in cluster_run (not myapp_run \u2014 moved there so the 1<->0 lifecycle is co-located in the function that receives the shutdown signal).", + "root_cause": "SOLVED (reproduced locally via /xce-build + 3-node deploy-ha, ASAN). cluster_start() — the ONLY place that sets c->running=1 — is ORPHANED (zero callers; grep confirms). The HA startup path (myapp_run) starts iomgr/vtaskmgr/delaymgr/persist_start itself but never calls cluster_start, so c->running stays 0 (calloc default) on the primary. check_controller() (the streaming gate passed to xce_reply_get for joblist/workerlist) returns -EINVAL when !c->running; xce_reply_get then sets nc=0 (chunk count), shipping an EMPTY list to the client even though collect_jobs/key_find found the job. reslist passes checkf=NULL so it was immune. DIAG proof: 'reply_get send: table=2 pos=0 nc=0 nrows=1 checkstatus=-22' (pre-fix) -> 'nc=1 ... checkstatus=1' (post-fix). Single-node/legacy cluster_init path called cluster_start, which is why non-replicated worked.", + "fix": "Set c->running=1 at the TOP of cluster_run() (src/controller/cluster.c), symmetric with the existing c->running=0 on shutdown — the orphaned cluster_start() used to set it. c->running is a liveness/shutdown flag: check_controller() is polled PER CHUNK inside xce_reply_get()'s streaming loop so a long multi-chunk fetch (e.g. thousands of jobs) ABORTS mid-stream (nc=0) if a shutdown request arrives during it. The bug: running started 0 and was never set under HA, so the gate fired on every reply, not just during shutdown. Fix restores running=1 for the run-loop lifetime; the shutdown abort path (running=0 when cluster_run's cbuf_get loop breaks on the stop signal) is unchanged. Verified E2E on a local 3-node deploy-ha (ASAN): jobs now appear in joblist; DIAG showed checkstatus/nc flip -22/0 -> 1/1. Final fix is in cluster_run (not myapp_run — moved there so the 1<->0 lifecycle is co-located in the function that receives the shutdown signal).", "tags": [ "joblist", "workerlist", @@ -5516,362 +1005,27 @@ "cluster_start", "orphaned-function", "xce_reply_get", - "SOLVED", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "SOLVED" ], "related_bugs": [ "bug-562" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-570", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-15T11:10:10.553Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/product.md", - "root_cause": "Had `raft.c`", - "fix": "Changed to `src/lib/raft/`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-572", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-15T11:10:26.312Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/tech.md", - "root_cause": "Had `raft.c`", - "fix": "Changed to `src/lib/raft/`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-575", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T00:10:49.295Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-prevote/research.md", - "root_cause": "Had 't understand PreVote (older binary) \u2014 does the ca", - "fix": "Changed to 's \"", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-576", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T00:18:28.793Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-prevote/research.md", - "root_cause": "Had 's \"", - "fix": "Changed to `ha_heartbeat_is_fresh()`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-578", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T03:00:48.818Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-prevote/design.md", - "root_cause": "Had `ha_elect_ctx_t`", - "fix": "Changed to `raft_on_vote_reply`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 9, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-579", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T03:01:07.659Z", - "error_message": "Wrong return value", - "file": ".kiro/specs/ha-prevote/design.md", - "root_cause": "Was returning: codes (e.g. `RAFT_PRE_VOTE_WON`, `RAFT_PRE_VOTE_LO", - "fix": "Now returns: codes (e.g. `RAFT_PRE_VOTE_WON`, `RAFT_PRE_VOTE_LO", - "tags": [ - "auto-detected", - "return-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-584", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T04:27:50.039Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->cbs->on_trace", - "fix": "Added guard clause: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-585", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T04:36:28.796Z", - "error_message": "Missing guard clause", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "No early return/throw for edge case: rc != 0", - "fix": "Added guard clause: if (rc != 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-586", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T04:50:38.855Z", - "error_message": "Missing error handling in struct", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 3.1 \u2014 voter-side PreVote dispatch gate (stickiness +; * no mutation, reply term unchanged) */", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-591", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T05:27:09.380Z", - "error_message": "Wrong reference: test_commit_engine should be test_ha_prevote", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_prevote\"", - "fix": "Changed test_commit_engine \u2192 test_ha_prevote", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-592", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T05:28:01.236Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c || !c->myconf", - "fix": "Added guard clause: if (!c || !c->myconf)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-593", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T05:44:57.430Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-prevote/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-594", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T05:50:25.729Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_commit_engine.c'", - "fix": "Changed to 'testsuite/test_ha_failover_prevote.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-595", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T05:50:35.123Z", - "error_message": "Wrong reference: test_commit_engine should be test_ha_failover_prevote", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_failover_prevote\"", - "fix": "Changed test_commit_engine \u2192 test_ha_failover_prevote", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-597", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T05:59:43.939Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-598", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T06:33:05.644Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: role == CTRL_STANDBY && ha->raft.pre_vote_in_progress", - "fix": "Added guard clause: if (role == CTRL_STANDBY && ha->raft.pre_vot)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-603", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T08:18:48.567Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-563.md" }, { "id": "bug-604", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-16T15:55:00Z", - "error_message": "controller unit test TIMEOUT 300.01s killed by SIGTERM; suite exceeds 300s meson cap", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller unit test TIMEOUT 300.01s killed by SIGTERM; suite exceeds 300s meson cap\n\ncontroller unit test TIMEOUT 300.01s killed by SIGTERM; suite exceeds 300s meson cap", "file": "src/controller/persist.c", - "root_cause": "fsync_worker slept the full wal_sync_ms in one uninterruptible nanosleep() then checked fsync_stop. persist_stop sets the flag and immediately pthread_join()s, so the join blocked until the in-progress sleep returned \u2014 up to a full wal_sync_ms per teardown. Tests set wal_sync_ms=5000 ('slow fsync, no races'), so each persist_stop cost ~5s; across ~7 suites with many fixtures the cumulative teardown time blew past the 300s controller-test cap. The 'commit: WAIT deadline expired' warnings in the log were unrelated intentional negative tests (50-300ms).", + "root_cause": "fsync_worker slept the full wal_sync_ms in one uninterruptible nanosleep() then checked fsync_stop. persist_stop sets the flag and immediately pthread_join()s, so the join blocked until the in-progress sleep returned — up to a full wal_sync_ms per teardown. Tests set wal_sync_ms=5000 ('slow fsync, no races'), so each persist_stop cost ~5s; across ~7 suites with many fixtures the cumulative teardown time blew past the 300s controller-test cap. The 'commit: WAIT deadline expired' warnings in the log were unrelated intentional negative tests (50-300ms).", "fix": "Replaced the nanosleep with pthread_cond_timedwait on a new fsync_mutex/fsync_cond (added to persist_t, init in persist_init, destroyed in persist_fini). persist_stop now sets fsync_stop and pthread_cond_signal(fsync_cond) under fsync_mutex before join, so the worker wakes immediately. fdatasync still runs under p->lock, released before re-acquiring fsync_mutex (no lock nesting). Full controller suite now completes in ~10s (was >300s); 0 failures.", "tags": [ "controller", @@ -5882,20 +1036,23 @@ "pthread_join", "nanosleep", "condvar", - "ha", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-604.md" }, { "id": "bug-605", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-16T16:30:00Z", - "error_message": "3-node fresh HA deploy never elects a primary: all nodes stay CTRL_STANDBY, config-holder (ctrl-1) loops PreVote LOST every ~11s forever; config-less peers idle as voter-only. REPRODUCED via /xce-deploy-ha + clean restart 2026-06-16.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "3-node fresh HA deploy never elects a primary: all nodes stay CTRL_STANDBY, config-holder (ctrl-1)…\n\n3-node fresh HA deploy never elects a primary: all nodes stay CTRL_STANDBY, config-holder (ctrl-1) loops PreVote LOST every ~11s forever; config-less peers idle as voter-only. REPRODUCED via /xce-deploy-ha + clean restart 2026-06-16.", "file": "src/controller/ha.c (ha_dispatch_gate); test: src/controller/testsuite/test_ha_role.c", "root_cause": "ha_dispatch_gate() admitted OPCode_requestVote in CTRL_INIT/CTRL_ELECTING/CTRL_STANDBY but NEVER OPCode_preVoteRequest (opcode 38). ha-prevote added the new opcode + handler (cluster.c:440) but missed wiring it into the role-admission gate. So a STANDBY voter rejected an incoming PreVote with -XCE_ERR_STANDBY (-10014) BEFORE the handler ran -> no preVoteReply sent -> candidate's failed/empty exchange tallied as a denial (pre_vote_reply_decode_and_dispatch counts ok=0 as non-grant; vote_ep_err_handler synthesizes a denied vote). In a cold cluster (config-holder + config-less voter-only peers) every PreVote 'lost' with NO voter ever making a liveness decision (zero 'PreVote DENIED' logs), so no primary was ever elected and the config-holder looped PreVote forever. Confirmed via gdb: myreq_handler ha_dispatch_gate rc=-10014 on opcode 38. Design invariant: a PreVote (strictly non-mutating) must be at least as admissible as a real requestVote in every role.", "fix": "Added OPCode_preVoteRequest alongside OPCode_requestVote in ha_dispatch_gate() for CTRL_INIT, CTRL_ELECTING, and the CTRL_STANDBY switch (src/controller/ha.c). PreVote now reaches the cluster.c handler and a proper preVoteReply is sent. Added TDD test test_dispatch_gate_allows_prevote_in_voter_roles (test_ha_role.c) asserting gate(preVoteRequest)==gate(requestVote)==0 across voter roles. VERIFIED: controller unit suite OK (new test PASS); fresh /xce-deploy-ha 3-node cold start now elects ctrl-1 PRIMARY (term=2), standbys establish primary channel + config committed, prevote-LOST=0, connections 96->8.", @@ -5907,25 +1064,28 @@ "cold-start", "bootstrap-deadlock", "FIXED", - "XCE_ERR_STANDBY", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "XCE_ERR_STANDBY" ], "related_bugs": [ "bug-339" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-605.md" }, { "id": "bug-606", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-06-16T17:46:00Z", - "error_message": "After killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in synchronized elections: PreVote win \u2192 CTRL_ELECTING \u2192 become_follower (term bumped) \u2192 ~10s \u2192 repeat, term climbing unbounded (3\u21929\u2192\u221e), no new primary ever elected. Reproduce: deploy 3-node HA, elect primary, submit 3-5 jobs, kill ctrl-1.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "After killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in…\n\nAfter killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in synchronized elections: PreVote win → CTRL_ELECTING → become_follower (term bumped) → ~10s → repeat, term climbing unbounded (3→9→∞), no new primary ever elected. Reproduce: deploy 3-node HA, elect primary, submit 3-5 jobs, kill ctrl-1.", "file": "src/controller/ha.c (ha_begin_failover, ha_election_stagger_ms, ha_failover_stagger_fn); src/controller/ha.h; test: src/controller/testsuite/test_ha_role.c", - "root_cause": "Dueling-candidates split-vote livelock \u2014 Raft \u00a75.2 randomized election timeout is missing on the FAILOVER path. Both survivors lose the primary at the same instant; their liveness polls fire on the identical fixed heartbeat_timeout, so they start PreVote+real-election in perfect lock-step (ms-aligned timestamps). Both win PreVote (neither sees a primary, so each grants the other \u2014 PreVote cannot break this tie), then both campaign at the same term, each votes for itself and denies the other (quorum 2, only 2 live nodes) \u2192 split vote \u2192 election times out (fixed election_timeout_ms, raft.c:105, NO jitter) \u2192 become_follower, term+1 \u2192 repeat forever. The priority/jitter config (election_jitter_max_ms, election_priority_step_ms) is applied ONLY at startup (ha_wait_for_heartbeat_or_timeout); the failover re-election path (liveness_poll_fn \u2192 ha_on_primary_ctrl_fd_rdhup \u2192 PreVote \u2192 ha_start_election_async \u2192 raft_start_election) has no de-synchronization. raft_on_leader_lost DOES have a priority stagger (raft.c:421) but the failover loop does not go through it.", - "fix": "Added ha_election_stagger_ms() (ha.c/ha.h): per-node failover delay = my_priority*election_priority_step_ms + random[0,election_jitter_max_ms], re-randomized each call (mirrors the startup stagger). ha_begin_failover() now schedules the PreVote start via delaymgr after that stagger (production only; c==NULL/no-delaymgr stays synchronous so existing tests are unaffected). The delaymgr callback ha_failover_stagger_fn guards with role==STANDBY && ha_heartbeat_is_fresh \u2192 skips PreVote if a primary connected during the stagger. Effect: two survivors no longer campaign in lock-step; the higher-priority survivor wins and its heartbeats reset the other. TDD: test_election_stagger_desyncs_by_priority (bounds + step>jitter ordering). VERIFIED on kill-primary repro: ctrl-2 (stagger 630ms) won \u2192 PRIMARY term=1; ctrl-3 (stagger 1072ms) received ctrl-2's ctrlhello first \u2192 STANDBY, then its stagger timer hit the guard and skipped PreVote ('a primary is alive \u2014 skipping PreVote'). No term inflation, no livelock. NOTE: optional raft.c:105 give-up-watchdog randomization NOT needed \u2014 the failover-start stagger alone resolves it; left as future defense-in-depth.", + "root_cause": "Dueling-candidates split-vote livelock — Raft §5.2 randomized election timeout is missing on the FAILOVER path. Both survivors lose the primary at the same instant; their liveness polls fire on the identical fixed heartbeat_timeout, so they start PreVote+real-election in perfect lock-step (ms-aligned timestamps). Both win PreVote (neither sees a primary, so each grants the other — PreVote cannot break this tie), then both campaign at the same term, each votes for itself and denies the other (quorum 2, only 2 live nodes) → split vote → election times out (fixed election_timeout_ms, raft.c:105, NO jitter) → become_follower, term+1 → repeat forever. The priority/jitter config (election_jitter_max_ms, election_priority_step_ms) is applied ONLY at startup (ha_wait_for_heartbeat_or_timeout); the failover re-election path (liveness_poll_fn → ha_on_primary_ctrl_fd_rdhup → PreVote → ha_start_election_async → raft_start_election) has no de-synchronization. raft_on_leader_lost DOES have a priority stagger (raft.c:421) but the failover loop does not go through it.", + "fix": "Added ha_election_stagger_ms() (ha.c/ha.h): per-node failover delay = my_priority*election_priority_step_ms + random[0,election_jitter_max_ms], re-randomized each call (mirrors the startup stagger). ha_begin_failover() now schedules the PreVote start via delaymgr after that stagger (production only; c==NULL/no-delaymgr stays synchronous so existing tests are unaffected). The delaymgr callback ha_failover_stagger_fn guards with role==STANDBY && ha_heartbeat_is_fresh → skips PreVote if a primary connected during the stagger. Effect: two survivors no longer campaign in lock-step; the higher-priority survivor wins and its heartbeats reset the other. TDD: test_election_stagger_desyncs_by_priority (bounds + step>jitter ordering). VERIFIED on kill-primary repro: ctrl-2 (stagger 630ms) won → PRIMARY term=1; ctrl-3 (stagger 1072ms) received ctrl-2's ctrlhello first → STANDBY, then its stagger timer hit the guard and skipped PreVote ('a primary is alive — skipping PreVote'). No term inflation, no livelock. NOTE: optional raft.c:105 give-up-watchdog randomization NOT needed — the failover-start stagger alone resolves it; left as future defense-in-depth.", "tags": [ "ha", "raft", @@ -5937,22 +1097,25 @@ "failover", "stagger", "jitter", - "FIXED", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "FIXED" ], "related_bugs": [ "bug-605" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-606.md" }, { "id": "bug-608", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-16T10:36:00Z", - "error_message": "libxce client never fails over after primary killed; xce_joblist returns 'Invalid argument' (EINVAL) indefinitely", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "libxce client never fails over after primary killed; xce_joblist returns 'Invalid argument'…\n\nlibxce client never fails over after primary killed; xce_joblist returns 'Invalid argument' (EINVAL) indefinitely", "file": "src/lib/request.c", "root_cause": "Two gaps: (1) xce_request_with_retry() (request.c:1956) is implemented+unit-tested but NO production path calls it - every xce_*_req wrapper calls plain xce_request() (request.c:1385 etc). (2) The retry primitive keys on E_XCE_STANDBY + reply.redirect_host/port (request.c:2010), but the real controller emits a dedicated E_XCE_REDIRECT status with data_which=nodata (join.c:264-266) and never populates those trailer fields. Controller non-primary taxonomy is E_XCE_REDIRECT / E_XCE_NOMASTER / E_XCE_NOT_READY / E_XCE_STANDBY (errnox.h:22-25). So the client both bypasses the wrapper AND, even if wired, would not detect the controller's actual redirect signal.", "fix": "Spec libxce-failover: wire all production request paths through transparent failover; classify on actual controller statuses (REDIRECT=fast-rotate to find existing primary, address-if-present else rotate master list; NOMASTER/NOT_READY=backoff then cluster-not-ready; STANDBY=rotate); realign the existing retry unit tests to the emitted statuses. Implementation pending (requirements phase).", @@ -5963,20 +1126,23 @@ "E_XCE_REDIRECT", "retry", "client", - "protocol-mismatch", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "protocol-mismatch" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-608.md" }, { "id": "bug-609", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-16T10:52:01.800Z", - "error_message": "After primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on every fresh call \u2014 transparent failover did not engage", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "After primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on…\n\nAfter primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on every fresh call — transparent failover did not engage", "file": "src/lib/client_failover.c", "root_cause": "handle_disconnect leaves xce_ctx_t.state=XCE_CTX_ERR after the primary's connection drops; the per-op xceapi guard (xceapi.c:14) returns -EINVAL when state!=XCE_CTX_RUN, BEFORE the op reaches a primary. xcecli_retry_decide classified -EINVAL as TERMINAL (non-retriable), so the failover loop bailed without rotating/recovering. The ctx->state ERR->RUN recovery only ran AFTER a rotate, which a terminal classification never triggered. Surfaced by the live 3-node E2E (task 6.2); the unit tests fed status codes directly to decide() and bypassed the real guard, so they missed it.", "fix": "In xcecli_retry_decide, before classifying *rc, detect ac->ctx.state==XCE_CTX_ERR (known-dead connection) and treat it as a prompt transport-style failover cause (XCE_FO_RETRY_NOW, no backoff) regardless of *rc, so the loop rotates+recovers. On budget exhaustion surface -E_XCE_NOT_READY (cluster-not-ready), never -EINVAL. Regression: test_decide_known_dead_ctx_retries_not_terminal.", @@ -5987,25 +1153,28 @@ "EINVAL", "XCE_CTX_ERR", "reconnect", - "E2E", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "E2E" ], "related_bugs": [ "bug-608" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-609.md" }, { "id": "bug-610", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-06-16T10:52:36.535Z", - "error_message": "ASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate \u2192 xcecli_retry_next \u2192 xcecli_joblist during live master failover (NULL ep->iomgr deref)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "ASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate → xcecli_retry_next →…\n\nASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate → xcecli_retry_next → xcecli_joblist during live master failover (NULL ep->iomgr deref)", "file": "src/lib/client_failover.c", - "root_cause": "xce_failover_rotate reconnected via endpoint_fini + endpoint_init + endpoint_connect, but endpoint_init (do_endpoint_init) resets the endpoint struct including ep->iomgr, and the rotate never re-bound it. The client's endpoint gets ep->iomgr = &ctx->iomgr only via xce_ctx_set_endpoint (called once in xcecli_start). endpoint_connect derefs iomgr->ssl_ctx when ep->ssl==NULL (true after endpoint_fini frees the SSL) \u2192 NULL deref crash. Also the rotate never re-set owner/msg_handler/err_handler nor iomgr_monitor'd the new fd, so even without the crash the reconnected endpoint would not route replies. Pure unit tests could not exercise the live socket/iomgr path; only the live failover E2E hit it.", - "fix": "Added xce_ctx_reconnect(ctx,host,port) in xcectx.c (where static handle_msg/handle_disconnect live) that mirrors the connection-establishment of xcecli_start/xce_ctx_start without restarting iomgr/vtaskmgr or touching ctx->state: endpoint_fini \u2192 endpoint_init \u2192 endpoint_set_peer \u2192 re-bind ep->iomgr=&ctx->iomgr \u2192 endpoint_connect \u2192 re-set owner+msg/err handlers \u2192 iomgr_monitor. xce_failover_rotate now delegates the reconnect to it.", + "root_cause": "xce_failover_rotate reconnected via endpoint_fini + endpoint_init + endpoint_connect, but endpoint_init (do_endpoint_init) resets the endpoint struct including ep->iomgr, and the rotate never re-bound it. The client's endpoint gets ep->iomgr = &ctx->iomgr only via xce_ctx_set_endpoint (called once in xcecli_start). endpoint_connect derefs iomgr->ssl_ctx when ep->ssl==NULL (true after endpoint_fini frees the SSL) → NULL deref crash. Also the rotate never re-set owner/msg_handler/err_handler nor iomgr_monitor'd the new fd, so even without the crash the reconnected endpoint would not route replies. Pure unit tests could not exercise the live socket/iomgr path; only the live failover E2E hit it.", + "fix": "Added xce_ctx_reconnect(ctx,host,port) in xcectx.c (where static handle_msg/handle_disconnect live) that mirrors the connection-establishment of xcecli_start/xce_ctx_start without restarting iomgr/vtaskmgr or touching ctx->state: endpoint_fini → endpoint_init → endpoint_set_peer → re-bind ep->iomgr=&ctx->iomgr → endpoint_connect → re-set owner+msg/err handlers → iomgr_monitor. xce_failover_rotate now delegates the reconnect to it.", "tags": [ "async-fix", "md", @@ -6016,360 +1185,26 @@ "endpoint_connect", "iomgr", "reconnect", - "E2E", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "E2E" ], "related_bugs": [ "bug-609", "bug-608" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-613", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T13:15:11.048Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-614", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T13:18:29.113Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Had `xce_ctx_call`", - "fix": "Changed to `xcecli_retry_next`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-617", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T23:38:24.000Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Had `redirect_host/port`", - "fix": "Changed to `host/port`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-619", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-16T23:40:22.003Z", - "error_message": "Wrong reference: address should be hint", - "file": ".kiro/specs/libxce-failover/tasks.md", - "root_cause": "Used \"address\" instead of \"hint\"", - "fix": "Changed address \u2192 hint", - "tags": [ - "auto-detected", - "wrong-reference", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-623", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T00:04:51.775Z", - "error_message": "Wrong reference: XCE_MAXHOST_LEN should be XCE_MAXHOSTNAME_LEN", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Used \"XCE_MAXHOST_LEN\" instead of \"XCE_MAXHOSTNAME_LEN\"", - "fix": "Changed XCE_MAXHOST_LEN \u2192 XCE_MAXHOSTNAME_LEN", - "tags": [ - "wrong-reference", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-624", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T02:35:29.695Z", - "error_message": "Incorrect value in code", - "file": "src/lib/meson.build", - "root_cause": "Had 'dataq.c'", - "fix": "Changed to 'client_failover.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-626", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T02:48:27.547Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: ac == NULL || host_out == NULL || host_sz == 0 || port_out =", - "fix": "Added guard clause: if (ac == NULL || host_out == NULL || host_s)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-629", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T03:18:22.365Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: base >= (double", - "fix": "Added guard clause: if (base >= (double)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-631", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T03:31:29.120Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: ac == NULL", - "fix": "Added guard clause: if (ac == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-633", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T03:40:10.853Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: ac == NULL", - "fix": "Added guard clause: if (ac == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-635", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T03:51:08.767Z", - "error_message": "Wrong return value", - "file": "src/lib/client.c", - "root_cause": "Was returning: xce_reslist(&(ctx->ctx), res);", - "fix": "Now returns: XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,", - "tags": [ - "auto-detected", - "return-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 8, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-637", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T03:57:33.232Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Had `nop`", - "fix": "Changed to `jobinfo`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-638", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T04:02:05.572Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: ac == NULL", - "fix": "Added guard clause: if (ac == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-639", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T04:11:59.759Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_p1_integration.c'", - "fix": "Changed to 'testsuite/test_join_redirect.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-640", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T04:12:01.203Z", - "error_message": "Wrong reference: test_p1_integration should be test_join_redirect", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_p1_integration\" instead of \"test_join_redirect\"", - "fix": "Changed test_p1_integration \u2192 test_join_redirect", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-641", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T04:19:58.697Z", - "error_message": "Wrong reference: test_recover_resets_err_state_to_run should be test_decide_jitter_staggers_many_clients", - "file": "src/lib/testsuite/test_libxce_retry.c", - "root_cause": "Used \"test_recover_resets_err_state_to_run\" instead of \"test_decide_jitter_staggers_many_clients\"", - "fix": "Changed test_recover_resets_err_state_to_run \u2192 test_decide_jitter_staggers_many_clients", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-644", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T08:15:04.978Z", - "error_message": "Incorrect value in code", - "file": "src/lib/meson.build", - "root_cause": "Had 'testsuite/test_raft.c'", - "fix": "Changed to 'testsuite/test_netio_arm_failure.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-610.md" }, { "id": "bug-646", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "hang", "timestamp": "2026-06-17T08:21:32.099Z", - "error_message": "xce_test jobsub hangs forever after a failover; main thread blocked in cbuf_get (xceapi.c:304) while io_worker sits idle in epoll_wait. Logs: 'rotate failed (rc=-111)' then 'endpoint_update_epoll rc1=-1 < 0'.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "xce_test jobsub hangs forever after a failover; main thread blocked in cbuf_get (xceapi.c:304)…\n\nxce_test jobsub hangs forever after a failover; main thread blocked in cbuf_get (xceapi.c:304) while io_worker sits idle in epoll_wait. Logs: 'rotate failed (rc=-111)' then 'endpoint_update_epoll rc1=-1 < 0'.", "file": "src/lib/netio.c", "root_cause": "endpoint_update_epoll() swallowed iomgr_update_poll() failures: it WARN'd 'rc1=-1' but returned 0 and left ep->flags unchanged, so the poll interest was never armed. iomgr_update_poll uses EPOLL_CTL_MOD, which fails with ENOENT when the fd is no longer registered (e.g. after a failed reconnect: xce_ctx_reconnect endpoint_fini's the old fd but only iomgr_monitor/EPOLL_CTL_ADDs on a SUCCESSFUL connect). The next endpoint_enq() send then MOD-failed silently, request bytes were never flushed, no reply arrived, and the synchronous client blocked forever in cbuf_get (no timeout) with no socket-drop event to wake it (handle_disconnect only fires on a broken socket). Server side was a red herring: DEFAULT_SYNC_TIMEOUT_MS=500 bounds the commit barrier, and ha_dispatch_gate already replies STANDBY/NOT_READY for non-serving roles.", "fix": "endpoint_update_epoll() captures errno and RETURNS -errno (not 0) on iomgr_update_poll failure. endpoint_enq(): on negative arm result, run handle_error(ep) (iomgr_unmonitor + handle_disconnect -> state=ERR + inject -ECONNRESET into wbuf, waking the blocked cbuf_get) and return the error so the failover loop rotates. handle_output(): propagates the arm failure so io_handler runs the error path. TDD: test_netio_arm_failure.c drives EPOLL_CTL_MOD ENOENT via endpoint_enq on an unregistered fd; asserts loud rc<0 + err_handler fired (verified red without fix, green with).", @@ -6385,43 +1220,26 @@ "silent-failure", "hang", "cbuf_get", - "reconnect", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "reconnect" ], "related_bugs": [ "bug-610", "bug-609" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-647", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T08:21:56.735Z", - "error_message": "Wrong reference: 611 should be 646", - "file": "src/lib/netio.c", - "root_cause": "Used \"611\" instead of \"646\"", - "fix": "Changed 611 \u2192 646", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-646.md" }, { "id": "bug-648", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-17T17:30:00Z", - "error_message": "Peer-close/EOF on an SSL connection was swallowed: io_fsm_do() set fsm->state=FSM_STATE_ERROR but returned rc=0 on a clean EOF, so handle_input() returned >=0, handle_error/handle_disconnect never ran, and a blocked cbuf_get() reply-waiter was not woken on the first close event.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Peer-close/EOF on an SSL connection was swallowed: io_fsm_do() set fsm->state=FSM_STATE_ERROR but…\n\nPeer-close/EOF on an SSL connection was swallowed: io_fsm_do() set fsm->state=FSM_STATE_ERROR but returned rc=0 on a clean EOF, so handle_input() returned >=0, handle_error/handle_disconnect never ran, and a blocked cbuf_get() reply-waiter was not woken on the first close event.", "file": "src/lib/netio.c", "root_cause": "io_fsm_do() computed rc = -errno when a closed/reset connection was detected. By OpenSSL 3.x contract (verified against docs + a live probe on OpenSSL 3.6.0), a peer close makes SSL_read() return 0 and is reported via SSL_get_error(), NOT via the return value or errno: clean close_notify -> SSL_ERROR_ZERO_RETURN (errno 0); unexpected EOF (FIN, no close_notify) -> SSL_ERROR_SSL since OpenSSL 3.0 (pre-3.0 it was SSL_ERROR_SYSCALL with errno 0 -- documented); RST -> SSL_ERROR_SYSCALL errno=ECONNRESET. errno is only meaningful for SSL_ERROR_SYSCALL; for SSL_ERROR_SSL the error queue is authoritative and any errno (we observed EPIPE=32) is a leaked side effect of OpenSSL's internal alert-write (write() on a broken connection returns EPIPE; read() never does). So -errno is 0 for the documented EOF cases -> the close looked like success. NOTE: on this 3.6 build a process-death FIN surfaced as SSL_ERROR_SSL/errno=32, which the OLD code propagated -- so this swallow is NOT necessarily the cause of the observed controller-kill permanent hang (that still implicates the reconnect/SSL-handshake path); it is a real, separate, errno-dependent non-determinism.", "fix": "Extracted pure io_classify_io_error(ssl_err, ecode, io_rc): returns 0 only for WANT_READ/WANT_WRITE and EAGAIN/EWOULDBLOCK syscall; otherwise returns a NEGATIVE error ((ecode!=0)?-ecode:-ECONNRESET) so a detected close NEVER returns 0 regardless of the unreliable errno. io_fsm_do() now sets FSM_STATE_ERROR iff the classifier returns <0. TDD: test_netio_close_propagation.c asserts ZERO_RETURN/errno0 and SSL_ERROR_SSL/EOF map to <0 (verified red with old -ecode, green with fix). libxce suite 168/0; controller_unit_test links clean.", @@ -6436,45 +1254,28 @@ "SSL_ERROR_ZERO_RETURN", "SSL_ERROR_SSL", "silent-failure", - "hang", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "hang" ], "related_bugs": [ "bug-646", "bug-610" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-651", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T10:44:37.350Z", - "error_message": "Wrong return value", - "file": "src/lib/request.c", - "root_cause": "Was returning: 0;", - "fix": "Now returns: (enq_rc < 0) ? enq_rc : 0;", - "tags": [ - "auto-detected", - "return-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-648.md" }, { "id": "bug-650", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "hang", "timestamp": "2026-06-17T18:30:00Z", - "error_message": "xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed. Log shows 'endpoint marked dead' (bug-646 firing) then a permanent hang; target primary (ctrl-2) is fully idle (no vtask in handle_jobsubmit) and the client fd is an unconnected TCP socket \u2014 the request was never transmitted yet the client waits for a reply forever.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed\n\nxce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed. Log shows 'endpoint marked dead' (bug-646 firing) then a permanent hang; target primary (ctrl-2) is fully idle (no vtask in handle_jobsubmit) and the client fd is an unconnected TCP socket — the request was never transmitted yet the client waits for a reply forever.", "file": "src/lib/request.c", - "root_cause": "xce_request() and the 5 other request.c senders (xce_reply, xce_replconfigpush_req_raw, xce_wal_batch_req_raw, xce_snapshot_chunk_req_raw, xce_replconfig_reply_raw) called endpoint_enq(ep,&pkt->link) and IGNORED its return, always `return 0`. So when the transport was dead, endpoint_enq() correctly returned <0 and ran handle_error() (bug-646 fix) \u2014 but xce_request() swallowed that <0 and reported success. xce_jobsubmit() (xceapi.c:299-304) then saw rc>=0 and fell through to the BLOCKING cbuf_get(); the request was never flushed (server idle) and when err_handler was NULL after a failed reconnect, handle_disconnect never injected -ECONNRESET, so nothing woke the waiter. The bug-646 netio fix was defeated one layer up: the request layer reported success on a failed send. This was the missing link making every prior netio fix insufficient for the live hang.", + "root_cause": "xce_request() and the 5 other request.c senders (xce_reply, xce_replconfigpush_req_raw, xce_wal_batch_req_raw, xce_snapshot_chunk_req_raw, xce_replconfig_reply_raw) called endpoint_enq(ep,&pkt->link) and IGNORED its return, always `return 0`. So when the transport was dead, endpoint_enq() correctly returned <0 and ran handle_error() (bug-646 fix) — but xce_request() swallowed that <0 and reported success. xce_jobsubmit() (xceapi.c:299-304) then saw rc>=0 and fell through to the BLOCKING cbuf_get(); the request was never flushed (server idle) and when err_handler was NULL after a failed reconnect, handle_disconnect never injected -ECONNRESET, so nothing woke the waiter. The bug-646 netio fix was defeated one layer up: the request layer reported success on a failed send. This was the missing link making every prior netio fix insufficient for the live hang.", "fix": "All six request.c senders propagate the enq result: `int enq_rc = endpoint_enq(ep,&pkt->link); return (enq_rc < 0) ? enq_rc : 0;`. A failed send now returns <0 through xce_jobsubmit_req -> xce_jobsubmit, which returns before cbuf_get; the failover loop classifies the transport error and rotates instead of blocking on a reply that never arrives. TDD: test_request_send_propagation.c builds an OPCode_nop request (encodes OK) on a dead endpoint (EPOLL_CTL_MOD ENOENT) and asserts xce_request returns <0 (verified red with old `return 0`, green with fix). libxce 169/0; controller_unit_test links clean.", "tags": [ "libxce", @@ -6485,64 +1286,28 @@ "hang", "cbuf_get", "endpoint_enq", - "send-propagation", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "send-propagation" ], "related_bugs": [ "bug-646", "bug-648" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-654", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T11:21:51.571Z", - "error_message": "Wrong return value", - "file": "src/lib/testsuite/test_netio_arm_failure.c", - "root_cause": "Was returning: a negative error and (2) invoke the endpoint's err", - "fix": "Now returns: a negative, RETRIABLE error so the caller propagat", - "tags": [ - "auto-detected", - "return-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-656", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T11:22:17.916Z", - "error_message": "Wrong reference: test_enq_arm_failure_is_loud_and_fires_errhandler should be test_enq_arm_failure_is_loud_no_errhandler", - "file": "src/lib/testsuite/test_netio_arm_failure.c", - "root_cause": "Used \"test_enq_arm_failure_is_loud_and_fires_errhandler\" instead of \"test_enq_arm_failure_is_loud_no_errhandler\"", - "fix": "Changed test_enq_arm_failure_is_loud_and_fires_errhandler \u2192 test_enq_arm_failure_is_loud_no_errhandler", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-650.md" }, { "id": "bug-652", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "hang", "timestamp": "2026-06-17T19:10:00Z", - "error_message": "Controller (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but the controller freezes. gdb shows Thread 3 in primary_heartbeat_fn holding ha->lock (owner=self) and re-acquiring it via xce_nop_req -> xce_request -> endpoint_enq -> handle_error -> vote_ep_err_handler (ha.c:3478); Thread 9 (io_worker) also blocked on the same ha->lock. Whole controller frozen.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Controller (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but…\n\nController (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but the controller freezes. gdb shows Thread 3 in primary_heartbeat_fn holding ha->lock (owner=self) and re-acquiring it via xce_nop_req -> xce_request -> endpoint_enq -> handle_error -> vote_ep_err_handler (ha.c:3478); Thread 9 (io_worker) also blocked on the same ha->lock. Whole controller frozen.", "file": "src/lib/netio.c", - "root_cause": "REGRESSION from bug-646: endpoint_enq() invoked handle_error() synchronously on the send path when the poll arm failed. handle_error() calls the endpoint's err_handler, which on a controller vote endpoint is vote_ep_err_handler() \u2014 it locks ha->lock. But endpoint_enq runs on the CALLER's thread, and the controller's primary_heartbeat_fn already holds ha->lock when it sends heartbeat NOPs. Non-recursive mutex -> self-deadlock. The err_handler is contractually an io_worker-thread callback (off an epoll error event, holding no caller locks); calling it from the synchronous send path violated that. With bug-650 now propagating the enq error up the return chain, the handle_error call in endpoint_enq was both redundant and harmful.", + "root_cause": "REGRESSION from bug-646: endpoint_enq() invoked handle_error() synchronously on the send path when the poll arm failed. handle_error() calls the endpoint's err_handler, which on a controller vote endpoint is vote_ep_err_handler() — it locks ha->lock. But endpoint_enq runs on the CALLER's thread, and the controller's primary_heartbeat_fn already holds ha->lock when it sends heartbeat NOPs. Non-recursive mutex -> self-deadlock. The err_handler is contractually an io_worker-thread callback (off an epoll error event, holding no caller locks); calling it from the synchronous send path violated that. With bug-650 now propagating the enq error up the return chain, the handle_error call in endpoint_enq was both redundant and harmful.", "fix": "endpoint_enq() no longer calls handle_error(); on arm failure it just returns a RETRIABLE -ECONNRESET. The caller propagates it (request.c senders return it -> client failover classifies ECONNRESET as RETRY_NOW and rotates; controller heartbeat path simply gets the error, no lock re-entry). The real teardown stays on the io_worker thread (handle_input/handle_output -> io_handler -> handle_error), which holds no caller locks. Updated test_netio_arm_failure.c: endpoint_enq must return <0 AND must NOT invoke the err_handler (g_err_count==0). libxce 169/0; controller_unit_test links clean.", "tags": [ "libxce", @@ -6554,66 +1319,29 @@ "reentrancy", "regression", "controller", - "vote_ep_err_handler", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "vote_ep_err_handler" ], "related_bugs": [ "bug-646", "bug-650" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-658", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T14:46:54.135Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)", - "fix": "Changed to: if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-659", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T14:47:25.446Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/cluster.c", - "root_cause": "Condition was: if (role == CTRL_STANDBY &&\n )", - "fix": "Changed to: if (role == CTRL_STANDBY &&\n )", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-652.md" }, { "id": "bug-660", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-17T22:00:00Z", - "error_message": "Failover election takes minutes (11+ PreVote rounds) to converge. A standby keeps denying a legitimately-higher-term candidate's PreVote with reason=fresh_heartbeat for ~2.5 min AFTER its primary died (ctrl-3 killed 21:44:14, ctrl-2 logs 'PreVote DENIED ... reason=fresh_heartbeat (my_term=3)' until 21:46:58).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Failover election takes minutes (11+ PreVote rounds) to converge\n\nFailover election takes minutes (11+ PreVote rounds) to converge. A standby keeps denying a legitimately-higher-term candidate's PreVote with reason=fresh_heartbeat for ~2.5 min AFTER its primary died (ctrl-3 killed 21:44:14, ctrl-2 logs 'PreVote DENIED ... reason=fresh_heartbeat (my_term=3)' until 21:46:58).", "file": "src/controller/ha.c", - "root_cause": "The PreVote/vote eligibility gate raft_cb_vote_eligible (ha.c:311) denied on `role==CTRL_STANDBY && ha_heartbeat_is_fresh(ha)` \u2014 checking freshness ALONE. ha_heartbeat_is_fresh derives from ha->last_activity, but ha_arm_heartbeat_timeout_vtask resets last_activity=now on EVERY watchdog (re-)arm, and ha_prevote_rearm_liveness re-arms it after every lost PreVote. So a node with NO live primary perpetually refreshes its own 'fresh heartbeat' each failover cycle and denies all other candidates (reason=fresh_heartbeat), blocking convergence until it happens to restart. The sibling gate in handle_request_vote (ha.c:2205) already required `primary_ctrl_fd >= 0 && is_fresh`; raft_cb_vote_eligible was missing the primary_ctrl_fd guard. The existing fresh-deny tests even documented the intent ('the deny gate requires a live connection AND a fresh heartbeat', set primary_ctrl_fd=999), but production didn't enforce it.", - "fix": "Added `ha->primary_ctrl_fd >= 0 &&` to the ha.c:311 eligibility gate (mirrors ha.c:2205). primary_ctrl_fd is -1 on init / after the primary RDHUPs / after the liveness watchdog fires, and a real fd only via config_repl.c:1451 (first ctrlhello from the live primary) \u2014 so a dead/absent primary no longer masquerades as fresh and the standby GRANTS, converging in ~1 round. Kept cluster.c:526 denial-reason logging in sync. TDD: test_pre_vote_dead_primary_grants_despite_fresh_activity (fresh last_activity + primary_ctrl_fd=-1 -> GRANT); verified RED (Expected 1 Was 0) without the guard, GREEN with it, and the 3 existing fresh-deny tests (fd=999) unaffected. Full controller suite + libxce 169/0, zero regressions.", + "root_cause": "The PreVote/vote eligibility gate raft_cb_vote_eligible (ha.c:311) denied on `role==CTRL_STANDBY && ha_heartbeat_is_fresh(ha)` — checking freshness ALONE. ha_heartbeat_is_fresh derives from ha->last_activity, but ha_arm_heartbeat_timeout_vtask resets last_activity=now on EVERY watchdog (re-)arm, and ha_prevote_rearm_liveness re-arms it after every lost PreVote. So a node with NO live primary perpetually refreshes its own 'fresh heartbeat' each failover cycle and denies all other candidates (reason=fresh_heartbeat), blocking convergence until it happens to restart. The sibling gate in handle_request_vote (ha.c:2205) already required `primary_ctrl_fd >= 0 && is_fresh`; raft_cb_vote_eligible was missing the primary_ctrl_fd guard. The existing fresh-deny tests even documented the intent ('the deny gate requires a live connection AND a fresh heartbeat', set primary_ctrl_fd=999), but production didn't enforce it.", + "fix": "Added `ha->primary_ctrl_fd >= 0 &&` to the ha.c:311 eligibility gate (mirrors ha.c:2205). primary_ctrl_fd is -1 on init / after the primary RDHUPs / after the liveness watchdog fires, and a real fd only via config_repl.c:1451 (first ctrlhello from the live primary) — so a dead/absent primary no longer masquerades as fresh and the standby GRANTS, converging in ~1 round. Kept cluster.c:526 denial-reason logging in sync. TDD: test_pre_vote_dead_primary_grants_despite_fresh_activity (fresh last_activity + primary_ctrl_fd=-1 -> GRANT); verified RED (Expected 1 Was 0) without the guard, GREEN with it, and the 3 existing fresh-deny tests (fd=999) unaffected. Full controller suite + libxce 169/0, zero regressions.", "tags": [ "controller", "ha", @@ -6623,24 +1351,26 @@ "fresh_heartbeat", "raft_cb_vote_eligible", "primary_ctrl_fd", - "slow-election", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "slow-election" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-660.md" }, { "id": "bug-661", "status": "fixed", "status_updated": "2026-09-24", "status_note": "f139e1c0", + "component": "controller", + "severity": "data-loss", "timestamp": "2026-06-17T23:30:00Z", - "error_message": "3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and survived first failover, permanently gone after a restarted node with near-empty WAL won election. joblist dropped from 10 to 3.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and…\n\n3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and survived first failover, permanently gone after a restarted node with near-empty WAL won election. joblist dropped from 10 to 3.", "file": "src/controller/persist.c, src/controller/commit.c, src/controller/repl.c", "root_cause": "'committed' does not imply 'durable'. WAL fsync is asynchronous (persist.c:173 fsync_worker, timer wal_sync_ms); persist_append and the standby WalAck path do NOT fdatasync before ACK. commit_on_ack (commit.c:303) advances committed_lsn on majority ACK = OS page cache only, durable on zero disks. On kill+restart, killed nodes lose the not-yet-fsynced tail; persist_recover hits CRC break at zeroed region (prev_crc=00000000, offset 40) and truncates replay to ~0 entries, yet the hollow node stays eligible, wins election under degraded required=1 quorum (PreVote+freshness gate cannot fence it because the committed entries were durable on nobody), self-promotes advertising committed_lsn=21 over an empty job table, and the node holding the real data converges DOWN. Violates Leader-Completeness assumption (majority-ACK != durable-on-majority) in src/controller/CLAUDE.md.", - "fix": "NOT YET FIXED. Discovery -> new spec ha-commit-durability (Path C, brief written). R1 fsync-before-ACK (standby fdatasync before WalAck, primary self-count only after local fdatasync, committed_lsn advances only on durable majority, group-commit to bound latency). R2 promotion fence (WAL CRC/replay failure => ineligible to lead until caught up, FATAL not silent truncate; never lead below known committed high-water even under degraded quorum). Live E2E reproduces via xce_test jobsub + primary kills on /xce-deploy-ha. DETERMINISTIC REPRO (task 1.3, 2026-06-18): `bash src/tools/test-commit-durability.sh --deterministic` launches the cluster with --wal-sync-ms 30000 (plumbed via deploy-ha.sh start.sh `$*` passthrough, EXTRA_CTRL_ARGS) so the primary's fsync_worker never fires in-window -> every committed entry is page-cache-only on the primary (majority_candidate_locked counts cap=highest_appended_lsn=current_lsn-1, NOT durable). It then drives rolling PRIMARY kills (each right after committing to that primary) + restart, so hollow nodes re-lead. RED CONFIRMED: 6-round run lost the 3 EARLIEST committed JobIds permanently (rounds 4 and 6 + final check), Results 5 passed/3 failed, exit=1 \u2014 matching the incident signature (earliest jobs gone). NOTE: a single double-kill does NOT repro \u2014 this codebase enforces strict majority (lone survivor will NOT self-elect: log shows 'no single-voter primary within 40s'); loss needs the ROLLING/term-accumulating path the incident used. Evidence log: dur-det-run.log in worktree.", + "fix": "NOT YET FIXED. Discovery -> new spec ha-commit-durability (Path C, brief written). R1 fsync-before-ACK (standby fdatasync before WalAck, primary self-count only after local fdatasync, committed_lsn advances only on durable majority, group-commit to bound latency). R2 promotion fence (WAL CRC/replay failure => ineligible to lead until caught up, FATAL not silent truncate; never lead below known committed high-water even under degraded quorum). Live E2E reproduces via xce_test jobsub + primary kills on /xce-deploy-ha. DETERMINISTIC REPRO (task 1.3, 2026-06-18): `bash src/tools/test-commit-durability.sh --deterministic` launches the cluster with --wal-sync-ms 30000 (plumbed via deploy-ha.sh start.sh `$*` passthrough, EXTRA_CTRL_ARGS) so the primary's fsync_worker never fires in-window -> every committed entry is page-cache-only on the primary (majority_candidate_locked counts cap=highest_appended_lsn=current_lsn-1, NOT durable). It then drives rolling PRIMARY kills (each right after committing to that primary) + restart, so hollow nodes re-lead. RED CONFIRMED: 6-round run lost the 3 EARLIEST committed JobIds permanently (rounds 4 and 6 + final check), Results 5 passed/3 failed, exit=1 — matching the incident signature (earliest jobs gone). NOTE: a single double-kill does NOT repro — this codebase enforces strict majority (lone survivor will NOT self-elect: log shows 'no single-voter primary within 40s'); loss needs the ROLLING/term-accumulating path the incident used. Evidence log: dur-det-run.log in worktree.", "tags": [ "controller", "ha", @@ -6654,363 +1384,29 @@ "promotion-fence", "async-fsync", "deterministic-repro", - "wal-sync-ms", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "wal-sync-ms" ], "related_bugs": [ "bug-339", "bug-447", "bug-396" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-662", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T23:34:32.115Z", - "error_message": "Function not marked async", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "f139e1c0" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-664", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-17T23:59:44.577Z", - "error_message": "Incorrect value in code", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Had \"${2:-6}\"", - "fix": "Changed to \"${2:-8}\"", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-666", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T02:32:07.670Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-durability/brief.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-667", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T02:35:08.157Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-668", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T03:12:26.259Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Had `persist.c:515`", - "fix": "Changed to `fsync_worker`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-671", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T03:15:13.287Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `fsync_worker`", - "fix": "Changed to `/kiro-validate-design`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-673", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T03:50:26.761Z", - "error_message": "Function not marked async", - "file": "src/controller/CLAUDE.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-676", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T04:00:06.872Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Had `src/include/persist.h`", - "fix": "Changed to `persist_wal_sync_append`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-677", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T04:39:24.460Z", - "error_message": "Wrong reference: myconf_t should be uint32_t", - "file": "src/include/myconf.h", - "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t \u2192 uint32_t", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-678", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T04:50:36.884Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'testsuite/test_commit_durability.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-679", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T04:51:30.430Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-682", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T05:02:07.556Z", - "error_message": "Incorrect value in code", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Had \"$(mktemp /tmp/xce-durability-acked.XXXXXX)\"", - "fix": "Changed to \"$(mktemp \"", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-684", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T05:28:05.352Z", - "error_message": "Function not marked async", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-686", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T05:43:02.761Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-688", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T05:54:46.908Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-690", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T05:55:33.474Z", - "error_message": "Wrong reference: else should be RUN_TEST", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Used \"else\" instead of \"RUN_TEST\"", - "fix": "Changed else \u2192 RUN_TEST", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-692", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T05:59:17.869Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-661.md" }, { "id": "bug-693", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-18T06:05:00.000Z", - "error_message": "durable_lsn advanced even when fdatasync FAILED (EIO/ENOSPC) at two of three sites (flusher_flush_once, persist_stop), letting a non-durable LSN count toward the commit majority", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "durable_lsn advanced even when fdatasync FAILED (EIO/ENOSPC) at two of three sites…\n\ndurable_lsn advanced even when fdatasync FAILED (EIO/ENOSPC) at two of three sites (flusher_flush_once, persist_stop), letting a non-durable LSN count toward the commit majority", "file": "src/controller/persist.c", "root_cause": "ha-commit-durability task 2.1 flush sites called fdatasync(p->wal_fd) without capturing/checking the return code, then advanced durable_lsn unconditionally. On sync error the bytes are not on stable storage, violating Req 1.1/1.2 and the fail-loud preference.", "fix": "Capture int rc = fdatasync(...) at every durable_lsn advance site (flusher_flush_once gating + fallback paths, persist_stop) and advance durable_lsn ONLY when rc == 0; on error leave durable_lsn unchanged and log loudly via ERROR(). persist_flush already gated on rc==0 (reference). Added XCE_TESTING fault-injection seam (test_force_fsync_fail + persist_test_set_force_fsync_fail + flusher_fsync_forced_fail helper, constant-false in production) and error-path test test_durable_lsn_holds_on_fsync_failure proving durable_lsn holds at baseline under injected failure and recovers when cleared.", @@ -7020,141 +1416,24 @@ "fdatasync", "fail-loud", "remediation", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "c" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-694", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:09:08.298Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", - "root_cause": "Had `/kiro-spec-init ha-commit-durability`", - "fix": "Changed to 't fence the stale node because the entries were d", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-695", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:12:59.924Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-696", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:13:32.891Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-697", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:13:40.659Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-699", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:14:16.158Z", - "error_message": "Function not marked async", - "file": "src/controller/commit.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-701", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:15:45.248Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Had \"config.h\"", - "fix": "Changed to \"commit.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-693.md" }, { "id": "bug-704", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-18", - "error_message": "controller suite regressions after capping commit self-contribution at durable_lsn: test_repl_majority/test_commit_barrier_rollout/test_coldstart_lsn0/test_demotion_convergence got -ETIMEDOUT / committed_lsn=0 (Expected N Was 0)", - "file": "src/controller/testsuite/{test_repl_majority,test_commit_barrier_rollout,test_coldstart_lsn0,test_demotion_convergence}.c", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller suite regressions after capping commit self-contribution at durable_lsn…\n\ncontroller suite regressions after capping commit self-contribution at durable_lsn: test_repl_majority/test_commit_barrier_rollout/test_coldstart_lsn0/test_demotion_convergence got -ETIMEDOUT / committed_lsn=0 (Expected N Was 0)", + "file": "src/controller/testsuite/{test_repl_majority, test_commit_barrier_rollout, test_coldstart_lsn0, test_demotion_convergence}.c", "root_cause": "ha-commit-durability 2.2 changed majority_candidate_locked self-cap from highest_appended_lsn to persist_durable_lsn. Existing commit-path fixtures prime current_lsn (appended tip) but never set durable_lsn, so under the new cap self contributed 0 and committed_lsn never advanced.", "fix": "In every commit-path stub/in-memory-primed fixture, set durable_lsn to the appended tip (appended_lsn / current_lsn-1) right after committed_lsn=0, preserving the pre-durability 'self covers every appended entry' model. The durability precondition itself is exercised by dedicated new tests.", "tags": [ @@ -7164,466 +1443,29 @@ "self-cap", "test-fixture", "ha-commit-durability", - "task-2.2", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-2.2" ], "related_bugs": [ "bug-661", "bug-693" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-705", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:33:12.523Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-707", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:34:06.875Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: p", - "fix": "Added guard clause: if (p)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-708", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:34:36.356Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-711", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T06:50:49.987Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: written_lsn > durable", - "fix": "Added guard clause: if (written_lsn > durable)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-714", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T07:00:36.008Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: written_lsn > durable", - "fix": "Added guard clause: if (written_lsn > durable)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-720", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T07:52:20.595Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* task 3.2 \u2014 Boot Policy Decision Table skeletons (IGNORED,; * 3.3/4.3). Registered so each row is reported as IGNORED, n", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-721", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T08:04:18.857Z", - "error_message": "Missing error handling in enum", - "file": "src/include/persist.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Bounded catch-up window (ha-commit-durability task 3.3, R; * The FINITE deadline a recovery-truncated / behind member i", - "tags": [ - "auto-detected", - "error-handling", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-726", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T08:19:47.036Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-728", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T08:39:54.889Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Had `durable_floor \u2264 applied_lsn`", - "fix": "Changed to `applied_lsn`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-732", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T08:51:34.635Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "No early return/throw for edge case: !mkdtemp(tmp", - "fix": "Added guard clause: if (!mkdtemp(tmp)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-733", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T08:52:12.840Z", - "error_message": "Wrong return value", - "file": "src/controller/ha.c", - "root_cause": "Was returning: persist_durable_floor(p);", - "fix": "Now returns: floor > 8ULL ? floor : 8ULL; /* BUG: pretends lear", - "tags": [ - "auto-detected", - "return-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-735", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T09:08:09.215Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* \u2500\u2500 ha-commit-durability task 4.3: pre-promotion eligibili; * Refuse promotion BEFORE the node becomes authoritative CTR", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-736", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T09:09:28.348Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fr; * member is not behind anything and must be promotable (no f", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-738", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T09:45:40.163Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-739", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T09:46:18.019Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-740", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T09:46:46.814Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-741", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T09:49:58.840Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Test-injectable send hook (defined in repl.c under XCE_TE; * repl_broadcast from dereferencing the dummy endpoints as r", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-743", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T10:25:02.673Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* ha_cancel_promotion_fence_timer: cancel a pending bounded; * (idempotent; no-op when none is armed). Called when the m", - "tags": [ - "auto-detected", - "error-handling", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-745", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T10:26:34.227Z", - "error_message": "Wrong operator: != should be ==", - "file": "src/controller/ha.c", - "root_cause": "Used \"!=\" instead of \"==\"", - "fix": "Changed operator != \u2192 ==", - "tags": [ - "auto-detected", - "operator-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-746", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T10:30:18.073Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* task 4.4 \u2014 bounded-window catch-up before FATAL (no synch; * deferred-timer FATAL only after a real bounded failure; fe", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-755", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T10:55:31.898Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Had \"bounded window\"", - "fix": "Changed to `DEFAULT_RECOVERY_CATCHUP_WINDOW_MS`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-759", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T11:21:44.844Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", - "root_cause": "Had `--wal-sync-ms`", - "fix": "Changed to `:FAIL:`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-704.md" }, { "id": "bug-760", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-18T13:00:00Z", - "error_message": "test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected 1 Was 0 \u2014 a covering ACK must advance committed_lsn over the retained fact (6/6 deterministic)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 6, + "error_message": "test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected…\n\ntest_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected 1 Was 0 — a covering ACK must advance committed_lsn over the retained fact (6/6 deterministic)", "file": "src/controller/testsuite/test_commit_barrier_rollout.c", "root_cause": "ha-commit-durability task 2.2 capped the primary commit self-contribution at durable_lsn (advanced asynchronously by the demand-driven group-commit flusher) and wired a durable-advance commit-wake callback. This adjacent-spec (ha-commit-barrier-rollout) test uses a REAL persist+flusher and checks committed_lsn SYNCHRONOUSLY the instant after repl_on_ack(fact_lsn); the flusher has not yet fdatasync'd LSN1, so durable_lsn==0, self-cap==0, majority candidate==0, committed_lsn stays 0. Pre-spec the test passed (self counted the appended tip immediately). Task 2.2 primed durable_lsn in the in-memory stub fixtures (barrier_setup_vcpus) but did NOT update this real-flusher test. The implementation-note claim that this is a pre-existing /tmp-overlay fdatasync-EIO artifact is FALSE in this env: fdatasync returns rc=0 on /tmp here, and the test passed pre-spec.", - "fix": "FIXED \u2014 test_commit_barrier_rollout.c:445: insert persist_flush(g_fp) + assert persist_durable_lsn>=fact_lsn BEFORE repl_on_ack, so the covering ACK (commit_on_ack runs synchronously inside repl_on_ack) deterministically completes the durable majority. Mirrors task 2.2's durable_lsn priming of the in-memory barrier_setup_vcpus stubs. Verified: was 6/6 deterministic FAIL, now fact_hook PASS across 4 runs + meson controller suite Ok:1 Fail:0, all groups 0 Failures. Production was never affected (RETAIN facts have no waiter; WAIT-mode replies block on commit barrier commit.c:334).", + "fix": "FIXED — test_commit_barrier_rollout.c:445: insert persist_flush(g_fp) + assert persist_durable_lsn>=fact_lsn BEFORE repl_on_ack, so the covering ACK (commit_on_ack runs synchronously inside repl_on_ack) deterministically completes the durable majority. Mirrors task 2.2's durable_lsn priming of the in-memory barrier_setup_vcpus stubs. Verified: was 6/6 deterministic FAIL, now fact_hook PASS across 4 runs + meson controller suite Ok:1 Fail:0, all groups 0 Failures. Production was never affected (RETAIN facts have no waiter; WAIT-mode replies block on commit barrier commit.c:334).", "tags": [ "ha-commit-durability", "commit-barrier-rollout", @@ -7632,86 +1474,29 @@ "test-race", "async-durability", "validation", - "NO-GO", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "NO-GO" ], "related_bugs": [ "bug-661", "bug-704" ], - "occurrences": 6, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-761", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T13:13:49.968Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_barrier_rollout.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-762", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T13:18:43.638Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-durability/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-763", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-18T14:54:25.478Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-760.md" }, { "id": "bug-765", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-06-18T14:30:00Z", - "error_message": "scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify path hits the bulk CREATE path (sched_update_existing_job is dead code) \u2192 corrupts/duplicates live job state, no -ENOENT; Req 3 broken at dequeue, Req 6 stores nothing (rusage never stored).", - "file": "src/scheduler/ (sched_update_existing_job dead; schedUpdateJob \u2192 CREATE path)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify…\n\nscheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify path hits the bulk CREATE path (sched_update_existing_job is dead code) → corrupts/duplicates live job state, no -ENOENT; Req 3 broken at dequeue, Req 6 stores nothing (rusage never stored).", + "file": "src/scheduler/ (sched_update_existing_job dead; schedUpdateJob → CREATE path)", "root_cause": "Tasks were checked [x] without the dequeue/update path and rusage storage actually being wired; gap surfaced only by cross-task re-audit (per-task review missed it).", - "fix": "FIXED 2026-06-18 (commits 5c1f715,93124c5,080ff78,276e26c): G-8/G-4 (route notify job-update to in-place sched_update_existing_job, all mutable fields, -ENOENT no-create); G-5 (store rusage on xce_sched_job_t via sched_apply_job_rusage + free_JobRusages_ptr); G-3 (free all 5 killer op payloads with generated deep-freers); G-7 (doc reconciliation \u2014 sched_op_name stays static in scheduler.c, lib must not depend on scheduler headers); G-6 (real white-box tests: test_handlers.c 6/6 for 9.2/9.3, test_job_update.c 10/10 for 9.4/9.5/9.7, registered as meson tests). Scheduler suite 31/31 GREEN, ASAN-clean, boundary-clean. validate-impl GO. RESIDUAL (documented, non-blocking): 9.6 dequeue worker-resvals APPLY unit test (dormant path R-3 \u2014 enqueue tested), 9.8 documented compile-time absence, 9.9 log-capture gated, 9.2 -ENOMEM fault-injection.", + "fix": "FIXED 2026-06-18 (commits 5c1f715,93124c5,080ff78,276e26c): G-8/G-4 (route notify job-update to in-place sched_update_existing_job, all mutable fields, -ENOENT no-create); G-5 (store rusage on xce_sched_job_t via sched_apply_job_rusage + free_JobRusages_ptr); G-3 (free all 5 killer op payloads with generated deep-freers); G-7 (doc reconciliation — sched_op_name stays static in scheduler.c, lib must not depend on scheduler headers); G-6 (real white-box tests: test_handlers.c 6/6 for 9.2/9.3, test_job_update.c 10/10 for 9.4/9.5/9.7, registered as meson tests). Scheduler suite 31/31 GREEN, ASAN-clean, boundary-clean. validate-impl GO. RESIDUAL (documented, non-blocking): 9.6 dequeue worker-resvals APPLY unit test (dormant path R-3 — enqueue tested), 9.8 documented compile-time absence, 9.9 log-capture gated, 9.2 -ENOMEM fault-injection.", "tags": [ "scheduler", "controller", @@ -7719,122 +1504,31 @@ "validate-gap", "not-implemented", "HIGH", - "job-state-corruption", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "job-state-corruption" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-766", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-19T02:55:49.212Z", - "error_message": "Wrong return value", - "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", - "root_cause": "Was returning: `-ENOENT` if job not found", - "fix": "Now returns: `-ENOENT` if job not found. _Remediation (bug-765,", - "tags": [ - "auto-detected", - "return-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "5c1f715", + "93124c5", + "080ff78", + "276e26c" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-768", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-19T03:30:28.056Z", - "error_message": "Wrong condition in logic", - "file": "src/scheduler/src/schedobj.c", - "root_cause": "Condition was: if (job->rusages != NULL && job->rusages != rusages)", - "fix": "Changed to: if (job->rusages != rusages)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-772", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-19T03:49:23.624Z", - "error_message": "Wrong reference: xce_sched_op_name should be sched_op_name", - "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", - "root_cause": "Used \"xce_sched_op_name\" instead of \"sched_op_name\"", - "fix": "Changed xce_sched_op_name \u2192 sched_op_name", - "tags": [ - "wrong-reference", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-773", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-19T03:49:39.373Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", - "root_cause": "Had `xce_sched_op_name()`", - "fix": "Changed to `sched_op_name()`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-774", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-19T03:50:12.460Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", - "root_cause": "Had `xce_sched_op_name()`", - "fix": "Changed to `sched_op_name()`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-765.md" }, { "id": "bug-777", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "data-loss", "timestamp": "2026-06-19T13:00:00Z", - "error_message": "3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS 5/5 jobs vanished'. Both surviving nodes recovered RECOVER_TRUNCATED after SIGKILL (torn WAL tail) and stay permanently fenced/non-promotable even though durable_floor==cluster_hwm (they hold ALL committed data).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS…\n\n3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS 5/5 jobs vanished'. Both surviving nodes recovered RECOVER_TRUNCATED after SIGKILL (torn WAL tail) and stay permanently fenced/non-promotable even though durable_floor==cluster_hwm (they hold ALL committed data).", "file": "src/controller/ha.c (raft_cb_become_leader promotion fence release) + persist.c (recovery_truncated verdict stickiness); design ha-commit-durability Req 2.2/2.6 + Boot Policy Decision Table", - "root_cause": "A SIGKILL tears the WAL tail; on restart replay_wal hits a CRC-break/short-read at the tail -> RECOVER_TRUNCATED (sticky flag). The promotion-fence release condition requires `!recovery_truncated`, but the truncated flag is NEVER cleared by catch-up \u2014 so a node whose intact durable prefix (durable_floor) already reaches/exceeds the cluster committed high-water (durable_floor>=hwm, i.e. holds ALL committed entries; the torn tail is entirely UNCOMMITTED) stays permanently non-promotable. The design assumed 'TRUNCATED => provably below HWM', which is FALSE for a torn TAIL above the committed prefix. When all survivors are in this state (after multiple rolling failovers), none can promote => election/availability WEDGE. Observed: ctrl-1 recovery_truncated=1 durable_floor=13 cluster_hwm=13, PreVote-DENY peers reason=catch_up, indefinitely.", - "fix": "DATA-LOSS dimension RESOLVED by the bug-778 fix (commit 64ab210): the cold-restart loss was the CRC-chain clobber; with the chain intact, cold restart recovers all committed jobs (soak 5/5, joblist 6/6 after cold restart). The AVAILABILITY-WEDGE dimension was a CONSEQUENCE of the spurious RECOVER_TRUNCATED from the broken chain \u2014 no longer triggered (nodes now recover CLEAN; round-3 re-elects normally). RESIDUAL HARDENING (latent, not currently triggered): the promotion fence keeps recovery_truncated sticky even when durable_floor>=HWM, so a GENUINE torn-tail that leaves a node RECOVER_TRUNCATED while holding all committed data could still over-fence. Optional follow-up: clear the truncated fence when durable_floor>=cluster_hwm (the torn tail is provably uncommitted). Not blocking \u2014 the systematic trigger is gone.", + "root_cause": "A SIGKILL tears the WAL tail; on restart replay_wal hits a CRC-break/short-read at the tail -> RECOVER_TRUNCATED (sticky flag). The promotion-fence release condition requires `!recovery_truncated`, but the truncated flag is NEVER cleared by catch-up — so a node whose intact durable prefix (durable_floor) already reaches/exceeds the cluster committed high-water (durable_floor>=hwm, i.e. holds ALL committed entries; the torn tail is entirely UNCOMMITTED) stays permanently non-promotable. The design assumed 'TRUNCATED => provably below HWM', which is FALSE for a torn TAIL above the committed prefix. When all survivors are in this state (after multiple rolling failovers), none can promote => election/availability WEDGE. Observed: ctrl-1 recovery_truncated=1 durable_floor=13 cluster_hwm=13, PreVote-DENY peers reason=catch_up, indefinitely.", + "fix": "DATA-LOSS dimension RESOLVED by the bug-778 fix (commit 64ab210): the cold-restart loss was the CRC-chain clobber; with the chain intact, cold restart recovers all committed jobs (soak 5/5, joblist 6/6 after cold restart). The AVAILABILITY-WEDGE dimension was a CONSEQUENCE of the spurious RECOVER_TRUNCATED from the broken chain — no longer triggered (nodes now recover CLEAN; round-3 re-elects normally). RESIDUAL HARDENING (latent, not currently triggered): the promotion fence keeps recovery_truncated sticky even when durable_floor>=HWM, so a GENUINE torn-tail that leaves a node RECOVER_TRUNCATED while holding all committed data could still over-fence. Optional follow-up: clear the truncated fence when durable_floor>=cluster_hwm (the torn tail is provably uncommitted). Not blocking — the systematic trigger is gone.", "tags": [ "ha", "convergence", @@ -7847,23 +1541,28 @@ "ha-commit-durability", "ha-convergence-recovery", "live-E2E", - "false-positive-data-loss", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "false-positive-data-loss" ], "related_bugs": [ "bug-661", "bug-339" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "64ab210" + ], + "record": ".wolf/bugs/bug-777.md" }, { "id": "bug-778", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "data-loss", "timestamp": "2026-06-19T13:40:00Z", - "error_message": "persist_recover RECOVER_TRUNCATED: 'CRC chain break ... prev_crc=00000000 expected=' at the first replicated job entry on cold restart; committed jobs unrecoverable (joblist=0). ROOT CAUSE of bug-777's cold-restart data loss.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "persist_recover RECOVER_TRUNCATED: 'CRC chain break ... prev_crc=00000000 expected=' at the…\n\npersist_recover RECOVER_TRUNCATED: 'CRC chain break ... prev_crc=00000000 expected=' at the first replicated job entry on cold restart; committed jobs unrecoverable (joblist=0). ROOT CAUSE of bug-777's cold-restart data loss.", "file": "src/controller/persist.c:1336 (apply_wal_entry) + src/controller/ha.c ha_apply_single_entry (~1570-1588)", "root_cause": "On the standby replicated-apply path, ha_apply_single_entry first calls persist_wal_write_replicated() which correctly sets p->last_crc to the standby's SELF-computed chain crc (the stream carries no crc; the on-disk CRC chain is node-local, prev_crc==last_crc). It THEN builds a synthetic wal_entry_hdr with hdr.crc=0 and calls apply_wal_entry(), whose line 1336 `p->last_crc = hdr->crc` OVERWRITES the correct chain link with 0. The next replicated write computes prev_crc=(seg_offset==0?WAL_CRC_NONE:last_crc)=0 and stamps prev_crc=0 on disk. So the on-disk CRC chain breaks at the 2nd replicated entry. Invisible while live (standby serves in-memory entities; never replays its own WAL), detonates on COLD RESTART where replay_wal validates the chain -> RECOVER_TRUNCATED -> recovers only the clean prefix -> committed jobs lost + the truncated verdict drives the promotion-fence availability wedge (the other half of bug-777). NOT an fd-position/offset race (pwrite+p->lock are correct); it is shared CRC-chain state (p->last_crc) written by two paths where the logical-apply step clobbers the physical-writer's link with 0.", "fix": "FIXED 2026-06-19 (commit 64ab210). apply_wal_entry is now LOGICAL entity-restore only (signature (p,type,lsn,payload,payload_len); no header); it advances the current_lsn high-water but NEVER touches p->last_crc. The CRC chain has a single owner: persist_wal_write_replicated (live) + replay_wal/persist_recover (recovery). ha_apply_single_entry drops the fabricated crc=0 header. Verified: RED->GREEN test test_replicated_apply_chain_survives_cold_restart (real ha_apply_wal_catchup_entry path, cold-replay -> RECOVER_CLEAN); controller suite 0 failures; 3-node rolling-SIGKILL soak on real ext2/3 fs 5/5 (was wedge+loss at round 3); full cold restart recovers all 6 jobs (was 0), CLEAN verdicts, normal election. Test callers updated: test_repl_apply, test_ha_role_lifecycle, test_egress_filter.", @@ -7879,66 +1578,31 @@ "bug-661", "standby-replication", "recovery", - "root-caused", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "root-caused" ], "related_bugs": [ "bug-777", "bug-661" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-781", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-19T14:02:49.235Z", - "error_message": "Wrong reference: hdr should be payload_len", - "file": "src/controller/persist.c", - "root_cause": "Used \"hdr\" instead of \"payload_len\"", - "fix": "Changed hdr \u2192 payload_len", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "64ab210" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-785", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-19T14:08:12.866Z", - "error_message": "Wrong reference: NULL should be type", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "Used \"NULL\" instead of \"type\"", - "fix": "Changed NULL \u2192 type", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-778.md" }, { "id": "bug-786", "status": "fixed", "status_updated": "2026-09-24", "status_note": "fae0bc0f, 86099763", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-19T15:00:00Z", - "error_message": "xce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on repeated/mid-failover submits; controller then logs 'handle_jobsubmit fail to lock job 63-15--1-...'. Client claims success with a malformed id it never got a clean ack for.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "xce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on…\n\nxce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on repeated/mid-failover submits; controller then logs 'handle_jobsubmit fail to lock job 63-15--1-...'. Client claims success with a malformed id it never got a clean ack for.", "file": "src/controller/testsuite/test.c (cmd_jobsub)", "root_cause": "On the 2nd+ submit (and during failover windows) the jobsub reply/jobid is not cleanly populated, leaving an uninitialized jobid struct; cmd_jobsub still prints 'jobsubmit ok' with the garbage id instead of verifying the controller reply status and failing. Forces E2E scripts (test-commit-durability.sh) to strict-regex-filter the 4-field id to avoid chasing phantom jobs.", - "fix": "PENDING \u2014 jobsub must report success ONLY on a clean ack with a valid jobid; otherwise print a typed error (REDIRECT/NOMASTER/NOT_READY/TIMEOUT) and set non-zero status. Part of the xce-test-e2e-primitives work (exit-code-as-verdict + $jobid auto-var). | STRENGTHENED 2026-06-19 (commit 8609976): `jobid != 0` gate was insufficient \u2014 a stale non-zero reply id (63-15--1-1015808) from a non-jobid reply during failover slipped through. New gate requires the echoed jobid == the locally idgen-generated jobsubmit.jobid (controller commits/echoes exactly that id, job.c:616). Surfaced by the live durability soak (task 7.2).", + "fix": "PENDING — jobsub must report success ONLY on a clean ack with a valid jobid; otherwise print a typed error (REDIRECT/NOMASTER/NOT_READY/TIMEOUT) and set non-zero status. Part of the xce-test-e2e-primitives work (exit-code-as-verdict + $jobid auto-var). | STRENGTHENED 2026-06-19 (commit 8609976): `jobid != 0` gate was insufficient — a stale non-zero reply id (63-15--1-1015808) from a non-jobid reply during failover slipped through. New gate requires the echoed jobid == the locally idgen-generated jobsubmit.jobid (controller commits/echoes exactly that id, job.c:616). Surfaced by the live durability soak (task 7.2).", "tags": [ "xce_test", "jobsub", @@ -7948,63 +1612,28 @@ "test-tooling", "xce-test-e2e-primitives", "stale-reply-union", - "live-soak", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "live-soak" ], "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-788", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-19T15:34:41.623Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/testsuite/test.c", - "root_cause": "Condition was: if (active_ctx->parent != NULL)", - "fix": "Changed to: if (i > 1)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "fae0bc0f" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-792", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-19T16:13:02.715Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test.c", - "root_cause": "Had \"shutdown\"", - "fix": "Changed to \"expect-jobs\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-786.md" }, { "id": "bug-795", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-19", - "error_message": "Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job not found). Recovery logs show WAL truncate.", - "file": "src/controller/{persist.c recovery/truncate, commit.c commit-gate, ha.c promotion-fence/election} \u2014 UPSTREAM ha-commit-durability Phase 6.2", - "root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) \u2014 must be reconfirmed on real hardware. UPDATE: with the hardened (trustworthy) soak, the loss ALSO reproduces in ROLLING single-node failover with the cluster confirmed serving (~1/3 runs) \u2014 broader than cold-restart-all; bug-661 not fully closed for rolling failover. Separate finding: >60s post-kill election (liveness) every run, always recovers. CORRECTION: NOT an overlay-fdatasync artifact \u2014 reproduces OUTSIDE the container (user's real host) AND on an in-sandbox real ext2/3 fs with ZERO fdatasync EIO. Real controller durability bug + a coupled liveness stall (>60s / stuck election after killing ctrl-1). Phase 6.1 answered (REAL); now at 6.2 root-cause.", - "fix": "FIXED AND VERIFIED LIVE -- ha-commit-durability was reopened specifically for this bug (Phase 6, tasks 6.1-6.4). Task 6.1 attempted to reproduce the cold-restart-all loss against a REAL fsync-able filesystem (ext4, not this sandbox's overlayfs) and it did not reproduce -- the original loss was an overlayfs fdatasync artifact, not a genuine product defect (tasks 6.2/6.3 root-cause/fix were correctly marked N/A as a direct consequence). Task 6.4 re-validated on real fs and closed bug-795. spec.json's validate_impl (decision: GO, 2026-07-18) independently reconfirms this with a later, fresh live re-validation: a 4-round rolling SIGKILL+restart soak plus a full cold-restart-all cycle, 8/8 PASS, 0 job loss -- all 20 committed jobs survived both rolling failover and cold-restart-all. Distinct root cause from bug-828 (a premature-truncate bug during a live election under rolling single-node failover, fixed separately in ha-term-safety commit b77d412) -- not the same defect, despite early investigation notes grouping them together as 'bug-828/795'. One adjacent, still-open, low-priority item surfaced during the same validation: an extra/duplicate committed job appeared once during the original 2026-07-02 validation run (over-counting, the opposite symptom, not yet root-caused, does not block the GO) -- filed separately, not part of this entry.\n\n[Original entry]\nNOT FIXED HERE \u2014 out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a…\n\nCold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job not found). Recovery logs show WAL truncate.", + "file": "src/controller/{persist.c recovery/truncate, commit.c commit-gate, ha.c promotion-fence/election} — UPSTREAM ha-commit-durability Phase 6.2", + "root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) — must be reconfirmed on real hardware. UPDATE: with the hardened (trustworthy) soak, the loss ALSO reproduces in ROLLING single-node failover with the cluster confirmed serving (~1/3 runs) — broader than cold-restart-all; bug-661 not fully closed for rolling failover. Separate finding: >60s post-kill election (liveness) every run, always recovers. CORRECTION: NOT an overlay-fdatasync artifact — reproduces OUTSIDE the container (user's real host) AND on an in-sandbox real ext2/3 fs with ZERO fdatasync EIO. Real controller durability bug + a coupled liveness stall (>60s / stuck election after killing ctrl-1). Phase 6.1 answered (REAL); now at 6.2 root-cause.", + "fix": "FIXED AND VERIFIED LIVE -- ha-commit-durability was reopened specifically for this bug (Phase 6, tasks 6.1-6.4). Task 6.1 attempted to reproduce the cold-restart-all loss against a REAL fsync-able filesystem (ext4, not this sandbox's overlayfs) and it did not reproduce -- the original loss was an overlayfs fdatasync artifact, not a genuine product defect (tasks 6.2/6.3 root-cause/fix were correctly marked N/A as a direct consequence). Task 6.4 re-validated on real fs and closed bug-795. spec.json's validate_impl (decision: GO, 2026-07-18) independently reconfirms this with a later, fresh live re-validation: a 4-round rolling SIGKILL+restart soak plus a full cold-restart-all cycle, 8/8 PASS, 0 job loss -- all 20 committed jobs survived both rolling failover and cold-restart-all. Distinct root cause from bug-828 (a premature-truncate bug during a live election under rolling single-node failover, fixed separately in ha-term-safety commit b77d412) -- not the same defect, despite early investigation notes grouping them together as 'bug-828/795'. One adjacent, still-open, low-priority item surfaced during the same validation: an extra/duplicate committed job appeared once during the original 2026-07-02 validation run (over-counting, the opposite symptom, not yet root-caused, does not block the GO) -- filed separately, not part of this entry.\n\n[Original entry]\nNOT FIXED HERE — out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict.", "tags": [ "ha-commit-durability", "commit-durability", @@ -8016,223 +1645,30 @@ "needs-real-fs", "rolling-failover", "liveness", - "non-deterministic", - "fixed" + "non-deterministic" ], "related_bugs": [ "bug-661", "bug-778", "bug-786" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-796", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T00:32:43.144Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/spec.json", - "root_cause": "Had \"2026-06-18T04:00:00Z\"", - "fix": "Changed to \"2026-06-19T00:00:00Z\"", - "tags": [ - "auto-detected", - "wrong-value", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "b77d412" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-799", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T01:05:25.091Z", - "error_message": "Incorrect value in code", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Had \"$label: DATA LOSS \u2014 a committed job vanished (bug", - "fix": "Changed to \"$label: DATA LOSS \u2014 a committed job vanished from", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-804", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T03:29:22.678Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/controller-lsid/requirements.md", - "root_cause": "Had \"none\"", - "fix": "Changed to \"unknown\"", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-805", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T03:29:29.570Z", - "error_message": "Wrong return value", - "file": ".kiro/specs/controller-lsid/requirements.md", - "root_cause": "Was returning: a well-defined \"unknown\" or \"none\" value for that ", - "fix": "Now returns: a well-defined \"unknown\" value for that field rath", - "tags": [ - "auto-detected", - "return-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-808", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T04:08:49.684Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/controller-status/design.md", - "root_cause": "Had `committed_config_tag`", - "fix": "Changed to `committedConfigVersion`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-809", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T04:40:36.648Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'ctrlstatus.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-810", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T04:45:48.985Z", - "error_message": "Wrong reference: Data should be Text", - "file": "src/schema/controller.capnp", - "root_cause": "Used \"Data\" instead of \"Text\"", - "fix": "Changed Data \u2192 Text", - "tags": [ - "auto-detected", - "wrong-reference", - "capnp", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-811", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T04:45:49.790Z", - "error_message": "Wrong reference: xce_data_t should be char", - "file": "src/include/typemap.h", - "root_cause": "Used \"xce_data_t\" instead of \"char\"", - "fix": "Changed xce_data_t \u2192 char", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-812", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T05:08:41.030Z", - "error_message": "Incorrect value in code", - "file": "src/lib/meson.build", - "root_cause": "Had 'raft/raft_sim.c'", - "fix": "Changed to 'testsuite/test_ctrlstatus.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-813", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T05:42:31.707Z", - "error_message": "Wrong operator: >= should be ==", - "file": "src/controller/testsuite/test.c", - "root_cause": "Used \">=\" instead of \"==\"", - "fix": "Changed operator >= \u2192 ==", - "tags": [ - "auto-detected", - "operator-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-795.md" }, { "id": "bug-817", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-19", - "error_message": "client op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to reach the live primary when a DEAD controller is FIRST in the masters list; with a live-only masters list the same op succeeds in ~46ms.", - "file": "src/lib/client_failover.c / client.c failover-rotation (libxce) \u2014 UPSTREAM, not controller-status", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "client op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to…\n\nclient op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to reach the live primary when a DEAD controller is FIRST in the masters list; with a live-only masters list the same op succeeds in ~46ms.", + "file": "src/lib/client_failover.c / client.c failover-rotation (libxce) — UPSTREAM, not controller-status", "root_cause": "libxce client failover does not cleanly advance past a dead FIRST master to the live nodes; surfaces a typed EINVAL instead of rotating. Pre-existing (controller-status consumes failover unchanged). Surfaced by controller-status task 5.1 E2E (authoritative wait-primary timed out after a primary kill while a dead node led the masters list, even though a new primary was serving).", "fix": "FIXED -- bug-817 and bug-854 are the same defect (client-side positional primary-discovery: a client fails to reach the live primary when the dead controller happens to be first in the configured masters list). libxce-failover's own spec.json validation note states this was 'found and fixed in prior follow-up commits, already present in the validated build.' Independently corroborated by component-failover-adoption/research.md (written 2026-08 this same project week): 'xcecli_start (client.c:125-188) already rotates through ALL configured masters on the very first connect (bug-854 fix, comment at line 136)' -- cited there as current, standing production behavior. Three later specs (ha-term-safety, raft-ha-core-safety, raft-adapter-conformance) all isolate/route-around bug-854 in their own soak verdicts rather than re-fixing it, consistent with it already being fixed elsewhere.\n\n[Original entry]\nNOT fixed in controller-status (out of boundary). Route to libxce-failover. Confirmed affects plain joblist identically. Workaround: ensure the masters list does not lead with a known-dead node, or restart killed nodes (as the durability soak does).", "tags": [ @@ -8242,121 +1678,25 @@ "dead-first-master", "EINVAL", "controller-status", - "upstream", - "fixed" + "upstream" ], "related_bugs": [ "bug-646" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-818", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T06:11:49.084Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/testsuite/test.c", - "root_cause": "Condition was: if (!g_continue_on_error)", - "fix": "Changed to: if (!g_continue_on_error && !g_interactive)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-820", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T06:16:34.685Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/xce-test-e2e-primitives/tasks.md", - "root_cause": "Had `g_interactive = isatty(STDIN_FILENO)`", - "fix": "Changed to `-e`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-821", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T08:14:43.577Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/controller-status/requirements.md", - "root_cause": "Had 's identity and role, the configured controller to", - "fix": "Changed to 's identity and role, the configured controller to", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-824", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T09:30:23.040Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/controller-status/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-825", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T09:31:24.076Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/controller-status/spec.json", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-817.md" }, { "id": "bug-827", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-20T18:30:00.000Z", - "error_message": "AddressSanitizer: global-buffer-overflow READ in xce_strerror_r (lib/utils.c:335), triggered from cmd_ctrlstatus (test.c:1273) when ctrlstatus failover exhausts and returns E_XCE_NOT_READY (-10013) on a primary-less cluster", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: global-buffer-overflow READ in xce_strerror_r (lib/utils.c:335), triggered from…\n\nAddressSanitizer: global-buffer-overflow READ in xce_strerror_r (lib/utils.c:335), triggered from cmd_ctrlstatus (test.c:1273) when ctrlstatus failover exhausts and returns E_XCE_NOT_READY (-10013) on a primary-less cluster", "file": "src/lib/utils.c", "root_cause": "Enum/table drift: errnox.h E_XCE_* enum gained NOMASTER(10011)/REDIRECT(10012)/NOT_READY(10013)/STANDBY(10014), bumping E_XCE_MAX to 10015, but xce_err_str[] still had only 10 strings + a NULL (11 slots, 88 bytes). xce_strerror_r's only upper guard was errnum>=E_XCE_MAX, so 10011-10014 passed the gate and indexed past the table: NOMASTER->xce_err_str[10]==NULL (strncpy NULL deref); NOT_READY->xce_err_str[12] = 8 bytes past the 88-byte array (the ASAN OOB). Pre-existing libxce foundation bug; controller-status ctrlstatus was the first caller to format these HA codes on a failing cluster.", "fix": "Filled xce_err_str[] with strings for all 14 codes (enum order); hardened xce_strerror_r to bound against the ACTUAL array size (idx>=nstr) + reject NULL slots (degrade to 'unknown error', never OOB) + null-terminate; added a _Static_assert tying sizeof(xce_err_str) to (E_XCE_MAX-XCE_ERRNO_BASE) so future enum additions without a string fail the build. RED: new test_strerror_all_xce_codes SEGV'd at utils.c:335 under ASAN; GREEN: lib suite Ok:1 Fail:0. Commit 1aa0761.", @@ -8369,24 +1709,28 @@ "libxce", "controller-status", "ctrlstatus", - "E_XCE_NOT_READY", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "E_XCE_NOT_READY" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "1aa0761" + ], + "record": ".wolf/bugs/bug-827.md" }, { "id": "bug-828", "status": "fixed", "status_updated": "2026-09-24", "status_note": "91b9a445, b77d4122", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-20T10:19:56.678Z", - "error_message": "Clean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ... (from 1 granted voters)' (self only) and truncates committed entries down to its own boot floor. Post-election safe-commit-from-votes plumbing was DEAD in production.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "Clean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ...…\n\nClean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ... (from 1 granted voters)' (self only) and truncates committed entries down to its own boot floor. Post-election safe-commit-from-votes plumbing was DEAD in production.", "file": "src/controller/ha.c (vote_reply_dispatch), src/controller/persist.c (persist_wal_write_replicated)", "root_cause": "vote_reply_decode_and_dispatch decoded the granting voter's advertised peerLastLogLsn (vr_lsn at ha.c ~4254) but vote_reply_dispatch DISCARDED it ('(void)peer_lsn; raft_on_vote_reply(...)'). raft_on_vote_reply only bumps s->votes_granted; it never records the peer LSN. The only writer of ctx->votes[idx].vote_granted/peer_lsn (handle_request_vote_reply) was, in production, called ONLY from the connect/error failure paths with (granted=0, lsn=0); the real-grant caller was gated behind a test-only hook (xce_vote_reply_hook, never set in production). So ctx->votes[*] stayed calloc'd zero -> ha_compute_safe_committed_lsn saw n_granted=1 (self) -> safe_committed_lsn = self_lsn. A standby promoting with its own durable_floor=1 then persist_truncate_wal_above(1) destroyed committed LSN 2..5.", - "fix": "PRIMARY: in vote_reply_dispatch production branch, route the decoded reply into the election context via a NEW record-only seam ha_record_vote_reply(ha,ctx,idx,term,granted,tag,lsn) BEFORE raft_on_vote_reply. Factored ha_record_vote_reply_locked out of handle_request_vote_reply: it records ctx->votes[idx] under ctx->lock with the idempotency guard but does NOT call ha_elect_evaluate_and_signal -> no second/racing legacy promotion path (raft_on_vote_reply remains the SOLE production promotion driver; the two quorum tallies (raft s->votes_granted / legacy ctx->votes_granted) and their guards (s->election_decided / ctx->done_posted) stay independent and idempotent). idx == vctx->peer_idx == the local ctx->votes[] index (absolute->local map done in raft_cb_send_vote_request), exactly the index ha_compute_safe_committed_lsn + the test hook use. SECONDARY: persist_wal_write_replicated now advances durable_lsn to the synced tip immediately after the inline fdatasync (mirror persist_flush; monotonic max; fail-loud rc-gated) so a standby advertises its own synced durable floor without waiting for the 10ms cadence flusher. VERIFIED: unit test_safe_commit_reflects_granting_voter_lsn RED (safe=1 'from 1 granted voters') -> GREEN (safe=5 'from 3 granted voters'); 177/177 controller_unit_test pass. NOTE: the LIVE clean_failover_preservation harness still FAILS, but the failure is PRE-EXISTING on baseline 14c41cb (verified by stash+rerun: baseline runs also diverge / lose jobs) \u2014 it is the SEPARATE bug-795/828 replication+catch-up gap (standbys' WALs physically contain only LSN 1..2, durable_floor=1; committed entries 2..5 never reach the standby WAL; primary advanced committed_lsn to 5 without a standby durable at 5). The fix DID change the live fingerprint from 'from 1 granted voters' to 'from 2 granted voters', proving the plumbing is now alive. safe-commit-from-votes is correct; the residual live job loss is out of scope (needs the catch-up/snapshot-backfill fix tracked under ha-commit-durability).", + "fix": "PRIMARY: in vote_reply_dispatch production branch, route the decoded reply into the election context via a NEW record-only seam ha_record_vote_reply(ha,ctx,idx,term,granted,tag,lsn) BEFORE raft_on_vote_reply. Factored ha_record_vote_reply_locked out of handle_request_vote_reply: it records ctx->votes[idx] under ctx->lock with the idempotency guard but does NOT call ha_elect_evaluate_and_signal -> no second/racing legacy promotion path (raft_on_vote_reply remains the SOLE production promotion driver; the two quorum tallies (raft s->votes_granted / legacy ctx->votes_granted) and their guards (s->election_decided / ctx->done_posted) stay independent and idempotent). idx == vctx->peer_idx == the local ctx->votes[] index (absolute->local map done in raft_cb_send_vote_request), exactly the index ha_compute_safe_committed_lsn + the test hook use. SECONDARY: persist_wal_write_replicated now advances durable_lsn to the synced tip immediately after the inline fdatasync (mirror persist_flush; monotonic max; fail-loud rc-gated) so a standby advertises its own synced durable floor without waiting for the 10ms cadence flusher. VERIFIED: unit test_safe_commit_reflects_granting_voter_lsn RED (safe=1 'from 1 granted voters') -> GREEN (safe=5 'from 3 granted voters'); 177/177 controller_unit_test pass. NOTE: the LIVE clean_failover_preservation harness still FAILS, but the failure is PRE-EXISTING on baseline 14c41cb (verified by stash+rerun: baseline runs also diverge / lose jobs) — it is the SEPARATE bug-795/828 replication+catch-up gap (standbys' WALs physically contain only LSN 1..2, durable_floor=1; committed entries 2..5 never reach the standby WAL; primary advanced committed_lsn to 5 without a standby durable at 5). The fix DID change the live fingerprint from 'from 1 granted voters' to 'from 2 granted voters', proving the plumbing is now alive. safe-commit-from-votes is correct; the residual live job loss is out of scope (needs the catch-up/snapshot-backfill fix tracked under ha-commit-durability).", "tags": [ "async-fix", "md", @@ -8406,250 +1750,75 @@ "leader-completeness", "ha-repl-hardening", "PARTIAL-FIX", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "c" ], "related_bugs": [ "bug-795", "bug-851" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "91b9a445", + "b77d4122" + ], + "record": ".wolf/bugs/bug-828.md" }, { - "id": "bug-829", + "id": "bug-828-part2", "status": "fixed", "status_updated": "2026-09-24", - "timestamp": "2026-06-20T11:44:25.496Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'ctrlstatus.c'", - "fix": "Changed to 'testsuite/test_ha_heartbeat_committed.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], + "status_note": "b77d4122; upstream gap fixed as bug-661 (f139e1c0)", + "component": "controller", + "severity": "unknown", + "timestamp": "2026-06-23T03:05:00.000Z", + "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-830", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T11:45:26.943Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", + "error_message": "Clean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2…\n\nClean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2 granted voters)' and walcompare reports committed-prefix DIVERGENCE at LSN 2; jobs at LSN 2..5 lost.", + "file": "src/controller/testsuite/test_ha_vote_gate.c (test added); ha.c/persist.c advertise path investigated (no change needed)", + "root_cause": "INVESTIGATED, NOT an advertisement-staleness bug. The advertise path already reflects the live durable tip: ha_candidate_advertise_lsn -> persist_durable_floor, and the voter floor = max(persist_durable_floor, learned_committed); persist_durable_floor == persist_durable_lsn (both read p->durable_lsn, the fsync watermark), and part-1's secondary fix advances durable_lsn after the standby's inline fdatasync in persist_wal_write_replicated. LIVE evidence (fix828c_r1..r3, STRICT_TERM clean failover x3) proves the surviving voters advertise 1 because they are GENUINELY durable only at LSN 1: their WAL physically holds 1..2 (LSN2 = their own term-3 NOOP), recovered tip=2, durable_floor=1. The original primary ctrl-1 (down at election time, WAL 1..5 tip=5) advanced committed_lsn 1->2->3->4->5 in ~250ms while standbys had only converged applied_lsn=1 and catch-up streaming was still in flight. So committed advanced past what the surviving majority durably held — the UPSTREAM replication / commit-barrier durability gap (bug-795/661), not the vote advertisement. safe_committed_lsn=1 is the CORRECT, SAFE Raft answer given the real durable state; advertising the true durable tip cannot conjure data never replicated.", + "fix": "No production code change required for part-2 — the durable-tip advertisement is already correct & safe after part-1. Added unit test test_live_durable_tip_advertised_over_lagging_learned (test_ha_vote_gate.c): a standby durable at LSN5 (real persist, flush -> durable_lsn=5) with learned_committed lagging at 1 must advertise 5 as BOTH candidate self-LSN (ha_candidate_advertise_lsn) and voter floor (ha_vote_self_freshness over persist_durable_floor, dominating learned=1), and ha_compute_safe_committed_lsn from two such granting voters must return 5 — locking in the in-scope contract. Explicitly did NOT fold learned_committed into the CANDIDATE advertise (the requested max(...,learned_committed)) because that re-opens the truncated-candidate-over-advertises-and-wins hole ha-commit-durability task 4.2 deliberately closed (candidate must advertise only a DURABLE position; learned_committed is volatile/not necessarily durable on this node). The residual live job loss is bug-795/661, out of scope.", "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "safety", + "failover", + "safe-committed-lsn", + "durable-tip", + "advertise", + "leader-completeness", + "bug-795", + "bug-661", + "commit-durability", + "INVESTIGATION", + "c" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-831", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T11:54:30.060Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_heartbeat_committed.c", - "root_cause": "Had \"xce/cluster_def.h\"", - "fix": "Changed to \"typemap.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "related_bugs": [ + "bug-828", + "bug-795", + "bug-661", + "bug-851", + "bug-924" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-832", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T11:56:28.240Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c || !hb", - "fix": "Added guard clause: if (!c || !hb)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "b77d4122", + "f139e1c0" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-834", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T12:22:39.011Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", - "root_cause": "Had `heartbeatmsg`", - "fix": "Changed to `xce-build.sh controller`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-835", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T12:26:15.999Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'ctrlstatus.c'", - "fix": "Changed to 'testsuite/test_ha_vote_floor_heartbeat.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-836", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T12:31:51.359Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-837", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T12:32:25.549Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-heartbeat-committed-propagation/spec.json", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-840", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T14:05:49.912Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test.c", - "root_cause": "Had \"xce/cluster.h\"", - "fix": "Changed to \"waldump.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-841", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T14:06:11.393Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/testsuite/test.c", - "root_cause": "Condition was: if (active_ctx->parent != NULL)", - "fix": "Changed to: if (argc < 2)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-843", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T14:15:35.599Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `!`", - "fix": "Changed to `xce_test`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-828-part2.md" }, { "id": "bug-845", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-20T14:30:00.000Z", - "error_message": "Every committed job appears as TWO consecutive WAL_JOB_SUBMIT entries with the SAME jobid (e.g. ctrl-1 LSN 2,3 = jobid 1868520205844480). Surfaced by waldump on /tmp/xce-ha-deploy.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Every committed job appears as TWO consecutive WAL_JOB_SUBMIT entries with the SAME jobid (e.g\n\nEvery committed job appears as TWO consecutive WAL_JOB_SUBMIT entries with the SAME jobid (e.g. ctrl-1 LSN 2,3 = jobid 1868520205844480). Surfaced by waldump on /tmp/xce-ha-deploy.", "file": "src/lib/hook.c (xce_add_hook idempotency); root path src/controller/{persist.c persist_start, ha.c ha_register_persist_hooks}", - "root_cause": "NOT array expansion \u2014 job_submit() assigns array members DISTINCT ids (base_id+i, job.c) and the Job capnp schema has a single jobid field, so identical-jobid pairs cannot be two array tasks. A single client submit is producing two WAL_JOB_SUBMIT appends of the same job. EXACTLY 2 entries per job (never 1 or 3), always at ADJACENT LSNs, for every job on every node. CONFIRMED via enhanced waldump (payload-CRC + subtime columns): each pair is BYTE-IDENTICAL (same PCRC, e.g. LSN 2,3 both 6d79074e; same subtime 1781959730) \u2014 proving a literal double-append of the SAME encoded job_t in one submit flow; a failover retry would re-stamp subtime=time(NULL) and differ, so retry is DEFINITIVELY ruled out. Remaining candidates: AFTER_EVENT jlist contains the job twice, or the JOB_NEW hook/handler persists twice per submit. So 'ctrl-1 has 12 submits' is really 6 jobs (halve when reconciling failover-test job totals). Doubles LSN consumption; consistent across nodes here (F@14,15 on both ctrl-1 and ctrl-3) so not itself divergence, but a latent cross-node LSN-divergence risk if the duplicate count ever varies by primary.", - "fix": "FIXED 2026-06-23 (src/lib/hook.c). Root cause: persist_job_hook_fun was registered on XCE_CONTROLLER_HOOK_JOB by TWO owners \u2014 persist_start() (unconditional, boot) AND ha_register_persist_hooks() (on PRIMARY promotion) \u2014 with the SAME persist_t closure. xce_add_hook did not dedup, xce_run_hooks fires ALL slots, so a promoted primary had 2 active slots => persist_job_hook_fun ran twice per submit => 2 byte-identical adjacent WAL_JOB_SUBMIT appends. Fix: made xce_add_hook idempotent on exact (fn,data) \u2014 re-registering an active (fn,data) is a no-op; same fn with DISTINCT data still registers (legitimate multi-closure). Preserves the persist_start test contract many unit tests rely on. Verified: WAL now single-entry per job (waldump); 5/5 hook unit tests; controller suite 343 tests 0 new failures; 3/3 post-fix HA soaks zero job loss. NOTE: an earlier post-fix soak hit the KNOWN intermittent failover durability race (bug-795/828 class, LSN-10 cross-node divergence) \u2014 confirmed pre-existing (baseline passes/fails on timing too), NOT caused by this dedup.", + "root_cause": "NOT array expansion — job_submit() assigns array members DISTINCT ids (base_id+i, job.c) and the Job capnp schema has a single jobid field, so identical-jobid pairs cannot be two array tasks. A single client submit is producing two WAL_JOB_SUBMIT appends of the same job. EXACTLY 2 entries per job (never 1 or 3), always at ADJACENT LSNs, for every job on every node. CONFIRMED via enhanced waldump (payload-CRC + subtime columns): each pair is BYTE-IDENTICAL (same PCRC, e.g. LSN 2,3 both 6d79074e; same subtime 1781959730) — proving a literal double-append of the SAME encoded job_t in one submit flow; a failover retry would re-stamp subtime=time(NULL) and differ, so retry is DEFINITIVELY ruled out. Remaining candidates: AFTER_EVENT jlist contains the job twice, or the JOB_NEW hook/handler persists twice per submit. So 'ctrl-1 has 12 submits' is really 6 jobs (halve when reconciling failover-test job totals). Doubles LSN consumption; consistent across nodes here (F@14,15 on both ctrl-1 and ctrl-3) so not itself divergence, but a latent cross-node LSN-divergence risk if the duplicate count ever varies by primary.", + "fix": "FIXED 2026-06-23 (src/lib/hook.c). Root cause: persist_job_hook_fun was registered on XCE_CONTROLLER_HOOK_JOB by TWO owners — persist_start() (unconditional, boot) AND ha_register_persist_hooks() (on PRIMARY promotion) — with the SAME persist_t closure. xce_add_hook did not dedup, xce_run_hooks fires ALL slots, so a promoted primary had 2 active slots => persist_job_hook_fun ran twice per submit => 2 byte-identical adjacent WAL_JOB_SUBMIT appends. Fix: made xce_add_hook idempotent on exact (fn,data) — re-registering an active (fn,data) is a no-op; same fn with DISTINCT data still registers (legitimate multi-closure). Preserves the persist_start test contract many unit tests rely on. Verified: WAL now single-entry per job (waldump); 5/5 hook unit tests; controller suite 343 tests 0 new failures; 3/3 post-fix HA soaks zero job loss. NOTE: an earlier post-fix soak hit the KNOWN intermittent failover durability race (bug-795/828 class, LSN-10 cross-node divergence) — confirmed pre-existing (baseline passes/fails on timing too), NOT caused by this dedup.", "tags": [ "double-register", "duplicate", - "fixed", "ha", "hook-dedup", "investigation", @@ -8662,18 +1831,52 @@ "bug-795", "bug-828" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-845.md" + }, + { + "id": "bug-846", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", + "timestamp": "2026-06-20T15:20:00.000Z", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "After the primary dies, xce_test client ops (ctrlstatus/joblist/jobsub) fail PERSISTENTLY with…\n\nAfter the primary dies, xce_test client ops (ctrlstatus/joblist/jobsub) fail PERSISTENTLY with 'Invalid argument' (EINVAL) and wait-primary times out, even though a new primary was elected (ctrl-2 term 3). Blocks failover testing from the client.", + "file": "src/lib/xceapi.c (per-op state guard) + src/lib/client_failover.c (xce_failover_classify / xcecli_retry_decide)", + "root_cause": "Client-side, NOT server. ha_dispatch_gate ALLOWS OPCode_ctrlstatus in every role and rejects client writes with the RETRYABLE -XCE_ERR_NOT_READY (never EINVAL), so a reachable controller would answer. The EINVAL is the xceapi per-op guard: `if (ctx->state != XCE_CTX_RUN) return -EINVAL`. The failover loop only AUTO-RECOVERS ctx.state==XCE_CTX_ERR (xcecli_retry_decide rotates+recovers); xce_failover_classify maps EINVAL -> XCE_FO_TERMINAL ('everything else -> terminal'), so a bare EINVAL is surfaced and the sticky master cursor never advances to the live new primary. So once the ctx lands in a non-ERR non-RUN state (INIT/DOWN) — or yields a bare EINVAL while RUN — failover gives up permanently. Exact stuck state needs a live repro to confirm.", + "fix": "FIXED (verified 2026-07-02, no direct code trail to when — landed under libxce-failover). src/lib/client_failover.c xcecli_retry_decide() now checks `ac->ctx.state == XCE_CTX_ERR` BEFORE classifying rc, bypassing the -EINVAL misclassification entirely (comment there literally reads 'the reported bug'). xcecli_ctx_recover() resets ERR->RUN post-reconnect. This is exactly recommendation (B) from this bug's own original fix note.", + "tags": [ + "client", + "stale-report", + "einval", + "blocker", + "failover", + "ha", + "libxce" + ], + "related_bugs": [ + "bug-646" + ], + "fix_commits": [], + "record": ".wolf/bugs/bug-846.md" }, { "id": "bug-847", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-20T14:37:08.863Z", - "error_message": "Catch-up WAL streaming silently DROPS entry payloads at scale: a rejoining controller (ctrl-1 restarted) received [1..14] with correct LSN/type/CRC but PLEN=0 / empty job blobs (waldump PCRC=00000000, jobid=?). Structurally-intact but content-empty WAL = silent data corruption. THE committed-job-loss mechanism.", - "file": "src/lib/request.c (xce_wal_batch_encode_raw / wal_entry_to_wire / xce_decode extract loop) \u2014 capnp multi-segment Data pointers", - "root_cause": "Multi-segment capnp bug. xce_wal_batch_encode_raw builds a WalCatchupEntry struct-list where each element's payload is a capnp Data (List(UInt8)) allocated via capn_new_list8(cs=root_seg,...). When the message outgrows its first segment (large batch), the per-element payload Data pointer is lost across the segment boundary \u2014 on decode extract_data_bytes(we.payload) reads CAPN_NULL and returns len 0. write_WalCatchupEntry uses capn_setp (far-pointer-capable) so the fault is in the overflow/segment handling as used here. REPRODUCED deterministically: src/lib/testsuite/test_wal_wire.c test_wal_batch_roundtrip_large_payloads (14 entries x 872B) FAILS 'Expected 872 Was 0'; the 2-entry test_wal_batch_roundtrip_payload_bytes PASSES. Threshold is size, not count. Live replication (repl_broadcast) carries payloads fine \u2014 ONLY the catch-up/rejoin path corrupts, which is why loss only appears after a node restarts and catches up.", - "fix": "FIXED \u2014 CONFIRMED via un-ignoring its own purpose-built RED reproducer (strongest possible evidence). Removed the TEST_IGNORE_MESSAGE guard from test_wal_batch_roundtrip_large_payloads (src/lib/testsuite/test_wal_wire.c); rebuilt and ran it 2x under ASAN with leak detection ON \u2014 clean PASS both times, all 14 entries' 872-byte payloads byte-exact, zero leaks (285/285 lib tests, 0 failures, 0 ignored). Root-cause investigation traced the encode path (xce_wal_batch_encode_raw / wal_entry_to_wire in src/lib/request.c) and the underlying c-capnproto far-pointer machinery (write_ptr/capn_setp in capn.c) \u2014 the cross-segment struct->Data far-pointer path is correct by construction (new_object relocates and re-tags .seg on segment overflow; write_ptr emits a proper far/double-far pointer when segments differ). capn-malloc.c's create() confirms genuine multi-segment allocation occurs at this test's scale (4096-byte segments, 12.2KB of payload), so the test IS exercising the real segment-boundary path, not a false negative from staying single-segment. No production code changed \u2014 whatever regressed this (library version bump via the documented nix flake repin, or an incidental request.c change) already fixed it; the TEST_IGNORE was simply never removed. This is the 6th bug in this sweep found to be already-fixed-but-untracked (846/855/1096/1147/1198/847) \u2014 buglog.json has significant staleness debt from fixes landing without their originating bug entries being closed.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "Catch-up WAL streaming silently DROPS entry payloads at scale: a rejoining controller (ctrl-1…\n\nCatch-up WAL streaming silently DROPS entry payloads at scale: a rejoining controller (ctrl-1 restarted) received [1..14] with correct LSN/type/CRC but PLEN=0 / empty job blobs (waldump PCRC=00000000, jobid=?). Structurally-intact but content-empty WAL = silent data corruption. THE committed-job-loss mechanism.", + "file": "src/lib/request.c (xce_wal_batch_encode_raw / wal_entry_to_wire / xce_decode extract loop) — capnp multi-segment Data pointers", + "root_cause": "Multi-segment capnp bug. xce_wal_batch_encode_raw builds a WalCatchupEntry struct-list where each element's payload is a capnp Data (List(UInt8)) allocated via capn_new_list8(cs=root_seg,...). When the message outgrows its first segment (large batch), the per-element payload Data pointer is lost across the segment boundary — on decode extract_data_bytes(we.payload) reads CAPN_NULL and returns len 0. write_WalCatchupEntry uses capn_setp (far-pointer-capable) so the fault is in the overflow/segment handling as used here. REPRODUCED deterministically: src/lib/testsuite/test_wal_wire.c test_wal_batch_roundtrip_large_payloads (14 entries x 872B) FAILS 'Expected 872 Was 0'; the 2-entry test_wal_batch_roundtrip_payload_bytes PASSES. Threshold is size, not count. Live replication (repl_broadcast) carries payloads fine — ONLY the catch-up/rejoin path corrupts, which is why loss only appears after a node restarts and catches up.", + "fix": "FIXED — CONFIRMED via un-ignoring its own purpose-built RED reproducer (strongest possible evidence). Removed the TEST_IGNORE_MESSAGE guard from test_wal_batch_roundtrip_large_payloads (src/lib/testsuite/test_wal_wire.c); rebuilt and ran it 2x under ASAN with leak detection ON — clean PASS both times, all 14 entries' 872-byte payloads byte-exact, zero leaks (285/285 lib tests, 0 failures, 0 ignored). Root-cause investigation traced the encode path (xce_wal_batch_encode_raw / wal_entry_to_wire in src/lib/request.c) and the underlying c-capnproto far-pointer machinery (write_ptr/capn_setp in capn.c) — the cross-segment struct->Data far-pointer path is correct by construction (new_object relocates and re-tags .seg on segment overflow; write_ptr emits a proper far/double-far pointer when segments differ). capn-malloc.c's create() confirms genuine multi-segment allocation occurs at this test's scale (4096-byte segments, 12.2KB of payload), so the test IS exercising the real segment-boundary path, not a false negative from staying single-segment. No production code changed — whatever regressed this (library version bump via the documented nix flake repin, or an incidental request.c change) already fixed it; the TEST_IGNORE was simply never removed. This is the 6th bug in this sweep found to be already-fixed-but-untracked (846/855/1096/1147/1198/847) — buglog.json has significant staleness debt from fixes landing without their originating bug entries being closed.", "tags": [ "c", "reproducer-unignored", @@ -8687,23 +1890,27 @@ "replication", "capnp", "critical", - "wal", - "fixed" + "wal" ], "related_bugs": [ "bug-828", "bug-795", "bug-661" ], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-847.md" }, { "id": "bug-848", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-20T14:40:02.211Z", - "error_message": "On a PRESERVED-DATA restart (no wipe \u2014 user confirmed), a controller recovers its WAL [1..7] from disk but rejoins advertising applied_lsn=0, so the primary re-streams the WHOLE log [1..committed] and O_TRUNC-overwrites the node's already-correct WAL. Logs: 'HA: learned committed_lsn=10 ... (was 0)'. THE root trigger that forces the full re-stream (then bug-847 empties it).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "On a PRESERVED-DATA restart (no wipe — user confirmed), a controller recovers its WAL [1..7] from…\n\nOn a PRESERVED-DATA restart (no wipe — user confirmed), a controller recovers its WAL [1..7] from disk but rejoins advertising applied_lsn=0, so the primary re-streams the WHOLE log [1..committed] and O_TRUNC-overwrites the node's already-correct WAL. Logs: 'HA: learned committed_lsn=10 ... (was 0)'. THE root trigger that forces the full re-stream (then bug-847 empties it).", "file": "src/controller/cluster.c (cluster_init_phase2, after persist_recover) + src/controller/ha.c (ha_init applied_lsn=0; state report sr->applied_lsn)", "root_cause": "ha->applied_lsn is NEVER reconstructed from the recovered WAL on boot. ha_init zeros it (ha.c:1367); persist_recover (cluster.c:1352) replays the WAL and advances persist->current_lsn/durable_lsn, but nothing copies that recovered position into ha->applied_lsn (persist_t and ha_state_t are separate; the only setter besides ha_init is the SNAPSHOT receive path ha.c:1771 = checkpoint_lsn). So the standby state report advertises sr->applied_lsn = ha->applied_lsn = 0 (ha.c:1787) and the primary's ha_evaluate_repl_action streams [0+1 .. committed]. ASYMMETRY proving it's a bug: ELECTION advertise uses ha_candidate_advertise_lsn = persist_durable_floor (the RECOVERED position, correct), but CATCH-UP advertise uses the un-recovered applied_lsn=0. The system knows its durable tip for voting but not for catch-up. Defeats Raft incremental catch-up for every restarted node; also the upstream cause of the earlier missing-prefix cases (full re-stream from a trimmed primary -> prefix gap).", "fix": "FIXED 2026-06-20 (HA-layer). Added ha_sync_applied_lsn_from_recovery(ha,p) (ha.c/ha.h) = monotonic store of persist_durable_floor(p) into ha->applied_lsn; wired into cluster_init_phase2 right after persist_recover, plus a diagnostic INFO log of recovered durable_floor/current_lsn/applied_lsn. TDD: test_recovered_member_advertises_real_applied_lsn (test_persist_recovery.c) write->recover->sync asserts applied_lsn==durable_floor>0 and monotonic. controller_unit_test 6/6 + adjacent HA 8/8, daemon builds. CAVEAT: live ctrl-1 also showed current_lsn=1 (seg O_TRUNC'd from LSN 1) implying persist_recover may not have loaded the WAL on that restart at all; if so this HA-layer fix is necessary but not sufficient. The new INFO log reveals on the next live restart whether persist_recover is effective (durable_floor>0) or not (bug-849 candidate: recovery not running/loading on the rejoin path).", @@ -8716,27 +1923,30 @@ "applied_lsn", "data-loss", "root-trigger", - "critical", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "critical" ], "related_bugs": [ "bug-847", "bug-828", "bug-795" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-848.md" }, { "id": "bug-849", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-20T14:42:09.400Z", - "error_message": "After bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL [15..16] only \u2014 missing [1..14] (waldump 'GAP: missing prefix LSN 1..14'). Payloads now full (bug-848 era), but the prefix is gone. Loss mode changed (empty-payload -> missing-prefix), loss remains.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "After bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL…\n\nAfter bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL [15..16] only — missing [1..14] (waldump 'GAP: missing prefix LSN 1..14'). Payloads now full (bug-848 era), but the prefix is gone. Loss mode changed (empty-payload -> missing-prefix), loss remains.", "file": "src/controller/myapp.c (standby-rejoin path ~317-320) + src/controller/cluster.c cluster_init_phase2 (holds persist_recover)", - "root_cause": "CONFIRMED from live logs (/tmp/xce-ha-deploy/logs/ctrl-1.log). persist_recover (and the bug-848 applied_lsn sync, and catch-up) run ONLY inside cluster_init_phase2, which is ELECTION-GATED \u2014 its real callers are all ha.c election/promotion paths (ha.c:1021,1166,2943,3022) + cluster_init. First boot ran it (ctrl-1 won an election; logged 'persist_recover done \u2014 durable_floor=0 current_lsn=1'). The 11:07:20 RESTART came up as a follower ('myapp_run: CTRL_STANDBY \u2014 waiting for primary to connect and push config') with NO election \u2192 cluster_init_phase2 NEVER ran \u2192 NO 'persist_recover done' line for pid 304897. So the rejoining follower never recovered its on-disk [1..7], operated with current_lsn=1/applied_lsn=0, then received LIVE replication of the new job [15..16] (repl_broadcast), O_TRUNC'd seg-001 and wrote [15..16] there. [1..7] never recovered + [1..14] never caught up = permanent prefix gap. Ordering corollary: live replication advances applied past the gap so the primary's catch-up never backfills it.", - "fix": "FIXED 2026-06-21 (design B \u2014 boot-path-universal recovery). Refined root cause: phase2 is invoked ONLY from the election-result path; a rejoining follower that SKIPS its election (goto skip_election in myapp_run because a primary already exists) never reaches it. Fix: in myapp.c the skip-election standby branch now calls cluster_init_phase2(&c) (build resources + persist_recover + bug-848 ha_sync_applied_lsn) when !ha.phase2_done, BEFORE cluster_run starts serving \u2014 so every boot path recovers its own WAL and advertises its real applied_lsn before accepting replication. Idempotent via phase2_done (same once-per-process contract as the election callers); recovery decoupled from the primary config push (a restart already loaded config from disk pre-election). Daemon builds; controller suite incl. test_persist_recovery/test_ha_startup/test_repl_catchup/test_ha_failover 8/8 green. LIVE VALIDATION PENDING: redeploy + kill/restart ctrl-1; expect a 'HA: persist_recover done \u2014 durable_floor=7 current_lsn=8' line for the restarted pid and ctrl-1 keeping [1..7] + catching up only the suffix.", + "root_cause": "CONFIRMED from live logs (/tmp/xce-ha-deploy/logs/ctrl-1.log). persist_recover (and the bug-848 applied_lsn sync, and catch-up) run ONLY inside cluster_init_phase2, which is ELECTION-GATED — its real callers are all ha.c election/promotion paths (ha.c:1021,1166,2943,3022) + cluster_init. First boot ran it (ctrl-1 won an election; logged 'persist_recover done — durable_floor=0 current_lsn=1'). The 11:07:20 RESTART came up as a follower ('myapp_run: CTRL_STANDBY — waiting for primary to connect and push config') with NO election → cluster_init_phase2 NEVER ran → NO 'persist_recover done' line for pid 304897. So the rejoining follower never recovered its on-disk [1..7], operated with current_lsn=1/applied_lsn=0, then received LIVE replication of the new job [15..16] (repl_broadcast), O_TRUNC'd seg-001 and wrote [15..16] there. [1..7] never recovered + [1..14] never caught up = permanent prefix gap. Ordering corollary: live replication advances applied past the gap so the primary's catch-up never backfills it.", + "fix": "FIXED 2026-06-21 (design B — boot-path-universal recovery). Refined root cause: phase2 is invoked ONLY from the election-result path; a rejoining follower that SKIPS its election (goto skip_election in myapp_run because a primary already exists) never reaches it. Fix: in myapp.c the skip-election standby branch now calls cluster_init_phase2(&c) (build resources + persist_recover + bug-848 ha_sync_applied_lsn) when !ha.phase2_done, BEFORE cluster_run starts serving — so every boot path recovers its own WAL and advertises its real applied_lsn before accepting replication. Idempotent via phase2_done (same once-per-process contract as the election callers); recovery decoupled from the primary config push (a restart already loaded config from disk pre-election). Daemon builds; controller suite incl. test_persist_recovery/test_ha_startup/test_repl_catchup/test_ha_failover 8/8 green. LIVE VALIDATION PENDING: redeploy + kill/restart ctrl-1; expect a 'HA: persist_recover done — durable_floor=7 current_lsn=8' line for the restarted pid and ctrl-1 keeping [1..7] + catching up only the suffix.", "tags": [ "wrong-value", "md", @@ -8747,9 +1957,7 @@ "election-gated", "catchup", "data-loss", - "critical", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "critical" ], "related_bugs": [ "bug-848", @@ -8757,18 +1965,23 @@ "bug-828", "bug-795" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-849.md" }, { "id": "bug-850", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-20T15:14:36.135Z", - "error_message": "A rejoining standby receives LIVE replication of new entries before its historical catch-up, advancing current_lsn past the gap so the idempotency guard skips the catch-up of the prefix. Explains ctrl-1 ending with [15..16] only (gap [1..14]). Distinct from bug-849; persists even after recovery.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "A rejoining standby receives LIVE replication of new entries before its historical catch-up…\n\nA rejoining standby receives LIVE replication of new entries before its historical catch-up, advancing current_lsn past the gap so the idempotency guard skips the catch-up of the prefix. Explains ctrl-1 ending with [15..16] only (gap [1..14]). Distinct from bug-849; persists even after recovery.", "file": "src/controller/repl.c repl_add_standby (line ~119) + repl_broadcast in_catch_up gate", "root_cause": "repl_add_standby initializes a freshly-registered standby with in_catch_up=0, so it is IMMEDIATELY eligible for repl_broadcast (which correctly skips in_catch_up standbys at repl.c:27). Mechanism (confirmed from ctrl-1.log 11:07:21): ctrl-1 reconnects -> registered in_catch_up=0 -> the new job [15..16] is live-broadcast to it (not gated) -> persist_wal_write_replicated writes them, current_lsn jumps to 17 -> THEN the catch-up stream of [1..14] arrives but persist_wal_write_replicated's idempotency guard (lsn+1 <= current_lsn) skips ALL of them (1+1<=17 ... 14+1<=17) -> permanent prefix gap [1..14]. Independent of bug-849: even with recovery (current_lsn=8), a live [15..16] arriving first jumps current_lsn to 17 and the catch-up of [8..14] is skipped (gap [8..14]). Root architectural cause: catch-up streaming and live broadcast are SEPARATE unordered paths (unlike Raft AppendEntries which is one ordered per-follower stream).", - "fix": "FIXED 2026-06-21 via a CONTIGUITY GUARD at the persist layer (Raft log-matching), NOT in_catch_up. First tried defaulting in_catch_up=1 in repl_add_standby \u2014 REVERTED: it broke the whole controller suite (commit-WAIT tests TIMEOUT 300s because a fresh standby no longer counts toward write quorum until convergence, and the unit tests don't run the heartbeat dispatch that would clear it). Correct fix: persist_wal_write_replicated now REJECTS a forward gap (lsn > current_lsn) with -EAGAIN instead of writing it \u2014 so a live broadcast that arrives ahead of the in-order catch-up is DROPPED (no write, no ack, no current_lsn jump), and the ACK-driven catch-up continuation re-streams from current_lsn so the log fills in order. ha_apply_single_entry + ha_apply_wal_catchup_entry treat -EAGAIN as a quiet skip (TRACE, no ack, no ERROR spam). Combined with bug-848 (advertise durable_floor == current_lsn-1) the catch-up always starts AT current_lsn so it is never spuriously rejected; only genuine forward gaps are. Files: src/controller/persist.c (guard), src/controller/ha.c (quiet -EAGAIN). TDD: test_repl_write_rejects_forward_gap (test_repl_apply.c) \u2014 contiguous write ok, forward-gap lsn 5 @ current_lsn 2 -> -EAGAIN + current_lsn unchanged, fill 2..4 then 5 ok, recovery clean. FULL controller suite green (129/0/0, no timeout); daemon builds. LIVE VALIDATION PENDING with bug-848+849+850 together.", + "fix": "FIXED 2026-06-21 via a CONTIGUITY GUARD at the persist layer (Raft log-matching), NOT in_catch_up. First tried defaulting in_catch_up=1 in repl_add_standby — REVERTED: it broke the whole controller suite (commit-WAIT tests TIMEOUT 300s because a fresh standby no longer counts toward write quorum until convergence, and the unit tests don't run the heartbeat dispatch that would clear it). Correct fix: persist_wal_write_replicated now REJECTS a forward gap (lsn > current_lsn) with -EAGAIN instead of writing it — so a live broadcast that arrives ahead of the in-order catch-up is DROPPED (no write, no ack, no current_lsn jump), and the ACK-driven catch-up continuation re-streams from current_lsn so the log fills in order. ha_apply_single_entry + ha_apply_wal_catchup_entry treat -EAGAIN as a quiet skip (TRACE, no ack, no ERROR spam). Combined with bug-848 (advertise durable_floor == current_lsn-1) the catch-up always starts AT current_lsn so it is never spuriously rejected; only genuine forward gaps are. Files: src/controller/persist.c (guard), src/controller/ha.c (quiet -EAGAIN). TDD: test_repl_write_rejects_forward_gap (test_repl_apply.c) — contiguous write ok, forward-gap lsn 5 @ current_lsn 2 -> -EAGAIN + current_lsn unchanged, fill 2..4 then 5 ok, recovery clean. FULL controller suite green (129/0/0, no timeout); daemon builds. LIVE VALIDATION PENDING with bug-848+849+850 together.", "tags": [ "wrong-value", "c", @@ -8780,9 +1993,7 @@ "race", "in_catch_up", "data-loss", - "critical", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "critical" ], "related_bugs": [ "bug-849", @@ -8791,120 +2002,22 @@ "bug-828", "bug-795" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-846", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T15:20:00.000Z", - "error_message": "After the primary dies, xce_test client ops (ctrlstatus/joblist/jobsub) fail PERSISTENTLY with 'Invalid argument' (EINVAL) and wait-primary times out, even though a new primary was elected (ctrl-2 term 3). Blocks failover testing from the client.", - "file": "src/lib/xceapi.c (per-op state guard) + src/lib/client_failover.c (xce_failover_classify / xcecli_retry_decide)", - "root_cause": "Client-side, NOT server. ha_dispatch_gate ALLOWS OPCode_ctrlstatus in every role and rejects client writes with the RETRYABLE -XCE_ERR_NOT_READY (never EINVAL), so a reachable controller would answer. The EINVAL is the xceapi per-op guard: `if (ctx->state != XCE_CTX_RUN) return -EINVAL`. The failover loop only AUTO-RECOVERS ctx.state==XCE_CTX_ERR (xcecli_retry_decide rotates+recovers); xce_failover_classify maps EINVAL -> XCE_FO_TERMINAL ('everything else -> terminal'), so a bare EINVAL is surfaced and the sticky master cursor never advances to the live new primary. So once the ctx lands in a non-ERR non-RUN state (INIT/DOWN) \u2014 or yields a bare EINVAL while RUN \u2014 failover gives up permanently. Exact stuck state needs a live repro to confirm.", - "fix": "FIXED (verified 2026-07-02, no direct code trail to when \u2014 landed under libxce-failover). src/lib/client_failover.c xcecli_retry_decide() now checks `ac->ctx.state == XCE_CTX_ERR` BEFORE classifying rc, bypassing the -EINVAL misclassification entirely (comment there literally reads 'the reported bug'). xcecli_ctx_recover() resets ERR->RUN post-reconnect. This is exactly recommendation (B) from this bug's own original fix note.", - "tags": [ - "client", - "stale-report", - "einval", - "blocker", - "failover", - "ha", - "fixed", - "libxce" - ], - "related_bugs": [ - "bug-646" - ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-853", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-20T15:27:41.698Z", - "error_message": "Wrong reference: test_wal_batch_entry_without_payload should be test_wal_batch_roundtrip_large_payloads", - "file": "src/lib/testsuite/test_wal_wire.c", - "root_cause": "Used \"test_wal_batch_entry_without_payload\" instead of \"test_wal_batch_roundtrip_large_payloads\"", - "fix": "Changed test_wal_batch_entry_without_payload \u2192 test_wal_batch_roundtrip_large_payloads", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-861", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T00:20:55.974Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_persist_recovery.c", - "root_cause": "Had \"persist.h\"", - "fix": "Changed to \"ha.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-864", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T03:39:07.483Z", - "error_message": "Missing guard clause", - "file": "src/controller/repl.c", - "root_cause": "No early return/throw for edge case: r->standbys[i].ep == ep", - "fix": "Added guard clause: if (r->standbys[i].ep == ep)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-871", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T04:01:07.089Z", - "error_message": "Wrong reference: test_repl_write_rejects_checkpoint should be test_repl_write_rejects_forward_gap", - "file": "src/controller/testsuite/test_repl_apply.c", - "root_cause": "Used \"test_repl_write_rejects_checkpoint\" instead of \"test_repl_write_rejects_forward_gap\"", - "fix": "Changed test_repl_write_rejects_checkpoint \u2192 test_repl_write_rejects_forward_gap", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-850.md" }, { "id": "bug-851", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "data-loss", "timestamp": "2026-06-21T13:30:00.000Z", - "error_message": "CHAOTIC leadership churn (3x kill-the-primary) causes SILENT committed-prefix DIVERGENCE: all 3 nodes report committedLsn=54 with lsn range 1..54 / 0 gaps / CRC intact, but WAL CONTENT diverges at LSN 45-54 (ctrl-1/ctrl-2/ctrl-3 each hold DIFFERENT jobs at the same LSNs; fps bfeb26ad/216af434/865304ec). Violates Raft committed-prefix invariant (a committed entry must be byte-identical on every member). Originally seen as a liveness 'commit stall'; the deeper defect is divergence + acked-job loss under churn.", - "file": "src/controller/persist.c:588/726/1040 (hdr.term=p->term, p->term never set) + src/controller/ha.c (election term = local+1, PreVote no-persist; two leaders per term) + src/controller/ha_convergence.c (reconcile cannot (term,lsn) log-match)", - "root_cause": "ROOT-CAUSED 2026-06-21 (owned-cluster repro, see .kiro/specs/ha-commit-durability/repro-851/). THREE compounding defects: (1) Election term not monotonic across cluster: term = locally-persisted+1, PreVote does NOT persist/propagate ('PreVote round ... no persist'); under chaotic kills a leader wins term N, appends own-term noop+jobs, and is killed BEFORE term N propagates to a majority; a restarted node recovers a STALE term and elects at the SAME N+1 => TWO leaders share a term (logs: term3 led by ctrl-3@08:40 AND ctrl-1@08:47:39; term4 led by ctrl-2@08:47:52 AND ctrl-3@08:48:04), each appending its own-term noop at the SAME lsn 45. (2) WAL entry term field is DEAD (always 0): wal_entry_hdr_t.term exists (wal_types.h:30) but writer does hdr.term=p->term (persist.c:588/726/1040) and p->term is NEVER assigned from ha->term anywhere in the controller (zero matches) => every entry persists term=0; waldump confirms term=0 cluster-wide. (3) No (term,lsn) reconcile: with no usable per-entry term, ha_convergence.c reconcile cannot Raft-log-match, so a divergent same-lsn suffix (45@term3 vs 45@term4) looks already-present and is never truncated; ha_compute_safe_committed_lsn then advances committed 'from 1 granted voter' over content other members never received. NOTE: a CLEAN single failover fully converges (all WALs identical 1..22, 848/849/850 validated) \u2014 only CHAOTIC churn triggers it.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "CHAOTIC leadership churn (3x kill-the-primary) causes SILENT committed-prefix DIVERGENCE: all 3…\n\nCHAOTIC leadership churn (3x kill-the-primary) causes SILENT committed-prefix DIVERGENCE: all 3 nodes report committedLsn=54 with lsn range 1..54 / 0 gaps / CRC intact, but WAL CONTENT diverges at LSN 45-54 (ctrl-1/ctrl-2/ctrl-3 each hold DIFFERENT jobs at the same LSNs; fps bfeb26ad/216af434/865304ec). Violates Raft committed-prefix invariant (a committed entry must be byte-identical on every member). Originally seen as a liveness 'commit stall'; the deeper defect is divergence + acked-job loss under churn.", + "file": "src/controller/persist.c:588/726/1040 (hdr.term=p->term, p->term never set) + src/controller/ha.c (election term = local+1, PreVote no-persist; two leaders per term) + src/controller/ha_convergence.c (reconcile cannot (term, lsn) log-match)", + "root_cause": "ROOT-CAUSED 2026-06-21 (owned-cluster repro, see .kiro/specs/ha-commit-durability/repro-851/). THREE compounding defects: (1) Election term not monotonic across cluster: term = locally-persisted+1, PreVote does NOT persist/propagate ('PreVote round ... no persist'); under chaotic kills a leader wins term N, appends own-term noop+jobs, and is killed BEFORE term N propagates to a majority; a restarted node recovers a STALE term and elects at the SAME N+1 => TWO leaders share a term (logs: term3 led by ctrl-3@08:40 AND ctrl-1@08:47:39; term4 led by ctrl-2@08:47:52 AND ctrl-3@08:48:04), each appending its own-term noop at the SAME lsn 45. (2) WAL entry term field is DEAD (always 0): wal_entry_hdr_t.term exists (wal_types.h:30) but writer does hdr.term=p->term (persist.c:588/726/1040) and p->term is NEVER assigned from ha->term anywhere in the controller (zero matches) => every entry persists term=0; waldump confirms term=0 cluster-wide. (3) No (term,lsn) reconcile: with no usable per-entry term, ha_convergence.c reconcile cannot Raft-log-match, so a divergent same-lsn suffix (45@term3 vs 45@term4) looks already-present and is never truncated; ha_compute_safe_committed_lsn then advances committed 'from 1 granted voter' over content other members never received. NOTE: a CLEAN single failover fully converges (all WALs identical 1..22, 848/849/850 validated) — only CHAOTIC churn triggers it.", "fix": "FIXED AND VERIFIED LIVE -- three distinct contributing root causes were found and fixed, all owned by ha-term-safety (spec.json validate_impl decision: GO, 2026-07-17): (1) Term propagation -- repl_broadcast (persist.c:747, the live replicator) built its wire entry without setting entry.term, so live traffic wire-encoded term=0 even though the primary's own header had the real term (same root cause as bug-990, fixed together in task 3.4: entry.term=hdr->term now carried through the live send+intake path, term-0 legacy direct-apply retired). 175/175 unit tests, live-confirmed on real fs. (2) Election-Safety violation (bug-1095/bug-1096, the 'bug-851 family', found by the raft-sim-conformance oracle): raft_on_vote_reply in libxce's raft.c double-counted duplicate vote replies and never dropped stale-term replies, allowing two leaders at the same term -- fixed with a per-peer vote-reply dedup + stale-term drop, unit-tested at both the raft-sim and controller level. (3) Premature election-time truncation (bug-828, closely related): ha_enter_real_election was truncating the WAL to a stale last_known_committed_lsn before computing the safe commit point, destroying already-durable committed entries -- fixed (commit b77d412), live-verified: clean failover preserves committed jobs, walcompare committed-prefix identical across all 3 members across multiple runs. ha-term-safety's GO decision explicitly cites a fresh 6-round LIVE chaotic-multi-kill soak (bug-851's exact repro scenario) passing with zero committed divergence/job loss -- superseding an earlier in-task note (task 5.4) that had called this env-blocked before that later live run completed.\n\n[Original entry]\nNOT FIXED (HA-core + persistence => full spec workflow per CLAUDE.md, not a quick patch). Keystone lever: stamp hdr.term from live ha->term on the primary append path + carry term through the repl envelope + persist it, THEN make ha_convergence reconcile truncate any local suffix whose (term,lsn) != leader's (standard Raft log-matching). MUST pair with durably-establishing the election term before a new leader acts (no two leaders per term: persist/propagate the won term to a majority before appending the own-term noop, or bump term beyond any (term,lsn) a voter advertises). Route to ha-consistency-invariants / ha-convergence-recovery as a new sub-spec.", "tags": [ "ha", @@ -8916,8 +2029,7 @@ "wal-term", "committed-divergence", "data-loss", - "spec-needed", - "fixed" + "spec-needed" ], "related_bugs": [ "bug-850", @@ -8925,19 +2037,25 @@ "bug-848", "bug-646" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z", - "root_caused": "2026-06-21T09:10:00.000Z" + "fix_commits": [ + "b77d412" + ], + "record": ".wolf/bugs/bug-851.md" }, { "id": "bug-854", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-21T09:10:00.000Z", - "error_message": "Client primary-discovery is POSITIONAL, not failover-correct: with masters=[m0,m1,...] the client only finds the primary if m0 IS the primary. wait-primary: 9993(primary-first)=ok 45ms; 9992,9993(standby-first)=no primary; 9991,9993(dead-first)=no primary; 9993,9991=ok.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Client primary-discovery is POSITIONAL, not failover-correct: with masters=[m0,m1,...] the client…\n\nClient primary-discovery is POSITIONAL, not failover-correct: with masters=[m0,m1,...] the client only finds the primary if m0 IS the primary. wait-primary: 9993(primary-first)=ok 45ms; 9992,9993(standby-first)=no primary; 9991,9993(dead-first)=no primary; 9993,9991=ok.", "file": "src/lib/client.c:403 xcecli_ctrlstatus_primary + src/controller/ctrlstatus.c:104 (standby primaryName=\"\")", - "root_cause": "Two compounding defects in libxce: (1) XCECLI_FAILOVER_CALL classifies a standby's status-0 ctrlstatus reply as SUCCESS and never rotates the endpoint (ctrlstatus is answerable in every role, so ANY reachable standby answers 0 \u2014 the inner call never rotates off it). (2) xcecli_ctrlstatus_primary captures a redirect hint from the non-authoritative reply but does not force rotation to consume it \u2014 it re-queries the same standby until the retry budget exhausts, never traversing the master ring. No server-side primaryName source existed on a standby (voted_for_name is not the leader, and primary_host is vestigial/never-written). Verified: baseline (no fix) with standby-first masters does NOT find the primary (pins to standby).", - "fix": "FIXED (2026-06-23). xcecli_ctrlstatus_primary: force xce_failover_rotate(ctx) + xcecli_ctx_recover(ctx) under failover.lock on every non-authoritative reply (hint-first then next-master rotation). Loop floor raised to max(max_retries, nmasters+1). Verified: wait-primary 'ctrl-3' serving after 92 ms with standby-first masters list; end-to-end clean-failover discover pass. Deliberately NOT adding server-side primaryName source \u2014 it is only a latency optimization; the client rotation fix alone resolves the hang.", + "root_cause": "Two compounding defects in libxce: (1) XCECLI_FAILOVER_CALL classifies a standby's status-0 ctrlstatus reply as SUCCESS and never rotates the endpoint (ctrlstatus is answerable in every role, so ANY reachable standby answers 0 — the inner call never rotates off it). (2) xcecli_ctrlstatus_primary captures a redirect hint from the non-authoritative reply but does not force rotation to consume it — it re-queries the same standby until the retry budget exhausts, never traversing the master ring. No server-side primaryName source existed on a standby (voted_for_name is not the leader, and primary_host is vestigial/never-written). Verified: baseline (no fix) with standby-first masters does NOT find the primary (pins to standby).", + "fix": "FIXED (2026-06-23). xcecli_ctrlstatus_primary: force xce_failover_rotate(ctx) + xcecli_ctx_recover(ctx) under failover.lock on every non-authoritative reply (hint-first then next-master rotation). Loop floor raised to max(max_retries, nmasters+1). Verified: wait-primary 'ctrl-3' serving after 92 ms with standby-first masters list; end-to-end clean-failover discover pass. Deliberately NOT adding server-side primaryName source — it is only a latency optimization; the client rotation fix alone resolves the hang.", "tags": [ "libxce", "client", @@ -8948,44 +2066,59 @@ "FIXED", "client", "rotation", - "xcecli_ctrlstatus_primary", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "xcecli_ctrlstatus_primary" ], "related_bugs": [ "bug-846", "bug-851", "bug-646" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-854.md" }, { - "id": "bug-875", + "id": "bug-855", "status": "fixed", "status_updated": "2026-09-24", - "timestamp": "2026-06-21T10:04:37.876Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/requirements.md", - "root_cause": "Had `OPCode_replConfigPush`", - "fix": "Changed to `snapshot_transfer_s`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], + "status_note": "", + "component": "controller", + "severity": "unknown", + "timestamp": "2026-06-22T03:10:06.653962Z", + "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "error_message": "controller_unit_test: 6 test_wal_catchup_* tests FAIL (Expected ... Was -11/-EAGAIN or wrong…\n\ncontroller_unit_test: 6 test_wal_catchup_* tests FAIL (Expected ... Was -11/-EAGAIN or wrong applied_lsn) on the raft_tla checkpoint branch", + "file": "src/controller/testsuite/test_config_repl.c (tests) / src/controller/persist.c:705 (guard)", + "root_cause": "PRE-EXISTING on checkpoint 20e59d0, NOT caused by ha-term-safety 1.1/1.2. The bug-850 contiguity guard (persist.c:705 rejects lsn>current_lsn with -EAGAIN) is committed at HEAD, but the test_wal_catchup_* tests feed forward-gap LSNs (30, 42) into a FRESH persist (current_lsn~=1) and still expect them to apply -> guard returns -EAGAIN -> applied_lsn does not advance. ha-term-safety changes touch only vote/schema/typemap/stubs; git diff --stat shows ZERO changes to persist.c/config_repl.c/test_config_repl.c. Also a separate run TIMED OUT 300s (the bug-850 DNR /tmp fsync-EIO + commit-WAIT signature; environment artifact on overlayfs).", + "fix": "FIXED (verified 2026-07-02): ran controller_unit_test directly — all test_wal_catchup_* tests (append_term_fields_round_trip, entry_advances_applied_lsn, entry_error_skipped_no_crash, entry_null_entry_no_crash, batch_advances_applied_lsn, partial_batch, batch_error_entry_skipped, batch_zero_entries_fallback, batch_256_entries) PASS. Landed as part of ha-term-safety's WAL term-index work (owner this bug deferred to).", + "tags": [ + "controller-test", + "stale-report", + "ha-term-safety", + "wal-catchup", + "EAGAIN", + "not-a-regression", + "contiguity-guard", + "bug-850", + "pre-existing" + ], + "related_bugs": [ + "bug-850", + "bug-851" + ], + "fix_commits": [], + "record": ".wolf/bugs/bug-855.md" }, { "id": "bug-857", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-21T10:30:00.000Z", - "error_message": "Config upgrade can leave members with DIVERGENT materialized state: do_upgrade_config_with_bytes pushes config bytes only then restart_requested; each node re-derives state by replaying its WAL under the NEW config, with no snapshot pinning an identical post-upgrade state. Replay is config-dependent (lost_and_found re-homes orphaned jobs), so nodes can diverge; amplified by bug-851 WAL tail divergence.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Config upgrade can leave members with DIVERGENT materialized state: do_upgrade_config_with_bytes…\n\nConfig upgrade can leave members with DIVERGENT materialized state: do_upgrade_config_with_bytes pushes config bytes only then restart_requested; each node re-derives state by replaying its WAL under the NEW config, with no snapshot pinning an identical post-upgrade state. Replay is config-dependent (lost_and_found re-homes orphaned jobs), so nodes can diverge; amplified by bug-851 WAL tail divergence.", "file": "src/controller/config_repl.c do_upgrade_config_with_bytes (~1001-1163)", "root_cause": "Config is a separate consistency domain (committed_config_tag, no term, tag anti-entropy) outside both the WAL log and raft.c. Upgrade activation = config push + restart + per-node WAL replay under new config; no snapshot transfers an identical materialized state, and state is interpreted against config. Identified via TLA+ gap analysis (config should be log-entry/snapshot-anchored). NOT empirically reproduced yet.", "fix": "FIXED AND VERIFIED LIVE -- config-upgrade-snapshot (the spec this was descoped into) is fully task-complete: every task 1 through 8.9 is done, including task 8.5 ('Reproduce bug-857 on the pre-change behavior and confirm it no longer occurs') and task 8.6 (a live chaotic multi-kill soak, 2026-07-29, confirming zero materialized-state/config divergence after a config upgrade). The fix materializes a full entity-bearing state snapshot at the config-change marker (term,lsn) boundary and installs it on every member, exactly as originally planned. Note: config-upgrade-snapshot/spec.json's own `phase` field was left stale at 'tasks-generated' despite the spec being fully implemented -- corrected separately alongside this entry.\n\n[Original entry]\nNOT FIXED. Descoped 2026-06-21 from ha-term-safety R8 into the independent `config-upgrade-snapshot` spec (.kiro/specs/config-upgrade-snapshot/, requirements generated). Fix = upgrade materializes a FULL entity-bearing state snapshot at the config-change marker (term,lsn) boundary, embeds new config, installs on every member; no per-node replay-under-new-config. HARD dependency: entity-bearing-snapshots (snapshots are a 16-byte LSN sentinel today; COW xzarray_freeze/thaw built but dormant). Verify by reproducing bug-857 (upgrade during chaotic multi-kill, diff materialized state).", @@ -9000,122 +2133,26 @@ "entity-bearing-snapshots", "state-divergence", "raft-safety", - "upgrade", - "fixed" + "upgrade" ], "related_bugs": [ "bug-851", "bug-828" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-878", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T10:34:30.593Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-879", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T10:59:29.212Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/TLA-gap-analysis.md", - "root_cause": "Had `raft.c`", - "fix": "Changed to 's split architecture, the chosen direction is to ", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-880", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T12:56:07.740Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/requirements.md", - "root_cause": "Had `snapshot_transfer_s`", - "fix": "Changed to `(term, lsn)`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-881", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T13:00:46.210Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/design.md", - "root_cause": "Had `xce_test`", - "fix": "Changed to `OPCode_replConfigPush`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-883", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T13:05:44.284Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/research.md", - "root_cause": "Had `config_repl.c:1857-1944`", - "fix": "Changed to `xzarray_freeze`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-857.md" }, { "id": "bug-884", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-21T15:20:00Z", - "error_message": "test_persistence_constants:FAIL Expected TRUE Was FALSE (DEFAULT_SYNC_TIMEOUT_MS==50)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_persistence_constants:FAIL Expected TRUE Was FALSE (DEFAULT_SYNC_TIMEOUT_MS==50)\n\ntest_persistence_constants:FAIL Expected TRUE Was FALSE (DEFAULT_SYNC_TIMEOUT_MS==50)", "file": "src/lib/testsuite/test_config.c", "root_cause": "commit ca1d9ae raised DEFAULT_SYNC_TIMEOUT_MS 50->500 in config.h but did not update the test assertion; pre-existing red on HEAD, unrelated to raft", "fix": "synced assertion to ==500 (commit 7b6cbb8)", @@ -9123,468 +2160,28 @@ "test", "config", "stale-assertion", - "ha-commit-engine", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha-commit-engine" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-885", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T15:21:59.689Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: index < 1 || index > RAFT_SIM_MAX_LOG", - "fix": "Added guard clause: if (index < 1 || index > RAFT_SIM_MAX_LOG)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "7b6cbb8" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-886", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T15:41:42.176Z", - "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_oversize_peers_returns_einval", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_oversize_peers_returns_einval\"", - "fix": "Changed test_raft_init_zero_peers_returns_einval \u2192 test_raft_init_oversize_peers_returns_einval", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-887", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T15:45:29.785Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->role != RAFT_LEADER", - "fix": "Added guard clause: if (s->role != RAFT_LEADER)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-888", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T15:52:45.272Z", - "error_message": "Missing guard clause", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "No early return/throw for edge case: index >= 1 && index <= RAFT_SIM_MAX_LOG", - "fix": "Added guard clause: if (index >= 1 && index <= RAFT_SIM_MAX_LOG)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-889", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T15:53:14.276Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->cbs->on_trace", - "fix": "Added guard clause: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-890", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T15:59:59.295Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: rc != 0", - "fix": "Added guard clause: if (rc != 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-891", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T16:06:37.923Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->cbs->on_trace", - "fix": "Added guard clause: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-893", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T16:16:47.908Z", - "error_message": "Wrong reference: g_te_spy should be g_pe_spy", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"g_te_spy\" instead of \"g_pe_spy\"", - "fix": "Changed g_te_spy \u2192 g_pe_spy", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-894", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T16:20:39.535Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->cbs->on_trace", - "fix": "Added guard clause: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-895", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T16:29:35.778Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: c->nodes[src].partition || c->nodes[dst].partition", - "fix": "Added guard clause: if (c->nodes[src].partition || c->nodes[dst])", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-899", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T16:42:30.241Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: node_idx >= 0 && node_idx < c->n", - "fix": "Added guard clause: if (node_idx >= 0 && node_idx < c->n)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-900", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T16:51:25.960Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: RUN_TEST(test_sim_9_1_replication_and_partition_catch_up);; RUN_TEST(test_sim_9_2_divergent_suffix_truncation);", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-903", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T17:04:17.830Z", - "error_message": "Wrong operator: <= should be >=", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"<=\" instead of \">=\"", - "fix": "Changed operator <= \u2192 >=", - "tags": [ - "auto-detected", - "operator-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-904", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T21:44:26.804Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: node_idx < 0 || node_idx >= c->n", - "fix": "Added guard clause: if (node_idx < 0 || node_idx >= c->n)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-905", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T21:49:10.796Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-log-replication/tasks.md", - "root_cause": "Had `initial_log_len`", - "fix": "Changed to `raft_sim_restart_node`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-907", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T23:14:37.837Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->role != RAFT_LEADER", - "fix": "Added guard clause: if (s->role != RAFT_LEADER)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-908", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T23:14:55.547Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: node_idx >= 0 && node_idx < c->n", - "fix": "Added guard clause: if (node_idx >= 0 && node_idx < c->n)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-909", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T23:30:04.636Z", - "error_message": "Missing guard clause", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "No early return/throw for edge case: g_mock.get_entry_term_log_len != 0 &&\n index == g_moc", - "fix": "Added guard clause: if (g_mock.get_entry_term_log_len != 0 &&\n )", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-911", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-21T23:32:38.225Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft.c", - "root_cause": "Condition was: if (s->cbs->on_vote_eligible != NULL)", - "fix": "Changed to: if (!raft_log_ok(s, cand_last_log_term, cand_last_log_)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-913", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T00:19:56.768Z", - "error_message": "Wrong reference: TEST_ASSERT_NOT_EQUAL_INT should be TEST_ASSERT_NOT_EQUAL", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"TEST_ASSERT_NOT_EQUAL_INT\" instead of \"TEST_ASSERT_NOT_EQUAL\"", - "fix": "Changed TEST_ASSERT_NOT_EQUAL_INT \u2192 TEST_ASSERT_NOT_EQUAL", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-914", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T01:24:34.541Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/spec.json", - "root_cause": "Had \"design-generated\"", - "fix": "Changed to \"tasks-generated\"", - "tags": [ - "auto-detected", - "wrong-value", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-855", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T03:10:06.653962Z", - "error_message": "controller_unit_test: 6 test_wal_catchup_* tests FAIL (Expected ... Was -11/-EAGAIN or wrong applied_lsn) on the raft_tla checkpoint branch", - "file": "src/controller/testsuite/test_config_repl.c (tests) / src/controller/persist.c:705 (guard)", - "root_cause": "PRE-EXISTING on checkpoint 20e59d0, NOT caused by ha-term-safety 1.1/1.2. The bug-850 contiguity guard (persist.c:705 rejects lsn>current_lsn with -EAGAIN) is committed at HEAD, but the test_wal_catchup_* tests feed forward-gap LSNs (30, 42) into a FRESH persist (current_lsn~=1) and still expect them to apply -> guard returns -EAGAIN -> applied_lsn does not advance. ha-term-safety changes touch only vote/schema/typemap/stubs; git diff --stat shows ZERO changes to persist.c/config_repl.c/test_config_repl.c. Also a separate run TIMED OUT 300s (the bug-850 DNR /tmp fsync-EIO + commit-WAIT signature; environment artifact on overlayfs).", - "fix": "FIXED (verified 2026-07-02): ran controller_unit_test directly \u2014 all test_wal_catchup_* tests (append_term_fields_round_trip, entry_advances_applied_lsn, entry_error_skipped_no_crash, entry_null_entry_no_crash, batch_advances_applied_lsn, partial_batch, batch_error_entry_skipped, batch_zero_entries_fallback, batch_256_entries) PASS. Landed as part of ha-term-safety's WAL term-index work (owner this bug deferred to).", - "tags": [ - "controller-test", - "stale-report", - "ha-term-safety", - "wal-catchup", - "EAGAIN", - "not-a-regression", - "contiguity-guard", - "fixed", - "bug-850", - "pre-existing" - ], - "related_bugs": [ - "bug-850", - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-884.md" }, { "id": "bug-918", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-06-22T03:30:00Z", - "error_message": "nix develop fails: hash mismatch for c-capnproto fixed-output derivation (expected sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM=, got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk=)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "nix develop fails: hash mismatch for c-capnproto fixed-output derivation (expected…\n\nnix develop fails: hash mismatch for c-capnproto fixed-output derivation (expected sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM=, got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk=)", "file": "src/flake.nix", "root_cause": "flake pins c-capnproto to the MUTABLE github archive/main.tar.gz; upstream main branch advanced so the tarball content/hash drifted, breaking the fixed-output derivation. Only pre-cached nix stores still build.", - "fix": "RESOLVED: user refreshed src/flake.lock; fresh nix develop builds (capnproto 1.2.0). A clean controller rebuild made the full controller test suite green (Ok:1 Fail:0, 12.7s) \u2014 the earlier wal_catchup -EAGAIN/decode failures were a STALE incremental-build artifact (schema compiled against the drifted toolchain), not fsync or a source regression. RECURRENCE 2026-07-23 (config worktree, /xce-build controller build+report task for config-upgrade-snapshot Task 1 RED-phase capture): identical hash mismatch reproduced verbatim (same specified sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM= vs got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk= for capnpc-1.0.1.drv source.drv) on FIRST invocation and again on an identical immediate RETRY (deterministic, not transient -- unlike the unrelated ninja-race bugs, retrying does not help here). Build never reached compilation; no xce_controller, controller_unit_test, or xce_test binaries produced; controller.capnp.c/h were not regenerated. This worktree (config, at .../worktrees/config) still has the pre-fix flake.lock/pin state -- the 2026-06-22 fix (refreshing src/flake.lock) was apparently not carried into this worktree, or upstream c-capnproto main.tar.gz drifted again since. NOT fixed this run (build+report only, no source/flake edits authorized by the task). Next step: refresh src/flake.lock in this worktree (nix flake update / lock the xbuild input) the same way the 2026-06-22 fix did, then retry the controller build. RECURRENCE 3 (2026-07-23, config worktree, config-upgrade-snapshot Task 1 implementer session): identical hash mismatch reproduced again AFTER a prior subagent had already logged occurrence 2 (build+report task, no fix attempted). This session went further: (1) confirmed `nix flake lock --update-input xbuild` in src/ is a NO-OP -- xbuild is already pinned to the latest available shen390s/my-nix-channel@master commit (073f527e), so re-locking flake.lock does NOT fix this recurrence (the previous 2026-06-22 fix's mechanism -- refresh flake.lock -- no longer applies once xbuild is already current); (2) confirmed via direct `nix develop` in the CANONICAL non-worktree repo (/home/rshen/projects/xce/xce/src) that it fails IDENTICALLY (same two sha256 hashes, same capnpc-1.0.1.drv path) -- this is a live, machine-wide, upstream c-capnproto main.tar.gz drift, NOT a worktree-isolation problem, and is not fixable from within this repo at all (the broken pin lives inside the external shen390s/my-nix-channel flake, at whatever commit is currently latest); (3) WORKAROUND discovered and used successfully: this machine's /nix/store already has REALIZED (fully built, not just .drv) output paths for capnpc-1.0.1 and capnproto-1.2.0 from a build that completed before the drift -- `find /nix/store -maxdepth 1 -iname '*capnp*'` surfaces them; invoking their bin/capnp and bin/capnpc-c binaries DIRECTLY (bypassing `nix develop`'s full devShell evaluation entirely) still works and successfully compiled a controller.capnp schema change end-to-end, generating correct bindings. This unblocks schema-level capnp verification even while the full `nix develop` shell is broken, but does NOT unblock a full project build/link -- controller_unit_test additionally needs `librtti` (fetched via a private SSH git flake input), which has NEVER been built/cached on this machine at all (no .drv or realized store path exists for it), so a full build+test run remains genuinely infeasible until either upstream fixes the c-capnproto pin or a machine with a pre-cached librtti+full toolchain is used.", + "fix": "RESOLVED: user refreshed src/flake.lock; fresh nix develop builds (capnproto 1.2.0). A clean controller rebuild made the full controller test suite green (Ok:1 Fail:0, 12.7s) — the earlier wal_catchup -EAGAIN/decode failures were a STALE incremental-build artifact (schema compiled against the drifted toolchain), not fsync or a source regression. RECURRENCE 2026-07-23 (config worktree, /xce-build controller build+report task for config-upgrade-snapshot Task 1 RED-phase capture): identical hash mismatch reproduced verbatim (same specified sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM= vs got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk= for capnpc-1.0.1.drv source.drv) on FIRST invocation and again on an identical immediate RETRY (deterministic, not transient -- unlike the unrelated ninja-race bugs, retrying does not help here). Build never reached compilation; no xce_controller, controller_unit_test, or xce_test binaries produced; controller.capnp.c/h were not regenerated. This worktree (config, at .../worktrees/config) still has the pre-fix flake.lock/pin state -- the 2026-06-22 fix (refreshing src/flake.lock) was apparently not carried into this worktree, or upstream c-capnproto main.tar.gz drifted again since. NOT fixed this run (build+report only, no source/flake edits authorized by the task). Next step: refresh src/flake.lock in this worktree (nix flake update / lock the xbuild input) the same way the 2026-06-22 fix did, then retry the controller build. RECURRENCE 3 (2026-07-23, config worktree, config-upgrade-snapshot Task 1 implementer session): identical hash mismatch reproduced again AFTER a prior subagent had already logged occurrence 2 (build+report task, no fix attempted). This session went further: (1) confirmed `nix flake lock --update-input xbuild` in src/ is a NO-OP -- xbuild is already pinned to the latest available shen390s/my-nix-channel@master commit (073f527e), so re-locking flake.lock does NOT fix this recurrence (the previous 2026-06-22 fix's mechanism -- refresh flake.lock -- no longer applies once xbuild is already current); (2) confirmed via direct `nix develop` in the CANONICAL non-worktree repo (/home/rshen/projects/xce/xce/src) that it fails IDENTICALLY (same two sha256 hashes, same capnpc-1.0.1.drv path) -- this is a live, machine-wide, upstream c-capnproto main.tar.gz drift, NOT a worktree-isolation problem, and is not fixable from within this repo at all (the broken pin lives inside the external shen390s/my-nix-channel flake, at whatever commit is currently latest); (3) WORKAROUND discovered and used successfully: this machine's /nix/store already has REALIZED (fully built, not just .drv) output paths for capnpc-1.0.1 and capnproto-1.2.0 from a build that completed before the drift -- `find /nix/store -maxdepth 1 -iname '*capnp*'` surfaces them; invoking their bin/capnp and bin/capnpc-c binaries DIRECTLY (bypassing `nix develop`'s full devShell evaluation entirely) still works and successfully compiled a controller.capnp schema change end-to-end, generating correct bindings. This unblocks schema-level capnp verification even while the full `nix develop` shell is broken, but does NOT unblock a full project build/link -- controller_unit_test additionally needs `librtti` (fetched via a private SSH git flake input), which has NEVER been built/cached on this machine at all (no .drv or realized store path exists for it), so a full build+test run remains genuinely infeasible until either upstream fixes the c-capnproto pin or a machine with a pre-cached librtti+full toolchain is used.", "tags": [ "nix", "flake", @@ -9592,100 +2189,23 @@ "env", "build", "worktree-isolation", - "worktree", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "worktree" ], "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z", - "recurrence_4_RESOLVED": "2026-07-24, config-upgrade-snapshot task 1 resumption session: RESOLVED (repo-local fix, no longer machine-wide-blocked as recurrence 3 concluded). Added a local override in src/nixSupport/xce.nix's capnpc derivation -- `src = pkgs.fetchFromGitHub { owner = \"shen390s\"; repo = \"c-capnproto\"; rev = \"e45ff682c4756ac5067cbac6d0f282a8ea5f0155\"; sha256 = \"sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk=\"; }` -- pinning capnpc's source to an IMMUTABLE commit (the exact content the drifted `main` tarball happened to resolve to, so the hash matches what was already realized in the local /nix/store from recurrence 3's workaround) instead of upstream xbuild's package.nix pinning the MUTABLE `main` ref. This makes the fix independent of upstream's moving target permanently, not just until the next drift. Verified end-to-end: `nix develop --command meson setup xce.build --buildtype=debug -Db_sanitize=address --reconfigure && ninja -C xce.build controller/controller_unit_test` built clean (librtti and all other deps resolved fine -- recurrence 3's librtti-never-cached concern did not reproduce, this machine already had it), and the full `controller_unit_test` binary ran to completion: 978 Tests 0 Failures 0 Ignored, OK. Do-not-repeat: when a fixed-output-derivation hash mismatch traces to an upstream flake pinning a MUTABLE branch/ref (not a tag/commit), the durable fix is a local override.overrideAttrs pinning `src` to an explicit immutable commit + matching sha256 in the CONSUMING repo's own nix file -- do not wait on upstream or keep re-locking flake.lock (recurrence 3 already proved that path is a dead end once xbuild is already at its latest available commit)." - }, - { - "id": "bug-920", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T03:58:08.538Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_term_safety_adapter.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-921", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T03:58:19.392Z", - "error_message": "Wrong reference: test_waldump should be test_term_safety_adapter", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_waldump\" instead of \"test_term_safety_adapter\"", - "fix": "Changed test_waldump \u2192 test_term_safety_adapter", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-922", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T03:59:43.099Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p || lsn == 0", - "fix": "Added guard clause: if (!p || lsn == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-923", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T04:01:37.467Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/persist.c", - "root_cause": "Condition was: if (hdr.lsn <= from_lsn)", - "fix": "Changed to: if (hdr.type < WAL_CHECKPOINT)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-918.md" }, { "id": "bug-924", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-06-22", - "error_message": "controller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11 EAGAIN) during ha-term-safety 1.3 validation", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "controller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11…\n\ncontroller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11 EAGAIN) during ha-term-safety 1.3 validation", "file": "environment (/ overlay filesystem)", "root_cause": "Disk filled to 100% (7466 leftover /tmp/xce_* WAL dirs from repeated ASAN test runs). make_fresh_persist's persist_init mkdir_p hit ENOSPC -> returned NULL -> cascade of 'Expected Non-NULL'. Also overlayfs returned fdatasync EIO under disk pressure. NOT a code regression: baseline full suite passes; test_config_repl catchup tests are order-dependent and fail in isolation on baseline too.\n\n[UPDATE 2026-09-01, worker-rejoin-resync task 7.2] Re-observed a FOURTH time, with a NEW manifestation shape not previously documented: 14367 leftover /tmp/xce_* dirs, 876.7G, filesystem at 99% used / 12.4G available. This time it did NOT surface as 'Expected Non-NULL' assertion cascades or a silent exit(1) -- it surfaced as controller_unit_test/`meson test` simply HANGING (TIMEOUT, killed by signal 15 after 300s) with no error output at all, reproducible across multiple independent invocations (both via `meson test` and the raw binary directly), always stalled inside persist_fini()/fsync_worker's pthread_cond_timedwait/pthread_cond_destroy teardown path (confirmed via gdb `thread apply all bt` on the wedged process -- Thread 2 stuck in fsync_worker's pthread_cond_timedwait at persist.c:847, Thread 1 stuck in persist_fini's pthread_cond_destroy at persist.c:1362), at a DIFFERENT test each time (not always the same call site) -- consistent with overlayfs fdatasync degrading under near-full disk rather than erroring outright, causing the flusher thread to sit in a slow/stuck wait instead of the previously-documented fast ENOSPC/EIO failure. `find /tmp -maxdepth 1 -name 'xce_*' -exec rm -rf {} +` (no live xce process holding any of them open, confirmed via `ps aux` first) freed 612G (99% -> 30% used); the exact same test run (`controller_unit_test -t test_worker_resync`) that had twice hung/timed out at 90s then completed cleanly in under a second. General lesson extended: an unexplained TIMEOUT/hang in this suite (not just an assertion-failure cascade) is an equally likely symptom of this same recurring /tmp exhaustion -- check `df -h /tmp` before spending time bisecting a hang as a suspected new concurrency bug in the code under test.", "fix": "rm -rf /tmp/xce_* freed 327GB; full controller suite then Ok:1 Fail:0. For future runs: clean /tmp/xce_* between heavy ASAN test cycles.\n\n[UPDATE 2026-08-26, xce-test-rc-file task 1.1] Re-observed at much larger scale (14731 leftover /tmp/xce_* dirs, 653.6G, filesystem at 100% used / 0 available) and a MORE SEVERE manifestation than the original 'Expected Non-NULL' assertion failures: mkdtemp() failing inside test_repl_apply.c's make_test_persist_for_apply()/make_test_persist_in() hit ENOSPC, and that pre-existing test file's own hard exit(1) on mkdtemp failure silently killed the ENTIRE controller_unit_test process mid-run (right after test_repl_unit's last test, well before test_repl_apply's own output ever printed) -- exit code 1, no crash/ASAN report, no visible error in the redirected log until grepped for the literal 'mkdtemp failed' stderr line. A leftover orphaned controller_unit_test process (PID from an earlier invocation, blocked in futex_do_wait, presumably stalled retrying a WAL fsync against the full disk) was also found and killed. Fixed for this session via `find /tmp -maxdepth 1 -name 'xce_*' -exec rm -rf {} +` (630G reclaimed, 100% -> 29% used) -- confirmed no live xce process had any of those dirs open before deleting. General lesson: when controller_unit_test dies silently/instantly with exit code 1 and the log just stops mid-file with no crash signature, check `df -h /tmp` and grep for 'mkdtemp failed'/'calloc failed' before assuming a real regression -- this sandbox's /tmp accumulates leftover unit-test mkdtemp() fixture dirs across EVERY session indefinitely (no test file in this suite cleans up its own /tmp/xce_* dirs), so this will keep recurring until either the testsuite adds its own cleanup or something periodically clears /tmp.\n\n[UPDATE 2026-08-30, bug-1735 verification build] Re-observed a THIRD time, at the largest scale yet: 16238 leftover /tmp/xce_* dirs, 696.1G, filesystem at 100% used / 0 available. This time it manifested as a `controller_unit_test` run reporting `1220 Tests 282 Failures` (mostly test_dispatch_commit.c/test_dispatch_fail_terminal.c/test_dispatch_fail_retry.c/test_jobdispatch_no_double_notify.c 'Expected Non-NULL' -- the exact same fixture-setup-ENOSPC cascade signature as both prior occurrences, on files wholly unrelated to whatever the run was actually verifying) plus a meson-internal `OSError: [Errno 28] No space left on device` while serializing testlog.json/testlog.junit.xml (corrupting those log artifacts, though the console summary line survived). No live xce process was running at the time (`ps aux` confirmed clean) so `find /tmp -maxdepth 1 -name 'xce_*' -exec rm -rf {} +` was safe; reclaimed 696.1G (100% -> 22% used). Re-running the full suite after cleanup is required before trusting any failure count from a run that coincided with disk exhaustion. Still true three occurrences in: nothing in this test suite cleans up its own /tmp/xce_* dirs, so this WILL keep recurring -- the durable fix is adding cleanup to the shared test fixture helpers, not another one-off `rm -rf`.", @@ -9699,488 +2219,23 @@ "ha-term-safety", "recurring", "hang", - "timeout", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "timeout" ], "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-925", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T04:42:28.897Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-926", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T04:45:07.018Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", - "fix": "Added guard clause: if (!ha || !ha->raft_c)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-928", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T04:55:58.789Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-929", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T04:57:03.605Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", - "fix": "Added guard clause: if (!ha || !ha->raft_c)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-930", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T04:57:25.141Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-931", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T05:45:09.373Z", - "error_message": "Function not marked async", - "file": "src/include/repl.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-933", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T05:47:50.037Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !d", - "fix": "Added guard clause: if (!d)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-937", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T06:39:51.351Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", - "fix": "Added guard clause: if (!ha || !ha->raft_c)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-941", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T07:53:06.457Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-942", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T07:54:52.956Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-943", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T07:55:30.714Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-944", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T07:56:08.623Z", - "error_message": "Wrong reference: ha_arm_own_term_gate should be ha_arm_own_term_gate_for_test", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Used \"ha_arm_own_term_gate\" instead of \"ha_arm_own_term_gate_for_test\"", - "fix": "Changed ha_arm_own_term_gate \u2192 ha_arm_own_term_gate_for_test", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-945", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T08:09:41.508Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-948", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T08:21:57.062Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat ; * library as a 0-entry AppendEntries so the LIBRARY performs", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-953", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T08:48:45.792Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: rc == 0 && out_lsn", - "fix": "Added guard clause: if (rc == 0 && out_lsn)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-957", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T09:08:41.177Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ep_is_vote", - "fix": "Added guard clause: if (ep_is_vote)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-960", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T09:20:00.000Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "ha-term-safety", - "raft", - "peer-index", - "match_index", - "oob", - "false-quorum", - "task-3.3", - "remediation", - "c", - "auto-detected", - "guard-clause", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [ - "bug-851" - ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-959", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T09:31:11.227Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !sha256 || !buf || !out_len", - "fix": "Added guard clause: if (!sha256 || !buf || !out_len)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-961", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T09:45:31.779Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !c || !sha256", - "fix": "Added guard clause: if (!c || !sha256)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-962", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T10:04:06.960Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: committed == 0", - "fix": "Added guard clause: if (committed == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-964", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T10:05:22.007Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: acked_out", - "fix": "Added guard clause: if (acked_out)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-966", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T10:10:11.668Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-967", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T10:10:17.811Z", - "error_message": "Wrong reference: persist_roll_segment should be persist_test_force_new_segment", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Used \"persist_roll_segment\" instead of \"persist_test_force_new_segment\"", - "fix": "Changed persist_roll_segment \u2192 persist_test_force_new_segment", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-924.md" }, { "id": "bug-970", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-22T10:24:35.905518Z", - "error_message": "ha-term-safety task 4.2 rejected (review r1): live config-change path (do_upgrade_config_with_bytes) never appended WAL_CONFIG_MARKER \u2014 config_committed_marker had ZERO production callers (dead primitive); and checkpoint/compaction could trim a committed marker segment, regressing the derive to -ENOENT.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ha-term-safety task 4.2 rejected (review r1): live config-change path…\n\nha-term-safety task 4.2 rejected (review r1): live config-change path (do_upgrade_config_with_bytes) never appended WAL_CONFIG_MARKER — config_committed_marker had ZERO production callers (dead primitive); and checkpoint/compaction could trim a committed marker segment, regressing the derive to -ENOENT.", "file": "src/controller/config_repl.c, src/controller/persist.c", "root_cause": "Finding1: do_upgrade_config_with_bytes still drove the term-blind committed_config_tag push/ack/advance and never called config_append_marker, so the derive primitive was dead code. Finding2: persist_checkpoint->trim_wal_segments unlinks older segments and the snapshot sentinel carried no config reference, so a committed marker in a trimmed segment vanished and config_committed_marker regressed/returned -ENOENT.", "fix": "Added config_commit_change() wired into do_upgrade_config_with_bytes step 7: appends WAL_CONFIG_MARKER as the consensus record (config_append_marker), then delegates byte delivery + the existing majority barrier to push_config_to_standbys (mechanism unchanged); committed_config_tag is now only a cache, config_committed_marker (committed-prefix derive) is authoritative. Compaction safety: persist_checkpoint scans the committed prefix for the latest marker {version_tag,sha256} (scan_latest_committed_config_ref) BEFORE trim and stamps it into the snapshot sentinel (41-byte optional tail); load_snapshot reads it back into persist.cfgref_*; config_committed_marker falls back to it when no marker survives the live WAL. Added integration test (drives config_commit_change) + checkpoint-trim test + persist_test_force_new_segment seam. cb5 upgrade pins made persist-backed so the marker append succeeds. RED confirmed (Expected 1 Was 0; Expected 0 Was -110); GREEN 0 failures direct-run.", @@ -10192,85 +2247,28 @@ "checkpoint", "compaction", "review-remediation", - "controller", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "controller" ], "related_bugs": [ "bug-851" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-971", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T10:39:44.072Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: !mc", - "fix": "Added guard clause: if (!mc)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-972", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T10:40:31.131Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-975", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T11:00:42.661Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha", - "fix": "Added guard clause: if (!ha)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-970.md" }, { "id": "bug-976", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-22T11:15:00.000Z", - "error_message": "ha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path (handle_repl_config_push pre-activated via load_primary_config before the hash gate); recovery-activate had NO production call site (boot loaded blindly); serve-fence had NO production reader (a fenced member still served)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path…\n\nha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path (handle_repl_config_push pre-activated via load_primary_config before the hash gate); recovery-activate had NO production call site (boot loaded blindly); serve-fence had NO production reader (a fenced member still served)", "file": "src/controller/config_repl.c, src/controller/ha.c, src/controller/myapp.c", "root_cause": "config_activate_committed (verify-before-activate + fence) was unit-correct but not integrated: (A) handle_repl_config_push called load_primary_config UNCONDITIONALLY before config_activate_committed, so mismatched bytes went live before the gate; (B) myapp.c boot step 7 + on_commit_advanced never called the consumer, so a recovering member loaded config blind against the committed marker; (C) ha_dispatch_gate / pre-promotion gate never read config_serve_fence, so a fenced member still served + could promote", - "fix": "(A) handle_repl_config_push now SAVES bytes (conf_save_config) then calls config_activate_committed ONLY \u2014 removed premature load_primary_config; activate is the sole activation point (hash-verify gate). (B) config_activate_committed -ENOENT branch falls back to load_primary_config (absent marker != mismatch, no false fence); myapp.c step 7 calls config_activate_committed (latest committed marker, verified). on_commit_advanced documents why activation is deferred off the iomgr thread (async-I/O invariant: scan/load do disk I/O). (C) ha_dispatch_gate CTRL_PRIMARY now refuses config-dependent state-changing ops when config_serve_fenced (replication/read-only pass), pre-promotion gate in raft_cb_become_leader refuses promotion when fenced via new ha_config_promotion_fenced; member stays ALIVE, never self-terminates; fence clears on correct bytes re-verify", + "fix": "(A) handle_repl_config_push now SAVES bytes (conf_save_config) then calls config_activate_committed ONLY — removed premature load_primary_config; activate is the sole activation point (hash-verify gate). (B) config_activate_committed -ENOENT branch falls back to load_primary_config (absent marker != mismatch, no false fence); myapp.c step 7 calls config_activate_committed (latest committed marker, verified). on_commit_advanced documents why activation is deferred off the iomgr thread (async-I/O invariant: scan/load do disk I/O). (C) ha_dispatch_gate CTRL_PRIMARY now refuses config-dependent state-changing ops when config_serve_fenced (replication/read-only pass), pre-promotion gate in raft_cb_become_leader refuses promotion when fenced via new ha_config_promotion_fenced; member stays ALIVE, never self-terminates; fence clears on correct bytes re-verify", "tags": [ "ha-term-safety", "config-replication", @@ -10280,62 +2278,26 @@ "promotion-gate", "safety", "task-4.3", - "remediation", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "remediation" ], "related_bugs": [ "bug-851", "bug-924" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-977", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T11:21:09.742Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-980", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T11:32:35.966Z", - "error_message": "Wrong reference: ClusterConfig should be cluster_config_t", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Used \"ClusterConfig\" instead of \"cluster_config_t\"", - "fix": "Changed ClusterConfig \u2192 cluster_config_t", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-976.md" }, { "id": "bug-981", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-22T11:35:00Z", - "error_message": "ha-term-safety 4.3 Finding D: on a standby, config MARKER (WAL) and config BYTES (out-of-band push) arrive on independent transports; on a 2nd+ upgrade the NEW bytes are saved before the NEW marker enters the committed prefix, so config_activate_committed hash-mismatches the OLDER committed marker and sets the serve-fence. When the NEW marker later commits, nothing re-runs activation (on_commit_advanced does no disk I/O on the iomgr ACK thread; primary's config-behind re-push net does not fire because conf_save_config already advanced the advertised config_version_tag) -> standby PERMANENTLY serve-fenced + non-promotable until reboot. Liveness violation of R7.9.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ha-term-safety 4.3 Finding D: on a standby, config MARKER (WAL) and config BYTES (out-of-band…\n\nha-term-safety 4.3 Finding D: on a standby, config MARKER (WAL) and config BYTES (out-of-band push) arrive on independent transports; on a 2nd+ upgrade the NEW bytes are saved before the NEW marker enters the committed prefix, so config_activate_committed hash-mismatches the OLDER committed marker and sets the serve-fence. When the NEW marker later commits, nothing re-runs activation (on_commit_advanced does no disk I/O on the iomgr ACK thread; primary's config-behind re-push net does not fire because conf_save_config already advanced the advertised config_version_tag) -> standby PERMANENTLY serve-fenced + non-promotable until reboot. Liveness violation of R7.9.", "file": "src/controller/config_repl.c, src/controller/ha.c", "root_cause": "No post-commit retry trigger for a config serve-fence set under the push-before-commit ordering. config_committed_marker returns the older committed marker until the new marker is in the committed prefix; the two activation call sites (boot, handle_repl_config_push) both run BEFORE the new marker commits, and ha_cb_commit_advanced_stub deliberately did nothing (async-I/O invariant forbids disk reads on the iomgr ACK thread).", "fix": "Added config_schedule_activate_retry(c) (non-blocking ordq_put via delay_task_ms, no-op unless serve-fenced AND delaymgr running) and config_activate_retry_fn(arg) (delaymgr task body that re-runs config_activate_committed off the iomgr thread). ha_cb_commit_advanced_stub now schedules the retry whenever config_serve_fenced(c) is set after commit advances. Once the NEW marker is in the committed prefix and the saved bytes hash-verify, the retry clears the fence and activates; -ENOENT still falls back without fencing; no busy-spin (no-op once unfenced). RED captured by stubbing config_activate_retry_fn to a no-op (fence stayed set, NEW config not loaded); GREEN with the real off-thread retry.", @@ -10348,120 +2310,23 @@ "delaymgr", "commit-advance", "push-before-commit", - "R7.9", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "R7.9" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-983", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T11:53:36.582Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: commit_index", - "fix": "Added guard clause: if (commit_index)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-984", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T12:03:15.865Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-985", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T12:06:43.530Z", - "error_message": "Wrong return value", - "file": "src/controller/testsuite/waldump.c", - "root_cause": "Was returning: -EINVAL;", - "fix": "Now returns: waldump_path_visit(path, out, stats, NULL, NULL);", - "tags": [ - "auto-detected", - "return-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-986", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T12:08:51.529Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test.c", - "root_cause": "Had \"waldump.h\"", - "fix": "Changed to \"walcompare.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-987", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T12:09:28.306Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test.c", - "root_cause": "No early return/throw for edge case: st.n_gaps > 0 || st.crc_broken", - "fix": "Added guard clause: if (st.n_gaps > 0 || st.crc_broken)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-981.md" }, { "id": "bug-990", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-22T13:24:07.476480", - "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term (e.g. primary LSN1..5 term=1, ctrl-2/ctrl-3 term=0); a clean failover's committed prefix is byte-identical in payload (pcrc) but NOT in term, so R5.1 'byte-identical WAL including terms' cannot pass. walcompare passes only via legacy-0 -> WAL_LEGACY_TERM tolerance, which masks it on a fresh cluster.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the…\n\nha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term (e.g. primary LSN1..5 term=1, ctrl-2/ctrl-3 term=0); a clean failover's committed prefix is byte-identical in payload (pcrc) but NOT in term, so R5.1 'byte-identical WAL including terms' cannot pass. walcompare passes only via legacy-0 -> WAL_LEGACY_TERM tolerance, which masks it on a fresh cluster.", "file": "src/controller/repl.c (live send repl_broadcast persist.c:747) + src/controller/ha.c (ha_apply_wal_catchup_entry term!=0 legacy direct-apply)", "root_cause": "The live replication SEND path was deferred from tasks 3.2/3.3 and still uses the legacy term-0 repl_broadcast; the standby applies via the legacy direct-apply path, NOT through raft_on_append_entries_request, so it never receives/persists the leader's term. The raft library send loop (on_send_append_entries / raft_send_append_entries) has ZERO live callers. on_persist_entry stamps the real term on the PRIMARY only.", "fix": "FIXED AND VERIFIED LIVE -- root-caused and fixed together with bug-851 in ha-term-safety task 3.4: repl_broadcast (persist.c:747, the live replicator) built its wire entry without setting entry.term, so standbys persisted term=0 for every live-replicated entry even though the primary's own WAL header carried the real term. Fix carries entry.term=hdr->term/prev_log_term through the live send+intake path (repl_broadcast, persist_wal_async, ha_stream_wal_to_standby, ha_apply_wal_catchup_entry all updated), retiring the term-0 legacy direct-apply branch. 175/175 unit tests, reviewer-approved. Live-confirmed on a real fs ($HOME/tmp/xce-ha-deploy, ext4, 2026-06-22): 'ctrl-2/ctrl-3 persisting the leader's REAL term... walcompare OK over the committed prefix.'\n\n[Original entry]\nNOT FIXED (out of task 5.2 testsuite-only boundary). Remaining integration work: retire the term-0 repl_broadcast live SEND and drive raft_send_append_entries/on_send_append_entries on the live primary so standbys persist the leader's term via raft_on_append_entries_request. Built clean_failover_preservation.sh (XCE_FAILOVER_STRICT_TERM=1) as the ready-to-run assertion this retirement must satisfy.", @@ -10473,21 +2338,56 @@ "repl_broadcast", "committed-prefix-divergence", "bug-851", - "BLOCKED", - "fixed" + "BLOCKED" ], "related_bugs": [ "bug-851" ], + "fix_commits": [], + "record": ".wolf/bugs/bug-990.md" + }, + { + "id": "bug-990-fix", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", + "timestamp": "2026-06-22T16:00:00.000000", + "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the…\n\nha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term; clean-failover committed prefix not byte-identical in TERM (R5.1 term-identity gap). Original bug-990.", + "file": "src/controller/repl.c (repl_broadcast), src/controller/persist.c (persist_wal_async call site + entry_term_locked), src/controller/config_repl.c (ha_stream_wal_to_standby catch-up), src/controller/ha.c (ha_apply_wal_catchup_entry receive route), src/lib/request.c (wal_entry_to_wire per-entry prev tuple encode)", + "root_cause": "The LIVE replication SEND paths (repl_broadcast live append + ha_stream_wal_to_standby catch-up) built wal_catchup_entry_t via memset(0) and never set entry.term/prev_log_index/prev_log_term, so standbys received term=0 entries and applied them via the LEGACY term-0 direct-apply branch (persist_wal_write_replicated stamps the STANDBY's local p->term, not the leader's). The term-carrying library receive route (raft_on_append_entries_request, task 3.2) only fired for term!=0 entries, which the live send never produced. Additionally wal_entry_to_wire only encoded per-entry term, not the per-entry prev tuple, so multi-entry catch-up batches could not be routed through the library one-at-a-time.", + "fix": "Task 3.4 minimal cutover (chosen over driving raft_send_append_entries live, which would rip out the repl model and risk 848/849/850): (1) repl_broadcast gains a prev_log_term param and sets entry.term=hdr->term, entry.prev_log_index=hdr->lsn-1, entry.prev_log_term (sourced under p->lock via new entry_term_locked accessor in persist_wal_async). (2) ha_stream_wal_to_standby sets each catch-up entry's term + prev tuple (tracking the prior streamed entry's lsn/term in ascending order). (3) wal_entry_to_wire now encodes prevLogIndex/prevLogTerm/leaderCommit PER SUB-ENTRY so a multi-entry batch is library-routable. (4) ha_apply_wal_catchup_entry routes every term-carrying entry (single + each batch sub-entry) through new ha_apply_term_entry_via_library -> ha_consume_append_entries -> raft_on_append_entries_request -> on_persist_entry, which persist_set_term(leader_term) before persist_wal_write_replicated (bug-850 contiguity guard preserved beneath). The term-0 legacy direct-apply is retired for live/catch-up entries. ACK/commit path (3.3) unchanged.\n\nVerification:\nUnit RED->GREEN: test_broadcast_carries_leader_term_and_prev_tuple (repl: neuter term -> Expected 7 Was 0) + test_live_broadcast_receive_persists_leader_term (adapter: neuter receive route -> Expected 7 Was 3, i.e. standby persisted its LOCAL term). Full controller suite 175 Tests 0 Failures (clean /tmp direct binary) + meson Ok:1 Fail:0. Live STRICT_TERM gate (clean_failover_preservation.sh) NOT runnable in this overlayfs sandbox: baseline-confirmed no failover completes (split-vote no-promotion within 20s + fdatasync errno=5) — bug-924 env limitation, requires real-fs. The 7 test_wal_catchup_* (test_config_repl.c) + occasional durability FAILs on the direct binary are the SAME pre-existing bug-924 overlayfs fdatasync artifact (identical on baseline).", + "tags": [ + "ha-term-safety", + "task-3.4", + "live-send-retirement", + "term-0", + "repl_broadcast", + "bug-990", + "FIXED", + "catch-up-term" + ], + "related_bugs": [ + "bug-990", + "bug-851" + ], + "fix_commits": [], + "record": ".wolf/bugs/bug-990-fix.md" }, { "id": "bug-991", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-22T16:30:00.000Z", - "error_message": "Standby WAL catch-up advanced applied_lsn to lsn+1 (next-to-apply) instead of lsn (last-held). The standby then advertised one LSN too high; the leader's catch-up streamer streamed (lsn+1, committed] and SKIPPED the real next-needed entry lsn+1, wedging the library logOk gate forever (no WalAck -> match_index never reaches -> raft commit_index frozen -> committed_lsn pinned at 1). Catch-up-path-specific; live repl_broadcast (absolute-LSN, no skip) commits fine, so failure was nondeterministic.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Standby WAL catch-up advanced applied_lsn to lsn+1 (next-to-apply) instead of lsn (last-held)\n\nStandby WAL catch-up advanced applied_lsn to lsn+1 (next-to-apply) instead of lsn (last-held). The standby then advertised one LSN too high; the leader's catch-up streamer streamed (lsn+1, committed] and SKIPPED the real next-needed entry lsn+1, wedging the library logOk gate forever (no WalAck -> match_index never reaches -> raft commit_index frozen -> committed_lsn pinned at 1). Catch-up-path-specific; live repl_broadcast (absolute-LSN, no skip) commits fine, so failure was nondeterministic.", "file": "src/controller/ha.c (ha_apply_term_entry_via_library ~2115; batch ~2205; single-entry ~2225)", "root_cause": "applied_lsn carried TWO conflicting semantics. Recovery (=durable_floor), checkpoint (=checkpoint_lsn), init (=0) and the leader's catch-up streamer + ha_evaluate_repl_action all treat it as 'last contiguously-held LSN' and stream (standby_lsn, committed]. But the three live/catch-up apply setters advanced to entry->lsn + 1 ('next-to-apply'). After a standby applied catch-up entry N it advertised N+1, so the leader's next stream began at N+2 and skipped N+1 -> logOk reject -> no WalAck.", "fix": "fix-1: the three live-apply setters now advance applied_lsn to the LAST-HELD LSN (e->lsn / last_good_lsn / entry->lsn), not +1, so applied_lsn means 'last contiguously-held LSN' everywhere. Reader audit (all consistent with last-held, no +1 reader needed): state report (advertised LSN, ha.c:2461) now matches streamer contract; convergence reported_lsn>=committed_lsn (2536) correct; snapshot/stream boundary standby_lsn+1 routed through the library, so raft.log_len already == applied_lsn after fix-1; raft.log_len legitimately differs from applied_lsn on the legacy term-0 direct-apply and recovery paths, so re-sourcing would regress bug-848 incremental catch-up -> fix-1 alone delivers the value the streamer+logOk need. Unit RED->GREEN: test_catchup_applied_lsn_is_last_held_no_skip (Expected 1 Was 2 pre-fix, PASS post-fix). 7 legacy test_wal_catchup_* expected values updated lsn+1 -> last-held. NOTE: the LIVE clean_failover_preservation STRICT_TERM commit=5 target was NOT reachable in the sandbox: committed_lsn stalls at 1 IDENTICALLY with this fix stashed (baseline experiment) -> that stall is a SEPARATE upstream blocker (library raft.log_len / commit-advance not advancing past the own-term noop; ha-term-safety task 3.1 area), NOT the bug-991 applied_lsn skip. waldump confirms all 3 members hold LSN 1..5 byte-identical (term 1) -> the replication half (which this fix corrects) is sound.", @@ -10500,9 +2400,7 @@ "off-by-one", "term-safety", "bug-991", - "streamer-skip", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "streamer-skip" ], "related_bugs": [ "bug-851", @@ -10511,223 +2409,20 @@ "bug-924", "bug-960" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-993", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T15:10:39.717Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p || lsn == 0", - "fix": "Added guard clause: if (!p || lsn == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-998", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T15:27:10.703Z", - "error_message": "Wrong reference: test_repl_ack_currentterm_quorum_advances_commit should be test_live_broadcast_receive_persists_leader_term", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Used \"test_repl_ack_currentterm_quorum_advances_commit\" instead of \"test_live_broadcast_receive_persists_leader_term\"", - "fix": "Changed test_repl_ack_currentterm_quorum_advances_commit \u2192 test_live_broadcast_receive_persists_leader_term", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-990-fix", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T16:00:00.000000", - "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term; clean-failover committed prefix not byte-identical in TERM (R5.1 term-identity gap). Original bug-990.", - "file": "src/controller/repl.c (repl_broadcast), src/controller/persist.c (persist_wal_async call site + entry_term_locked), src/controller/config_repl.c (ha_stream_wal_to_standby catch-up), src/controller/ha.c (ha_apply_wal_catchup_entry receive route), src/lib/request.c (wal_entry_to_wire per-entry prev tuple encode)", - "root_cause": "The LIVE replication SEND paths (repl_broadcast live append + ha_stream_wal_to_standby catch-up) built wal_catchup_entry_t via memset(0) and never set entry.term/prev_log_index/prev_log_term, so standbys received term=0 entries and applied them via the LEGACY term-0 direct-apply branch (persist_wal_write_replicated stamps the STANDBY's local p->term, not the leader's). The term-carrying library receive route (raft_on_append_entries_request, task 3.2) only fired for term!=0 entries, which the live send never produced. Additionally wal_entry_to_wire only encoded per-entry term, not the per-entry prev tuple, so multi-entry catch-up batches could not be routed through the library one-at-a-time.", - "fix": "Task 3.4 minimal cutover (chosen over driving raft_send_append_entries live, which would rip out the repl model and risk 848/849/850): (1) repl_broadcast gains a prev_log_term param and sets entry.term=hdr->term, entry.prev_log_index=hdr->lsn-1, entry.prev_log_term (sourced under p->lock via new entry_term_locked accessor in persist_wal_async). (2) ha_stream_wal_to_standby sets each catch-up entry's term + prev tuple (tracking the prior streamed entry's lsn/term in ascending order). (3) wal_entry_to_wire now encodes prevLogIndex/prevLogTerm/leaderCommit PER SUB-ENTRY so a multi-entry batch is library-routable. (4) ha_apply_wal_catchup_entry routes every term-carrying entry (single + each batch sub-entry) through new ha_apply_term_entry_via_library -> ha_consume_append_entries -> raft_on_append_entries_request -> on_persist_entry, which persist_set_term(leader_term) before persist_wal_write_replicated (bug-850 contiguity guard preserved beneath). The term-0 legacy direct-apply is retired for live/catch-up entries. ACK/commit path (3.3) unchanged.", - "tags": [ - "ha-term-safety", - "task-3.4", - "live-send-retirement", - "term-0", - "repl_broadcast", - "bug-990", - "FIXED", - "catch-up-term", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [ - "bug-990", - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "validation": "Unit RED->GREEN: test_broadcast_carries_leader_term_and_prev_tuple (repl: neuter term -> Expected 7 Was 0) + test_live_broadcast_receive_persists_leader_term (adapter: neuter receive route -> Expected 7 Was 3, i.e. standby persisted its LOCAL term). Full controller suite 175 Tests 0 Failures (clean /tmp direct binary) + meson Ok:1 Fail:0. Live STRICT_TERM gate (clean_failover_preservation.sh) NOT runnable in this overlayfs sandbox: baseline-confirmed no failover completes (split-vote no-promotion within 20s + fdatasync errno=5) \u2014 bug-924 env limitation, requires real-fs. The 7 test_wal_catchup_* (test_config_repl.c) + occasional durability FAILs on the direct binary are the SAME pre-existing bug-924 overlayfs fdatasync artifact (identical on baseline)." - }, - { - "id": "bug-1000", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T15:44:38.466Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1001", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T15:44:54.113Z", - "error_message": "Missing error handling in errno", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - _Blocked (live portion only \u2014 build+unit gate CONFIRMED GR", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1002", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T21:10:31.345Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Had 's term and standbys persist it, proven at the int", - "fix": "Changed to 's REAL term (2) for the committed prefix (pre-3.4", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1003", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T22:09:23.064Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft.c", - "root_cause": "Condition was: if (agreed == 0)", - "fix": "Changed to: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1005", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T22:31:03.315Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1006", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T22:32:53.722Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Advance applied_lsn to last_good_lsn after the full batch; * last contiguously-held LSN, NOT last_good_lsn + 1 \u2014 see th", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1009", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-22T22:44:50.365Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-991.md" }, { "id": "bug-1013", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-22T00:00:00Z", - "error_message": "Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes \u2014 STRICT_TERM live failover gate fails 'committed_lsn>=5'", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes…\n\nStable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes — STRICT_TERM live failover gate fails 'committed_lsn>=5'", "file": "src/controller/commit.c", "root_cause": "ha-term-safety task 2.4 added a min(durable-majority, raft_commit_index) ceiling in majority_candidate_locked. raft_commit_index is fed only via the library ACK path (ha_raft_on_repl_ack -> raft_advance_commit -> on_commit_advanced), which advances from match_index[] driven by the asynchronous catch-up streamer and routinely LAGS the durable majority. When delivery goes via catch-up, raft_commit_index stays low and PINS committed_lsn below the safe durable watermark (a redundant, starvation-prone gate). The durable-majority barrier was already current-term-safe under single-writer + the own-term NOOP writable gate, making the ceiling redundant.", "fix": "Option A: removed the raft_commit_index fold from majority_candidate_locked (commit.c). The durable-majority barrier + own-term writable_lsn gate are now authoritative for commit-advance. commit_set_raft_commit_index / ce->raft_commit_index made INERT-ADVISORY (recorded for observability/conformance-oracle only; still fed by on_commit_advanced so the library half keeps a sink). Kept durable self-cap (persist_durable_lsn), persist-before-ACK, and assert(cand<=persist_durable_lsn). Verified live: commit advances 4->5 via durable barrier (commit.c:184); walcompare OK committed prefix byte-identical across 3 nodes up to LSN 5.", @@ -10740,62 +2435,30 @@ "commit-ceiling", "optA", "liveness", - "catch-up", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "catch-up" ], "related_bugs": [ "bug-991", "bug-661", "bug-851" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-828-part2", - "status": "fixed", - "status_updated": "2026-09-24", - "status_note": "b77d4122; upstream gap fixed as bug-661 (f139e1c0)", - "timestamp": "2026-06-23T03:05:00.000Z", - "error_message": "Clean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2 granted voters)' and walcompare reports committed-prefix DIVERGENCE at LSN 2; jobs at LSN 2..5 lost.", - "file": "src/controller/testsuite/test_ha_vote_gate.c (test added); ha.c/persist.c advertise path investigated (no change needed)", - "root_cause": "INVESTIGATED, NOT an advertisement-staleness bug. The advertise path already reflects the live durable tip: ha_candidate_advertise_lsn -> persist_durable_floor, and the voter floor = max(persist_durable_floor, learned_committed); persist_durable_floor == persist_durable_lsn (both read p->durable_lsn, the fsync watermark), and part-1's secondary fix advances durable_lsn after the standby's inline fdatasync in persist_wal_write_replicated. LIVE evidence (fix828c_r1..r3, STRICT_TERM clean failover x3) proves the surviving voters advertise 1 because they are GENUINELY durable only at LSN 1: their WAL physically holds 1..2 (LSN2 = their own term-3 NOOP), recovered tip=2, durable_floor=1. The original primary ctrl-1 (down at election time, WAL 1..5 tip=5) advanced committed_lsn 1->2->3->4->5 in ~250ms while standbys had only converged applied_lsn=1 and catch-up streaming was still in flight. So committed advanced past what the surviving majority durably held \u2014 the UPSTREAM replication / commit-barrier durability gap (bug-795/661), not the vote advertisement. safe_committed_lsn=1 is the CORRECT, SAFE Raft answer given the real durable state; advertising the true durable tip cannot conjure data never replicated.", - "fix": "No production code change required for part-2 \u2014 the durable-tip advertisement is already correct & safe after part-1. Added unit test test_live_durable_tip_advertised_over_lagging_learned (test_ha_vote_gate.c): a standby durable at LSN5 (real persist, flush -> durable_lsn=5) with learned_committed lagging at 1 must advertise 5 as BOTH candidate self-LSN (ha_candidate_advertise_lsn) and voter floor (ha_vote_self_freshness over persist_durable_floor, dominating learned=1), and ha_compute_safe_committed_lsn from two such granting voters must return 5 \u2014 locking in the in-scope contract. Explicitly did NOT fold learned_committed into the CANDIDATE advertise (the requested max(...,learned_committed)) because that re-opens the truncated-candidate-over-advertises-and-wins hole ha-commit-durability task 4.2 deliberately closed (candidate must advertise only a DURABLE position; learned_committed is volatile/not necessarily durable on this node). The residual live job loss is bug-795/661, out of scope.", - "tags": [ - "safety", - "failover", - "safe-committed-lsn", - "durable-tip", - "advertise", - "leader-completeness", - "bug-795", - "bug-661", - "commit-durability", - "INVESTIGATION", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [ - "bug-828", - "bug-795", - "bug-661", - "bug-851", - "bug-924" - ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1013.md" }, { "id": "bug-1023", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-23T03:45:00Z", - "error_message": "Clean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite a standby being genuinely durable at LSN 5; walcompare reports committed-prefix divergence at LSN 2 (jobs gone on the new leader).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Clean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite…\n\nClean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite a standby being genuinely durable at LSN 5; walcompare reports committed-prefix divergence at LSN 2 (jobs gone on the new leader).", "file": "src/controller/ha.c", - "root_cause": "ha_enter_real_election() PRE-TRUNCATED the candidate's WAL above ha->last_known_committed_lsn BEFORE the vote. last_known_committed_lsn is the LOCALLY-LEARNED commit watermark, which LAGS the true cluster commit point (the primary commits an entry on a durable majority, then dies before a heartbeat carries that committed LSN to the standby). persist_truncate_wal_above clamps durable_lsn down to the cutoff (clamp_durable_lsn_locked). So a standby durably holding committed entries [committed+1..tip] discarded them, then advertised the stale-low cutoff as its durable floor (ha_candidate_advertise_lsn / ha_vote_self_freshness both read persist_durable_floor). Every voter doing the same collapsed safe-committed-from-votes to the stale watermark; the AUTHORITATIVE post-election truncate (raft_cb_become_leader -> persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale prefix, losing committed jobs. Live trace (DBG828) PROVED the standbys wrote+fdatasync'd LSN 2-5 (durable_lsn=5) and only LOST them at the pre-election truncate clamp \u2014 NOT a durability/ack-before-fsync violation. Final bug-828/795 slice.", - "fix": "Removed the pre-election persist_truncate_wal_above(last_known_committed_lsn) in ha_enter_real_election (src/controller/ha.c). Raft model: a candidate does NOT truncate its log before campaigning \u2014 it keeps its full durable log and advertises its real durable tip so voters holding more committed entries deny a colder candidate and the winner has every committed entry (Leader Completeness). Divergent UNCOMMITTED tails are removed only by the authoritative post-election safe-commit truncate (above the majority-held point) and by the library's per-entry (term,lsn) reconcile (on_truncate_from) during AppendEntries. RED: live STRICT_TERM harness safe=1 + walcompare divergence at LSN 2. GREEN: every run now logs safe_committed_lsn=5 (from 2 granted voters) and walcompare 'committed prefixes identical'. Unit: new test_repl_catchup_durable_floor_survives_failover (test_repl_apply.c) + 177/177 unit GREEN. NOTE: live EXIT still 1 due to a PRE-EXISTING libxce client-failover bug (bug: 'client op fails to reach live primary when a DEAD controller is FIRST in the masters list') surfacing now that the durability bug no longer masks the post-walcompare expect-job/wait-primary step \u2014 committed jobs are physically present on disk (walcompare proves it).", + "root_cause": "ha_enter_real_election() PRE-TRUNCATED the candidate's WAL above ha->last_known_committed_lsn BEFORE the vote. last_known_committed_lsn is the LOCALLY-LEARNED commit watermark, which LAGS the true cluster commit point (the primary commits an entry on a durable majority, then dies before a heartbeat carries that committed LSN to the standby). persist_truncate_wal_above clamps durable_lsn down to the cutoff (clamp_durable_lsn_locked). So a standby durably holding committed entries [committed+1..tip] discarded them, then advertised the stale-low cutoff as its durable floor (ha_candidate_advertise_lsn / ha_vote_self_freshness both read persist_durable_floor). Every voter doing the same collapsed safe-committed-from-votes to the stale watermark; the AUTHORITATIVE post-election truncate (raft_cb_become_leader -> persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale prefix, losing committed jobs. Live trace (DBG828) PROVED the standbys wrote+fdatasync'd LSN 2-5 (durable_lsn=5) and only LOST them at the pre-election truncate clamp — NOT a durability/ack-before-fsync violation. Final bug-828/795 slice.", + "fix": "Removed the pre-election persist_truncate_wal_above(last_known_committed_lsn) in ha_enter_real_election (src/controller/ha.c). Raft model: a candidate does NOT truncate its log before campaigning — it keeps its full durable log and advertises its real durable tip so voters holding more committed entries deny a colder candidate and the winner has every committed entry (Leader Completeness). Divergent UNCOMMITTED tails are removed only by the authoritative post-election safe-commit truncate (above the majority-held point) and by the library's per-entry (term,lsn) reconcile (on_truncate_from) during AppendEntries. RED: live STRICT_TERM harness safe=1 + walcompare divergence at LSN 2. GREEN: every run now logs safe_committed_lsn=5 (from 2 granted voters) and walcompare 'committed prefixes identical'. Unit: new test_repl_catchup_durable_floor_survives_failover (test_repl_apply.c) + 177/177 unit GREEN. NOTE: live EXIT still 1 due to a PRE-EXISTING libxce client-failover bug (bug: 'client op fails to reach live primary when a DEAD controller is FIRST in the masters list') surfacing now that the durability bug no longer masks the post-walcompare expect-job/wait-primary step — committed jobs are physically present on disk (walcompare proves it).", "tags": [ "ha", "raft", @@ -10806,9 +2469,7 @@ "durability", "truncate", "election", - "durable_lsn", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "durable_lsn" ], "related_bugs": [ "bug-661", @@ -10818,37 +2479,23 @@ "bug-854", "bug-339" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1025", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T04:09:25.647Z", - "error_message": "Wrong reference: $MASTERS should be $MASTERS_LIVE", - "file": "src/controller/testsuite/clean_failover_preservation.sh", - "root_cause": "Used \"$MASTERS\" instead of \"$MASTERS_LIVE\"", - "fix": "Changed $MASTERS \u2192 $MASTERS_LIVE", - "tags": [ - "wrong-reference", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1023.md" }, { "id": "bug-1028", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-23", - "error_message": "Post-failover AVAILABILITY: after a clean failover the new primary never becomes available to serve clients (wait-primary times out). New primary logs 'ha_dispatch_repl_action_for_standby: standby[0] lsn=5 < committed_lsn=6 -> stream WAL batches' every ~3s indefinitely; the lagging standby never advances to the new primary's own-term tip, and the client cannot reach a serving primary. Surfaced only AFTER bug-828 was fixed (the committed-job-loss previously masked this by failing the harness earlier at walcompare).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Post-failover AVAILABILITY: after a clean failover the new primary never becomes available to…\n\nPost-failover AVAILABILITY: after a clean failover the new primary never becomes available to serve clients (wait-primary times out). New primary logs 'ha_dispatch_repl_action_for_standby: standby[0] lsn=5 < committed_lsn=6 -> stream WAL batches' every ~3s indefinitely; the lagging standby never advances to the new primary's own-term tip, and the client cannot reach a serving primary. Surfaced only AFTER bug-828 was fixed (the committed-job-loss previously masked this by failing the harness earlier at walcompare).", "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby / catch-up streamer on the NEW term), config_repl.c (ha_stream_wal_to_standby), persist.c (standby apply) -- NOT data loss; committed prefix is byte-identical on disk (walcompare OK).", - "root_cause": "ha_stream_wal_to_standby (config_repl.c) stamped each streamed entry's term and the AppendEntries prev_log_term from raw on-disk hdr.term (the WAL header), but the standby's logOk (raft_on_append_entries_request) and the library's own send path both source terms from persist_entry_term (the authoritative in-memory per-LSN term index), which maps a legacy-0 entry to WAL_LEGACY_TERM(1). So for a legacy-0 predecessor entry (raw hdr.term==0), the streamer sent prev_log_term=0 while the standby computed its own term as 1 \u2192 logOk mismatch \u2192 the new-term NOOP (the post-promotion first entry) is perpetually rejected \u2192 no WalAck \u2192 applied_lsn stuck at pre-promotion value \u2192 the heartbeat state report advertises the stale lsn \u2192 the primary re-streams [lsn+1..committed] every heartbeat cycle. The committed prefix stays byte-identical because only term metadata diverges, not data. CONFIRMED: baseline walcompare shows committed-prefix DIVERGENCE (term mismatch \u2192 CRC-chain breaks on some members); with the fix the committed prefix is IDENTICAL.", - "fix": "FIXED (2026-06-23). ha_stream_wal_to_standby gained a const persist_t* p parameter (decl in config_repl.h). All three hdr.term source sites in the streamer body (entry->term, prev_streamed_term on the main path, prev_streamed_term on the OOM-skip path) replaced with p ? persist_entry_term(p, hdr.lsn) : hdr.term (NULL-p path for count-only unit tests). Callers (ha_dispatch_repl_action_for_standby in ha.c, catchup_stream_range in config_repl.c) pass &c->persist. Unit test (test_wal_stream_term_from_index_not_raw_hdr) legacy-appends LSN 1 at term 0 + LSN 2 at term 3; streams standby_lsn=1,committed=2; asserts streamed entry term==3 and prev_log_term==WAL_LEGACY_TERM(1). RED\u2192GREEN proven (Expected 1 Was 0 \u2192 PASS). Live: standby converge + committed prefix identical.", + "root_cause": "ha_stream_wal_to_standby (config_repl.c) stamped each streamed entry's term and the AppendEntries prev_log_term from raw on-disk hdr.term (the WAL header), but the standby's logOk (raft_on_append_entries_request) and the library's own send path both source terms from persist_entry_term (the authoritative in-memory per-LSN term index), which maps a legacy-0 entry to WAL_LEGACY_TERM(1). So for a legacy-0 predecessor entry (raw hdr.term==0), the streamer sent prev_log_term=0 while the standby computed its own term as 1 → logOk mismatch → the new-term NOOP (the post-promotion first entry) is perpetually rejected → no WalAck → applied_lsn stuck at pre-promotion value → the heartbeat state report advertises the stale lsn → the primary re-streams [lsn+1..committed] every heartbeat cycle. The committed prefix stays byte-identical because only term metadata diverges, not data. CONFIRMED: baseline walcompare shows committed-prefix DIVERGENCE (term mismatch → CRC-chain breaks on some members); with the fix the committed prefix is IDENTICAL.", + "fix": "FIXED (2026-06-23). ha_stream_wal_to_standby gained a const persist_t* p parameter (decl in config_repl.h). All three hdr.term source sites in the streamer body (entry->term, prev_streamed_term on the main path, prev_streamed_term on the OOM-skip path) replaced with p ? persist_entry_term(p, hdr.lsn) : hdr.term (NULL-p path for count-only unit tests). Callers (ha_dispatch_repl_action_for_standby in ha.c, catchup_stream_range in config_repl.c) pass &c->persist. Unit test (test_wal_stream_term_from_index_not_raw_hdr) legacy-appends LSN 1 at term 0 + LSN 2 at term 3; streams standby_lsn=1,committed=2; asserts streamed entry term==3 and prev_log_term==WAL_LEGACY_TERM(1). RED→GREEN proven (Expected 1 Was 0 → PASS). Live: standby converge + committed prefix identical.", "tags": [ "failover", "availability", @@ -10862,9 +2509,7 @@ "catch-up", "ha_stream_wal_to_standby", "persist_entry_term", - "logOk", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "logOk" ], "related_bugs": [ "bug-828", @@ -10872,195 +2517,23 @@ "bug-795", "bug-991" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1033", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T08:05:43.916Z", - "error_message": "Incorrect value in code", - "file": "repro854.sh", - "root_cause": "Had 'set masters %s\\nwait-primary 6000\\nctrlstatus\\nqu", - "fix": "Changed to 'set masters %s\\nset myname failover-test\\nclient\\", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1035", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T09:53:37.812Z", - "error_message": "Incorrect value in code", - "file": "src/lib/xcectx.c", - "root_cause": "Had \"xcectx.h\"", - "fix": "Changed to \"controller.capnp.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1036", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T10:44:48.074Z", - "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", - "file": "src/lib/netio.c", - "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", - "fix": "Changed handle_ssl_handshake \u2192 endpoint_ssl_handshake", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1037", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T10:48:18.493Z", - "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", - "file": "src/controller/cluster.c", - "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", - "fix": "Changed handle_ssl_handshake \u2192 endpoint_ssl_handshake", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1040", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T10:54:14.342Z", - "error_message": "Wrong reference: endpoint_ssl_handshake should be handle_ssl_handshake", - "file": "src/lib/netio.c", - "root_cause": "Used \"endpoint_ssl_handshake\" instead of \"handle_ssl_handshake\"", - "fix": "Changed endpoint_ssl_handshake \u2192 handle_ssl_handshake", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1045", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T14:08:05.466Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: # \u2500\u2500 Steps 7\u201310 looped: kill primary, verify, submit, restar; #", - "tags": [ - "auto-detected", - "error-handling", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1049", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T14:39:01.881Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (0 && recovered_tip > ha->raft.log_len)", - "fix": "Changed to: if (recovered_tip > ha->raft.log_len)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1051", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T14:48:18.998Z", - "error_message": "Wrong operator: >= should be ==", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Used \">=\" instead of \"==\"", - "fix": "Changed operator >= \u2192 ==", - "tags": [ - "auto-detected", - "operator-fix", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1054", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-23T16:02:42.489Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/SKILL.md", - "root_cause": "Had \"End-to-end HA failover soak test for an XCE contr", - "fix": "Changed to \"End-to-end HA failover soak test for an XCE contr", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1028.md" }, { "id": "bug-1060", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-06-24T12:05:00.000Z", - "error_message": "After soak test coverage maneuver, ctrl-1 and ctrl-2 enter permanent prevote deadlock: ctrl-1 denies ctrl-2 (freshness: LSN=45 < floor=47), ctrl-2 denies ctrl-1 (log_stale: cand_llt=12 < local_last_log_term=13). Neither can ever win.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "After soak test coverage maneuver, ctrl-1 and ctrl-2 enter permanent prevote deadlock: ctrl-1…\n\nAfter soak test coverage maneuver, ctrl-1 and ctrl-2 enter permanent prevote deadlock: ctrl-1 denies ctrl-2 (freshness: LSN=45 < floor=47), ctrl-2 denies ctrl-1 (log_stale: cand_llt=12 < local_last_log_term=13). Neither can ever win.", "file": "src/controller/ha.c (raft_cb_become_leader / safe_committed_lsn), src/controller/cluster.c (OPCode_preVoteRequest)", - "root_cause": "ctrl-3 computed safe_committed_lsn=44 from voter peerLastLogLsn (ctrl-3=47, ctrl-2=44 \u2192 majority at 44) and truncated committed entries 45-47 (term=12, acked by ctrl-3 itself as standby under ctrl-1). ctrl-3 then wrote leadership NOP at LSN=45 (term=13) and replicated it to ctrl-2. Result: ctrl-1 has LSN=45=JOB_SUBMIT/term=12, ctrl-2 has LSN=45=NOOP/term=13 \u2014 divergent WALs. The safe_committed_lsn formula ignores the candidate's own durable knowledge of committed entries, violating Leader Completeness: the new leader must never truncate entries it has that were committed by a prior leader.", - "fix": "IMPLEMENTED (2026-06-24): added pure helper ha_promotion_truncate_floor(safe_lsn, durable_tip)=max(safe_lsn,durable_tip) in ha.c/ha.h; raft_cb_become_leader now truncates at max(safe_committed_lsn, persist_durable_floor(&c->persist)) instead of bare safe_committed_lsn. Since the cutoff is never below the leader's own durable tip, persist_truncate_wal_above is a no-op on the leader's durable log \u2014 the new leader NEVER truncates entries it holds (Raft \u00a75.4 Leader Completeness). safe_committed_lsn is still the commit lower-bound and promotion-gate HWM (NOT clamped \u2014 clamping it would defeat the behind-node fence). The leadership NOP now appends at the true tip (current_lsn) so replication converges peers to the leader's WAL with zero job loss. Unit test test_promotion_never_truncates_own_committed_entries in test_ha_election.c (GREEN). Steering src/controller/CLAUDE.md updated (Leader Completeness section + Post-Election Safe Commit Point + verification checklist). Recovery of an already-diverged cluster still requires restarting the over-truncated node (loses the cut jobs); this fix prevents the divergence.", + "root_cause": "ctrl-3 computed safe_committed_lsn=44 from voter peerLastLogLsn (ctrl-3=47, ctrl-2=44 → majority at 44) and truncated committed entries 45-47 (term=12, acked by ctrl-3 itself as standby under ctrl-1). ctrl-3 then wrote leadership NOP at LSN=45 (term=13) and replicated it to ctrl-2. Result: ctrl-1 has LSN=45=JOB_SUBMIT/term=12, ctrl-2 has LSN=45=NOOP/term=13 — divergent WALs. The safe_committed_lsn formula ignores the candidate's own durable knowledge of committed entries, violating Leader Completeness: the new leader must never truncate entries it has that were committed by a prior leader.", + "fix": "IMPLEMENTED (2026-06-24): added pure helper ha_promotion_truncate_floor(safe_lsn, durable_tip)=max(safe_lsn,durable_tip) in ha.c/ha.h; raft_cb_become_leader now truncates at max(safe_committed_lsn, persist_durable_floor(&c->persist)) instead of bare safe_committed_lsn. Since the cutoff is never below the leader's own durable tip, persist_truncate_wal_above is a no-op on the leader's durable log — the new leader NEVER truncates entries it holds (Raft §5.4 Leader Completeness). safe_committed_lsn is still the commit lower-bound and promotion-gate HWM (NOT clamped — clamping it would defeat the behind-node fence). The leadership NOP now appends at the true tip (current_lsn) so replication converges peers to the leader's WAL with zero job loss. Unit test test_promotion_never_truncates_own_committed_entries in test_ha_election.c (GREEN). Steering src/controller/CLAUDE.md updated (Leader Completeness section + Post-Election Safe Commit Point + verification checklist). Recovery of an already-diverged cluster still requires restarting the over-truncated node (loses the cut jobs); this fix prevents the divergence.", "tags": [ "prevote", "deadlock", @@ -11069,46 +2542,29 @@ "leader-completeness", "data-loss", "coverage-maneuver", - "FIXED", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "FIXED" ], "related_bugs": [ "bug-795", "bug-828" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1066", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T04:36:50.602Z", - "error_message": "Incorrect value in code", - "file": "src/controller/CLAUDE.md", - "root_cause": "Had `persist_truncate_wal_above(safe_committed_lsn)`", - "fix": "Changed to 's own durable log**: the cutoff is `", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1060.md" }, { "id": "bug-1067", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-24T00:00:00.000Z", - "error_message": "Cross-term WAL catch-up permanently stalls: ctrl-2 stuck at lsn=8, all walCatchupEntry rejected (ae_reject_stale + ae_reject_logmismatch)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Cross-term WAL catch-up permanently stalls: ctrl-2 stuck at lsn=8, all walCatchupEntry rejected…\n\nCross-term WAL catch-up permanently stalls: ctrl-2 stuck at lsn=8, all walCatchupEntry rejected (ae_reject_stale + ae_reject_logmismatch)", "file": "src/controller/config_repl.c, src/controller/ha.c", - "root_cause": "AE term conflation: entry->term is the WAL entry's stored term (e.g. 4 for entries written by ctrl-1 at term=4), but ha_consume_append_entries uses it as the AE header ae_term. For live appends this is correct (leader appends at current term), but for catch-up streaming of historical entries from prior terms, ae_term = entry_term < leader.current_term. The Raft AE stale-term check (if term < s->term \u2192 ae_reject_stale) correctly rejects, but it should never fire \u2014 the AE header term must always be the leader's current term. There is no separate leader_term field in wal_catchup_entry_t. Consequence: historical entries at entry_term < raft.term are permanently rejected; standby_reported_lsn never advances; streamer retransmits forever.", - "fix": "IMPLEMENTED (2026-06-24): separated the AppendEntries HEADER term (leader's CURRENT term) from the per-entry STORED term. Added wire field WalCatchupEntry.leaderCurrentTerm @8 (controller.capnp) + C field wal_catchup_entry_t.leader_current_term (typemap.h, hand-defined struct \u2014 capnp generates encode/decode against it). Encoder (request.c wal_entry_to_wire + top container) writes it; decode auto-populates (generated, scalar mapname field). The catch-up streamer ha_stream_wal_to_standby gained a leader_term param (callers pass atomic_load(&c->ha.term)) and stamps entry->leader_current_term=leader_term on every streamed entry. Receiver ha_consume_append_entries now uses ae_term = e->leader_current_term ? e->leader_current_term : entry_term (single-entry branch) \u2014 the leader term rides on the sub-entry e by reference through ha_apply_term_entry_via_library, so no extra threading needed. entry->term (@4) remains the per-entry stored term for logOk/on_persist_entry. 0 = unset (legacy/live sender) \u2192 falls back to stored term (live appends have entry_term==leader_term, so unchanged). repl.c live paths UNTOUCHED. Tests (test_term_safety_adapter.c): test_intake_crossterm_catchup_uses_leader_current_term (historical entry@term4 streamed by leader@term5 to follower@term5 is ACCEPTED, not stale-rejected) + test_intake_stale_leader_term_still_rejected (leader_current_term=3 < follower term 5 still REJECTED \u2014 safety preserved). GREEN: 185 PASS term-safety+config-repl, 6 PASS wal_wire wire round-trip, 80 PASS failover/repl/durability, 0 failures.", + "root_cause": "AE term conflation: entry->term is the WAL entry's stored term (e.g. 4 for entries written by ctrl-1 at term=4), but ha_consume_append_entries uses it as the AE header ae_term. For live appends this is correct (leader appends at current term), but for catch-up streaming of historical entries from prior terms, ae_term = entry_term < leader.current_term. The Raft AE stale-term check (if term < s->term → ae_reject_stale) correctly rejects, but it should never fire — the AE header term must always be the leader's current term. There is no separate leader_term field in wal_catchup_entry_t. Consequence: historical entries at entry_term < raft.term are permanently rejected; standby_reported_lsn never advances; streamer retransmits forever.", + "fix": "IMPLEMENTED (2026-06-24): separated the AppendEntries HEADER term (leader's CURRENT term) from the per-entry STORED term. Added wire field WalCatchupEntry.leaderCurrentTerm @8 (controller.capnp) + C field wal_catchup_entry_t.leader_current_term (typemap.h, hand-defined struct — capnp generates encode/decode against it). Encoder (request.c wal_entry_to_wire + top container) writes it; decode auto-populates (generated, scalar mapname field). The catch-up streamer ha_stream_wal_to_standby gained a leader_term param (callers pass atomic_load(&c->ha.term)) and stamps entry->leader_current_term=leader_term on every streamed entry. Receiver ha_consume_append_entries now uses ae_term = e->leader_current_term ? e->leader_current_term : entry_term (single-entry branch) — the leader term rides on the sub-entry e by reference through ha_apply_term_entry_via_library, so no extra threading needed. entry->term (@4) remains the per-entry stored term for logOk/on_persist_entry. 0 = unset (legacy/live sender) → falls back to stored term (live appends have entry_term==leader_term, so unchanged). repl.c live paths UNTOUCHED. Tests (test_term_safety_adapter.c): test_intake_crossterm_catchup_uses_leader_current_term (historical entry@term4 streamed by leader@term5 to follower@term5 is ACCEPTED, not stale-rejected) + test_intake_stale_leader_term_still_rejected (leader_current_term=3 < follower term 5 still REJECTED — safety preserved). GREEN: 185 PASS term-safety+config-repl, 6 PASS wal_wire wire round-trip, 80 PASS failover/repl/durability, 0 failures.", "tags": [ "catch-up", "replication", @@ -11117,65 +2573,29 @@ "ae_reject_logmismatch", "walCatchupEntry", "cross-term", - "FIXED", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "FIXED" ], "related_bugs": [ "bug-1028", "bug-991" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1070", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T08:18:27.510Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: if (c && c->persist.wal_path[0]) {; /* bug-1067: pass the leader's CURRENT term so historical ca", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1075", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T08:23:04.980Z", - "error_message": "Wrong reference: standby_lsn should be leader_term", - "file": "src/controller/testsuite/test_coldstart_lsn0.c", - "root_cause": "Used \"standby_lsn\" instead of \"leader_term\"", - "fix": "Changed standby_lsn \u2192 leader_term", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1067.md" }, { "id": "bug-1077", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-06-24T17:30:00.000Z", - "error_message": "AddressSanitizer: SEGV / stack-use-after-return in free_CtrlStatus (lib/controller.capnp.c:11638) via handle_reply -> free_CtrlStatus_ptr (lib/xcectx.c:91) on `ctrlstatus` after failover; READ derefs the dead-stack `getreply` of xce_get_with_condition (xceapi.c)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: SEGV / stack-use-after-return in free_CtrlStatus (lib/controller.capnp.c:11638)…\n\nAddressSanitizer: SEGV / stack-use-after-return in free_CtrlStatus (lib/controller.capnp.c:11638) via handle_reply -> free_CtrlStatus_ptr (lib/xcectx.c:91) on `ctrlstatus` after failover; READ derefs the dead-stack `getreply` of xce_get_with_condition (xceapi.c)", "file": "src/lib/xcectx.c", "root_cause": "ctx->reply.u is an UNTAGGED union shared across opcodes (debugmsg/get/jobid/clusterconf/ctrlstatus alias the same 8 bytes). Commit 61e1391 ('fix bug-1060 bug-1067', unrelated Raft work) added speculative free-guards `if (ctx->reply.u.ctrlstatus) free_CtrlStatus_ptr(&ctx->reply.u.ctrlstatus)` (and the clusterconf twin). A non-NULL slot does NOT imply a CtrlStatus*: a failover-aborted reslist/joblist leaves ctx->reply.u.get == &getreply (a dead STACK pointer) because xce_get_with_condition/xce_resval only NULL the slot on the success path, not the xce_get_req<0 / cbuf_get<0 early-returns. The next ctrlstatus reply (worker thread) sees the foreign non-NULL slot and free_CtrlStatus walks it -> stack-use-after-return SEGV.", - "fix": "1) Revert the two unsafe free-guards in handle_reply (untagged union slot can never be safely freed by type) \u2014 restores prior MOVE_PTR-overwrite behavior. 2) Clear ctx->reply.u.get = NULL on ALL early-return paths of xce_get_with_condition and xce_resval (kill the dangling-stack-pointer leak at its source; also prevents a late OPCode_get reply writing through the dead stack via handle_get_reply). 3) Defense-in-depth: handle_get_reply drops a reply when ctx->reply.u.get is NULL. Regression test: test_handle_reply_ctrlstatus_does_not_free_foreign_slot (exposes xce_handle_reply, proves a foreign heap value in the shared union is not freed). VERIFIED 2026-06-24 (Opus): re-confirmed 3-layer fix present in working tree; RED proven by re-injecting the ctrlstatus free-guard -> ASan double-free at test_ctrlstatus.c:384 (free_CtrlStatus_ptr at controller.capnp.c:11892), then restored -> GREEN. Full libxce suite 291 Tests / 0 Failures / 0 ASan errors under ASAN_OPTIONS=detect_leaks=0.", + "fix": "1) Revert the two unsafe free-guards in handle_reply (untagged union slot can never be safely freed by type) — restores prior MOVE_PTR-overwrite behavior. 2) Clear ctx->reply.u.get = NULL on ALL early-return paths of xce_get_with_condition and xce_resval (kill the dangling-stack-pointer leak at its source; also prevents a late OPCode_get reply writing through the dead stack via handle_get_reply). 3) Defense-in-depth: handle_get_reply drops a reply when ctx->reply.u.get is NULL. Regression test: test_handle_reply_ctrlstatus_does_not_free_foreign_slot (exposes xce_handle_reply, proves a foreign heap value in the shared union is not freed). VERIFIED 2026-06-24 (Opus): re-confirmed 3-layer fix present in working tree; RED proven by re-injecting the ctrlstatus free-guard -> ASan double-free at test_ctrlstatus.c:384 (free_CtrlStatus_ptr at controller.capnp.c:11892), then restored -> GREEN. Full libxce suite 291 Tests / 0 Failures / 0 ASan errors under ASAN_OPTIONS=detect_leaks=0.", "tags": [ "asan", "use-after-return", @@ -11186,301 +2606,27 @@ "libxce", "memory-safety", "regression-61e1391", - "VERIFIED", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "VERIFIED" ], "related_bugs": [ "bug-646", "bug-1060", "bug-1067" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1078", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T09:18:52.605Z", - "error_message": "Wrong reference: handle_reply should be xce_handle_reply", - "file": "src/lib/xcectx.c", - "root_cause": "Used \"handle_reply\" instead of \"xce_handle_reply\"", - "fix": "Changed handle_reply \u2192 xce_handle_reply", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1079", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T09:27:59.772Z", - "error_message": "Wrong reference: libxce_failover_client_hang should be project_libxce_failover_hang", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1077_reply_union_type_confusion.md", - "root_cause": "Used \"libxce_failover_client_hang\" instead of \"project_libxce_failover_hang\"", - "fix": "Changed libxce_failover_client_hang \u2192 project_libxce_failover_hang", - "tags": [ - "wrong-reference", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1080", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T10:24:07.439Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-sim-conformance/design.md", - "root_cause": "Had \"on_get_entry_term -> node.log[idx].term | 0\"", - "fix": "Changed to \"on_get_entry_term -> node.log[idx].term | 0\"", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1081", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T10:30:45.585Z", - "error_message": "Wrong reference: apply should be advanced", - "file": ".kiro/specs/raft-sim-conformance/design.md", - "root_cause": "Used \"apply\" instead of \"advanced\"", - "fix": "Changed apply \u2192 advanced", - "tags": [ - "wrong-reference", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1082", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T10:30:56.237Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-sim-conformance/design.md", - "root_cause": "Had \"vote/AE requests + responses;
Drop/Duplicate/", - "fix": "Changed to \"vote/AE requests + responses,
Drop/Duplicate/", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1083", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T11:45:52.473Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: n > sizeof(v", - "fix": "Added guard clause: if (n > sizeof(v)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1085", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T12:00:01.924Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: i != last", - "fix": "Added guard clause: if (i != last)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1086", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T12:13:27.532Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: i < 0 || i >= c->bag_count", - "fix": "Added guard clause: if (i < 0 || i >= c->bag_count)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1087", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T12:22:22.094Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: sa.role != RAFT_LEADER", - "fix": "Added guard clause: if (sa.role != RAFT_LEADER)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1088", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T12:30:35.436Z", - "error_message": "Wrong reference: test_sim_check_election_safety_fail_on_two_leaders should be test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_sim_check_election_safety_fail_on_two_leaders\" instead of \"test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix\"", - "fix": "Changed test_sim_check_election_safety_fail_on_two_leaders \u2192 test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1089", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T12:38:40.360Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: sl.role != RAFT_LEADER", - "fix": "Added guard clause: if (sl.role != RAFT_LEADER)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1090", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T12:47:03.843Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: shared > RAFT_SIM_MAX_LOG", - "fix": "Added guard clause: if (shared > RAFT_SIM_MAX_LOG)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1091", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T13:04:27.057Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: c->trace_len >= RAFT_SIM_MAX_STEPS", - "fix": "Added guard clause: if (c->trace_len >= RAFT_SIM_MAX_STEPS)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1094", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T13:22:05.858Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: c->bag[i].kind != RAFT_SIM_MSG_VOTE_RSP", - "fix": "Added guard clause: if (c->bag[i].kind != RAFT_SIM_MSG_VOTE_RSP)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1077.md" }, { "id": "bug-1095", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-24", - "error_message": "Election Safety violation (two leaders at same term) surfaced by raft-sim-conformance step driver (task 4.2): a duplicated VOTE_RSP grant double-counts a single peer's vote", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "Election Safety violation (two leaders at same term) surfaced by raft-sim-conformance step driver…\n\nElection Safety violation (two leaders at same term) surfaced by raft-sim-conformance step driver (task 4.2): a duplicated VOTE_RSP grant double-counts a single peer's vote", "file": "src/lib/raft/raft.c", "root_cause": "raft_on_vote_reply tallies grants with a COUNTER (s->votes_granted++) and explicitly ignores from_idx ((void)from_idx;), so DuplicateMessage of one peer's grant counts that peer twice and a candidate reaches quorum with a single real vote. Diverges from raft.tla where votesGranted is a SET (idempotent). Proven: 3-node, single peer grant duplicated -> candidate becomes LEADER with votes_granted=2.", "fix": "FIXED (verified 2026-07-18 by validate-impl for raft-sim-conformance): src/lib/raft/raft.c raft_on_vote_reply now has a per-peer vote_replies[from_idx] idempotency guard (dedupes a DuplicateMessage grant) plus a term < s->term DropStaleResponse guard, landed together with bug-1096 in commit b5b7e01 (fix(raft): vote-reply per-peer dedup + stale-term drop). This buglog entry was stale (said NOT FIXED) even though the fix has been live since 2026-06-24 per tasks.md raft-sim-conformance Implementation Notes and bug-1096.fix; corrected here for consistency.", @@ -11494,26 +2640,33 @@ "needs-library-fix", "bug-851-family", "dedup", - "fixed", "stale-buglog-entry" ], "related_bugs": [ "bug-1096", "bug-851" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 29 + "fix_commits": [ + "b5b7e01" + ], + "record": ".wolf/bugs/bug-1095.md", + "gitea_issue": 29, + "gitea_synced_status": "fixed" }, { "id": "bug-1096", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-24", - "error_message": "raft_on_vote_reply counts a stale-term vote-reply (term < currentTerm) toward the current election instead of dropping it", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "raft_on_vote_reply counts a stale-term vote-reply (term < currentTerm) toward the current election…\n\nraft_on_vote_reply counts a stale-term vote-reply (term < currentTerm) toward the current election instead of dropping it", "file": "src/lib/raft/raft.c", "root_cause": "No DropStaleResponse: a reply with term < s->term fell through to replies_received++/votes_granted++, corrupting quorum accounting (raft.tla DropStaleResponse discards mterm < currentTerm).", - "fix": "FIXED (verified 2026-07-02): src/lib/raft/raft.c:832 has `if (reply_term != s->term)` guard before tallying, with an on_trace(\"vote_reply_stale_drop\", ...) at raft.c:455 \u2014 exactly the fix this bug's own second entry described. Landed under raft-sim-conformance / raft-commit-safety (both implemented, 18/18 + 23/23 tasks).", + "fix": "FIXED (verified 2026-07-02): src/lib/raft/raft.c:832 has `if (reply_term != s->term)` guard before tallying, with an on_trace(\"vote_reply_stale_drop\", ...) at raft.c:455 — exactly the fix this bug's own second entry described. Landed under raft-sim-conformance / raft-commit-safety (both implemented, 18/18 + 23/23 tasks).", "tags": [ "raft", "vote-reply", @@ -11523,7 +2676,6 @@ "conformance", "needs-library-fix", "stale-report", - "fixed", "bug-851-family", "stale-response" ], @@ -11531,116 +2683,20 @@ "bug-1095", "bug-851" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1099", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T14:08:44.250Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: failing != RAFT_SIM_INV_NONE", - "fix": "Added guard clause: if (failing != RAFT_SIM_INV_NONE)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1104", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T14:28:33.169Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Condition was: if (c->n >= 2 && c->nodes[0].commit_index >= 1 && c->n)", - "fix": "Changed to: if (c->n >= 2)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1108", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T14:38:06.432Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: chk < 0", - "fix": "Added guard clause: if (chk < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1110", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T14:51:30.855Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: i == leader", - "fix": "Added guard clause: if (i == leader)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1111", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T15:13:15.333Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: p == leader", - "fix": "Added guard clause: if (p == leader)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1096.md" }, { "id": "bug-1112", "status": "fixed", "status_updated": "2026-09-24", "status_note": "b1f289d2", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-24T15:15:03.882281Z", - "error_message": "ld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 \u2014 lib test binary fails to link", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 — lib test…\n\nld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 — lib test binary fails to link", "file": "src/lib/raft/raft_sim.c", "root_cause": "Commit e445404 (task 5.3 drop/dup/reorder) added the raft.h declaration + the test_sim_5_3 test but FORGOT to add the raft_sim_scenario_drop_dup_reorder implementation to raft_sim.c. Symbol is declared+referenced but never defined, breaking the link for the whole xcelib_test binary.", "fix": "Pre-existing, outside task 5.4 boundary. Surfaced while validating task 5.4 (figure8). Needs task 5.3 implementation added to raft_sim.c. NOT fixed by 5.4.", @@ -11649,201 +2705,28 @@ "link-error", "task-5.3", "undefined-reference", - "pre-existing", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pre-existing" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1113", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T15:16:52.683Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /*; * sim_scen_catch_up_commit: drive bounded AppendEntries-to-a | Also: * on a CONNECTED quorum, yet raft_advance_commit must NOT co; * count alone \u2014 the current-term Figure-8 rule (raft.c G4) f", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "b1f289d2" ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1115", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T15:18:59.248Z", - "error_message": "Wrong operator: != should be ==", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Used \"!=\" instead of \"==\"", - "fix": "Changed operator != \u2192 ==", - "tags": [ - "auto-detected", - "operator-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1116", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T15:25:17.886Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: i == leader", - "fix": "Added guard clause: if (i == leader)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1117", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T15:32:34.085Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft.c", - "root_cause": "Condition was: if (s->cbs->on_get_entry_term(s->ctx, agreed)", - "fix": "Changed to: if (0 && s->cbs->on_get_entry_term(s->ctx, agreed)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1120", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T16:09:04.749Z", - "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_1_fault_switch_mechanism", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_1_fault_switch_mechanism\"", - "fix": "Changed test_sim_6_1_seed_sweep \u2192 test_sim_7_1_fault_switch_mechanism", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1123", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T16:22:58.427Z", - "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_2_negative_both_directions", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_2_negative_both_directions\"", - "fix": "Changed test_sim_6_1_seed_sweep \u2192 test_sim_7_2_negative_both_directions", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1125", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T16:42:32.261Z", - "error_message": "Wrong reference: test_sim_4_2_step_liveness_and_coverage should be test_sim_8_1_export_trace_named_and_gated", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_sim_4_2_step_liveness_and_coverage\" instead of \"test_sim_8_1_export_trace_named_and_gated\"", - "fix": "Changed test_sim_4_2_step_liveness_and_coverage \u2192 test_sim_8_1_export_trace_named_and_gated", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1126", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-24T16:43:39.427Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Condition was: if (node_idx < 0 || node_idx >= c->n)", - "fix": "Changed to: if (getenv(\"RAFT_SIM_EXPORT\")", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1127", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T03:17:45.530Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1112.md" }, { "id": "bug-1128", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-25T03:20:00.000Z", - "error_message": "test_demotion_replay_yields_committed_only flakes ~16% on overlayfs: boot recovery over surviving WAL must yield the committed prefix ONLY", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "test_demotion_replay_yields_committed_only flakes ~16% on overlayfs: boot recovery over surviving…\n\ntest_demotion_replay_yields_committed_only flakes ~16% on overlayfs: boot recovery over surviving WAL must yield the committed prefix ONLY", "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "TWO compounding non-determinism sources, both fixture artifacts not product bugs: (1) overlayfs metadata incoherence \u2014 a freshly-opened persist_t fd over a just-ftruncate'd WAL intermittently reads the STALE pre-truncate size, so the cross-context fresh recovery replays the already-removed suffix; (2) DEEPER, fs-independent: the fixture wires a LIVE commit engine and on a single node (n_peers=1) the persist durable-advance callback (fsync worker -> on_durable_advance) commits the whole durable log, so persist_flush asynchronously raced committed_lsn UP from the test-set committed_boundary (5) to the appended tip (8) AFTER the test pinned it; the demotion seam reads committed_lsn at task time -> truncated to 8 not 5 -> current_lsn=9 > committed_boundary+1. DIAG proof: committed_boundary=5 live_committed=8 live_tip=9 on failing runs.", - "fix": "Test-only, two parts. (a) Detach the durable-advance commit wake (g_dc.persist.on_durable_advance = NULL) and re-pin committed_lsn = committed_boundary right before driving the demotion task, killing the commit-engine race deterministically (helps BOTH fs paths). (b) Hybrid gate by fs type via static dc_path_is_overlayfs() (statfs f_type == OVERLAYFS_SUPER_MAGIC 0x794c7630, defaults to real-fs path on statfs error): on a real fs keep the strong cross-context fresh boot-recovery assertion (fresh->current_lsn <= committed_boundary+1); on overlayfs skip that fd-incoherent step and assert the LIVE-context truncate effect (g_dc.persist.current_lsn <= committed_boundary+1, coherent under p->lock). Both branches prove the same property. Production persist.c truncate (ftruncate+fdatasync) untouched.", - "fix_validation": "Full controller suite 20/20 runs: 0 demotion_replay failures, 0 total failures. Group-only 60/60: 0 failures. Baseline (pre-fix) was 4/20. /tmp confirmed overlayfs (statfs magic 0x794c7630), so the overlayfs branch is the one exercised here.", + "root_cause": "TWO compounding non-determinism sources, both fixture artifacts not product bugs: (1) overlayfs metadata incoherence — a freshly-opened persist_t fd over a just-ftruncate'd WAL intermittently reads the STALE pre-truncate size, so the cross-context fresh recovery replays the already-removed suffix; (2) DEEPER, fs-independent: the fixture wires a LIVE commit engine and on a single node (n_peers=1) the persist durable-advance callback (fsync worker -> on_durable_advance) commits the whole durable log, so persist_flush asynchronously raced committed_lsn UP from the test-set committed_boundary (5) to the appended tip (8) AFTER the test pinned it; the demotion seam reads committed_lsn at task time -> truncated to 8 not 5 -> current_lsn=9 > committed_boundary+1. DIAG proof: committed_boundary=5 live_committed=8 live_tip=9 on failing runs.", + "fix": "Test-only, two parts. (a) Detach the durable-advance commit wake (g_dc.persist.on_durable_advance = NULL) and re-pin committed_lsn = committed_boundary right before driving the demotion task, killing the commit-engine race deterministically (helps BOTH fs paths). (b) Hybrid gate by fs type via static dc_path_is_overlayfs() (statfs f_type == OVERLAYFS_SUPER_MAGIC 0x794c7630, defaults to real-fs path on statfs error): on a real fs keep the strong cross-context fresh boot-recovery assertion (fresh->current_lsn <= committed_boundary+1); on overlayfs skip that fd-incoherent step and assert the LIVE-context truncate effect (g_dc.persist.current_lsn <= committed_boundary+1, coherent under p->lock). Both branches prove the same property. Production persist.c truncate (ftruncate+fdatasync) untouched.\n\nVerification:\nFull controller suite 20/20 runs: 0 demotion_replay failures, 0 total failures. Group-only 60/60: 0 failures. Baseline (pre-fix) was 4/20. /tmp confirmed overlayfs (statfs magic 0x794c7630), so the overlayfs branch is the one exercised here.", "tags": [ "test-flake", "demotion-convergence", @@ -11855,9 +2738,7 @@ "sandbox", "commit-engine-race", "deterministic", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "c" ], "related_bugs": [ "bug-343", @@ -11867,238 +2748,23 @@ "bug-9799", "bug-1060" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1133", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T07:12:01.309Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-log-replication/spec.json", - "root_cause": "Had \"2026-06-21T11:00:00Z\"", - "fix": "Changed to \"2026-06-25T00:00:00Z\"", - "tags": [ - "auto-detected", - "wrong-value", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1134", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T08:18:43.915Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1135", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T09:26:49.441Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1136", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T09:27:00.312Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-apply-decoupling/brief.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1138", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T11:36:52.942Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1139", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T14:02:51.225Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/research.md", - "root_cause": "Had `VTASK_IO`", - "fix": "Changed to `endpoint_block_tasks()`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1141", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T15:12:16.193Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-drive-async/research.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1142", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T15:13:00.445Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `.kiro/specs/raft-drive-async/brief.md`", - "fix": "Changed to `delay_task_ms`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1143", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T23:37:38.579Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - Commit-advance under pool load (R4.5): with the vtask pool; - Snapshot/checkpoint concurrency (R1.1/1.3/1.5): concurrent", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1144", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T23:37:49.697Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Had `persist_checkpoint`", - "fix": "Changed to `wait_all_with_timeout`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1145", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-25T23:38:08.746Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1128.md" }, { "id": "bug-1147", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-26T03:46:38.904419+00:00", - "error_message": "WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor \u2192 lagging/empty-rejoin standby loses its needed log range with no snapshot backfill (permanent committed-prefix gap)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor →…\n\nWAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor → lagging/empty-rejoin standby loses its needed log range with no snapshot backfill (permanent committed-prefix gap)", "file": "src/controller/persist.c:1374 (trim_wal_segments)", - "root_cause": "trim_wal_segments bounds deletion ONLY by active_seg_lsn (the currently-open segment): `if (seg_start_lsn < active_seg_lsn) unlink()`. It consults no standby acked_lsn, no committed_lsn, no in-flight snapshot boundary. Strategy-A (cursor-bounded retention) discipline is absent. Strategy-B (entity-bearing snapshot install) is also not live \u2014 write_snapshot (persist.c:1247) writes only a 16-byte P0 sentinel, not materialized state. So a standby that lags by more than ~one open segment (down across segment rolls, slow link, >64MB lag, or empty rejoin from cursor=0) hits a log range that exists nowhere and cannot catch up.", - "fix": "FIXED (verified 2026-07-02): src/controller/persist.c trim_wal_segments() computes `floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s)` (min of active segment, all standby acked_lsn, disconnected-slot guards) and only unlinks segments below that floor \u2014 comment inline explicitly cites 'bug-1147 guard'. Landed under raft-drive-async task 3.2 (all leaf tasks complete).", + "root_cause": "trim_wal_segments bounds deletion ONLY by active_seg_lsn (the currently-open segment): `if (seg_start_lsn < active_seg_lsn) unlink()`. It consults no standby acked_lsn, no committed_lsn, no in-flight snapshot boundary. Strategy-A (cursor-bounded retention) discipline is absent. Strategy-B (entity-bearing snapshot install) is also not live — write_snapshot (persist.c:1247) writes only a 16-byte P0 sentinel, not materialized state. So a standby that lags by more than ~one open segment (down across segment rolls, slow link, >64MB lag, or empty rejoin from cursor=0) hits a log range that exists nowhere and cannot catch up.", + "fix": "FIXED (verified 2026-07-02): src/controller/persist.c trim_wal_segments() computes `floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s)` (min of active segment, all standby acked_lsn, disconnected-slot guards) and only unlinks segments below that floor — comment inline explicitly cites 'bug-1147 guard'. Landed under raft-drive-async task 3.2 (all leaf tasks complete).", "tags": [ "durability", "permanent-gap", @@ -12109,7 +2775,6 @@ "latent", "catch-up", "wal", - "fixed", "snapshot", "bug-795" ], @@ -12118,859 +2783,25 @@ "bug-828", "bug-1060" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1148", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T03:46:51.038Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Had 's boundary or any standby'", - "fix": "Changed to `persist.c:1374`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1149", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T03:48:48.222Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `/kiro-spec-design raft-drive-async`", - "fix": "Changed to `approved: false`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1151", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:31:53.879Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Had `commit_submit`", - "fix": "Changed to `commit.c`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1152", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:32:20.960Z", - "error_message": "Wrong reference: PeerCtrlDispatch should be LivenessAdmission", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Used \"PeerCtrlDispatch\" instead of \"LivenessAdmission\"", - "fix": "Changed PeerCtrlDispatch \u2192 LivenessAdmission", - "tags": [ - "wrong-reference", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1153", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:33:10.286Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `COMMIT_WAIT_RESERVED=2`", - "fix": "Changed to `commit.c:16`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1154", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:37:18.052Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `VTASK_LIVENESS_RESERVED`", - "fix": "Changed to `R = clamp(n_peers \u2212 1, 1, n_vcpus \u2212 1)`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1155", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:47:49.937Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_drive_async_config.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1156", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:48:34.246Z", - "error_message": "Function not marked async", - "file": "src/include/config.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1157", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:48:50.005Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1158", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:49:04.568Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1159", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:49:44.821Z", - "error_message": "Function not marked async", - "file": "src/controller/cluster.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1160", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T04:57:57.877Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1161", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T05:02:36.720Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_slowlane.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1162", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T05:06:30.148Z", - "error_message": "Function not marked async", - "file": "src/include/xce/cluster_def.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1163", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T05:06:42.680Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/cluster.c", - "root_cause": "Condition was: if (ha_is_primary(&c->ha)", - "fix": "Changed to: if (c->slowlane.workers == NULL)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1164", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T05:06:53.129Z", - "error_message": "Function not marked async", - "file": "src/controller/cluster.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1165", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T06:51:00.808Z", - "error_message": "Wrong reference: test_request_send_propagation should be test_netio_bounded_connect", - "file": "src/lib/testsuite/test.c", - "root_cause": "Used \"test_request_send_propagation\" instead of \"test_netio_bounded_connect\"", - "fix": "Changed test_request_send_propagation \u2192 test_netio_bounded_connect", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1166", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T06:51:57.649Z", - "error_message": "Function not marked async", - "file": "src/lib/netio.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1168", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T06:52:21.608Z", - "error_message": "Function not marked async", - "file": "src/include/netio.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1169", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:16:36.377Z", - "error_message": "Function not marked async", - "file": "src/include/vtaskmgr.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1171", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:18:02.307Z", - "error_message": "Function not marked async", - "file": "src/lib/vtaskmgr.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1172", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:19:14.857Z", - "error_message": "Function not marked async", - "file": "src/controller/include/commit.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1173", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:19:45.059Z", - "error_message": "Function not marked async", - "file": "src/controller/commit.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1174", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:20:07.233Z", - "error_message": "Wrong operator: >= should be !=", - "file": "src/controller/commit.c", - "root_cause": "Used \">=\" instead of \"!=\"", - "fix": "Changed operator >= \u2192 !=", - "tags": [ - "auto-detected", - "operator-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1175", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:20:41.448Z", - "error_message": "Function not marked async", - "file": "src/controller/cluster.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1177", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:47:19.731Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1178", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:48:00.209Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1179", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:51:01.491Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "No early return/throw for edge case: deadline.tv_nsec >= 1000000000L", - "fix": "Added guard clause: if (deadline.tv_nsec >= 1000000000L)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1180", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:51:10.764Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1183", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T07:57:35.484Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1184", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T08:12:52.624Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1185", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T08:13:19.511Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1186", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T08:15:44.315Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx)", - "fix": "Changed to: if (0 && cur_sctx && cur_sctx->peer_idx == sctx->peer_)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1188", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T10:20:33.560Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1189", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T10:21:12.319Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1190", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T10:24:21.473Z", - "error_message": "Function not marked async", - "file": "src/controller/cluster.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1191", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T10:27:00.557Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "No early return/throw for edge case: atomic_load(&g_pc_gate_on", - "fix": "Added guard clause: if (atomic_load(&g_pc_gate_on)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1192", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T10:27:14.358Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1193", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T10:56:43.461Z", - "error_message": "Function not marked async", - "file": "src/include/vtaskmgr.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1194", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T10:57:18.280Z", - "error_message": "Function not marked async", - "file": "src/lib/vtaskmgr.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1195", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T10:57:44.750Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1196", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T11:00:15.213Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1147.md" }, { "id": "bug-1197", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-26T00:00:00.000Z", - "error_message": "raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint UAF \u2014 raft_cb_become_follower runs on a pool worker and endpoint_fini()s all peer vote-eps (frees ep->data) while the single io_worker can concurrently read ep_j->data in peer_ctrl_msg_handler_inline for a same-batch queued event", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint…\n\nraft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint UAF — raft_cb_become_follower runs on a pool worker and endpoint_fini()s all peer vote-eps (frees ep->data) while the single io_worker can concurrently read ep_j->data in peer_ctrl_msg_handler_inline for a same-batch queued event", "file": "src/controller/ha.c", "root_cause": "Moving peer-control off the sequential io_worker onto the pool co-locates teardown (frees ep->data) and inbound delivery (reads ep->data) on different threads with no unmonitor-before-free ordering. Safe today only because g_drive_async_peerctrl_dispatch_enabled defaults OFF.", - "fix": "SUPERSEDED 2026-06-30 by bug-1298 endpoint reference counting (commit 6e05dc1). The pool-reachable teardown-frees-peer-ep UAF class is now governed generally by endpoint_ref/endpoint_unref (refcnt + heap_alloc gate) acquired per ep-bound VTASK_IO in vtask_enq and released in run_task/taskq_item_killer \u2014 a queued vtask holds a ref so the io_worker cannot free the ep out from under the dispatcher. Re-validate when raft-drive-async flips g_drive_async_peerctrl_dispatch ON.", + "fix": "SUPERSEDED 2026-06-30 by bug-1298 endpoint reference counting (commit 6e05dc1). The pool-reachable teardown-frees-peer-ep UAF class is now governed generally by endpoint_ref/endpoint_unref (refcnt + heap_alloc gate) acquired per ep-bound VTASK_IO in vtask_enq and released in run_task/taskq_item_killer — a queued vtask holds a ref so the io_worker cannot free the ep out from under the dispatcher. Re-validate when raft-drive-async flips g_drive_async_peerctrl_dispatch ON.", "tags": [ "endpoint-quiescence", - "fixed", "c", "dispatch_inflight", "task-3.4a", @@ -12986,43 +2817,25 @@ "related_bugs": [ "bug-1198" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1199", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T00:00:00.000Z", - "error_message": "raft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its io_packet_t (only the executed entry transfers ownership via iod->pkt=NULL)", - "file": "src/lib/vtaskmgr.c", - "root_cause": "taskq_item_killer called vtask_fini (frees task->data wrapper only), never the carried io_packet_t. The generic killer could not free the pkt unconditionally because the libxce client path (xcectx.c vtask_handle_msg) does NOT clear ep->fsm_in.pkt \u2014 its pkt lifetime is the endpoint's, so an unconditional free would double-free.", - "fix": "Added int owns_io_pkt to vtask_t (default 0 in vtask_init). taskq_item_killer frees the pkt ONLY when owns_io_pkt && vtype==VTASK_IO && pkt!=NULL. Controller peer_ctrl_dispatch_to_pool sets owns_io_pkt=1. libxce path keeps owns_io_pkt=0 -> unchanged, no double-free. Verified: leak count 808 unchanged vs baseline, 0 double-free under ASan.", - "tags": [ - "raft-drive-async", - "task-3.1", - "memory-leak", - "vtaskmgr", - "packet-ownership", - "review-finding", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "6e05dc1" ], - "related_bugs": [ - "bug-1077" - ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1197.md" }, { "id": "bug-1198", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-06-26T00:00:00.000Z", - "error_message": "raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work VTASK_LIVENESS, but the CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT->persist_checkpoint (blocking fdatasync) inline \u2014 a blocking op under the never-gated liveness reservation breaks deadlock-freedom", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work…\n\nraft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work VTASK_LIVENESS, but the CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT->persist_checkpoint (blocking fdatasync) inline — a blocking op under the never-gated liveness reservation breaks deadlock-freedom", "file": "src/controller/ha.c", "root_cause": "repl_ack_decode_and_dispatch -> ha_dispatch_repl_action_for_standby can run the snapshot send inline. LIVENESS is only correct once that long op is offloaded off the pool.", - "fix": "FIXED/CLOSED (verified 2026-07-02): src/controller/ha.c:5647 carries an explicit in-code 'CLOSED' banner comment \u2014 the bug-1197 endpoint_ref/endpoint_unref bracket around the lane dispatch means the repl-ack path's snapshot send is ALWAYS offloaded before peer_ctrl_classify can tag it VTASK_LIVENESS, so bug-1198 cannot recur. Both flags (peerctrl dispatch + snapshot offload) default ON since bug-1376 was fixed.", + "fix": "FIXED/CLOSED (verified 2026-07-02): src/controller/ha.c:5647 carries an explicit in-code 'CLOSED' banner comment — the bug-1197 endpoint_ref/endpoint_unref bracket around the lane dispatch means the repl-ack path's snapshot send is ALWAYS offloaded before peer_ctrl_classify can tag it VTASK_LIVENESS, so bug-1198 cannot recur. Both flags (peerctrl dispatch + snapshot offload) default ON since bug-1376 was fixed.", "tags": [ "task-3.1", "review-finding", @@ -13030,121 +2843,54 @@ "raft-drive-async", "deferred", "flag-on-hazard", - "fixed", "deadlock-freedom", "liveness-admission" ], "related_bugs": [ "bug-1197" ], + "fix_commits": [], + "record": ".wolf/bugs/bug-1198.md" + }, + { + "id": "bug-1199", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "leak", + "timestamp": "2026-06-26T00:00:00.000Z", + "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1202", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T11:33:28.328Z", - "error_message": "Missing error handling in task", - "file": "src/controller/testsuite/test_e2e_checkpoint.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Helper: check if a file exists in a directory ; static int file_exists_in(const char* dir, const char* name)", + "error_message": "raft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its…\n\nraft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its io_packet_t (only the executed entry transfers ownership via iod->pkt=NULL)", + "file": "src/lib/vtaskmgr.c", + "root_cause": "taskq_item_killer called vtask_fini (frees task->data wrapper only), never the carried io_packet_t. The generic killer could not free the pkt unconditionally because the libxce client path (xcectx.c vtask_handle_msg) does NOT clear ep->fsm_in.pkt — its pkt lifetime is the endpoint's, so an unconditional free would double-free.", + "fix": "Added int owns_io_pkt to vtask_t (default 0 in vtask_init). taskq_item_killer frees the pkt ONLY when owns_io_pkt && vtype==VTASK_IO && pkt!=NULL. Controller peer_ctrl_dispatch_to_pool sets owns_io_pkt=1. libxce path keeps owns_io_pkt=0 -> unchanged, no double-free. Verified: leak count 808 unchanged vs baseline, 0 double-free under ASan.", "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "raft-drive-async", + "task-3.1", + "memory-leak", + "vtaskmgr", + "packet-ownership", + "review-finding" ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1204", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T11:35:37.597Z", - "error_message": "Wrong reference: file_exists_for_trim_test should be seg_file_exists", - "file": "src/controller/testsuite/test_e2e_checkpoint.c", - "root_cause": "Used \"file_exists_for_trim_test\" instead of \"seg_file_exists\"", - "fix": "Changed file_exists_for_trim_test \u2192 seg_file_exists", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "related_bugs": [ + "bug-1077" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1205", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T11:39:53.357Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1206", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T15:16:40.996Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1207", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T15:28:19.885Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ha_repl_send_snapshot_hook != NULL", - "fix": "Added guard clause: if (ha_repl_send_snapshot_hook != NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1199.md" }, { "id": "bug-1210", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-26", - "error_message": "raft-drive-async task 3.2 snapshot slow-lane offload flag g_drive_async_snapshot_offload defaulted ON (=1); header doc also said default ON", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-drive-async task 3.2 snapshot slow-lane offload flag g_drive_async_snapshot_offload defaulted…\n\nraft-drive-async task 3.2 snapshot slow-lane offload flag g_drive_async_snapshot_offload defaulted ON (=1); header doc also said default ON", "file": "src/controller/ha.c, src/controller/ha.h", "root_cause": "Snapshot offload borrows a standby endpoint shared with the io_worker epoll loop (not owned, unlike reconnect_lane_fn). With the flag ON the send runs on a slow-lane thread while io_worker can RDHUP-free that endpoint => cross-thread UAF (bug-1197 family). Defaulting ON would ship that UAF; the borrowed-endpoint lifetime discipline is not established until task 3.4.", "fix": "Default the flag OFF (dormant landing, same protocol as task 3.1 peerctrl dispatch). Inline path runs on io_worker, serialized with teardown -> no overlap. Rewrote both the ha.c definition comment and the ha.h accessor doc to state OFF is the production default and 3.4 is the gate that flips it ON. bug-1198 stays open by design until then.", @@ -13157,184 +2903,29 @@ "feature-flag", "bug-1197", "bug-1198", - "liveness", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "liveness" ], "related_bugs": [ "bug-1197", "bug-1198", "bug-1147" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1211", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T15:49:46.434Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1212", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T15:50:27.084Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1213", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T15:51:29.747Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ha_configpush_probe != NULL", - "fix": "Added guard clause: if (ha_configpush_probe != NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1214", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T15:54:05.463Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1216", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T16:54:57.384Z", - "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", - "file": "src/include/netio.h", - "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", - "fix": "Changed ssl_handshake_done \u2192 tearing_down", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1217", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T16:55:15.332Z", - "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", - "file": "src/lib/netio.c", - "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", - "fix": "Changed ssl_handshake_done \u2192 tearing_down", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1218", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T16:55:48.690Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ectx", - "fix": "Added guard clause: if (ectx)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1222", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-26T17:44:25.670Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_ha_dispatch_integration.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1210.md" }, { "id": "bug-1224", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-06-26", - "error_message": "raft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently \u2014 controller_unit_test never completes (timeout-killed)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently…\n\nraft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently — controller_unit_test never completes (timeout-killed)", "file": "src/controller/testsuite/test_ha_dispatch_integration.c", - "root_cause": "Test teardown race: P1/P2/P3 released a parked vtask body (gate_open) then immediately called vtaskmgr_fini, which tears down workers via pthread_cancel. The tests polled a run-counter incremented MID-body, so fini raced the just-released body still unwinding (esp. P2 REQ_BLOCK bodies in vtaskmgr_blocking_release racing blk_lock teardown) \u2192 flaky deadlock. slowlane_fini is graceful (join) so the lane side was safe; vtaskmgr_fini (cancel) was the hazard.", + "root_cause": "Test teardown race: P1/P2/P3 released a parked vtask body (gate_open) then immediately called vtaskmgr_fini, which tears down workers via pthread_cancel. The tests polled a run-counter incremented MID-body, so fini raced the just-released body still unwinding (esp. P2 REQ_BLOCK bodies in vtaskmgr_blocking_release racing blk_lock teardown) → flaky deadlock. slowlane_fini is graceful (join) so the lane side was safe; vtaskmgr_fini (cancel) was the hazard.", "fix": "Add end-of-body completion counters (g_pc_completed on both pc_probe paths incl. the park path; g_block_done at the end of block_task_entry) and poll them to the full expected count AFTER gate_open and BEFORE vtaskmgr_fini, so every pool body has fully returned (worker idle in cbuf_get) before the cancel-based fini. Verified deterministic 5/5 isolated runs + full suite EXIT=0, ASan-clean.", "tags": [ "raft-drive-async", @@ -13344,22 +2935,25 @@ "vtaskmgr_fini", "pthread_cancel", "teardown-race", - "deadlock", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "deadlock" ], "related_bugs": [ "bug-924" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1224.md" }, { "id": "bug-1225", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-06-26", - "error_message": "/xce-test-ha live failover soak (ASAN build, real fs ~/tmp): FAIL round 4 \"cluster did not reconverge to 3-of-3 after restarting ctrl-2\"", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "/xce-test-ha live failover soak (ASAN build, real fs ~/tmp): FAIL round 4 \"cluster did not…\n\n/xce-test-ha live failover soak (ASAN build, real fs ~/tmp): FAIL round 4 \"cluster did not reconverge to 3-of-3 after restarting ctrl-2\"", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh (wait_for_full_convergence)", "root_cause": "Pre-existing HA-core instability under 6-round CHAOTIC repeated failover, NOT a raft-drive-async regression. BASELINE-CONFIRMED: the same soak against pre-Phase-3 binaries (commit 21e26a1, release/non-ASAN) ALSO fails at round 4 (baseline symptom: no-new-primary election stall; current symptom: round-4 reconverge + committed-prefix divergence at LSN 9). Both builds preserve committed JOBS (zero job loss in all completed rounds). The committed-prefix divergence is the documented open bug-851/795 family (chaotic-failover silent divergence; separate HA-core spec). raft-drive-async Phase 3 flags are default-OFF (inert); the live trim-guard only preserves more WAL.", "fix": "Not a raft-drive-async bug (baseline-confirmed). 4.2 regression sweep PASSES for this spec: no NEW failure introduced. Pre-existing bug-851 chaotic-failover divergence + bug-1042 reconverge flakiness remain open for a separate HA-core effort.", @@ -13373,426 +2967,29 @@ "reconverge", "soak", "task-4.2", - "xce-test-ha", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "xce-test-ha" ], "related_bugs": [ "bug-1042", "bug-924" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1226", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T02:53:39.273Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Had \"$REPO/src/xce.build/controller/xce_controller\"", - "fix": "Changed to \"${XCE_HA_BIN:-$REPO/src/xce.build/controller/xce_", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1227", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T03:04:05.891Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1228", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T04:13:35.943Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "Had `; no silent retention of a stale-term entry.\n- In", - "fix": "Changed to `; no silent retention of a stale-term entry.\n- In", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1233", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T04:59:57.805Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: # \u2500\u2500 Chaotic-mode primitives (count outcomes into the verdic; # Run `walcompare --report` across all three members and ADD | Also: # \u2500\u2500 Chaotic mode: overlapping-kill rounds \u2192 isolated verdic; # Opt-in (--chaotic / XCE_HA_MODE=chaotic). Kills land insid", - "tags": [ - "auto-detected", - "error-handling", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1234", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T05:00:06.726Z", - "error_message": "Function not marked async", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1236", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T09:27:45.080Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "Had `raft_on_append_entries_request`", - "fix": "Changed to `research.md`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1237", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T09:28:07.944Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - **Technical debt addressed:** the silent empty-payload shi | Also: - `.claude/skills/xce-test-ha/run-ha-soak.sh` (and `SKILL.md; - `src/controller/` log init (only if the client-probe appro | Also: | Catch-up Payload Integrity | Adapter | Stream a non-empty ; | Persist Conflict Guard | Persist | `(term,lsn,payload-crc) | Also: - The live-repro gate (task 1.3) has **already run and refut; - A green synthetic unit test over a still-red soak is not a", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1243", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T10:02:56.820Z", - "error_message": "Wrong operator: == should be <<", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Used \"==\" instead of \"<<\"", - "fix": "Changed operator == \u2192 <<", - "tags": [ - "auto-detected", - "operator-fix", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1244", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T10:04:31.979Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Had \"no primary elected within 45s (check $TOP/logs)\"", - "fix": "Changed to \"no primary elected within ~90s via client probe (", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1246", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T10:07:37.692Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/SKILL.md", - "root_cause": "Had `no primary elected within 45s`", - "fix": "Changed to `no primary elected within ~90s via client probe`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1247", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T10:25:38.366Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p || lsn == 0", - "fix": "Added guard clause: if (!p || lsn == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1249", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T10:45:00.686Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/persist.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* (a) non-empty incoming whose payload differs at an existi; * \u2014 a genuine payload conflict. Branch on tip vs below-tip.", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1250", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T10:45:34.063Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1252", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T10:47:06.628Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/persist.c", - "root_cause": "Condition was: if (lsn != tip_lsn)", - "fix": "Changed to: if (trc0 != 0)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1254", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T11:14:27.133Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: * TASK 2.3 \u2014 cross-term catch-up streamer payload integrity ; * | Also: /* Task 2.3 cross-term catch-up streamer payload integrity:; * CORE RED \u2014 the streamer must never ship a non-empty cro", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1255", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-27T11:28:45.351Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !pbuf", - "fix": "Added guard clause: if (!pbuf)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1264", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-29T04:13:02.187Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_ha_partition.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1266", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-29T04:25:39.111Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_partition.c", - "root_cause": "No early return/throw for edge case: peer_idx == g_partition_peer_idx && g_reconnect_gate", - "fix": "Added guard clause: if (peer_idx == g_partition_peer_idx && g_re)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1269", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T01:47:07.635Z", - "error_message": "Missing await", - "file": "src/controller/ha.c", - "root_cause": "Async call without await \u2014 returned Promise instead of value", - "fix": "Added await to async call", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1271", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T02:25:32.055Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - _Depends: 1.5, 6.2, 5.2, 5.3_; - _Blocked: first soak run (HEAD 01419e1, Phase-2+5 fixes) i", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1225.md" }, { "id": "bug-1272", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-06-30T02:25:00Z", - "error_message": "raft-ha-core-safety 7.2 chaotic soak: round-4 'NO new primary within window (possible election stall)' after 3 overlapping kills", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "raft-ha-core-safety 7.2 chaotic soak: round-4 'NO new primary within window (possible election…\n\nraft-ha-core-safety 7.2 chaotic soak: round-4 'NO new primary within window (possible election stall)' after 3 overlapping kills", "file": "src/controller/ha.c (election/PreVote path)", "root_cause": "After 3 overlapping kills the only fully-up-to-date node is killed; the two survivors are both restarted-and-lagging (tips 15 and 11; leader was 19). Both hold the full committed prefix [1..11] (safe candidates) but their PreVote rounds repeatedly time out with no grant quorum (prospective terms frozen, never bumped). Distinct from the 5.2 spurious-failover flap (that is fixed and confirmed). Contributors: freshness mutual-denial between lagging survivors, in_catch_up-denies-all-votes (Req 2.5) with no live primary to catch up from, and/or peer vote connection not established between two nodes that only held a control channel to the dead primary.", - "fix": "FIXED at unit level by task 5.3 (commit bf88681): gate the in_catch_up PreVote deny in raft_cb_vote_eligible on a LIVE primary (primary_ctrl_fd>=0 && ha_heartbeat_is_fresh), so a catching-up survivor with a dead primary can grant/win a fresh PreVote (logOk preserves Leader Completeness). CORRECTION: this in_catch_up deadlock is a real latent defect (unit RED\u2192GREEN) but is NOT the soak's round-4 blocker. The soak stall is bug-924 (WAL fdatasync\u2192EINVAL wedges a survivor so it stops answering PreVotes; presents as an election stall). 7.2 live acceptance is blocked on that env defect (out of scope; owned by ha-commit-durability), not on consensus.", + "fix": "FIXED at unit level by task 5.3 (commit bf88681): gate the in_catch_up PreVote deny in raft_cb_vote_eligible on a LIVE primary (primary_ctrl_fd>=0 && ha_heartbeat_is_fresh), so a catching-up survivor with a dead primary can grant/win a fresh PreVote (logOk preserves Leader Completeness). CORRECTION: this in_catch_up deadlock is a real latent defect (unit RED→GREEN) but is NOT the soak's round-4 blocker. The soak stall is bug-924 (WAL fdatasync→EINVAL wedges a survivor so it stops answering PreVotes; presents as an election stall). 7.2 live acceptance is blocked on that env defect (out of scope; owned by ha-commit-durability), not on consensus.", "tags": [ "prevote", "ha", @@ -13806,9 +3003,7 @@ "fdatasync", "raft", "5.3", - "in_catch_up", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "in_catch_up" ], "related_bugs": [ "bug-924", @@ -13816,76 +3011,23 @@ "bug-1042", "bug-1060" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1273", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T02:41:41.017Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 5.3 lagging-but-safe survivor electability under ove; * CORE RED \u2014 a catching-up survivor with NO live primary m", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "bf88681" ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1275", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T02:55:23.808Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1276", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T02:55:37.693Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `research.md`", - "fix": "Changed to 's persist flusher jams on `", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1272.md" }, { "id": "bug-1281", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "crash", "timestamp": "2026-06-30T03:25:00Z", - "error_message": "Restart-stability bug: a restarted xce_controller becomes unable to participate at chaotic round 4. Release build SIGSEGVs ('Segmentation fault (core dumped) --myname ctrl-3'); under gdb the SAME corruption manifests as a WEDGED inbound-accept/iomgr path (last accept 03:42:03 while the election timer keeps firing) so two restarted survivors can't exchange PreVotes -> permanent round-N election stall.", - "file": "src/controller (restart/recovery/election path \u2014 exact site unknown, no backtrace)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "Restart-stability bug: a restarted xce_controller becomes unable to participate at chaotic round…\n\nRestart-stability bug: a restarted xce_controller becomes unable to participate at chaotic round 4. Release build SIGSEGVs ('Segmentation fault (core dumped) --myname ctrl-3'); under gdb the SAME corruption manifests as a WEDGED inbound-accept/iomgr path (last accept 03:42:03 while the election timer keeps firing) so two restarted survivors can't exchange PreVotes -> permanent round-N election stall.", + "file": "src/controller (restart/recovery/election path — exact site unknown, no backtrace)", "root_cause": "ROOT-CAUSED via gdb SIGQUIT all-thread bt: an ABBA lock-order deadlock in src/lib async dispatch. io_worker (netio.c:672) holds ep->lock then blocks on q->lock (dataq_enq); vdispatcher dataq_deq (dataq.c:65) holds q->lock then blocks on ep->lock via the vtask_eligible->endpoint_is_task_blocked (netio.c:1001) match callback. io_worker dies -> node stops accepting/answering PreVotes -> round-4 stall. In release the same ep/q race corrupts -> SIGSEGV. See bug-1282 (the actual dispatch defect).", "fix": "The crash/hang IS bug-1282 (UPSTREAM raft-drive-async dispatch deadlock). raft-ha-core-safety 7.2 stays blocked until bug-1282 is fixed upstream, then re-run the gate. No raft-ha-core-safety code change.", "tags": [ @@ -13901,66 +3043,29 @@ "accept-hang", "restart-stability", "segfault", - "chaotic-soak", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "chaotic-soak" ], "related_bugs": [ "bug-1272", "bug-1282", "bug-924" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1279", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T03:33:06.273Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had 's persist flusher jams on `", - "fix": "Changed to `xce_controller`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1280", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T03:57:32.784Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had \"Task 7.2 on ~/tmp\"", - "fix": "Changed to \"Task 7.2/5.4 \u2014 UNIFIED root cause\"", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1281.md" }, { "id": "bug-1282", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-06-30T04:17:08.329Z", - "error_message": "ABBA lock-order deadlock under chaotic churn: io_worker (ep->lock then dataq_enq wants q->lock) vs vdispatcher dataq_deq (q->lock then vtask_eligible->endpoint_is_task_blocked wants ep->lock). Wedges io_worker -> controller stops accepting connections; in release the race corrupts -> SIGSEGV.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "ABBA lock-order deadlock under chaotic churn: io_worker (ep->lock then dataq_enq wants q->lock) vs…\n\nABBA lock-order deadlock under chaotic churn: io_worker (ep->lock then dataq_enq wants q->lock) vs vdispatcher dataq_deq (q->lock then vtask_eligible->endpoint_is_task_blocked wants ep->lock). Wedges io_worker -> controller stops accepting connections; in release the race corrupts -> SIGSEGV.", "file": "src/lib/dataq.c:65 (dataq_deq holds q->lock across match cb) + src/lib/vtaskmgr.c:255 (vtask_eligible) + src/lib/netio.c:1001 (endpoint_is_task_blocked takes ep->lock) + src/lib/netio.c:672 (io_worker holds ep->lock across vtask_enq->dataq_enq)", - "root_cause": "dataq_deq runs its match callback while holding q->lock, and the callback (vtask_eligible->endpoint_is_task_blocked) acquires ep->lock \u2014 inverting the ep->lock->q->lock order io_worker uses. Two-thread ABBA deadlock, deterministic under round-4 chaotic churn. Confirmed via gdb all-thread backtrace on two independently-wedged nodes.", + "root_cause": "dataq_deq runs its match callback while holding q->lock, and the callback (vtask_eligible->endpoint_is_task_blocked) acquires ep->lock — inverting the ep->lock->q->lock order io_worker uses. Two-thread ABBA deadlock, deterministic under round-4 chaotic churn. Confirmed via gdb all-thread backtrace on two independently-wedged nodes.", "fix": "FIXED (commit 25e741d): promoted endpoint_t.busy from a 1-bit bitfield to _Atomic int; endpoint_block_tasks/endpoint_unblock_tasks/endpoint_is_task_blocked now lockless (release/acquire), removing the ep->lock acquire from the vtask-eligibility path that ran under q->lock. Verified: post-fix gdb soak has ZERO __lll_lock_wait threads + no endpoint_is_task_blocked in any lock path; controller_unit_test all green. Mirrors dispatch_inflight (bug-1197). NOTE: a SEPARATE release-only restart segfault (bug-1283) remains and still blocks 7.2.", "tags": [ "deadlock", @@ -13975,65 +3080,30 @@ "async-io", "upstream", "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "md" ], "related_bugs": [ "bug-1281" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1283", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T04:17:58.579Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `xce_controller`", - "fix": "Changed to `src/lib`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "25e741d" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1285", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T08:24:48.860Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `vtask_eligible`", - "fix": "Changed to `endpoint_is_task_blocked`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1282.md" }, { "id": "bug-1286", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-06-30T05:30:00Z", - "error_message": "xce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart \u2014 Thread received SIGSEGV at ha.c:3194 (atomic_load &ha->role) with ha=0x3ff0 (garbage). The release-only crash that blocks raft-ha-core-safety 7.2; DISTINCT from the bug-1282 deadlock (which is fixed).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "xce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart — Thread received…\n\nxce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart — Thread received SIGSEGV at ha.c:3194 (atomic_load &ha->role) with ha=0x3ff0 (garbage). The release-only crash that blocks raft-ha-core-safety 7.2; DISTINCT from the bug-1282 deadlock (which is fixed).", "file": "src/controller/cluster.c:251 (myreq_handler) + src/controller/ha.c:3194 (ha_dispatch_gate)", - "root_cause": "NULL ep->owner. myreq_handler does `cluster_t* c = (cluster_t*)ep->owner;` then `ha_dispatch_gate(&c->ha, opcode)`. On a chaotic restart an inbound endpoint dispatches a walCatchupEntry (op 37) vtask while ep->owner==NULL, so c=NULL and &c->ha = NULL+offsetof(cluster_t,ha) = 0x3ff0 \u2014 a garbage pointer ha_dispatch_gate dereferences (atomic_load &ha->role, ha.c:3194) -> SIGSEGV. Backtrace (debug -O0 -g3 under gdb): ha_dispatch_gate(ha=0x3ff0,opcode=37) <- myreq_handler cluster.c:251 <- vtask_entry cluster.c:693 <- run_task/vcpu_entry. ep->owner is NULL because the accept/dispatch path runs a request before owner is wired (or owner cleared mid-flight during teardown).", - "fix": "FIXED 2026-06-30 (commit 5db734a). NULL-guard `c = ep->owner` in myreq_handler (cluster.c): a chaotic-restart walCatchupEntry (op 37) vtask ran with ep->owner==NULL \u2192 &c->ha==0x3ff0 deref crash at ha.c:3194. Guard returns early when owner unset. Verified: round-4 restart SIGSEGV gone; controller_unit_test green.", + "root_cause": "NULL ep->owner. myreq_handler does `cluster_t* c = (cluster_t*)ep->owner;` then `ha_dispatch_gate(&c->ha, opcode)`. On a chaotic restart an inbound endpoint dispatches a walCatchupEntry (op 37) vtask while ep->owner==NULL, so c=NULL and &c->ha = NULL+offsetof(cluster_t,ha) = 0x3ff0 — a garbage pointer ha_dispatch_gate dereferences (atomic_load &ha->role, ha.c:3194) -> SIGSEGV. Backtrace (debug -O0 -g3 under gdb): ha_dispatch_gate(ha=0x3ff0,opcode=37) <- myreq_handler cluster.c:251 <- vtask_entry cluster.c:693 <- run_task/vcpu_entry. ep->owner is NULL because the accept/dispatch path runs a request before owner is wired (or owner cleared mid-flight during teardown).", + "fix": "FIXED 2026-06-30 (commit 5db734a). NULL-guard `c = ep->owner` in myreq_handler (cluster.c): a chaotic-restart walCatchupEntry (op 37) vtask ran with ep->owner==NULL → &c->ha==0x3ff0 deref crash at ha.c:3194. Guard returns early when owner unset. Verified: round-4 restart SIGSEGV gone; controller_unit_test green.", "tags": [ "segfault", "null-deref", @@ -14043,86 +3113,31 @@ "ha_dispatch_gate", "restart", "7.2", - "ROOT-CAUSED", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ROOT-CAUSED" ], "related_bugs": [ "bug-1282", "bug-1281", "bug-1197" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1290", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T09:15:12.861Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had 't crash) \u2014 STILL blocks the gate; likely an endpo", - "fix": "Changed to 't crash) \u2014 is now **ROOT-CAUSED** (via the new `", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "5db734a" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1292", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T09:33:08.918Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `1+2*njobs`", - "fix": "Changed to `c = ep->owner`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1295", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T13:19:42.706Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: jmax > 0", - "fix": "Added guard clause: if (jmax > 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1286.md" }, { "id": "bug-1296", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-06-30T13:21:00.000Z", - "error_message": "10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP\u2192PreVote\u2192timeout\u2192repeat livelock \u2014 zero pre_vote_reply_in events (not denials, silence). ctrl-3 sent 20+ PreVote rounds with NO reply received; ctrl-1 and ctrl-2 same loop. Rounds 1-6 PASS clean (all elections + job-verification succeeded); round 7 kills ctrl-2 (term=9 primary), restarts ctrl-3 (no convergence wait \u2014 chaotic). 2 surviving nodes (ctrl-1, ctrl-3) both start PreVote; neither processes the other's request \u2192 mutual timeout \u2192 RDHUP fires again \u2192 repeat forever.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP→PreVote→timeout→repeat livelock — zero…\n\n10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP→PreVote→timeout→repeat livelock — zero pre_vote_reply_in events (not denials, silence). ctrl-3 sent 20+ PreVote rounds with NO reply received; ctrl-1 and ctrl-2 same loop. Rounds 1-6 PASS clean (all elections + job-verification succeeded); round 7 kills ctrl-2 (term=9 primary), restarts ctrl-3 (no convergence wait — chaotic). 2 surviving nodes (ctrl-1, ctrl-3) both start PreVote; neither processes the other's request → mutual timeout → RDHUP fires again → repeat forever.", "file": "src/controller/ha.c", - "root_cause": "TASK 5.6 dueling-candidate PreVote cross-denial \u2014 DISTINCT from the earlier PreVote-WINS + real-election split-vote (which had both nodes winning PreVote then deadlocking in RequestVote). Here PreVote itself starves: each node's PreVote handler likely drops/ignores inbound PreVote requests while in its own PreVote round (no reply sent, not even denial). The task 5.6 RDHUP guard (commit 25e741d+) only prevents restarting failover while in CTRL_ELECTING \u2014 it does NOT fix the core problem where two nodes enter simultaneous PreVote rounds and each starves the other's reply. The priority stagger (bug-606) helps when 3 nodes are alive (one grants to break symmetry), but with only 2 survivors (the third restarting without convergence in chaotic mode), both deny each other and the stagger is insufficient.", + "root_cause": "TASK 5.6 dueling-candidate PreVote cross-denial — DISTINCT from the earlier PreVote-WINS + real-election split-vote (which had both nodes winning PreVote then deadlocking in RequestVote). Here PreVote itself starves: each node's PreVote handler likely drops/ignores inbound PreVote requests while in its own PreVote round (no reply sent, not even denial). The task 5.6 RDHUP guard (commit 25e741d+) only prevents restarting failover while in CTRL_ELECTING — it does NOT fix the core problem where two nodes enter simultaneous PreVote rounds and each starves the other's reply. The priority stagger (bug-606) helps when 3 nodes are alive (one grants to break symmetry), but with only 2 survivors (the third restarting without convergence in chaotic mode), both deny each other and the stagger is insufficient.", "fix": "FIXED 2026-06-30 (subsumed by tasks 5.6 commit 81cb09a + 5.7 commit 74784b2). The RDHUP->PreVote->timeout liveloc was the same fixed-watchdog resonance as bug covered by 5.7: per-campaign election watchdog re-randomized to [T,2T] (Ongaro 3.4) + don't-preempt-in-flight-election (5.6). CONFIRMED by a 10-round chaotic soak: 10/10 rounds elect a new primary in a single term increment (4->13), zero term escalation, round 7+ (the old liveloc point) all clean.", "tags": [ "raft-ha-core-safety", @@ -14131,9 +3146,7 @@ "dueling-candidates", "task-5.6", "chaotic-soak", - "round-7-stall", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "round-7-stall" ], "related_bugs": [ "bug-1281", @@ -14141,37 +3154,25 @@ "bug-1286", "bug-1292" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1297", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T13:28:01.170Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: jmax > 0", - "fix": "Added guard clause: if (jmax > 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "81cb09a", + "74784b2" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1296.md" }, { "id": "bug-1298", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-06-30T13:36:00-07:00", - "error_message": "AddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask dispatcher T3) \u2014 endpoint freed by endpoint_killer cluster.c:813 (io_worker T2); ctrl-3 ABORTs mid-chaotic-soak round 3\u21924, dropping the cluster to 1/3 (no quorum) so round 4 'NO new primary within window'", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask…\n\nAddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask dispatcher T3) — endpoint freed by endpoint_killer cluster.c:813 (io_worker T2); ctrl-3 ABORTs mid-chaotic-soak round 3→4, dropping the cluster to 1/3 (no quorum) so round 4 'NO new primary within window'", "file": "src/lib/netio.c / src/lib/vtaskmgr.c / src/controller/cluster.c", - "root_cause": "Teardown-vs-dispatch race: io_worker (T2) tears down a connection on error \u2014 handle_endpoint_error \u2192 zarray_del \u2192 endpoint_killer free()s the endpoint_t \u2014 while the vtask dispatcher (T3) is concurrently scanning the dataq (dataq_deq_first_match \u2192 vtask_eligible \u2192 endpoint_is_task_blocked) and reads ep->busy on that just-freed endpoint. The endpoint is freed while still referenced by a pending vtask in the dataq; nothing drains/guards the ep's queued vtasks before the free, and no refcount keeps it alive across the dispatcher read. Same netio surface as bug-1282 (ABBA, ep->busy made lockless) and the bug-1197 dispatch_inflight precedent; the bug-1282 lockless read removed the (incidental) lock contention so the dispatcher reaches the read faster, exposing the latent UAF.", + "root_cause": "Teardown-vs-dispatch race: io_worker (T2) tears down a connection on error — handle_endpoint_error → zarray_del → endpoint_killer free()s the endpoint_t — while the vtask dispatcher (T3) is concurrently scanning the dataq (dataq_deq_first_match → vtask_eligible → endpoint_is_task_blocked) and reads ep->busy on that just-freed endpoint. The endpoint is freed while still referenced by a pending vtask in the dataq; nothing drains/guards the ep's queued vtasks before the free, and no refcount keeps it alive across the dispatcher read. Same netio surface as bug-1282 (ABBA, ep->busy made lockless) and the bug-1197 dispatch_inflight precedent; the bug-1282 lockless read removed the (incidental) lock contention so the dispatcher reaches the read faster, exposing the latent UAF.", "fix": "FIXED (commit 6e05dc1): added endpoint_t reference count (refcnt _Atomic + heap_alloc flag). endpoint_get starts refcnt=1 (owner); vtask_enq takes a per-vtask ref before publishing the task to the dispatcher, run_task/taskq_item_killer release it; every heap-ep teardown site drops its owner ref via endpoint_unref (last unref fini+frees). The dispatcher can now read ep->busy on a queued vtask without the io_worker freeing it underneath. Embedded eps (heap_alloc=0) never freed by unref. Validated: 182 unit tests/0 fail; 6-round ASAN chaotic soak ZERO ASan reports, ctrl-3 survives round 4, all rounds elect.", "tags": [ "netio", @@ -14184,67 +3185,32 @@ "soak", "quorum", "bug-1282-family", - "FIXED", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "FIXED" ], "related_bugs": [ "bug-1282", "bug-1197", "bug-1286" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1299", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T13:39:09.194Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "6e05dc1" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1304", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T14:24:36.874Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/research.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1298.md" }, { "id": "bug-1306", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-06-30T14:31:00-07:00", - "error_message": "6-round chaotic-failover soak FAIL: committed_job_loss=4 committed_divergence=7 (rounds 4-6). Committed job 0-0-1782829181-1 permanently absent on reachable primary from round 4 on, across 3 elected primaries; rounds 1-3 clean.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "6-round chaotic-failover soak FAIL: committed_job_loss=4 committed_divergence=7 (rounds 4-6)\n\n6-round chaotic-failover soak FAIL: committed_job_loss=4 committed_divergence=7 (rounds 4-6). Committed job 0-0-1782829181-1 permanently absent on reachable primary from round 4 on, across 3 elected primaries; rounds 1-3 clean.", "file": "src/lib/hook.c (xce_run_hooks); job.c:603 gate is correct", "root_cause": "SHARPENED 2026-06-30 (waldump+log bisect of the 10-round soak): NOT the catch-up prefix-gap/entity-snapshot mechanism. xce_run_hooks (src/lib/hook.c:134-157) returns only the LAST-registered hook's rc (rc overwritten each loop iter; break is BEFORE_EVENT-only). The durable-majority gate works: a term-9 leader's job at LSN 30 timed out commit (peers mid-restart, repl_wait_majority_ack -> commit_submit -> -ETIMEDOUT), persist_job_hook_fun returned -ETIMEDOUT, but a later-registered JOB AFTER_EVENT hook (sched/apiserver) returned 0 and CLOBBERED it -> handle_jobsubmit saw hook_rc==0 -> replied 'jobsubmit ok' for an uncommitted entry -> term-10 leader legally overwrote LSN 30 (term-10 NOOP) -> committed-job-loss. Violates the Commit Durability invariant (reply-success => majority-durable). Evidence: all 3 WALs gapless/CRC-intact/full payloads; lost job only on ctrl-1@LSN30 term9 vs term10 NOOP on majority; ctrl-1 log has the WAIT-deadline-expired WARNs and NO 'rejecting job' line.", - "fix": "FIXED 2026-06-30 (raft-apply-decoupling item #1 down payment). Extracted the WAL append + majority-commit barrier out of persist_job_hook_fun's WAIT branch into an explicit persist_submit_wal_and_commit(p, jlist); handle_jobsubmit now calls it directly and reads the commit rc DIRECTLY (rejects on <0 before notifying), so the -ETIMEDOUT can no longer be clobbered by a later notify hook's 0 in xce_run_hooks. The JOB_NEW AFTER hook is now notification-only. PROVEN: 10-round chaotic soak committed_job_loss 5 -> 0 (same election code, A/B isolates the durability fix). Unit: +test_job_hook_new_is_notify_only; persist_hooks 19/0, e2e_hooks 3/0, commit_barrier_reply 22/0, commit_durability 38/0. Remaining soak FAIL is committed_divergence (transient catch-up lag, present=2) + a separate round-4 election escalation tail (5.7 class) \u2014 neither is job loss.", + "fix": "FIXED 2026-06-30 (raft-apply-decoupling item #1 down payment). Extracted the WAL append + majority-commit barrier out of persist_job_hook_fun's WAIT branch into an explicit persist_submit_wal_and_commit(p, jlist); handle_jobsubmit now calls it directly and reads the commit rc DIRECTLY (rejects on <0 before notifying), so the -ETIMEDOUT can no longer be clobbered by a later notify hook's 0 in xce_run_hooks. The JOB_NEW AFTER hook is now notification-only. PROVEN: 10-round chaotic soak committed_job_loss 5 -> 0 (same election code, A/B isolates the durability fix). Unit: +test_job_hook_new_is_notify_only; persist_hooks 19/0, e2e_hooks 3/0, commit_barrier_reply 22/0, commit_durability 38/0. Remaining soak FAIL is committed_divergence (transient catch-up lag, present=2) + a separate round-4 election escalation tail (5.7 class) — neither is job loss.", "tags": [ "ha-commit-durability", "pre-existing", @@ -14258,9 +3224,7 @@ "failover", "hook-rc-clobber", "ack-before-majority", - "data-loss", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "data-loss" ], "related_bugs": [ "bug-795", @@ -14268,18 +3232,23 @@ "bug-851", "bug-1298" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1306.md" }, { "id": "bug-1307", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "data-loss", "timestamp": "2026-06-30T15:05:00.000Z", - "error_message": "Task 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic soak shows 7 committed divergences (LSNs 8,12,16,19) + 4 committed job losses (job 0-0-1782831451-1). Previous run (pre-5.7, commit 81cb09a) had ZERO divergences and ZERO job losses through 6 rounds. LSN 19 divergence is present=3 (ALL 3 members have different content at same LSN \u2014 silent divergence, textbook bug-851). Election liveness improved (all 6 rounds complete vs stalled at round 7), but the jitter defeats the election-safety balance: faster-starting node wins before catching up on committed prefix \u2192 Leader Completeness violation.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Task 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic…\n\nTask 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic soak shows 7 committed divergences (LSNs 8,12,16,19) + 4 committed job losses (job 0-0-1782831451-1). Previous run (pre-5.7, commit 81cb09a) had ZERO divergences and ZERO job losses through 6 rounds. LSN 19 divergence is present=3 (ALL 3 members have different content at same LSN — silent divergence, textbook bug-851). Election liveness improved (all 6 rounds complete vs stalled at round 7), but the jitter defeats the election-safety balance: faster-starting node wins before catching up on committed prefix → Leader Completeness violation.", "file": "src/controller/ha.c", - "root_cause": "Task 5.7 jittered watchdog ([T,2T] randomized) lets a less-complete node start and win an election before catching up on the full committed prefix from the dead primary. With the original fixed watchdog, both survivors fired RDHUP simultaneously \u2192 cross-denied \u2192 the library's priority stagger resolved the tie safely (the fresher node won after deterministic desynchronization). With jitter, one node can fire much earlier (near T) and the other much later (near 2T) \u2192 the early node campaigns with an incomplete log \u2192 voters grant based on stale view \u2192 winner truncates committed entries it never held. The 5.3 in_catch_up release (which drops the vote-deny when the primary is dead) opens a window where a still-catching-up candidate can win \u2014 with the new jitter timings this window is hit frequently.", - "fix": "CONFIRMED FIXED via live re-soak (2026-07-03): ran the exact reproducing scenario \u2014 6-round chaotic soak (overlapping kill-the-primary, real ext4 at ~/tmp, ASAN controller binary) \u2014 that originally showed 7 committed divergences + 4 job losses. This run: 21 jobs submitted, 0 committed_divergence, 0 committed_job_loss, PASS across all 6 rounds (terms 3->9, all 3 nodes rotated through primary). Matches the earlier circumstantial evidence (raft-ha-core-safety Phase 5 fully complete + prior soak-clean sessions) with a fresh, direct confirmation run \u2014 no residual doubt.", + "root_cause": "Task 5.7 jittered watchdog ([T,2T] randomized) lets a less-complete node start and win an election before catching up on the full committed prefix from the dead primary. With the original fixed watchdog, both survivors fired RDHUP simultaneously → cross-denied → the library's priority stagger resolved the tie safely (the fresher node won after deterministic desynchronization). With jitter, one node can fire much earlier (near T) and the other much later (near 2T) → the early node campaigns with an incomplete log → voters grant based on stale view → winner truncates committed entries it never held. The 5.3 in_catch_up release (which drops the vote-deny when the primary is dead) opens a window where a still-catching-up candidate can win — with the new jitter timings this window is hit frequently.", + "fix": "CONFIRMED FIXED via live re-soak (2026-07-03): ran the exact reproducing scenario — 6-round chaotic soak (overlapping kill-the-primary, real ext4 at ~/tmp, ASAN controller binary) — that originally showed 7 committed divergences + 4 job losses. This run: 21 jobs submitted, 0 committed_divergence, 0 committed_job_loss, PASS across all 6 rounds (terms 3->9, all 3 nodes rotated through primary). Matches the earlier circumstantial evidence (raft-ha-core-safety Phase 5 fully complete + prior soak-clean sessions) with a fresh, direct confirmation run — no residual doubt.", "tags": [ "data-safety-regression", "task-5.7", @@ -14289,8 +3258,7 @@ "committed-job-loss", "election-liveness", "bug-1296-followup", - "committed-divergence", - "fixed" + "committed-divergence" ], "related_bugs": [ "bug-1296", @@ -14299,497 +3267,20 @@ "bug-1286", "bug-851" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1308", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T21:22:32.991Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `research.md`", - "fix": "Changed to 't-preempt-in-flight-election; reduced term churn)", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1310", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-06-30T18:40:00Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: p == NULL || jlist == NULL", - "fix": "Added guard clause: if (p == NULL || jlist == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "election-liveness", - "rdhup", - "term-escalation", - "debounce", - "5.10b", - "raft", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [ - "bug-1296", - "bug-1306" - ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1312", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T00:29:56.872Z", - "error_message": "Wrong reference: test_job_hook_start_writes_wal_entry should be test_job_hook_new_is_notify_only", - "file": "src/controller/testsuite/test_persist_hooks.c", - "root_cause": "Used \"test_job_hook_start_writes_wal_entry\" instead of \"test_job_hook_new_is_notify_only\"", - "fix": "Changed test_job_hook_start_writes_wal_entry \u2192 test_job_hook_new_is_notify_only", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1313", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T00:42:13.591Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1315", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T01:26:38.341Z", - "error_message": "Wrong reference: unistd should be time", - "file": "src/controller/testsuite/test_ha_rdhup_integration.c", - "root_cause": "Used \"unistd\" instead of \"time\"", - "fix": "Changed unistd \u2192 time", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1319", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T04:04:13.051Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-entity-snapshots/design.md", - "root_cause": "Had \"collect+encode jobs\\n(RTTI_LOCK_READ on jobmgr)\"", - "fix": "Changed to \"collect+encode jobs\\n(table READ lock + per-job R", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1320", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T04:47:19.662Z", - "error_message": "Wrong reference: test_ha_partition should be test_persist_snapshot_entities", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_ha_partition\" instead of \"test_persist_snapshot_entities\"", - "fix": "Changed test_ha_partition \u2192 test_persist_snapshot_entities", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1326", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T06:17:52.049Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_persist_snapshot_entities.c", - "root_cause": "No early return/throw for edge case: fd < 0", - "fix": "Added guard clause: if (fd < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1327", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T06:18:39.877Z", - "error_message": "Wrong reference: pthread should be fcntl", - "file": "src/controller/testsuite/test_persist_snapshot_entities.c", - "root_cause": "Used \"pthread\" instead of \"fcntl\"", - "fix": "Changed pthread \u2192 fcntl", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1328", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T06:33:25.117Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_persist_snapshot_entities.c", - "root_cause": "No early return/throw for edge case: fd < 0", - "fix": "Added guard clause: if (fd < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1330", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T07:24:22.541Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "No early return/throw for edge case: cleared == 0", - "fix": "Added guard clause: if (cleared == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1331", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T07:46:42.046Z", - "error_message": "Wrong return value", - "file": "src/controller/persist.c", - "root_cause": "Was returning: load_snapshot(p, path, lsn);", - "fix": "Now returns: rc;", - "tags": [ - "auto-detected", - "return-value", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1340", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T09:55:17.967Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p || checkpoint_lsn == 0", - "fix": "Added guard clause: if (!p || checkpoint_lsn == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1343", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T13:27:42.519Z", - "error_message": "Missing error handling in every", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 6.1 integration: leader->standby replication carries; * payload) faithfully end-to-end across BOTH live in-term re", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1348", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T14:10:48.987Z", - "error_message": "Wrong reference: test_pre_vote_catch_up_denies should be test_pre_vote_catch_up_grants_when_primary_unreachable", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "Used \"test_pre_vote_catch_up_denies\" instead of \"test_pre_vote_catch_up_grants_when_primary_unreachable\"", - "fix": "Changed test_pre_vote_catch_up_denies \u2192 test_pre_vote_catch_up_grants_when_primary_unreachable", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1349", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-01T14:24:21.310Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (atomic_load_explicit(&ha->in_catch_up, memory_orde)", - "fix": "Changed to: if (0 && atomic_load_explicit(&ha->in_catch_up, memory)", - "tags": [ - "auto-detected", - "logic-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1355", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T03:03:21.985Z", - "error_message": "Wrong reference: myconf_t should be uint32_t", - "file": "src/include/myconf.h", - "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t \u2192 uint32_t", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1357", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T03:03:51.842Z", - "error_message": "Function not marked async", - "file": "src/controller/main.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1362", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T03:50:14.650Z", - "error_message": "Wrong reference: persist_config_t should be uint32_t", - "file": "src/include/persist.h", - "root_cause": "Used \"persist_config_t\" instead of \"uint32_t\"", - "fix": "Changed persist_config_t \u2192 uint32_t", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1363", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T03:50:42.583Z", - "error_message": "Wrong reference: WAL_SEGMENT_SIZE_BYTES should be p", - "file": "src/controller/persist.c", - "root_cause": "Used \"WAL_SEGMENT_SIZE_BYTES\" instead of \"p\"", - "fix": "Changed WAL_SEGMENT_SIZE_BYTES \u2192 p", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1364", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T03:52:09.933Z", - "error_message": "Wrong reference: myconf_t should be uint32_t", - "file": "src/include/myconf.h", - "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t \u2192 uint32_t", - "tags": [ - "wrong-reference", - "h", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1369", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T03:57:52.302Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Had \"MODE=force-snapshot \u2014 forcing one full-checkpoint", - "fix": "Changed to \"MODE=force-snapshot \u2014 forcing one full-checkpoint", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1370", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T04:10:10.430Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1372", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T04:39:29.527Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: fd < 0", - "fix": "Added guard clause: if (fd < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1307.md" }, { "id": "bug-1376", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-07-02T05:10:00.000Z", - "error_message": "raft-drive-async: cluster hangs silently (no crash, no ASAN report) during a --force-snapshot soak with g_drive_async_snapshot_offload + g_drive_async_peerctrl_dispatch_enabled both ON: a freshly-wiped empty node cold-starts, self-elects, then goes silent after 'peer_ctrl_dispatch_to_pool: vtask pool not wired -- running inline' is immediately followed by cluster_init_phase2 wiring the pool while raft_cb_become_follower's deferred-teardown vtasks (bug-1197's mechanism) are being enqueued -- the process never logs another line and is eventually killed by the test harness timeout.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-drive-async: cluster hangs silently (no crash, no ASAN report) during a --force-snapshot soak…\n\nraft-drive-async: cluster hangs silently (no crash, no ASAN report) during a --force-snapshot soak with g_drive_async_snapshot_offload + g_drive_async_peerctrl_dispatch_enabled both ON: a freshly-wiped empty node cold-starts, self-elects, then goes silent after 'peer_ctrl_dispatch_to_pool: vtask pool not wired -- running inline' is immediately followed by cluster_init_phase2 wiring the pool while raft_cb_become_follower's deferred-teardown vtasks (bug-1197's mechanism) are being enqueued -- the process never logs another line and is eventually killed by the test harness timeout.", "file": "src/controller/ha.c", "root_cause": "peer_ctrl_msg_handler (ha.c) held ep->dispatch_inflight incremented across the ENTIRE peer_ctrl_dispatch_to_pool call, including its inline fallback (the common cold-start shape when ha->pool is not yet wired). That inline body can reach raft_cb_become_follower, which -- now that cluster_init_phase2 ran synchronously earlier in the SAME call chain and wired the pool -- tears down its own loser vote-ep (the very ep whose reply triggered this call) via the deferred path (vote_ep_teardown_async), whose endpoint_mark_tearing_down spins waiting for dispatch_inflight==0 on that SAME ep. That count is held only by this exact call, on this exact thread, and cannot reach 0 until the spin itself returns -- a permanent self-deadlock. Confirmed live via a gdb thread backtrace (GDB_WRAP debug seam added to run-ha-soak.sh, launching the target node directly under `gdb --batch` so it becomes ptrace-legal as gdb's own direct child, then SIGINT to gdb once the log went stale for a bounded interrupt + full backtrace): frame `peer_ctrl_dispatch_to_pool` showed pool=0x0 at dispatch time; the same ep pointer appeared in the nested vote_ep_teardown_async -> endpoint_mark_tearing_down frame, spinning on sched_yield().", "fix": "Moved the ep->dispatch_inflight release into peer_ctrl_dispatch_to_pool itself, at the exact right point per exit path: released BEFORE calling the inline body on every fallback path (pool==NULL, OOM, enqueue failure) -- safe because io_worker is single-threaded and the inline path never enqueues anything, so there is no pool-worker race left to protect against once inline execution is committed to -- and only AFTER the (non-blocking) enqueue completes on the success path, preserving the original protection for the genuinely concurrent pool-worker case. New unit test test_vep_self_teardown_does_not_deadlock (test_ha_vep_teardown.c) uses a bounded-timeout pthread_timedjoin_np so a regression fails loud instead of hanging the suite; RED confirmed via a temporary manual revert (Expected 0 Was 110 == ETIMEDOUT) before restoring the fix. Live-verified with both g_drive_async_snapshot_offload and g_drive_async_peerctrl_dispatch_enabled flipped ON: --force-snapshot soak (the exact repro shape) completes without hanging across repeated runs, confirmed real snapshot installs, committed_divergence=0, committed_job_loss=0; a 6-round --chaotic soak with both flags ON also passed clean; full controller suite green throughout. Both flags now default ON in production (raft-drive-async tasks 3.1/3.2 closed).", @@ -14797,43 +3288,27 @@ "raft-drive-async", "cold-start", "vtask-pool", - "self-deadlock", - "fixed" + "self-deadlock" ], "related_bugs": [ "bug-1197", "bug-1298", "bug-1310" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1380", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T07:16:35.420Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `, differential-verified: this specific soak scena", - "fix": "Changed to `, differential-verified). (4) Re-ran `", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1376.md" }, { "id": "bug-1381", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-02T00:35:00.000Z", - "error_message": "raft-drive-async task 3.3 (config-push slow-lane offload) blocked: push_config_transport_only has a pre-existing endpoint-lifetime race, independent of the g_drive_async_configpush_offload flag -- it grabs raw endpoint_t* pointers for every registered standby under c->ha.lock, releases the lock, then spawns one pthread per standby (push_thread) that reads/writes those endpoints unprotected for up to CONF_PUSH_TIMEOUT_MS. If a standby RDHUP-disconnects during that window, the io_worker's teardown path (endpoint_killer -> endpoint_unref, the bug-1298 refcount system) can free the endpoint while a push_thread is still using it -- a UAF. This is NOT specific to the lane offload: the transport fan-out spawns its own independent OS threads regardless of which caller (inline drive-thread or lane worker) invokes it, so the documented 'runs inline on io_worker so teardown and push are serialized on one thread' rationale for keeping the flag OFF does not actually hold -- the race exists in the inline path today too.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-drive-async task 3.3 (config-push slow-lane offload) blocked: push_config_transport_only has…\n\nraft-drive-async task 3.3 (config-push slow-lane offload) blocked: push_config_transport_only has a pre-existing endpoint-lifetime race, independent of the g_drive_async_configpush_offload flag -- it grabs raw endpoint_t* pointers for every registered standby under c->ha.lock, releases the lock, then spawns one pthread per standby (push_thread) that reads/writes those endpoints unprotected for up to CONF_PUSH_TIMEOUT_MS. If a standby RDHUP-disconnects during that window, the io_worker's teardown path (endpoint_killer -> endpoint_unref, the bug-1298 refcount system) can free the endpoint while a push_thread is still using it -- a UAF. This is NOT specific to the lane offload: the transport fan-out spawns its own independent OS threads regardless of which caller (inline drive-thread or lane worker) invokes it, so the documented 'runs inline on io_worker so teardown and push are serialized on one thread' rationale for keeping the flag OFF does not actually hold -- the race exists in the inline path today too.", "file": "src/controller/config_repl.c", "root_cause": "push_config_transport_only (config_repl.c ~line 1366) takes c->ha.lock only long enough to copy standby ep pointers into a push_ctx_t array, then releases the lock before pthread_create-ing one push_thread per standby. Those threads read/write the raw endpoint_t* for the full duration of the send + wait_all_with_timeout(CONF_PUSH_TIMEOUT_MS), with no ref taken on the endpoints and no coordination with concurrent teardown (handle_endpoint_error -> zarray_del -> endpoint_killer -> endpoint_unref). Unlike the snapshot-lane fix (bug-1197/bug-1376 session), no endpoint_ref/endpoint_unref bracket protects this fan-out.", "fix": "FIXED. push_config_transport_only (config_repl.c) now takes endpoint_ref() on each standby ep while the pointer is snapshotted under c->ha.lock, mirroring the snapshot-lane fix (bug-1197/bug-1376). Every ref has exactly one matching endpoint_unref(): from push_thread after it finishes using ctx->ep (new close in push_thread itself), or from push_config_transport_only inline if pthread_create fails for that standby, or from a cleanup loop if the threads[] allocation fails entirely. A concurrent RDHUP teardown (endpoint_killer -> endpoint_unref) can now only drop the OWNER reference while a push is in flight -- it can no longer free ep out from under push_thread. New regression test test_configpush_transport_holds_endpoint_ref_across_send (test_config_repl.c) uses a new push_thread_send_hook test seam (same pattern as xce_wal_stream_request_hook) to park a push_thread mid-send, simulate the owner teardown, and assert ep survives with refcnt intact. RED->GREEN verified via temporary revert (Expected 2 Was 1 with the ref-take removed); full controller suite green (230/230) after restore.", @@ -14841,197 +3316,87 @@ "raft-drive-async", "configpush", "endpoint-lifetime", - "uaf-suspect", - "fixed" + "uaf-suspect" ], "related_bugs": [ "bug-1197", "bug-1298", "bug-1376" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1382", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T07:23:50.910Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `snapshot_lane_fn`", - "fix": "Changed to `configpush_offload`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1383", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T07:31:28.737Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: waited >= timeout_ms", - "fix": "Added guard clause: if (waited >= timeout_ms)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1384", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T07:37:33.427Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had ` (test_ha_liveness.c). Blocked on **bug-1381** (o", - "fix": "Changed to ` (test_ha_liveness.c). **bug-1381 CLOSED 2026-07-", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1381.md" }, { "id": "bug-1386", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-02T08:15:00.000Z", - "error_message": "/xce-test-ha --force-snapshot round: after a wiped ctrl-2 rejoins empty and the primary sends a full-checkpoint snapshot, ctrl-2 sometimes never logs a completed install (\"HA: snapshot loaded/legacy\" line absent) though the cluster still reconverges to 3-of-3 -- OR the cluster fails to reconverge to 3-of-3 at all. Reproduced with g_drive_async_configpush_offload BOTH ON (0/3 pass) and OFF (1/2 pass) on the same freshly-built xce.build binary, so it is independent of that flag -- discovered while validating the bug-1381 configpush flip.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "/xce-test-ha --force-snapshot round: after a wiped ctrl-2 rejoins empty and the primary sends a…\n\n/xce-test-ha --force-snapshot round: after a wiped ctrl-2 rejoins empty and the primary sends a full-checkpoint snapshot, ctrl-2 sometimes never logs a completed install (\"HA: snapshot loaded/legacy\" line absent) though the cluster still reconverges to 3-of-3 -- OR the cluster fails to reconverge to 3-of-3 at all. Reproduced with g_drive_async_configpush_offload BOTH ON (0/3 pass) and OFF (1/2 pass) on the same freshly-built xce.build binary, so it is independent of that flag -- discovered while validating the bug-1381 configpush flip.", "file": "src/controller/config_repl.c", "root_cause": "ha_send_snapshot_to_standby (config_repl.c, ~line 2596) loops over up to 12 65536-byte chunks calling xce_snapshot_chunk_req_raw(ep, &chunk, buf, chunk_len) WITHOUT checking or propagating its return value. xce_snapshot_chunk_req_raw -> endpoint_enq can return -ECONNRESET (e.g. if the standby endpoint is mid-teardown or reconnecting during the empty-rejoin window) but the loop keeps incrementing offset/chunk_index regardless, and the function still logs \"transfer complete\" and returns rc=0 at the end. Confirmed via ctrl-2 log evidence: primary logs start+complete for a 735546-byte (12-chunk) snapshot within ~2ms, but ctrl-2 only ever receives a handful of leading chunks (2 in one run, 5 in another) before the stream silently stops -- no ERROR on either side.", - "fix": "FIXED 2026-07-02 (config_repl.c): ha_send_snapshot_to_standby's chunk loop now captures the return value of xce_snapshot_chunk_req_raw (production path) and xce_snapshot_chunk_hook (XCE_TESTING path) into send_rc, breaks the loop on send_rc != 0, and propagates it as the function's rc instead of silently continuing. The 'transfer complete' INFO log is now strictly gated on rc==0; a failure path now logs a WARN with checkpoint_lsn/total_size/sent-offset/chunk_index/rc for observability. No receive-side timeout/re-request mechanism was added (out of scope, no existing seam) \u2014 the existing per-peer snapshot_inflight[] flag already clears unconditionally on send-thread exit (snapshot_lane_fn, ha.c) and the next heartbeat's SEND_SNAPSHOT decision naturally re-triggers a fresh transfer to a peer whose applied_lsn never advanced, so the existing retry cadence recovers once the endpoint is healthy again. New RED->GREEN unit test test_snapshot_transfer_stops_and_propagates_send_failure (test_config_repl.c) injects a mid-loop -ECONNRESET via the existing xce_snapshot_chunk_hook seam (new snaptest_fail_after_hook) and asserts rc == -ECONNRESET and exactly 2 chunks attempted (not the false-'complete' 3). RED confirmed via temporary revert ('Expected -104 Was 0'); GREEN restored, full controller_unit_test suite clean (0 failures across all groups).", + "fix": "FIXED 2026-07-02 (config_repl.c): ha_send_snapshot_to_standby's chunk loop now captures the return value of xce_snapshot_chunk_req_raw (production path) and xce_snapshot_chunk_hook (XCE_TESTING path) into send_rc, breaks the loop on send_rc != 0, and propagates it as the function's rc instead of silently continuing. The 'transfer complete' INFO log is now strictly gated on rc==0; a failure path now logs a WARN with checkpoint_lsn/total_size/sent-offset/chunk_index/rc for observability. No receive-side timeout/re-request mechanism was added (out of scope, no existing seam) — the existing per-peer snapshot_inflight[] flag already clears unconditionally on send-thread exit (snapshot_lane_fn, ha.c) and the next heartbeat's SEND_SNAPSHOT decision naturally re-triggers a fresh transfer to a peer whose applied_lsn never advanced, so the existing retry cadence recovers once the endpoint is healthy again. New RED->GREEN unit test test_snapshot_transfer_stops_and_propagates_send_failure (test_config_repl.c) injects a mid-loop -ECONNRESET via the existing xce_snapshot_chunk_hook seam (new snaptest_fail_after_hook) and asserts rc == -ECONNRESET and exactly 2 chunks attempted (not the false-'complete' 3). RED confirmed via temporary revert ('Expected -104 Was 0'); GREEN restored, full controller_unit_test suite clean (0 failures across all groups).", "tags": [ "raft-drive-async", "snapshot-transfer", "silent-failure", - "force-snapshot-soak", - "fixed" + "force-snapshot-soak" ], "related_bugs": [ "bug-1197", "bug-1376", "bug-1381" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1388", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T08:19:04.015Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: g_snaptest_chunk_count >= 16", - "fix": "Added guard clause: if (g_snaptest_chunk_count >= 16)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1386.md" }, { "id": "bug-1389", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-07-02T09:05:00.000Z", - "error_message": "/xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on true completion), the force-snapshot soak still FAILs. ctrl-2's log shows all 12 chunks of a 735546-byte snapshot received, then `ha_handle_snapshot_chunk: persist_load_snapshot failed rc=-5` / `handle_snapshot_transfer: chunk 11 rc=-5` (rc=-5=-EIO). ctrl-1 (sender) correctly logs 'transfer complete' both times \u2014 the send genuinely succeeded, full byte count. This is NOT bug-1386 (that was a chunk DROP producing a truncated/short transfer). Byte-comparing the primary's source checkpoint file against the exact bytes ctrl-2 assembled into snap.part -> snap-{lsn}.bin (same run, two separate checkpoint LSNs 0x75 and 0x76, both reproduced) shows: identical total size (735546==735546) but DIFFERENT CONTENT \u2014 the received file diverges from the sender's file starting exactly at a SNAPSHOT_CHUNK_SIZE (65536-byte) chunk boundary, then the content is a byte-SHIFTED copy of the correct data (same repeating job-frame strings, same occurrence count, just offset) that re-synchronizes with the source at a later offset. The shift direction/magnitude differs between the two independent attempts (-912 bytes resyncing at 262144 vs +4684 bytes resyncing at ~458752) \u2014 nondeterministic, not a fixed structural bug.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "/xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on…\n\n/xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on true completion), the force-snapshot soak still FAILs. ctrl-2's log shows all 12 chunks of a 735546-byte snapshot received, then `ha_handle_snapshot_chunk: persist_load_snapshot failed rc=-5` / `handle_snapshot_transfer: chunk 11 rc=-5` (rc=-5=-EIO). ctrl-1 (sender) correctly logs 'transfer complete' both times — the send genuinely succeeded, full byte count. This is NOT bug-1386 (that was a chunk DROP producing a truncated/short transfer). Byte-comparing the primary's source checkpoint file against the exact bytes ctrl-2 assembled into snap.part -> snap-{lsn}.bin (same run, two separate checkpoint LSNs 0x75 and 0x76, both reproduced) shows: identical total size (735546==735546) but DIFFERENT CONTENT — the received file diverges from the sender's file starting exactly at a SNAPSHOT_CHUNK_SIZE (65536-byte) chunk boundary, then the content is a byte-SHIFTED copy of the correct data (same repeating job-frame strings, same occurrence count, just offset) that re-synchronizes with the source at a later offset. The shift direction/magnitude differs between the two independent attempts (-912 bytes resyncing at 262144 vs +4684 bytes resyncing at ~458752) — nondeterministic, not a fixed structural bug.", "file": "src/lib/request.c (suspected: xce_snapshot_chunk_encode_raw capnp Data encode) or src/controller/ha.c (ha_handle_snapshot_chunk per-chunk append)", - "root_cause": "NOT YET PINNED to an exact line \u2014 root cause is a chunk-CONTENT-corruption bug in the OPCode_snapshotTransfer chunk pipeline, most likely in the capnp raw-Data encode/decode path for the 64KB `payload :Data` field (xce_snapshot_chunk_encode_raw, request.c:630-671, uses the same capn_new_list8/capn_setv8 pattern as bug-847 \u2014 'WAL Catch-Up Streaming Silently Corrupts Payloads at capnp Segment Boundary' \u2014 a proven prior case of this exact capnp pattern corrupting raw-byte payloads at a segment boundary). Ruled out as NOT the cause: (1) receiver-side concurrent chunk processing \u2014 vtaskmgr's per-endpoint FIFO (endpoint_block_tasks/endpoint_unblock_tasks) guarantees chunk N+1 cannot dispatch until chunk N's ha_handle_snapshot_chunk (including its fopen/fwrite/fclose) fully returns; (2) sender buf-reuse race \u2014 xce_snapshot_chunk_encode_raw copies bytes into the capnp segment synchronously before ha_send_snapshot_to_standby's loop re-fread()s the shared buf; (3) fixed-size packet-pool truncation \u2014 alloc_packet (netio.c:416-432) mallocs fresh per packet, sized to capn_size(&c), no shared buffer to overflow into. persist_load_snapshot (persist.c:2440 -> load_snapshot at 2325 -> load_snapshot_entities at 2222) is NOT the bug \u2014 it correctly rejects the corrupted file via one of its length-sanity -EIO guards (most likely persist.c:2240-2241, the job-frame payload_len-vs-file_size guard); format_version/job_count field-order misparse was considered and ruled out (load_snapshot consumes and validates format_version at persist.c:2399-2413 before ever calling load_snapshot_entities).", - "fix": "FIXED 2026-07-02 (ha.c, ha.h). Root cause pinned via live crc32 instrumentation (TRACE logs at both ha_send_snapshot_to_standby's send point and ha_handle_snapshot_chunk's receive point, correlated by snapshot_lsn+chunk_index): every chunk's CONTENT crossed the wire byte-for-byte intact (crc32 matched exactly for all 12 chunks) \u2014 the corruption was never in capnp encode/decode/transport at all. The real bug: OPCode_snapshotTransfer chunks of one transfer are dispatched across the vtask pool and are NOT guaranteed to arrive/execute in chunk_index order (live soak log showed arrival order 0,1,3,4,2,5,6,7,8,9,10,11 across cpu#0-3 workers), but ha_handle_snapshot_chunk appended every chunk's bytes to snap.part in ARRIVAL order (fopen \"ab\"/fwrite) instead of at the chunk's own chunk_offset (a field already carried on the wire but never used for anything). Reordered chunks landed at the wrong file position: same total byte count, wrong content from the first reordered chunk's offset onward \u2014 exactly the observed 'byte-shifted, resyncing later, nondeterministic magnitude' signature. Fix: (1) write each chunk via pwrite(fd, payload, len, chunk->chunk_offset) instead of append \u2014 order-independent and safe for concurrent writers (positioned writes to disjoint byte ranges of the same fd don't race). (2) Track completion via a new ha_state_t.snapshot_recv_count/snapshot_recv_total pair (guarded by the existing ha->lock) instead of finalizing on 'this call happens to carry is_last' \u2014 under out-of-order arrival, is_last can be processed before an earlier chunk; finalize now fires only once every chunk 0..N-1 has actually been written, and snapshot_recv_total is claimed (reset to 0) by whichever thread crosses the completion threshold so a late straggler chunk can never double-finalize. chunk_index==0 still resets both the stale-file unlink and the new counters (Req 4.4 unchanged). New RED->GREEN unit test test_snapshot_recv_out_of_order_chunks_assemble_correctly (test_config_repl.c) replays the exact non-sequential delivery order and asserts byte-identical assembly; RED confirmed via temporary revert (Expected 0 Was -22, job_count field corrupted by the misordered chunk boundary); GREEN restored, full controller suite clean (0 failures). Also added 2 new deterministic full-size (65536-byte) capnp encode/decode round-trip tests (test_wal_wire.c) that definitively cleared the capnp/transport layer before the real cause was found \u2014 permanent regression coverage, the prior test only covered a 300-byte payload. New xce_test snapverify/snapdiff subcommands (built as part of this investigation) were instrumental in diagnosing this live. Live-validated: 3/3 --force-snapshot rounds PASS (genuine 'HA: snapshot loaded ... full job/queue entity restoration' installs confirmed, zero divergence/loss) + 6-round --chaotic PASS.", + "root_cause": "NOT YET PINNED to an exact line — root cause is a chunk-CONTENT-corruption bug in the OPCode_snapshotTransfer chunk pipeline, most likely in the capnp raw-Data encode/decode path for the 64KB `payload :Data` field (xce_snapshot_chunk_encode_raw, request.c:630-671, uses the same capn_new_list8/capn_setv8 pattern as bug-847 — 'WAL Catch-Up Streaming Silently Corrupts Payloads at capnp Segment Boundary' — a proven prior case of this exact capnp pattern corrupting raw-byte payloads at a segment boundary). Ruled out as NOT the cause: (1) receiver-side concurrent chunk processing — vtaskmgr's per-endpoint FIFO (endpoint_block_tasks/endpoint_unblock_tasks) guarantees chunk N+1 cannot dispatch until chunk N's ha_handle_snapshot_chunk (including its fopen/fwrite/fclose) fully returns; (2) sender buf-reuse race — xce_snapshot_chunk_encode_raw copies bytes into the capnp segment synchronously before ha_send_snapshot_to_standby's loop re-fread()s the shared buf; (3) fixed-size packet-pool truncation — alloc_packet (netio.c:416-432) mallocs fresh per packet, sized to capn_size(&c), no shared buffer to overflow into. persist_load_snapshot (persist.c:2440 -> load_snapshot at 2325 -> load_snapshot_entities at 2222) is NOT the bug — it correctly rejects the corrupted file via one of its length-sanity -EIO guards (most likely persist.c:2240-2241, the job-frame payload_len-vs-file_size guard); format_version/job_count field-order misparse was considered and ruled out (load_snapshot consumes and validates format_version at persist.c:2399-2413 before ever calling load_snapshot_entities).", + "fix": "FIXED 2026-07-02 (ha.c, ha.h). Root cause pinned via live crc32 instrumentation (TRACE logs at both ha_send_snapshot_to_standby's send point and ha_handle_snapshot_chunk's receive point, correlated by snapshot_lsn+chunk_index): every chunk's CONTENT crossed the wire byte-for-byte intact (crc32 matched exactly for all 12 chunks) — the corruption was never in capnp encode/decode/transport at all. The real bug: OPCode_snapshotTransfer chunks of one transfer are dispatched across the vtask pool and are NOT guaranteed to arrive/execute in chunk_index order (live soak log showed arrival order 0,1,3,4,2,5,6,7,8,9,10,11 across cpu#0-3 workers), but ha_handle_snapshot_chunk appended every chunk's bytes to snap.part in ARRIVAL order (fopen \"ab\"/fwrite) instead of at the chunk's own chunk_offset (a field already carried on the wire but never used for anything). Reordered chunks landed at the wrong file position: same total byte count, wrong content from the first reordered chunk's offset onward — exactly the observed 'byte-shifted, resyncing later, nondeterministic magnitude' signature. Fix: (1) write each chunk via pwrite(fd, payload, len, chunk->chunk_offset) instead of append — order-independent and safe for concurrent writers (positioned writes to disjoint byte ranges of the same fd don't race). (2) Track completion via a new ha_state_t.snapshot_recv_count/snapshot_recv_total pair (guarded by the existing ha->lock) instead of finalizing on 'this call happens to carry is_last' — under out-of-order arrival, is_last can be processed before an earlier chunk; finalize now fires only once every chunk 0..N-1 has actually been written, and snapshot_recv_total is claimed (reset to 0) by whichever thread crosses the completion threshold so a late straggler chunk can never double-finalize. chunk_index==0 still resets both the stale-file unlink and the new counters (Req 4.4 unchanged). New RED->GREEN unit test test_snapshot_recv_out_of_order_chunks_assemble_correctly (test_config_repl.c) replays the exact non-sequential delivery order and asserts byte-identical assembly; RED confirmed via temporary revert (Expected 0 Was -22, job_count field corrupted by the misordered chunk boundary); GREEN restored, full controller suite clean (0 failures). Also added 2 new deterministic full-size (65536-byte) capnp encode/decode round-trip tests (test_wal_wire.c) that definitively cleared the capnp/transport layer before the real cause was found — permanent regression coverage, the prior test only covered a 300-byte payload. New xce_test snapverify/snapdiff subcommands (built as part of this investigation) were instrumental in diagnosing this live. Live-validated: 3/3 --force-snapshot rounds PASS (genuine 'HA: snapshot loaded ... full job/queue entity restoration' installs confirmed, zero divergence/loss) + 6-round --chaotic PASS.", "tags": [ "raft-drive-async", "snapshot-transfer", "data-corruption", "capnp", - "force-snapshot-soak", - "fixed" + "force-snapshot-soak" ], "related_bugs": [ "bug-1386", "bug-847", "bug-1197" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1393", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T12:05:51.415Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `~/tmp/xce-ha-deploy/controller/ctrl-{1,2}/data/sn", - "fix": "Changed to `ha_send_snapshot_to_standby`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1395", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T13:25:19.081Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `[~]`", - "fix": "Changed to `[x]`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1389.md" }, { "id": "bug-1396", "status": "cannot-reproduce", "status_updated": "2026-09-24", "status_note": "extra committed job (20 vs 19) never reproduced after 9318f997 harness hardening", - "repro_attempts": "2 fresh repro attempts x3 rounds (2026-07-09), 0 reproduced", + "component": "build", + "severity": "unknown", "timestamp": "2026-07-02T14:00:00.000Z", - "error_message": "ha-commit-durability task 6.1 (cold-restart-all real-fs reproduction, src/tools/test-commit-durability.sh on a genuine ext4 deploy dir, ~/tmp not overlay): run 5 of 5 reported `expect-jobs: expected 19, actual 20` after a full cold-stop-all + cold-start-all cycle \u2014 the cluster had ONE MORE committed job than the harness's own ACKED_FILE (deduplicated via `sort -u`) tracked. All 19 tracked job IDs individually verified present (`expect-job` loop all `ok`); the 20th job's identity was not captured (cluster had already been torn down by the script's EXIT trap by the time this was investigated). This is the OPPOSITE of bug-661/795's symptom (loss) \u2014 an unexplained EXTRA job, not a missing one. 4 of the other 5 runs (and every prior round within run 5 itself, rounds 1-8) showed correct exact-count durability with zero anomalies.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "ha-commit-durability task 6.1 (cold-restart-all real-fs reproduction…\n\nha-commit-durability task 6.1 (cold-restart-all real-fs reproduction, src/tools/test-commit-durability.sh on a genuine ext4 deploy dir, ~/tmp not overlay): run 5 of 5 reported `expect-jobs: expected 19, actual 20` after a full cold-stop-all + cold-start-all cycle — the cluster had ONE MORE committed job than the harness's own ACKED_FILE (deduplicated via `sort -u`) tracked. All 19 tracked job IDs individually verified present (`expect-job` loop all `ok`); the 20th job's identity was not captured (cluster had already been torn down by the script's EXIT trap by the time this was investigated). This is the OPPOSITE of bug-661/795's symptom (loss) — an unexplained EXTRA job, not a missing one. 4 of the other 5 runs (and every prior round within run 5 itself, rounds 1-8) showed correct exact-count durability with zero anomalies.", "file": "src/tools/test-commit-durability.sh (harness) or controller job-recovery path (unconfirmed which)", - "root_cause": "NOT INVESTIGATED \u2014 deprioritized because it doesn't match the loss scenario task 6.1 was probing for, and reproducing it requires re-running with a 'joblist' capture at the exact expect-jobs failure point (the script doesn't dump the full list on a count mismatch, only the count). Two live hypotheses, unconfirmed: (a) harness-side race \u2014 ACKED_FILE's write (submit_n) and run_cold_restart_recovery's read (`committed=$(sort -u ACKED_FILE | grep -c .)`) are not synchronized against a submission whose ack arrives asynchronously after the harness moves on, so a genuinely-committed job simply never got recorded in ACKED_FILE \u2014 a test-harness bookkeeping gap, not a controller bug; (b) a genuine job-duplication bug on the controller's recovery/replay path creating an extra job entry with a fresh ID. Also observed in this run's ctrl-1.log: repeated unrelated `handle_jobsubmit fail to lock job 63-15--1-1015808[32751]` errors and one `job not durable (rc=-110) \u2014 rejecting submission` for a DIFFERENTLY-FORMATTED job ID (not this harness's `0-0--0` convention) \u2014 almost certainly a stray unrelated process hitting the same cluster ports in the shared sandbox, not this test's own traffic, but flagged here in case it's connected.", - "fix": "STILL NOT ROOT-CAUSED for the original exact symptom (successful query, count=20 vs expected=19) \u2014 2 fresh repro attempts (2026-07-09, 3 rounds each, real ext4 fs under ~/tmp) did not reproduce an extra job; both runs showed the tracked count exactly matched after the fix below. HOWEVER, both attempts DID hit a DIFFERENT, related bug in the exact same assert_job_count() check: right after the cold-restart-all election, the aggregate expect-jobs/joblist query itself FAILED outright with 'xce_joblist: Unknown error 10014' / 'could not determine job count' (a query FAILURE, not a wrong-but-successful count) \u2014 this is the identical client-side race as bug-1422 (libxce's failover retry loop, client_failover.c, can exhaust its 20-attempt budget in a rapid post-election retry storm and surface a stale E_XCE_STANDBY). Confirmed as a false alarm both times: the very next per-id assert_no_loss check (individual expect-job calls) found every job present. Fixed assert_job_count() with the same bounded-retry pattern used for bug-1422 (3 attempts, 2s settle) \u2014 re-run after the fix: 7 passed, 0 failed, clean. This closes the QUERY-FAILURE false-negative class in this check, and makes the harness reliable enough to actually catch the ORIGINAL extra-job symptom if it recurs (previously a query failure and a genuine extra-job count were indistinguishable at a glance; now the retry-exhausted case self-heals and no longer masks a real mismatch). The original 'extra job' phenomenon itself remains open/unconfirmed \u2014 needs a longer run (original was 1-of-5 full 8-round soaks) to have a realistic chance of firing again.", + "root_cause": "NOT INVESTIGATED — deprioritized because it doesn't match the loss scenario task 6.1 was probing for, and reproducing it requires re-running with a 'joblist' capture at the exact expect-jobs failure point (the script doesn't dump the full list on a count mismatch, only the count). Two live hypotheses, unconfirmed: (a) harness-side race — ACKED_FILE's write (submit_n) and run_cold_restart_recovery's read (`committed=$(sort -u ACKED_FILE | grep -c .)`) are not synchronized against a submission whose ack arrives asynchronously after the harness moves on, so a genuinely-committed job simply never got recorded in ACKED_FILE — a test-harness bookkeeping gap, not a controller bug; (b) a genuine job-duplication bug on the controller's recovery/replay path creating an extra job entry with a fresh ID. Also observed in this run's ctrl-1.log: repeated unrelated `handle_jobsubmit fail to lock job 63-15--1-1015808[32751]` errors and one `job not durable (rc=-110) — rejecting submission` for a DIFFERENTLY-FORMATTED job ID (not this harness's `0-0--0` convention) — almost certainly a stray unrelated process hitting the same cluster ports in the shared sandbox, not this test's own traffic, but flagged here in case it's connected.", + "fix": "STILL NOT ROOT-CAUSED for the original exact symptom (successful query, count=20 vs expected=19) — 2 fresh repro attempts (2026-07-09, 3 rounds each, real ext4 fs under ~/tmp) did not reproduce an extra job; both runs showed the tracked count exactly matched after the fix below. HOWEVER, both attempts DID hit a DIFFERENT, related bug in the exact same assert_job_count() check: right after the cold-restart-all election, the aggregate expect-jobs/joblist query itself FAILED outright with 'xce_joblist: Unknown error 10014' / 'could not determine job count' (a query FAILURE, not a wrong-but-successful count) — this is the identical client-side race as bug-1422 (libxce's failover retry loop, client_failover.c, can exhaust its 20-attempt budget in a rapid post-election retry storm and surface a stale E_XCE_STANDBY). Confirmed as a false alarm both times: the very next per-id assert_no_loss check (individual expect-job calls) found every job present. Fixed assert_job_count() with the same bounded-retry pattern used for bug-1422 (3 attempts, 2s settle) — re-run after the fix: 7 passed, 0 failed, clean. This closes the QUERY-FAILURE false-negative class in this check, and makes the harness reliable enough to actually catch the ORIGINAL extra-job symptom if it recurs (previously a query failure and a genuine extra-job count were indistinguishable at a glance; now the retry-exhausted case self-heals and no longer masks a real mismatch). The original 'extra job' phenomenon itself remains open/unconfirmed — needs a longer run (original was 1-of-5 full 8-round soaks) to have a realistic chance of firing again.", "tags": [ "ha-commit-durability", "low-priority", @@ -15039,86 +3404,30 @@ "job-count-mismatch", "harness-instrumented", "unconfirmed", - "related-fix-applied", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "related-fix-applied" ], "related_bugs": [ "bug-661", "bug-1422" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1397", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T13:57:56.610Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-durability/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1398", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T13:58:18.885Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/tasks.md", - "root_cause": "Had `/kiro-validate-impl`", - "fix": "Changed to `src/tools/test-commit-durability.sh ~/tmp/xce-dur", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1399", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T14:03:20.814Z", - "error_message": "Incorrect value in code", - "file": "src/controller/CLAUDE.md", - "root_cause": "Had `xce-test-e2e-primitives`", - "fix": "Changed to `ha-commit-durability`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1396.md", + "repro_attempts": "2 fresh repro attempts x3 rounds (2026-07-09), 0 reproduced" }, { "id": "bug-1400", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-07-02T14:20:00.000Z", - "error_message": "xce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` \u2014 positional args instead of required getopt flags (-u/-q/-c). getopt silently ignores unrecognized positional args, leaving submit.command/queue etc. NULL, and something downstream derefs a NULL field.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` — positional args instead of required getopt…\n\nxce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` — positional args instead of required getopt flags (-u/-q/-c). getopt silently ignores unrecognized positional args, leaving submit.command/queue etc. NULL, and something downstream derefs a NULL field.", "file": "src/controller/testsuite/test.c (cmd_jobsub, ~line 1748)", - "root_cause": "CONFIRMED STALE for the reported SEGV: git blame shows NULLSTR guards for submit.user/submit.queue/submit.start_info->cmdline were added 2025-11-05 (commit 41f56fa6), long before this bug was logged 2026-07-02 \u2014 'jobsub sleep 1' getopt-returns -1 immediately on the non-option 'sleep' token, then hits the -EINVAL guard at test.c:1838 before any NULL deref. No SEGV path exists in current code; the buglog entry was stale (fix landed under a different/untracked change, buglog never updated). A REAL, separate defect was found in the same function: submit.start_info (calloc'd test.c:1767) was never freed on any of the 4 early-return validation paths (getopt default case + the 3 NULLSTR checks) \u2014 a ~128-byte leak per malformed jobsub invocation.", - "fix": "SEGV claim: not applicable, already fixed pre-existing. Leak: refactored cmd_jobsub (test.c) to `goto cleanup` on all 4 early-return validation failures instead of a bare `return -EINVAL`, reusing the existing FREEUP(submit.user/queue/resreq/start_info->cmdline/exdata/start_info) block via a new `cleanup:` label \u2014 mirrors the `goto clean` idiom already used in queue_ctrl() (queue.c). Verified: controller ASAN build + full controller_unit_test suite green (0 failures) after the change.", + "root_cause": "CONFIRMED STALE for the reported SEGV: git blame shows NULLSTR guards for submit.user/submit.queue/submit.start_info->cmdline were added 2025-11-05 (commit 41f56fa6), long before this bug was logged 2026-07-02 — 'jobsub sleep 1' getopt-returns -1 immediately on the non-option 'sleep' token, then hits the -EINVAL guard at test.c:1838 before any NULL deref. No SEGV path exists in current code; the buglog entry was stale (fix landed under a different/untracked change, buglog never updated). A REAL, separate defect was found in the same function: submit.start_info (calloc'd test.c:1767) was never freed on any of the 4 early-return validation paths (getopt default case + the 3 NULLSTR checks) — a ~128-byte leak per malformed jobsub invocation.", + "fix": "SEGV claim: not applicable, already fixed pre-existing. Leak: refactored cmd_jobsub (test.c) to `goto cleanup` on all 4 early-return validation failures instead of a bare `return -EINVAL`, reusing the existing FREEUP(submit.user/queue/resreq/start_info->cmdline/exdata/start_info) block via a new `cleanup:` label — mirrors the `goto clean` idiom already used in queue_ctrl() (queue.c). Verified: controller ASAN build + full controller_unit_test suite green (0 failures) after the change.", "tags": [ "stale-report", "low-priority", @@ -15126,79 +3435,48 @@ "input-validation", "crash", "cmd_jobsub", - "fixed", "xce_test" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1400.md" }, { "id": "bug-1401", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "leak", "timestamp": "2026-07-02T14:25:00.000Z", - "error_message": "ASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr (lib/controller.capnp.c:11873) \u2014 observed during ha-heartbeat-committed-propagation task 4.1 live validation: 120 bytes/1 alloc after a single ctrlstatus call, 12360 bytes/103 allocs after ~a dozen calls (roughly proportional to call count).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr…\n\nASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr (lib/controller.capnp.c:11873) — observed during ha-heartbeat-committed-propagation task 4.1 live validation: 120 bytes/1 alloc after a single ctrlstatus call, 12360 bytes/103 allocs after ~a dozen calls (roughly proportional to call count).", "file": "src/lib/controller.capnp.c (decode_CtrlStatus_ptr, generated codec) or its caller in request.c/xcecli", - "root_cause": "CONFIRMED via codegraph + generated codec trace: src/include/client.h:119 defined `xce_free_ctrlstatus(s)` as `free_CtrlStatus(s)` \u2014 the generated free_CtrlStatus() only frees the FIELDS INSIDE the xce_ctrlstatus_t (clusterName/primaryName/responderName/configChecksum/peers list), matching decode_CtrlStatus(). But the real client reply path (xce_handle_reply -> MOVE_PTR from the decoded RequestReply -> xce_ctrlstatus) populates the struct via decode_CtrlStatus_PTR(), which calloc's the xce_ctrlstatus_t itself (controller.capnp.c:11873) before calling decode_CtrlStatus() to fill it. Only free_CtrlStatus_ptr() (which calls free_CtrlStatus() THEN free()s the struct and nulls the caller's pointer) matches that allocation. free_RequestReply/free_Request/MOVE_PTR were all confirmed symmetric and correct \u2014 the bug was isolated to this one macro definition. Existing unit tests never caught it because they call free_CtrlStatus(&out) directly on a stack struct, never exercising the xce_free_ctrlstatus() macro used by real client code.", + "root_cause": "CONFIRMED via codegraph + generated codec trace: src/include/client.h:119 defined `xce_free_ctrlstatus(s)` as `free_CtrlStatus(s)` — the generated free_CtrlStatus() only frees the FIELDS INSIDE the xce_ctrlstatus_t (clusterName/primaryName/responderName/configChecksum/peers list), matching decode_CtrlStatus(). But the real client reply path (xce_handle_reply -> MOVE_PTR from the decoded RequestReply -> xce_ctrlstatus) populates the struct via decode_CtrlStatus_PTR(), which calloc's the xce_ctrlstatus_t itself (controller.capnp.c:11873) before calling decode_CtrlStatus() to fill it. Only free_CtrlStatus_ptr() (which calls free_CtrlStatus() THEN free()s the struct and nulls the caller's pointer) matches that allocation. free_RequestReply/free_Request/MOVE_PTR were all confirmed symmetric and correct — the bug was isolated to this one macro definition. Existing unit tests never caught it because they call free_CtrlStatus(&out) directly on a stack struct, never exercising the xce_free_ctrlstatus() macro used by real client code.", "fix": "src/include/client.h:119 changed to `#define xce_free_ctrlstatus(s) free_CtrlStatus_ptr(&(s))`. Added regression test test_xce_free_ctrlstatus_frees_the_struct_itself (src/lib/testsuite/test_ctrlstatus.c), which decodes via decode_CtrlStatus_ptr (the real allocation path, not the stack-struct helper the other roundtrip tests use) and frees via the xce_free_ctrlstatus macro. RED confirmed: reverting the fix reproduces the exact reported leak signature (LeakSanitizer: 120 bytes in 1 object, decode_CtrlStatus_ptr calloc). GREEN after fix: 285 lib tests / 0 failures / 0 leaks. Controller ASAN build + controller_unit_test also green post-fix (client.h is shared).", "tags": [ "asan", - "fixed", "low-priority", "controller-status", "ctrlstatus", "memory-leak" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1402", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-02T14:21:35.080Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/controller-status/tasks.md", - "root_cause": "Had `/xce-deploy-ha`", - "fix": "Changed to `~/tmp`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1405", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-03T00:08:25.320Z", - "error_message": "Wrong reference: test_xcecli_ctrlstatus_null_ctx_typed_failure should be test_xce_free_ctrlstatus_frees_the_struct_itself", - "file": "src/lib/testsuite/test_ctrlstatus.c", - "root_cause": "Used \"test_xcecli_ctrlstatus_null_ctx_typed_failure\" instead of \"test_xce_free_ctrlstatus_frees_the_struct_itself\"", - "fix": "Changed test_xcecli_ctrlstatus_null_ctx_typed_failure \u2192 test_xce_free_ctrlstatus_frees_the_struct_itself", - "tags": [ - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1401.md" }, { "id": "bug-1406", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-07-03T08:00:00.000Z", - "error_message": "controller_unit_test SIGSEGV (full run, no --run filter): Thread 1 crashes in SSL_CTX_free() called from iomgr_fini() during test_ha_vep_teardown.c:271 (test_vep_teardown_runs_after_inflight), which runs at controller_test.c main() TEST(test_ha_vep_teardown). Kills the whole binary before any later-registered TEST() can run, so a bare `controller_unit_test` invocation with no --run filter never reaches a final summary and the exit code is 139 (SIGSEGV), not a Unity pass/fail count.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "controller_unit_test SIGSEGV (full run, no --run filter): Thread 1 crashes in SSL_CTX_free()…\n\ncontroller_unit_test SIGSEGV (full run, no --run filter): Thread 1 crashes in SSL_CTX_free() called from iomgr_fini() during test_ha_vep_teardown.c:271 (test_vep_teardown_runs_after_inflight), which runs at controller_test.c main() TEST(test_ha_vep_teardown). Kills the whole binary before any later-registered TEST() can run, so a bare `controller_unit_test` invocation with no --run filter never reaches a final summary and the exit code is 139 (SIGSEGV), not a Unity pass/fail count.", "file": "src/controller/testsuite/test_ha_vep_teardown.c", "root_cause": "ROOT-CAUSED 2026-07-09: iomgr_init() (src/lib/netio.c:1168-1184) only ever initialized iomgr_t's `lock` and `epoll_fd` fields -- it never set `ssl_ctx` (or `worker`). Every production caller embeds iomgr_t inside a calloc'd/static context struct (cluster_t.iomgr, xce_ctx_t.iomgr), which masked the gap. test_ha_vep_teardown.c stack-allocates `iomgr_t iomgr;` at 4 call sites (lines 235, 294, 349, 476) WITHOUT a memset -- unlike the sibling libxce tests (test_netio_arm_failure.c:52, test_request_send_propagation.c:34) which explicitly `memset(&iomgr, 0, sizeof(iomgr))` before iomgr_init, establishing the real (undocumented) contract. Left un-zeroed, `ssl_ctx` holds whatever garbage bit pattern occupies that stack slot -- non-deterministic across runs/process history (confirmed intermittent: some full-suite runs exit 0, others SIGSEGV, matching the reported non-strictly-deterministic behavior). iomgr_fini()'s `if (iomgr->ssl_ctx != NULL) SSL_CTX_free(...)` guard then passes on that garbage pointer, and SSL_CTX_free() dereferences a wild address -> SIGSEGV. Proven deterministically with a standalone repro (poisoned an iomgr_t with 0xAB, called the real iomgr_init()+iomgr_fini() from libxce.a) producing an identical backtrace (SSL_CTX_free <- iomgr_fini netio.c:1262 <- caller). gdb backtrace from the original report: SSL_CTX_free (libssl.so.3) <- iomgr_fini (netio.c:1262) <- test_vep_teardown_runs_after_inflight (test_ha_vep_teardown.c:271) <- UnityDefaultTestRun <- test_ha_vep_teardown (test_ha_vep_teardown.c:630) <- main (controller_test.c). Not a double-free/UAF and not a cross-test state leak -- iomgr_set_ssl_ctx() is never called anywhere in test_ha_vep_teardown.c, so no real SSL_CTX ownership is even in play; it is purely uninitialized-memory read.", "fix": "FIXED 2026-07-09: src/lib/netio.c iomgr_init() now explicitly sets `iomgr->ssl_ctx = NULL;` and `iomgr->worker = 0;` before touching lock/epoll_fd, so iomgr_init() fully establishes iomgr_t's invariants itself instead of assuming the caller pre-zeroed the struct -- mirrors the same file's do_endpoint_init(), which already fully initializes every endpoint_t field rather than relying on calloc. Verified safe for all production callers (xcectx.c, myapp.c, cluster.c): every one calls iomgr_set_ssl_ctx() AFTER iomgr_init(), never before, so clearing ssl_ctx in iomgr_init cannot clobber a caller-supplied value. No test file was modified -- the library-level fix protects every current and future caller, including the 4 unguarded stack declarations in test_ha_vep_teardown.c. Verified: (1) standalone ASAN repro poisoning the stack before iomgr_init now shows ssl_ctx == NULL post-init and iomgr_fini() returns cleanly instead of crashing; (2) rebuilt controller_unit_test and ran the full suite (no --run filter) repeatedly under ASAN (detect_leaks=0) with zero SIGSEGVs across all runs -- one full run captured 265 Tests / 0 Failures / 0 Ignored, with all 6 tests in the test_ha_vep_teardown group (including test_vep_teardown_runs_after_inflight) passing their own assertions, not just failing to crash.", @@ -15209,19 +3487,23 @@ "ssl", "iomgr", "pre-existing", - "raft-apply-decoupling", - "fixed" + "raft-apply-decoupling" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1406.md" }, { "id": "bug-1407", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-03T08:00:00.000Z", - "error_message": "controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence FAILS (\"Expected TRUE Was FALSE\") when running --run test_config_repl (RUN_TEST site: test_config_repl.c:4008). NOTE: Unity's own printed file:line prefix for this failure is misleading/stale (shows test_repl_unit.c:1543 or controller_test.c:1543 depending on which other suites ran earlier in the same process) -- Unity.TestFile is apparently not re-set per test_xxx(void) group in this harness, only Unity.CurrentTestLineNumber is accurate. The true source is test_config_repl.c.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 5, + "error_message": "controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence FAILS (\"Expected…\n\ncontroller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence FAILS (\"Expected TRUE Was FALSE\") when running --run test_config_repl (RUN_TEST site: test_config_repl.c:4008). NOTE: Unity's own printed file:line prefix for this failure is misleading/stale (shows test_repl_unit.c:1543 or controller_test.c:1543 depending on which other suites ran earlier in the same process) -- Unity.TestFile is apparently not re-set per test_xxx(void) group in this harness, only Unity.CurrentTestLineNumber is accurate. The true source is test_config_repl.c.", "file": "src/controller/testsuite/test_config_repl.c", "root_cause": "DEFINITIVELY ROOT-CAUSED 2026-07-03: this was never a source-code bug. It was a stale/corrupted incremental-build artifact in the main worktree's shared `build/` directory, which had been incrementally `ninja -C build`-rebuilt across 5 sequential tasks (1.1-1.5) without ever a fresh `meson setup`. Proof (four independent measurements, isolated via separate `git worktree add` + fresh `meson setup` + `ninja`, never mutating the main worktree in place): (1) pre-feature commit 17b210c, fresh build: 0 failures, 3/3 runs. (2) commit 9b4b805 (task-1.4 HEAD, IDENTICAL source to the failing main-worktree binary), fresh build in a separate worktree: 0 failures, 3/3 runs. (3) main worktree's stale incrementally-rebuilt `build/`, same 9b4b805 commit: 100% failure, 8/8 runs across two investigation sessions. (4) main worktree after `rm -rf build && meson setup --buildtype=debug -Db_sanitize=address && ninja -C build` (full clean rebuild, same source, same commit): 0 failures, 3/3 runs. This conclusively isolates the failure to the incremental build state, not the code. This is a DIFFERENT gotcha from bug-924 (which is about the `meson test` driver hanging on overlayfs fdatasync) -- this one is `ninja -C build`'s incremental compilation silently producing a wrong/stale binary despite reporting success, likely a header-dependency tracking gap on this overlayfs-backed worktree. The prior 'CLOSED' (task 1.4, 70/70) and 'REOPENED' (task 1.5, 10/10 fail) statuses were BOTH real, contradictory, correct measurements of the SAME source code in DIFFERENT incremental-build states -- neither implementer did anything wrong; the build itself was non-deterministic relative to source content.", "fix": "CLOSED 2026-07-03 (high confidence, this time backed by cross-checked fresh-build isolation, not just an in-place re-run). No source-code fix needed -- `test_commit_advance_retry_clears_push_before_commit_fence` has apparently been correct since at least commit 3ecb724 (pre-feature). ACTION ITEM for all future tasks in this series: do not trust `ninja -C build` incremental results alone when a test result looks surprising or contradicts a very recent prior measurement on the same commit -- rebuild with `rm -rf build && meson setup build --buildtype=debug -Db_sanitize=address && ninja -C build` (or isolate via a throwaway `git worktree add` + fresh setup) before concluding a regression exists. Added to `.wolf/cerebrum.md` Do-Not-Repeat.", @@ -15231,23 +3513,27 @@ "commit-engine", "build-environment", "false-positive", - "raft-apply-decoupling", - "closed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "raft-apply-decoupling" ], "related_bugs": [ "bug-924" ], - "occurrences": 5, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "3ecb724" + ], + "record": ".wolf/bugs/bug-1407.md" }, { "id": "bug-1408", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "security", "timestamp": "2026-07-03T21:00:00.000Z", - "error_message": "raft-apply-decoupling task 3.1 round-1 handle_jobsubmit(): when an array job submission has >1 element and ONE element fails post-commit construction (create_job()/makeJob_() returns NULL, e.g. a jobid collision that slipped past jobid_range_exists() pre-check TOCTOU, or a plain calloc failure), the function returned the construction error IMMEDIATELY after the per-element construct loop, before ever building the AFTER_EVENT notify jlist or firing HOOK_DETAIL_JOB_NEW -- even for SIBLING array elements that constructed successfully and are already durably committed on a majority.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-apply-decoupling task 3.1 round-1 handle_jobsubmit(): when an array job submission has >1…\n\nraft-apply-decoupling task 3.1 round-1 handle_jobsubmit(): when an array job submission has >1 element and ONE element fails post-commit construction (create_job()/makeJob_() returns NULL, e.g. a jobid collision that slipped past jobid_range_exists() pre-check TOCTOU, or a plain calloc failure), the function returned the construction error IMMEDIATELY after the per-element construct loop, before ever building the AFTER_EVENT notify jlist or firing HOOK_DETAIL_JOB_NEW -- even for SIBLING array elements that constructed successfully and are already durably committed on a majority.", "file": "src/controller/job.c", "root_cause": "handle_jobsubmit()'s post-commit construction loop set construct_rc<0 on any failed create_job() call but used continue (not break) to keep constructing remaining siblings correctly -- however, immediately AFTER the loop, \"if (construct_rc < 0) { reply error; return; }\" fired BEFORE the jlist-build + xce_run_hooks(HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW) block, so a single failed element suppressed notification for every successfully constructed sibling in the same array submission. Both the scheduler and apiserver are purely event-driven off this hook with no periodic reconciliation, so the stranded siblings (real, durable, present in c->jobmgr) were invisible until the next controller restart replayed the WAL.", "fix": "Moved the construct_rc<0 early-return in handle_jobsubmit() (src/controller/job.c) to AFTER the jlist-build + xce_run_hooks(HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, ...) block instead of before it. The jlist-build loop already only finds jobs that key_find() locates in c->jobmgr, so a failed-to-construct element is naturally and silently absent from jlist with no extra bookkeeping -- the hook now fires unconditionally for every successfully constructed element regardless of a sibling's construct_rc, and the client-facing reply still reports the construct_rc error afterward. Added regression test test_jobsubmit_array_partial_construct_failure_notifies_survivors in src/controller/testsuite/test_commit_barrier_reply.c: drives handle_jobsubmit with a 2-element array on a worker pthread, ACKs element 0's WAL entry, then deterministically reproduces the TOCTOU outcome by inserting a stand-in job directly into c->jobmgr for element 1's jobid (via the existing cbrp_make_job helper) strictly after jobid_range_exists() has already run and strictly before the construction loop can run (element 1's commit is still pending on a withheld ACK) -- makeJob_()'s own key_find() then legitimately returns EEXIST for element 1. A new test hook capture (cbrp_jobnew_capture_fn, registered via xce_add_hook) records every jobid delivered through HOOK_AFTER_EVENT/HOOK_DETAIL_JOB_NEW so the test can assert element 0 was both constructed AND notified despite element 1 failing. RED-phase confirmed against the pre-fix code (job 9201 construction failed with errno=17/EEXIST as designed, and the notification assertion for job 9200 failed as predicted). CORRECTION (round-2 review): under the standard verification invocation (--run test_commit_barrier_reply, the full 25-test group), the failing TEST_ASSERT_TRUE_MESSAGE at test_commit_barrier_reply.c longjmps past this test's own inline cbrp_teardown() call (Unity TEST_ABORT semantics) -- the SAME longjmp-past-teardown pathology already documented for round 1's withheld-commit test, not a clean assertion FAIL. It leaks live persist/commit-engine background threads into the next test's setup, producing a deterministic HANG under the full-group command (reproduced 3/3 times, both threads blocked in futex_do_wait). A clean, fast FAIL was only observed when this test was isolated from the other 24 tests in the group -- that isolated form is what the original write-up described. GREEN after the fix (reordering removes the failure entirely, so this hang mode never triggers in the fixed code): both pre-existing single-element jobsubmit tests and the full 25-test test_commit_barrier_reply group plus the full controller_unit_test binary pass (0 failures) via the standard --run invocation. Hardened afterward: the test now captures its assertion-relevant booleans (cbrp_job_exists/cbrp_jobnew_notified) into locals and calls cbrp_teardown() unconditionally before asserting on the captured locals, so a future regression in this exact test can no longer skip cleanup and hang sibling tests.", @@ -15260,20 +3546,23 @@ "commit-barrier", "toctou", "round-2-remediation", - "test-longjmp-past-teardown", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-longjmp-past-teardown" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1408.md" }, { "id": "bug-1409", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-07-04T17:00:00.000Z", - "error_message": "test_demotion_convergence hung/spun (18.5M+ log lines, timeout kill) once commit.c's commit_resolve_pending unconditionally called the new persist_apply_pump_kick", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_demotion_convergence hung/spun (18.5M+ log lines, timeout kill) once commit.c's…\n\ntest_demotion_convergence hung/spun (18.5M+ log lines, timeout kill) once commit.c's commit_resolve_pending unconditionally called the new persist_apply_pump_kick", "file": "src/controller/persist.c", "root_cause": "apply_pump_drain()'s recheck-then-clear retry loop ignored persist_apply_pump_run()'s return code: on a genuine read failure (e.g. committed_lsn advanced in a test fixture with no real WAL bytes on disk for that LSN, rc=-ENOENT), last_applied_lsn never advances, so the very next re-check inside the SAME drain loop sees identical unresolved work and immediately reclaims the scheduled flag and retries forever -- a tight busy loop with no backoff, hammering disk reads and the logger.", "fix": "apply_pump_drain() now checks persist_apply_pump_run()'s rc immediately after clearing apply_pump_scheduled; a negative rc returns immediately instead of re-checking for more work and reclaiming the flag. The failure is already logged loudly inside persist_apply_pump_run itself. A future, genuinely NEW kick (a later real commit-advance event) retries then, at that event's natural cadence, never in a busy spin.", @@ -15284,20 +3573,23 @@ "busy-loop", "infinite-loop", "commit-resolve-pending", - "regression-sweep", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "regression-sweep" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1409.md" }, { "id": "bug-1410", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-04T17:00:00.000Z", - "error_message": "test_apply_pump's spy hook recorded jobid=0 for HOOK_DETAIL_JOB_UPDATE/HOOK_DETAIL_JOB_NEW instead of the real jobid", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_apply_pump's spy hook recorded jobid=0 for HOOK_DETAIL_JOB_UPDATE/HOOK_DETAIL_JOB_NEW instead…\n\ntest_apply_pump's spy hook recorded jobid=0 for HOOK_DETAIL_JOB_UPDATE/HOOK_DETAIL_JOB_NEW instead of the real jobid", "file": "src/controller/testsuite/test_apply_pump.c", "root_cause": "The hook's per-subscriber `data`/cls pointer (bound at xce_add_hook registration time) is separate from the va_list's own variadic arguments. apply_committed_entry's notify calls pass (cluster_t*, payload) as the actual varargs -- the spy hook read the FIRST va_arg as the payload pointer directly, skipping the leading cluster_t* argument every real dispatch call (job.c, persist.c) always sends first.", "fix": "Added `(void)va_arg(ap, cluster_t*);` before extracting the real payload (xlist_t*/job_t*) in both HOOK_DETAIL_JOB_NEW and HOOK_DETAIL_JOB_UPDATE cases, mirroring persist_job_hook_fun's own established va_arg consumption order.", @@ -15306,20 +3598,23 @@ "task-6.1", "test-only", "va_arg", - "hook-system", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "hook-system" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1410.md" }, { "id": "bug-1411", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-04T17:00:00.000Z", - "error_message": "AddressSanitizer global-buffer-overflow in dataq_enq when test_apply_pump.c called persist_submit_job_fact with a real 3-member repl setup", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer global-buffer-overflow in dataq_enq when test_apply_pump.c called…\n\nAddressSanitizer global-buffer-overflow in dataq_enq when test_apply_pump.c called persist_submit_job_fact with a real 3-member repl setup", "file": "src/controller/testsuite/test_apply_pump.c", "root_cause": "repl_broadcast tried to enqueue onto the dummy standby `int` stand-ins (g_tap_hep_a/b, opaque identity handles only) as if they were real endpoint_t* objects, overflowing past the 4-byte int into adjacent globals -- the fixture omitted the repl_broadcast_send_hook interception test_commit_barrier_reply.c's cbrp_setup already uses to keep repl_broadcast from ever dereferencing those dummy pointers.", "fix": "Added the identical repl_broadcast_send_hook = tap_drop_send_hook (a no-op stub returning success) in tap_setup/tap_teardown, mirroring test_commit_barrier_reply.c's cbrp_drop_send_hook exactly.", @@ -15329,20 +3624,23 @@ "test-only", "asan", "repl-broadcast", - "fixture", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fixture" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1411.md" }, { "id": "bug-1412", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-04T17:00:00.000Z", - "error_message": "test_apply_pump.c failed to compile: 'unknown type name RETAIN', stray bytes, __gnuc_va_list errors cascading from an unrelated location", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_apply_pump.c failed to compile: 'unknown type name RETAIN', stray bytes, __gnuc_va_list…\n\ntest_apply_pump.c failed to compile: 'unknown type name RETAIN', stray bytes, __gnuc_va_list errors cascading from an unrelated location", "file": "src/controller/testsuite/test_apply_pump.c", "root_cause": "A doc comment contained the literal substring 'WAL_JOB_*/RETAIN' -- the '*' immediately followed by '/' forms a real C block-comment terminator, closing the file's top doc comment many lines early and turning the rest of the comment text into stray code tokens.", "fix": "Reworded to 'a WAL_JOB_SUBMIT WAIT-mode entry mixed with WAL_JOB_{START,FINISH} RETAIN-mode facts', removing the accidental */ sequence. Lesson: never write '*/' inside a C block comment, even when '*' is meant as a wildcard/glob character.", @@ -15351,20 +3649,23 @@ "task-6.1", "test-only", "comment-terminator", - "build-break", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "build-break" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1412.md" }, { "id": "bug-1413", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-07-04T17:00:00.000Z", - "error_message": "test_apply_pump's burst-ordering test hung the NEXT test (longjmp-past-teardown) after a wrong off-by-one assertion (expected lsn_before+3, actual watermark is lsn_before+2 for 3 appended entries) failed", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_apply_pump's burst-ordering test hung the NEXT test (longjmp-past-teardown) after a wrong…\n\ntest_apply_pump's burst-ordering test hung the NEXT test (longjmp-past-teardown) after a wrong off-by-one assertion (expected lsn_before+3, actual watermark is lsn_before+2 for 3 appended entries) failed", "file": "src/controller/testsuite/test_apply_pump.c", "root_cause": "A TEST_ASSERT failure mid-test longjmps past the rest of the function body, including the explicit tap_teardown() call at the end -- leaving the fsync thread/tmpdir/hooksys from the failed test alive when the next test's tap_setup() ran, hanging it. Root trigger was a genuine off-by-one in the test's own math (current_lsn before N appends is the FIRST assigned lsn, not one less than it; the watermark after N appends is first_lsn + N - 1, not first_lsn + N).", "fix": "Fixed the off-by-one (first_lsn + 2 == watermark for 3 appends starting at first_lsn). Restructured all three tests in the file to capture every assertion-relevant observable into local variables BEFORE any TEST_ASSERT call, and to call tap_teardown() unconditionally before running any assertion -- the exact pattern already established in this codebase for task 3.1's identical hazard (tasks.md Implementation Notes, task 3.1 round 2/3).", @@ -15374,22 +3675,25 @@ "test-only", "longjmp-past-teardown", "off-by-one", - "test-hygiene", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-hygiene" ], "related_bugs": [ "bug-1408" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1413.md" }, { "id": "bug-1414", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-04T18:00:00.000Z", - "error_message": "raft-apply-decoupling task 6.1 round 1 REJECTED: FINISH (LSN N) then a POST_EXEC-stage FAIL (LSN N+1) for the same job, with FAIL received before FINISH applies, corrupts the job to RUN|EXIT|PERR (0x106) instead of the correct DONE|PERR (0x108) once both apply in LSN order -- DONE is silently lost and a stray RUN bit appears.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-apply-decoupling task 6.1 round 1 REJECTED: FINISH (LSN N) then a POST_EXEC-stage FAIL (LSN…\n\nraft-apply-decoupling task 6.1 round 1 REJECTED: FINISH (LSN N) then a POST_EXEC-stage FAIL (LSN N+1) for the same job, with FAIL received before FINISH applies, corrupts the job to RUN|EXIT|PERR (0x106) instead of the correct DONE|PERR (0x108) once both apply in LSN order -- DONE is silently lost and a stray RUN bit appears.", "file": "src/controller/job.c, src/controller/persist.c, src/include/wal_types.h, src/include/persist.h, src/controller/include/cluster.h", "root_cause": "record_job_fail_fact (job.c) and record_job_update_fact (postdone) precomputed a FINAL absolute state at RECEIPT time (scratch-copy merge for FAIL, inline backstop+PDONE merge for postdone) and baked it into wal_jobstate_t.state -- correct only when receipt and apply are the same event. Once task 6.1 deferred apply past the commit barrier, a second in-flight fact for the same job (e.g. FINISH) could apply BETWEEN this fact's receipt and its own apply, making the receipt-time snapshot stale by the time apply_committed_entry's generic apply_job_update_transition() did its blind absolute overwrite -- silently clobbering whatever the other fact had just applied.", "fix": "Repurposed wal_jobstate_t._pad (previously always-zero, unused) as a discriminator: _pad==0 still means legacy absolute state (job_ctrl's mark_job_fail/mark_job_update live paths, which mutate+capture under one lock atomically -- no staleness risk, unchanged); a nonzero _pad (WAL_JOB_FAIL_STAGE_FLAG|stage for FAIL, WAL_JOB_UPDATE_POSTDONE_MARK for postdone) marks the entry for apply-time recomputation. apply_committed_entry (persist.c) now dispatches WAL_JOB_FAIL/WAL_JOB_UPDATE on this bit, calling the REAL apply_job_fail_transition() / new apply_job_postdone_transition() (job.c) against the job's ACTUAL current state at apply time instead of a stale baked value. New persist_submit_job_fact_ex(p,type,jobid,state,pad) primitive added; persist_submit_job_fact becomes a pad=0 thin wrapper so zero existing call sites needed changes. Also fixed 2 pre-existing test_commit_barrier_reply.c tests that asserted the old wire format directly (cbrp_count_jobstate_entries reading state bits) -- extended the helper to a _ex variant exposing _pad, updated assertions to check the new marker plus a direct call to the guard core on a scratch job_t. New regression test test_apply_pump_fail_racing_finish_merges_against_current_state in test_apply_pump.c drives FINISH then FAIL through the real receipt handlers with both entries durably appended but unapplied, then a single covering ACK; RED-confirmed byte-for-byte (Expected 264 Was 262) against the reverted round-1 code, GREEN after the fix.", @@ -15402,22 +3706,25 @@ "deferred-apply", "wal-wire-format", "wal_jobstate_t", - "_pad-discriminator", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "_pad-discriminator" ], "related_bugs": [ "bug-1413" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1414.md" }, { "id": "bug-1415", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-04T19:30:00.000Z", - "error_message": "raft-apply-decoupling task 6.2 review finding (NOT reproduced under the test suite's pool-less fixtures, NOT fixed): a real vtaskmgr pool (n_vcpus > 1) can run the commit continuation's dispatch (job_commit_resolve_adapter -> jobsubmit_commit_continuation -> persist_register_pending_apply) and the apply pump's kick (persist_apply_pump_kick -> apply_pump_drain -> persist_apply_pump_run) on two DIFFERENT worker threads with no completion-order guarantee, even though commit_resolve_pending() enqueues them in that order. If the apply-pump task happens to finish first, persist_take_pending_apply() finds nothing registered yet for that lsn (0, correct no-op per its own contract), applies the entry, and advances last_applied_lsn past it -- then when the continuation task finally runs and calls persist_register_pending_apply() for that now-already-applied lsn, the registration is permanently orphaned: the walk only ever revisits [last_applied_lsn+1..committed_lsn], so nothing will ever call persist_take_pending_apply() for that lsn again. Net effect: the job is correctly constructed and visible in c->jobmgr, but the client that submitted it never receives a reply -- silently, forever.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-apply-decoupling task 6.2 review finding (NOT reproduced under the test suite's pool-less…\n\nraft-apply-decoupling task 6.2 review finding (NOT reproduced under the test suite's pool-less fixtures, NOT fixed): a real vtaskmgr pool (n_vcpus > 1) can run the commit continuation's dispatch (job_commit_resolve_adapter -> jobsubmit_commit_continuation -> persist_register_pending_apply) and the apply pump's kick (persist_apply_pump_kick -> apply_pump_drain -> persist_apply_pump_run) on two DIFFERENT worker threads with no completion-order guarantee, even though commit_resolve_pending() enqueues them in that order. If the apply-pump task happens to finish first, persist_take_pending_apply() finds nothing registered yet for that lsn (0, correct no-op per its own contract), applies the entry, and advances last_applied_lsn past it -- then when the continuation task finally runs and calls persist_register_pending_apply() for that now-already-applied lsn, the registration is permanently orphaned: the walk only ever revisits [last_applied_lsn+1..committed_lsn], so nothing will ever call persist_take_pending_apply() for that lsn again. Net effect: the job is correctly constructed and visible in c->jobmgr, but the client that submitted it never receives a reply -- silently, forever.", "file": "src/controller/commit.c (commit_resolve_pending, commit_resolve/dispatch_continuation), src/controller/persist.c (persist_apply_pump_kick, persist_apply_pump_run)", "root_cause": "commit_resolve_pending() (commit.c ~793-835, landed by task 1.2/6.1) resolves every ready continuation via commit_resolve() -> dispatch_continuation(), which for a real pool (ce->pool != NULL) enqueues the continuation onto vtaskmgr via vtask_enq_classified(VTASK_LIVE_BLOCK) and returns WITHOUT waiting for it to run; commit_resolve_pending() then calls persist_apply_pump_kick(ce->p, ce->pool), which enqueues a SECOND, independent VTASK_LIVE_BLOCK task onto the SAME pool. vtaskmgr.c's dispatcher (vdispatcher_entry) hands queued tasks to whichever vcpu worker thread is next idle (pick_idle_vcpu/pick_eligible_task) -- with n_vcpus > 1 (the only realistic production configuration; every test fixture in this spec deliberately never calls commit_set_pool, so ce->pool stays NULL and both dispatches run synchronously inline instead, which is why no test in this suite reproduces the race), two tasks enqueued back-to-back can be picked up by two different idle vcpus at nearly the same time, and their COMPLETION order depends on each task's own runtime (a disk-reading WAL-scan-and-apply walk vs. a calloc+mutex-guarded list insert) -- not their enqueue order. This is a pre-existing structural gap from task 1.2 (async continuation dispatch) + task 6.1 (apply-pump-kick wired into the same choke point, same pool) that had no observable consequence until task 6.2 added a real consumer (persist_take_pending_apply) whose registration can now race the very entry it targets.", "fix": "FIXED round 2 (task 6.2): persist_register_pending_apply() (persist.c) now self-heals option (b) from this entry's own proposed fixes -- before linking a new node onto pending_applies, it checks `lsn <= persist_last_applied_lsn(p)`; if the apply pump's walk has already applied `lsn` (last_applied_lsn already at or past it), `on_applied(ctx, lsn)` fires immediately and synchronously on the CALLING thread instead of being linked for a walk pass that will never revisit that lsn. Race-safe in both interleavings: when the walk has not reached `lsn` yet, the check is false and registration proceeds exactly as before (unchanged behavior, verified against the full existing regression suite); when the walk already passed `lsn`, its own persist_take_pending_apply() call for that lsn already ran and found nothing (that is precisely how we know registration is late), so the immediate inline fire can never double-fire alongside it. New deterministic regression test `test_apply_pump_late_pending_apply_registration_self_heals` (test_apply_pump.c) wires a REAL multi-vcpu vtaskmgr_t (n_vcpus=4) via commit_set_pool -- the one test in this spec to do so -- and forces the exact bad interleaving via a barrier (the test's stand-in commit continuation polls persist_last_applied_lsn() until the apply pump has already applied the lsn, THEN registers), rather than relying on natural timing luck. Verified genuine RED (fails with 'Expected 1 Was 0' against the unfixed check, confirming the hang is real and reproducible) -> GREEN (fires exactly once, resource released exactly once, no orphaned registration) after the fix. Full regression sweep (test_apply_pump, test_commit_barrier_reply, test_apply_reapply_idempotency, test_commit_resolve, test_commit_admission_pool, test_demotion_convergence) shows zero new failures -- the only 2 failures present (test_jobsubmit_returns_before_commit_and_registers_pending_apply, test_jobsubmit_array_mixed_outcome_no_phantom_job_for_failed_elements, both in test_commit_barrier_reply.c) are pre-existing/expected per task 6.1's own Implementation Notes (their assertion text predates task 6's apply pump landing; task 7.3's job to update), confirmed identical before and after this fix by reverting to the pre-round-2 baseline and re-running.", @@ -15430,21 +3737,25 @@ "lost-reply", "commit_resolve_pending", "persist_apply_pump_kick", - "fixed", "self-healing" ], "related_bugs": [ "bug-1409" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1415.md" }, { "id": "bug-1416", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-05T06:15:00.000Z", - "error_message": "[xce-test-ha] FAIL: factstream round 1: cluster did not reconverge to 3-of-3 after restarting ctrl-1 -- root cause: persist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip (expected last_applied_lsn=72 new_job_count=23); persist_recover_loud_fence_if_unrecoverable: UNRECOVERABLE, staying alive/non-promotable", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 11, + "error_message": "[xce-test-ha] FAIL: factstream round 1: cluster did not reconverge to 3-of-3 after restarting…\n\n[xce-test-ha] FAIL: factstream round 1: cluster did not reconverge to 3-of-3 after restarting ctrl-1 -- root cause: persist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip (expected last_applied_lsn=72 new_job_count=23); persist_recover_loud_fence_if_unrecoverable: UNRECOVERABLE, staying alive/non-promotable", "file": "src/controller/persist.c (persist_verify_apply_correctness / persist_recover, cluster.c cluster_init_phase2)", "root_cause": "Found live via the new raft-apply-decoupling task 7.2 kill-mid-fact-stream soak scenario (--factstream), reproduced 4/4 runs (ASAN debug build, paced-wave ASAN build, release build, and a cleanup-suppressed debug postmortem run) in this sandboxed environment. cluster_init_phase2() calls cluster_open_listener() (which starts accepting network I/O, including replicated/catch-up WAL traffic from an already-serving primary) BEFORE persist_recover() runs its boot-time apply-correctness verification (task 5.1, Req 8.1/8.2). Under a slow/contended boot (observed: persist_recover firing ~36s after process start instead of the normal ~5s, plausibly due to heavy concurrent nix-develop/test-process CPU contention in this sandbox), a standby's listener can already be live and processing real traffic for tens of seconds before its own persist_recover() call finally executes. persist_verify_apply_correctness() (persist.c ~3597) reads p->last_applied_lsn and live-queries count_jobmgr_entries(p->cluster) AFTER replay_wal completes -- both of these are also mutated by the LIVE apply pump (raft-apply-decoupling task 6.1, wired through commit_resolve_pending) reacting to replicated commit-advance events. If live traffic is already flowing and being applied through the live pump DURING the extended pre-persist_recover window, the live-read 'actual' values used by persist_verify_apply_correctness can reflect state beyond what persist_recover's own single WAL-scan pass computed as 'expected', producing a DIVERGENT verdict even though no real apply defect exists -- confirmed via ctrlstatus probes showing committedLsn identical (79) and no actual data loss across all 3 nodes; the fence engaged correctly per its own fail-loud design (stayed alive, non-promotable, kept retrying) but two of three nodes (including the node that went on to win the next election, term=5, first_boot=1) were fenced at the exact moment persist_recover ran. This matches task 5.1's own documented residual-risk note (tasks.md Implementation Notes, task 5.1 entry): 'the position check is diagnostically tautological for THIS SPECIFIC call site today... this will change once task 6's live apply pump gives expectation and actual state a real chance to diverge across threads/time' -- that predicted future divergence has now materialized under a slow-boot + concurrent-live-traffic combination that only a scenario driving traffic this early/aggressively after cluster boot (the factstream scenario) exercises; existing soak scenarios (chaotic/sequential/force-snapshot) submit far fewer jobs, more slowly, and do not reliably land traffic inside a slow node's pre-recovery window. REFINED after further reproduction (10/10 runs total, across ASAN/release builds and job counts from 8 to 20): the trigger is specifically the RESTART/catch-up path, not the initial multi-node cold boot. A standalone control test (3 nodes, no kill, 20s settle before any traffic) submitted jobs and drove jstart/jfinish/jfail cleanly with zero APPLY_DIVERGENT -- confirming the race needs a node that was DOWN while the cluster kept moving (i.e. an actual failover), not merely a fast cold boot: the returning node's persist_recover()/cluster_init_phase2 vtask must complete while it is ALSO receiving a compressed catch-up replication burst covering everything it missed while down, and that burst's volume is inherent to 'kill mid-fact-stream' (the whole point of the scenario) -- a run-ha-soak.sh-side pacing fix cannot eliminate the trigger the way it did for the (separately real but distinct) initial-boot instance the settle-delay mitigation addresses. Reproduced identically down to 8 fact-stream jobs with 2-job/1s-gap waves, so the trigger threshold is low. This plausibly explains why no prior soak scenario in this spec's history (chaotic/sequential/force-snapshot, all using NJOBS=3 by default, un-paced but small) ever surfaced it: none combine a real kill-during-traffic failover with this much job+fact volume until raft-apply-decoupling task 7.2's factstream scenario. Also independently added test.c's print_jobinfo state_raw hex field (task 7.2) and confirmed via a live probe that apply_job_fail_transition correctly applies EXIT|PERR bits even when job_state_str's RUN-checked-first priority ordering hides them in the pretty-printed state string (job showed 'state: RUN' / 'state_raw: 0x106' after a jstart+jfail-without-jfinish sequence) -- a separate, minor, likely pre-existing (not introduced by raft-apply-decoupling) display-only quirk in job_state_str, also not fixed here (out of boundary), but which the factstream phantom-check was updated to work around by reading state_raw instead of the ambiguous pretty string.", "fix": "FIXED (round 2, raft-apply-decoupling task 7.2 review remediation). Root cause confirmed exactly as hypothesized in the prior entry's candidate #2: persist_verify_apply_correctness() compared a WAL-scan-derived 'expected' snapshot against a LIVE re-read of last_applied_lsn/count_jobmgr_entries(), while TWO live mutation paths could write those same values concurrently -- the live apply pump (persist_apply_pump_run, driven by persist_apply_pump_kick off any of commit.c's 6 committed_lsn-advance sites) and the standby live-replication apply path (ha_apply_single_entry / ha_apply_term_entry_via_library in ha.c). Fixed via a new _Atomic bool persist_t.recovery_complete gate (persist.h/persist.c): persist_apply_pump_kick() now no-ops while the gate is closed; ha_apply_single_entry/ha_apply_term_entry_via_library now skip ONLY the in-memory apply_wal_entry() call while closed (durability write + replication ACK + applied_lsn advance are never gated, preserving the existing persist-before-apply-before-ACK contract and bug-1043's durability/materialization decoupling precedent). persist_recover() closes the gate at entry and reopens it (plus issues one deliberate persist_apply_pump_kick catch-up pass) right before returning, regardless of verdict -- covering any direct unit-test caller. ROUND 2 FINDING during live verification: this alone was insufficient -- a node stuck retrying its own election as a losing/timing-out candidate can receive and durably apply live-replicated entries via the ungated (persist_init default=true) path for tens of seconds BEFORE cluster_init_phase2() ever calls persist_recover() for the first time, reproducing the identical false-DIVERGENT symptom via an earlier entry point that persist_recover()'s own bracket cannot see. Closed by adding persist_mark_recovery_pending(&c.persist) in myapp_run() (myapp.c), called immediately after persist_init()/persist_start() succeed and BEFORE ha_start_election_async()/cluster_run() can process any network traffic -- closing the gate from the earliest safe moment in the real boot sequence. persist_init()'s default stays TRUE (ungated) so none of this spec's ~20 existing unit-test fixtures that construct a persist_t directly (bypassing myapp_run entirely) are affected; persist_mark_recovery_pending is never called outside myapp.c. Verified: live 3-node factstream soak (2 rounds, kill+restart both rounds) now PASSes clean with zero APPLY_DIVERGENT/UNRECOVERABLE across 5 persist_recover() calls (2 initial-boot delayed by the SAME pre-existing election-instability condition that originally exposed the round-2 gap, 2 post-kill restarts, all APPLY_CLEAN); broader controller_unit_test sweep (23 groups incl. test_apply_correctness_verification's 3 real-divergence tests, test_apply_pump/_integration, test_ha_core_safety, test_demotion_convergence) shows zero new failures (only the 2 pre-existing, documented task-6.1/6.2 test_commit_barrier_reply.c failures remain, task 7.3's job). Real-divergence detection independently re-confirmed intact (test_apply_divergent_decision_forces_unconditional_fence, test_apply_divergent_loud_fence_sets_sticky_flag_and_stays_alive, test_real_replay_divergence_sets_unrecoverable_fence all still PASS).", @@ -15459,7 +3770,6 @@ "boot-race", "concurrency", "live-regression-found", - "fixed", "recovery-complete-gate", "myapp-boot-sequencing" ], @@ -15467,20 +3777,23 @@ "bug-1409", "bug-1415" ], - "occurrences": 11, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1416.md" }, { "id": "bug-1417", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-07T09:30:00.000Z", - "error_message": "[xce-test-ha --chaotic] FAIL round 4/10: election stall after killing ctrl-1 \u2014 ctrl-2 PreVote wins but real election times out despite ctrl-3 granting RequestVote", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "[xce-test-ha --chaotic] FAIL round 4/10: election stall after killing ctrl-1 — ctrl-2 PreVote wins…\n\n[xce-test-ha --chaotic] FAIL round 4/10: election stall after killing ctrl-1 — ctrl-2 PreVote wins but real election times out despite ctrl-3 granting RequestVote", "file": "src/controller/ha.c", - "root_cause": "vote_reply_dispatch passes local votes_idx (compressed, excludes self) directly to raft_on_vote_reply which expects ABSOLUTE peer index (includes self). For self_idx=1 (ctrl-2), ctrl-3's vote (absolute=2, local=1) lands in vote_replies[1]=ctrl-2's own self-slot \u2192 treated as duplicate \u2192 vote silently dropped. With ctrl-1 dead in chaotic mode, ctrl-2 only has self-vote (1 < quorum 2) \u2192 eternal timeout. Self_idx=0 and self_idx=2 are partially/fully immune due to index alignment luck. ha_repl_ack_abs_peer_idx() already exists (ha.c:3779) with correct mapping and passing tests \u2014 vote_reply_dispatch just doesn't call it.", - "fix": [ - "FIXED (src/controller/ha.c vote_reply_dispatch, ~line 5233): raft_on_vote_reply now receives ha_repl_ack_abs_peer_idx(ha, 1, idx) instead of the raw compressed local idx. ha_repl_ack_abs_peer_idx (ha.c:3779, already existed + unit-tested for the heartbeat-ACK path) performs the exact inverse of raft_cb_send_vote_request send-side remap: votes_idx < self_idx ? votes_idx : votes_idx + 1. ha_record_vote_reply (immediately preceding line) is UNCHANGED -- it correctly wants the compressed idx for ctx->votes[]. Confirmed pre_vote_reply_dispatch needs NO equivalent fix: raft_cb_send_pre_vote_request (ha.c:301) already stores the ABSOLUTE peer_idx in vctx->peer_idx for the PreVote path (explicit code comment there), matching raft_on_pre_vote_reply/pre_vote_replies[] absolute-index expectation -- only the real-vote send path (raft_cb_send_vote_request, ha.c:175) stores compressed. RED->GREEN: new unit test test_bug1417_middle_index_real_vote_from_peer2_not_dropped (test_ha_vote_send.c) drives a REAL 3-node election (self_idx=1) through raft_start_election then a genuine peer-2 grant via vote_reply_msg_handler_for_test (the actual decode/dispatch path, not the xce_vote_reply_hook bypass every other real-vote test in this file uses) -- reproduced Expected 2 Was 1 pre-fix, GREEN post-fix (votes_granted=2, role=RAFT_LEADER). Verified: full 79-group controller_unit_test sweep 0 new failures; 10-round --chaotic xce-test-ha soak PASS (committed_divergence=0, committed_job_loss=0, 18 jobs). Soak surfaced an UNRELATED, pre-existing --chaotic harness bug (run-ha-soak.sh run_chaotic_rounds: DOWN variable is not cleared on a skip-this-round no-primary-to-kill path, so a subsequent round calls start_node on the same already-running node without killing it first -- 6 stacked ctrl-2 processes observed piling up over rounds 5-10) -- confirmed via git diff this script was untouched by this fix, logged separately, NOT fixed here (out of bug-1417 boundary)." - ], + "root_cause": "vote_reply_dispatch passes local votes_idx (compressed, excludes self) directly to raft_on_vote_reply which expects ABSOLUTE peer index (includes self). For self_idx=1 (ctrl-2), ctrl-3's vote (absolute=2, local=1) lands in vote_replies[1]=ctrl-2's own self-slot → treated as duplicate → vote silently dropped. With ctrl-1 dead in chaotic mode, ctrl-2 only has self-vote (1 < quorum 2) → eternal timeout. Self_idx=0 and self_idx=2 are partially/fully immune due to index alignment luck. ha_repl_ack_abs_peer_idx() already exists (ha.c:3779) with correct mapping and passing tests — vote_reply_dispatch just doesn't call it.", + "fix": "FIXED (src/controller/ha.c vote_reply_dispatch, ~line 5233): raft_on_vote_reply now receives ha_repl_ack_abs_peer_idx(ha, 1, idx) instead of the raw compressed local idx. ha_repl_ack_abs_peer_idx (ha.c:3779, already existed + unit-tested for the heartbeat-ACK path) performs the exact inverse of raft_cb_send_vote_request send-side remap: votes_idx < self_idx ? votes_idx : votes_idx + 1. ha_record_vote_reply (immediately preceding line) is UNCHANGED -- it correctly wants the compressed idx for ctx->votes[]. Confirmed pre_vote_reply_dispatch needs NO equivalent fix: raft_cb_send_pre_vote_request (ha.c:301) already stores the ABSOLUTE peer_idx in vctx->peer_idx for the PreVote path (explicit code comment there), matching raft_on_pre_vote_reply/pre_vote_replies[] absolute-index expectation -- only the real-vote send path (raft_cb_send_vote_request, ha.c:175) stores compressed. RED->GREEN: new unit test test_bug1417_middle_index_real_vote_from_peer2_not_dropped (test_ha_vote_send.c) drives a REAL 3-node election (self_idx=1) through raft_start_election then a genuine peer-2 grant via vote_reply_msg_handler_for_test (the actual decode/dispatch path, not the xce_vote_reply_hook bypass every other real-vote test in this file uses) -- reproduced Expected 2 Was 1 pre-fix, GREEN post-fix (votes_granted=2, role=RAFT_LEADER). Verified: full 79-group controller_unit_test sweep 0 new failures; 10-round --chaotic xce-test-ha soak PASS (committed_divergence=0, committed_job_loss=0, 18 jobs). Soak surfaced an UNRELATED, pre-existing --chaotic harness bug (run-ha-soak.sh run_chaotic_rounds: DOWN variable is not cleared on a skip-this-round no-primary-to-kill path, so a subsequent round calls start_node on the same already-running node without killing it first -- 6 stacked ctrl-2 processes observed piling up over rounds 5-10) -- confirmed via git diff this script was untouched by this fix, logged separately, NOT fixed here (out of bug-1417 boundary).", "tags": [ "vote", "raft", @@ -15488,7 +3801,6 @@ "index-mapping", "chaotic-failover", "off-by-one", - "fixed", "ha", "ha_repl_ack_abs_peer_idx", "election" @@ -15496,15 +3808,55 @@ "related_bugs": [ "bug-828" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1417.md" + }, + { + "id": "bug-1418", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "9b173c9a + bug-1421 + raft-apply-decoupling", + "component": "controller", + "severity": "hang", + "timestamp": "2026-07-09T10:30:00.000Z", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 5, + "error_message": "Final resolution pass over bug-1418 (all 4 prior occurrences): the two contributing factors from…\n\nFinal resolution pass over bug-1418 (all 4 prior occurrences): the two contributing factors from the 2026-07-07T18:20 analysis (term desync + apply-divergence promotion-gate liveness deadlock) are both now resolved, not by the originally-proposed gate relaxation.", + "file": "src/controller/ha.c", + "root_cause": "Re-audited both factors against the CURRENT code (post raft-apply-decoupling 32/32, post bug-1421): (1) TERM DESYNC — ha_start_prevote_async (ha.c:6715) already computes its prospective term from ha->raft.term + 1, not the stale ha->term the original analysis described; ha.c:1009-1010 explicitly syncs ha->term from the raft term on every relevant transition. This half was already fixed independently of bug-1418's own tracked fix. (2) APPLY-DIVERGENCE LIVENESS DEADLOCK — the promotion gate (ha.c raft_cb_become_leader ~line 1130, and promotion_fence_timeout_fn ~line 876) unconditionally refuses promotion while persist_apply_verdict()==APPLY_DIVERGENT, by deliberate design (raft-apply-decoupling task 5.3, Req 8.4: a divergent local materialization must never be trusted to self-clear via any live/structural condition, including durable_floor>=cluster_hwm — only a fresh persist_recover() replay may clear it). The buglog's own previously-proposed fix (relax the gate to self-apply when durable_floor>=cluster_hwm) would have violated this intentional invariant and risked serving genuinely divergent state to clients. Instead, raft-apply-decoupling's tasks 5.1 (checkpoint_lsn mislabeled from WAL tip instead of the logical applied cursor, causing a false APPLY_DIVERGENT on an ordinary crash-during-checkpoint) and 7.2/bug-1416 (a boot-time TOCTOU where a returning node's persist_recover() replay-verify raced the live apply pump / live-replication apply path, producing a false APPLY_DIVERGENT on the restart/catch-up path specifically — the exact scenario bug-1418's residual occurrence hit: 'apply pump lagged behind WAL tip during kill') closed the false-positive detection paths that were producing spurious APPLY_DIVERGENT verdicts in the first place. A genuine (non-false-positive) divergence still correctly fences forever by design.", + "fix": "[Consolidated 2026-08-15: this ID previously had 4 duplicate entries in buglog.json (a pgrep-guard test-harness fix, a partial term-desync analysis, and a stale mid-investigation 'NOT FIXED' snapshot -- all superseded by this final resolution pass, which already accounted for all 3 in its own analysis). Collapsed to this single entry; the other 3 removed.]\n\nNo code change to the promotion gate itself — it is intentionally correct and must stay unconditional. The actual fix is the combination of already-shipped upstream work: bug-1421 (AE-reject term-discovery blindspot, this session) + raft-apply-decoupling tasks 5.1/5.3/7.2 (false-APPLY_DIVERGENT elimination, closed prior to this session). Validated: two 40-round soaks today (sequential 123 jobs zero-loss all-3-nodes-led; chaotic 123 jobs zero-committed-divergence zero-job-loss) — an order of magnitude more election churn than the original 10-round soak that first surfaced bug-1418 — produced zero apply-divergent promotion refusals and zero liveness deadlocks. Further corroborated by the 30-round sequential+chaotic regression run immediately after this entry.", + "tags": [ + "ha", + "raft", + "apply-divergence", + "promotion-gate", + "chaotic-failover", + "liveness", + "term-desync", + "resolved-via-upstream" + ], + "related_bugs": [ + "bug-1417", + "bug-1419", + "bug-1421", + "bug-1416" + ], + "fix_commits": [ + "9b173c9a" + ], + "record": ".wolf/bugs/bug-1418.md" }, { "id": "bug-1419", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-07-07T14:00:00.000Z", - "error_message": "[xce-test-ha --chaotic] genuine, reproducible election-liveness stall (not the bug-1418 harness process-stacking artifact, which was a separate, now-fixed issue): after 2+ overlapping kills, the surviving 2-node set can permanently deadlock -- neither node can ever become primary again for the rest of the run.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "[xce-test-ha --chaotic] genuine, reproducible election-liveness stall (not the bug-1418 harness…\n\n[xce-test-ha --chaotic] genuine, reproducible election-liveness stall (not the bug-1418 harness process-stacking artifact, which was a separate, now-fixed issue): after 2+ overlapping kills, the surviving 2-node set can permanently deadlock -- neither node can ever become primary again for the rest of the run.", "file": "src/controller/persist.c (apply-correctness verification) + src/controller/ha.c (PreVote freshness gate) + WAL duplicate-submit mechanism (unlocated)", "root_cause": "Direct log/WAL analysis (waldump on the affected node's data dir) during a live repro superseded two prior, less-precise theories filed as bug-1418 (a shell-script process-stacking artifact, unrelated and separately fixed; and an unverified ha->term/ha->raft.term \"term desync\" claim that does not match the code -- raft_begin_pre_vote/ha_start_prevote_async both read s->term == ha->raft.term directly, there is no separate stale-field read on the PreVote path).\n\nACTUAL mechanism observed: waldump on ctrl-3's WAL (the node with the longest/freshest log after the chaotic kill sequence) showed \"duplicate submits: 3 (byte-identical: 0)\" -- the SAME jobid appended to the WAL twice with DIFFERING payloads (different subtime, different CRC), e.g. lsn=13 and lsn=14 both jobid=0-0-1783418222-0 but distinct subtime/PCRC. This is NOT bug-845's mechanism (that fix, landed 2026-06-23, was for byte-IDENTICAL double-hook-registration dupes and is confirmed still working -- these are non-identical). count_jobmgr_entries() legitimately sees these as 2 distinct job-table mutations (or the entity count otherwise fails to match the WAL-replay-derived expectation), so persist_recover()'s post-replay apply-correctness check (task 5.1, Req 8.1/8.2) correctly fires APPLY_DIVERGENT -> the loud, sticky, non-promotable fence (raft-apply-decoupling Req 8.3/8.4) -- this fencing behavior is WORKING AS DESIGNED, not itself a bug.\n\nThe resulting LIVENESS bug: with the fenced node holding the only sufficiently-fresh (longest) log, the other surviving node can never win the PreVote freshness gate against it (correctly, per Raft/PreVote semantics -- it really is behind), while the fenced node itself can win Raft elections (quorum_reached/became_leader observed at terms 27/30/32 in logs) but its own raft_cb_become_leader promotion gate refuses CTRL_PRIMARY every time (apply_divergent=1). With the third node dead (killed by the chaotic driver) and never restarted long enough to catch up, NO node can ever become primary again -- a genuine, reproducible liveness deadlock, distinct from (and downstream of) the WAL duplicate-submit root cause. Confirmed via direct log grep across ctrl-1.log/ctrl-3.log at the exact stall timestamps (09:57:16 divergence detected, repeating \"PreVote DENIED ... reason=freshness\" and \"promotion REFUSED ... apply_divergent=1\" through 10:05+); NOT root-caused via speculation.", "fix": "FIXED (2026-07-07). Layer (1), the WAL duplicate-non-identical-submit mechanism: raft-apply-decoupling task 3.1 deferred job/entity construction in c->jobmgr until AFTER commit (apply_committed_entry, post-commit apply pump), but jobid_range_exists() -- the sole dup guard in handle_jobsubmit -- only checks c->jobmgr. A submission that is durably WAL-appended and awaiting its commit outcome does not yet exist in c->jobmgr, so a client-side retry for the SAME jobid (XCECLI_FAILOVER_CALL/xce_failover_classify's idempotent-replay contract resends on a transport error or failover-mid-flight) sailed straight past the guard and appended a second, non-identical WAL_JOB_SUBMIT entry (different subtime/CRC) for the same jobid -- exactly the 'duplicate submits: N (byte-identical: 0)' signature waldump showed. Fix: src/controller/job.c adds a process-lifetime in-flight-jobid-range registry (s_inflight_submits, guarded by s_inflight_submits_lock) that handle_jobsubmit atomically check-and-claims (inflight_range_try_add) immediately after the existing jobid_range_exists check, and releases (inflight_range_remove) from jobsubmit_async_ctx_release. Regression test test_jobsubmit_resubmit_before_resolution_rejected_no_dup_wal_entry added to test_commit_barrier_reply.c (RED->GREEN confirmed). Committed fe4352f.\n\nLayer (2), the PreVote-freshness/apply-divergence liveness deadlock: initially framed as needing a fundamental safety/liveness design tradeoff among 3 unpalatable options (force-promote a divergent node / truncate committed-but-unapplied entries / relax the freshness gate) -- this framing was WRONG, corrected after user pushback (\"kill-mid-apply should be recoverable per Raft protocol\", \"I do not think break FSM invariant is a good idea\"). Root-caused instead to a concrete, scoped bug: persist_checkpoint() (persist.c:2261-2263, now fixed) stamped p->checkpoint_lsn from p->current_lsn (the physical WAL append tip) instead of p->last_applied_lsn (the true 'jobmgr reflects up through here' cursor, persist.h:344-363 documents these as DISTINCT once apply is decoupled from commit). write_snapshot() serializes live c->jobmgr, so a checkpoint taken while the async apply pump lags behind the WAL tip mislabels the snapshot: on restart, last_applied_lsn is re-seeded FROM checkpoint_lsn and replay resumes strictly above it, silently skipping committed-but-not-yet-applied entries that jobmgr never actually received -- the real mechanism behind kill-mid-apply producing a permanent APPLY_DIVERGENT fence. Confirmed apply_committed_entry() is fully deterministic/idempotent (rules out non-determinism) and write_snapshot() is crash-atomic at the file level (temp+fdatasync+rename, rules out torn-file corruption) -- the bug is purely a mislabeled watermark, not a structural divergence. Fix: persist_checkpoint() now reads checkpoint_lsn from last_applied_lsn and holds apply_pump_lock across the label-capture + write_snapshot() call so the label and the jobmgr state it describes are captured atomically w.r.t. the live apply pump (the pump only ever moves last_applied_lsn/jobmgr forward, so freezing it here rules out the label ever claiming more than jobmgr actually holds). test_checkpoint_sets_checkpoint_lsn updated (now simulates apply catch-up explicitly, since a cluster-less unit-test persist_t never runs the apply pump); new regression test test_checkpoint_lsn_tracks_last_applied_not_current_lsn added to test_persist_checkpoint.c (RED confirmed pre-fix: checkpoint_lsn took the WAL tip=7 instead of the simulated applied cursor=2; GREEN after). Full 78-group/782-test controller_unit_test sweep: 0 failures.\n\nADDENDUM (2026-07-08, post-fix re-soak): a fresh chaotic soak on a rebuilt binary (both layers included) cleared rounds 1-4 (previously stalled at round 4) but still hit a fresh APPLY_DIVERGENT fence at round 5. Deep dive (waldump + log correlation + code trace) proved BOTH layer 1 and layer 2 are working correctly and are NOT the cause: added test_jobsubmit_resubmit_after_full_resolution_rejected_no_second_job (test_commit_barrier_reply.c) proving the post-resolution dedup window is also airtight (-EEXIST, no second WAL entry) -- both known dedup windows are provably race-free. The round-5 duplicate jobid was traced instead to a TEST-HARNESS-ONLY defect: xce_test (test.c:1868) hardcoded idgen(&idctx, 0, 0, ...) (clusterId=apiserverId=0) and run-ha-soak.sh's submit_jobs() spawns a fresh xce_test process per retry attempt, resetting idgen's static idctx.sequence to 0 each time -- two unrelated job submissions from two fresh processes landing in the same wall-clock second collide on an identical jobid (ts+tag+seq all equal). Confirmed NOT a production risk: apiserver's GenerateJobID (clib.go) holds one process-lifetime idgen_ctx_t via sync.Once and each deployed apiserver instance has its own distinct apiserverId, so config.h's jobid bit-layout (cluster/apiserver bits) already partitions production's namespace. Fixed by folding getpid() into the harness's apiserverId (test.c:1868) so cross-process collisions become structurally impossible there too; filed as bug-1420. bug-1419 itself (both production layers) stands as correctly fixed -- this addendum exists so a future session doesn't misread the round-5 re-soak stall as evidence layer 1/2 are insufficient.", @@ -15518,8 +3870,7 @@ "liveness", "wal-duplicate-submit", "checkpoint-lsn", - "kill-mid-apply", - "fixed" + "kill-mid-apply" ], "related_bugs": [ "bug-1417", @@ -15528,15 +3879,22 @@ "bug-1418", "bug-1420" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "fe4352f" + ], + "record": ".wolf/bugs/bug-1419.md" }, { "id": "bug-1420", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-08T04:00:00.000Z", - "error_message": "xce_test CLI harness generates colliding jobids across separate process invocations within the same wall-clock second, surfacing as a spurious APPLY_DIVERGENT fence in chaotic HA soak (misattributed at first to bug-1419's controller-side dedup logic).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_test CLI harness generates colliding jobids across separate process invocations within the…\n\nxce_test CLI harness generates colliding jobids across separate process invocations within the same wall-clock second, surfacing as a spurious APPLY_DIVERGENT fence in chaotic HA soak (misattributed at first to bug-1419's controller-side dedup logic).", "file": "src/controller/testsuite/test.c", "root_cause": "idgen() (src/lib/idgen.c) derives jobid from a per-context {timestamp (1s resolution), sequence (reset to 0 whenever the second rolls over or the context is freshly initialized)} pair, plus caller-supplied clusterId/apiserverId bits (src/include/config.h: 6+4 bits respectively) that are meant to partition the jobid namespace across independent generators. xce_test's cmd_jsub called idgen(&idctx, 0, 0, arsize, ...) -- both partition fields hardcoded to 0 -- so every xce_test process shares the SAME (degenerate, unpartitioned) jobid space. run-ha-soak.sh's submit_jobs() (run-ha-soak.sh:446) spawns a fresh xce_test process per submit/retry attempt (`printf script | xtest`, xtest() = `timeout 60 \"$XT\"`), and idgen_ctx_init() resets sequence=0 on every fresh process start. Two unrelated job submissions from two different xce_test process launches landing in the same wall-clock second therefore compute an IDENTICAL jobid (ts, tag, seq all equal) -- a genuine, reproducible collision between logically distinct jobs, not a retry of the same one. This is NOT a production defect: apiserver's GenerateJobID (src/frontend/apiserver/backend/clib.go) initializes its idgen_ctx_t exactly once per process via sync.Once (sequence only resets on an actual apiserver restart, far rarer than per-request), and each deployed apiserver instance is assigned its own distinct apiserverId, so config.h's bit layout already partitions the production jobid namespace across instances even in the rare restart-collision case.", "fix": "Folded getpid() into the harness's apiserverId at the idgen() call site (test.c:1868): idgen(&idctx, 0, (size_t)((uint64_t)getpid() & MAX_APISERVER_ID), arsize, &(jobsubmit.jobid)) -- gives every xce_test process invocation its own slice of the same namespace partitioning production already relies on, making a same-second cross-process collision structurally impossible. Mechanical, test-tool-only change; no production code touched. Verified via clean rebuild of both xce_test and controller_unit_test targets (no coupling, no regressions).", @@ -15545,23 +3903,27 @@ "jobid", "idgen", "chaotic-soak", - "false-positive", - "fixed" + "false-positive" ], "related_bugs": [ "bug-1419" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1420.md" }, { "id": "bug-1421", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-08T06:55:15.000Z", - "error_message": "Sequential HA soak regression: no primary elected at term>8 by round 6 (initially suspected to stem from the bug-1417 fix); root-caused 2026-07-09 as the primary never discovering a follower's higher term via AppendEntries-reject responses", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Sequential HA soak regression: no primary elected at term>8 by round 6 (initially suspected to…\n\nSequential HA soak regression: no primary elected at term>8 by round 6 (initially suspected to stem from the bug-1417 fix); root-caused 2026-07-09 as the primary never discovering a follower's higher term via AppendEntries-reject responses", "file": "src/controller/ha.c", - "root_cause": "ha_raft_on_repl_ack only routes AE-reply term-adoption for success=1 ACKs; AE rejects (success=0) never reach raft_on_append_entries_reply. That function additionally drops any reply with reply_term != s->term (raft.c:832-833, stale-term drop) before it can adopt a higher term. Net effect: once a follower is ahead in term (e.g. it steps down from PreVote to CTRL_STANDBY at a higher term while an existing primary keeps campaigning/replicating at its old term), every AppendEntries the stale primary sends is rejected with ae_reject_stale, the primary never learns the higher term, and it can never regain write-quorum (needs 2/3, only self-vote counts) \u2014 a permanent livelock. First surfaced as an election regression (sequential soak: no primary elected at term>8 by round 6, chaotic stalled at round 5) suspected to be caused by the bug-1417 vote-index-mapping fix; re-reproduced 2026-07-09 as a round-4 sequential stall (primary ctrl-2 term=6, follower ctrl-3 term=7, 490 ae_reject_stale events, primary writable_lsn stuck at 17), which pinned the exact mechanism above.", + "root_cause": "ha_raft_on_repl_ack only routes AE-reply term-adoption for success=1 ACKs; AE rejects (success=0) never reach raft_on_append_entries_reply. That function additionally drops any reply with reply_term != s->term (raft.c:832-833, stale-term drop) before it can adopt a higher term. Net effect: once a follower is ahead in term (e.g. it steps down from PreVote to CTRL_STANDBY at a higher term while an existing primary keeps campaigning/replicating at its old term), every AppendEntries the stale primary sends is rejected with ae_reject_stale, the primary never learns the higher term, and it can never regain write-quorum (needs 2/3, only self-vote counts) — a permanent livelock. First surfaced as an election regression (sequential soak: no primary elected at term>8 by round 6, chaotic stalled at round 5) suspected to be caused by the bug-1417 vote-index-mapping fix; re-reproduced 2026-07-09 as a round-4 sequential stall (primary ctrl-2 term=6, follower ctrl-3 term=7, 490 ae_reject_stale events, primary writable_lsn stuck at 17), which pinned the exact mechanism above.", "fix": "Added two fallback term-discovery vectors that fire regardless of AE-reject outcome. (1) StateReport heartbeat-reply now carries the standby's current term (controller.capnp +term field @2, typemap.h state_report_t +term, ha_nop_reply_state_report populates it from ha->term) so the primary learns a higher term on every convergence cycle. (2) New library entry point raft_on_higher_term_observed(s, observed_term) mirrors UpdateTerm (strictly-greater-than only; touches term/voted_for/role; on_persist before on_become_follower for durability ordering) and is called from repl_ack_decode_and_dispatch on both the heartbeat state-report branch and the WAL-NACK branch (ha.c). Primary now steps down instead of retrying AppendEntries against a follower that rejects forever. Validated: 265 controller + 287 lib unit tests pass (zero regressions); 10-round sequential soak PASS (33 jobs, zero loss, all 3 nodes led ctrl-1/ctrl-2/ctrl-3); 10-round chaotic soak PASS (33 jobs, zero committed divergence, zero committed job loss).", "tags": [ "ha", @@ -15570,81 +3932,59 @@ "liveness", "election-regression", "ae-reject", - "replication", - "fixed" + "replication" ], "related_bugs": [ "bug-1417", "bug-1418", "bug-851" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1421.md" }, { "id": "bug-1422", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-07-09T10:14:14.000Z", - "error_message": "40-round sequential HA soak FAIL at round 36: verify_no_loss reported '0/111 jobs confirmed present' right after a fresh election (new primary ctrl-1, term=40) \u2014 xce_test's expect-job got E_XCE_STANDBY (10014) from a fresh client session whose own wait-primary had JUST confirmed ctrl-1 as serving 45ms earlier", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "40-round sequential HA soak FAIL at round 36: verify_no_loss reported '0/111 jobs confirmed…\n\n40-round sequential HA soak FAIL at round 36: verify_no_loss reported '0/111 jobs confirmed present' right after a fresh election (new primary ctrl-1, term=40) — xce_test's expect-job got E_XCE_STANDBY (10014) from a fresh client session whose own wait-primary had JUST confirmed ctrl-1 as serving 45ms earlier", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "False-positive test-harness failure, NOT real committed-job loss: ctrl-1's own controller log confirms it stayed CTRL_PRIMARY throughout the window and successfully committed the 3 new post-failover jobs (log_append/ae_reply_success/commit_advanced at 10:14:22-23). ha_dispatch_gate's CTRL_PRIMARY branch (ha.c:3451-3531) unconditionally permits OPCode_get (joblist) even while fenced, so ctrl-1 itself could not have returned E_XCE_STANDBY for this query \u2014 the reply must have come from a standby (ctrl-2, just killed, or ctrl-3) that the client's failover retry loop landed on. Root cause traced to client_failover.c's xcecli_retry_next/xcecli_retry_decide (client_failover.c:305-319): E_XCE_STANDBY classifies as XCE_FO_RETRY_NOW (client_failover.c:59-60, prompt rotate, no backoff), and the 20-attempt retry budget (3 masters) should reliably reach the real primary \u2014 but when the WHOLE budget is exhausted while every recent classification was RETRY_NOW (as opposed to RETRY_BACKOFF), xcecli_retry_decide's budget-exhaustion branch (line 312-318) does NOT override *rc the way it does for RETRY_BACKOFF exhaustion (which maps to the distinct -E_XCE_NOT_READY, line 316-317) \u2014 it just returns false with *rc left at whatever the last attempt's raw status was (here, a stray E_XCE_STANDBY from a standby hit during the rapid post-election retry storm). verify_no_loss() (run-ha-soak.sh) then did a single one-shot check with zero retry at the driver level, unlike the job-submit loop just above it in the same file which already retries on this exact 'primary likely just promoted, regaining write-quorum' condition (run-ha-soak.sh:483-486). Only surfaced at round 36 of a 40-round run \u2014 narrow enough timing that shorter 6-10 round soaks across many prior sessions never hit it.", - "fix": "Added a bounded retry (3 total attempts, 2s settle sleep between) around the WHOLE verify_no_loss() check in run-ha-soak.sh, mirroring the existing submit-retry loop's rationale: a job still missing after a real settle window + 2 retries is a genuine loss, not this client-side retry-storm race. Did not touch client_failover.c (the battle-tested libxce-failover retry/rotation library with many other production consumers) \u2014 fixing at the test-driver level is safer and fully local. Re-ran both 40-round soaks after the fix: sequential PASS (123 jobs, zero loss, all 3 nodes led) and chaotic PASS (123 jobs, zero committed divergence, zero committed job loss).", + "root_cause": "False-positive test-harness failure, NOT real committed-job loss: ctrl-1's own controller log confirms it stayed CTRL_PRIMARY throughout the window and successfully committed the 3 new post-failover jobs (log_append/ae_reply_success/commit_advanced at 10:14:22-23). ha_dispatch_gate's CTRL_PRIMARY branch (ha.c:3451-3531) unconditionally permits OPCode_get (joblist) even while fenced, so ctrl-1 itself could not have returned E_XCE_STANDBY for this query — the reply must have come from a standby (ctrl-2, just killed, or ctrl-3) that the client's failover retry loop landed on. Root cause traced to client_failover.c's xcecli_retry_next/xcecli_retry_decide (client_failover.c:305-319): E_XCE_STANDBY classifies as XCE_FO_RETRY_NOW (client_failover.c:59-60, prompt rotate, no backoff), and the 20-attempt retry budget (3 masters) should reliably reach the real primary — but when the WHOLE budget is exhausted while every recent classification was RETRY_NOW (as opposed to RETRY_BACKOFF), xcecli_retry_decide's budget-exhaustion branch (line 312-318) does NOT override *rc the way it does for RETRY_BACKOFF exhaustion (which maps to the distinct -E_XCE_NOT_READY, line 316-317) — it just returns false with *rc left at whatever the last attempt's raw status was (here, a stray E_XCE_STANDBY from a standby hit during the rapid post-election retry storm). verify_no_loss() (run-ha-soak.sh) then did a single one-shot check with zero retry at the driver level, unlike the job-submit loop just above it in the same file which already retries on this exact 'primary likely just promoted, regaining write-quorum' condition (run-ha-soak.sh:483-486). Only surfaced at round 36 of a 40-round run — narrow enough timing that shorter 6-10 round soaks across many prior sessions never hit it.", + "fix": "Added a bounded retry (3 total attempts, 2s settle sleep between) around the WHOLE verify_no_loss() check in run-ha-soak.sh, mirroring the existing submit-retry loop's rationale: a job still missing after a real settle window + 2 retries is a genuine loss, not this client-side retry-storm race. Did not touch client_failover.c (the battle-tested libxce-failover retry/rotation library with many other production consumers) — fixing at the test-driver level is safer and fully local. Re-ran both 40-round soaks after the fix: sequential PASS (123 jobs, zero loss, all 3 nodes led) and chaotic PASS (123 jobs, zero committed divergence, zero committed job loss).", "tags": [ "xce-test-ha", "false-positive", "client-failover", "retry-budget-exhaustion", "E_XCE_STANDBY", - "test-harness", - "fixed" + "test-harness" ], "related_bugs": [ "bug-854", "bug-1421" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1418", - "status": "fixed", - "status_updated": "2026-09-24", - "status_note": "9b173c9a + bug-1421 + raft-apply-decoupling", - "timestamp": "2026-07-09T10:30:00.000Z", - "error_message": "Final resolution pass over bug-1418 (all 4 prior occurrences): the two contributing factors from the 2026-07-07T18:20 analysis (term desync + apply-divergence promotion-gate liveness deadlock) are both now resolved, not by the originally-proposed gate relaxation.", - "file": "src/controller/ha.c", - "root_cause": "Re-audited both factors against the CURRENT code (post raft-apply-decoupling 32/32, post bug-1421): (1) TERM DESYNC \u2014 ha_start_prevote_async (ha.c:6715) already computes its prospective term from ha->raft.term + 1, not the stale ha->term the original analysis described; ha.c:1009-1010 explicitly syncs ha->term from the raft term on every relevant transition. This half was already fixed independently of bug-1418's own tracked fix. (2) APPLY-DIVERGENCE LIVENESS DEADLOCK \u2014 the promotion gate (ha.c raft_cb_become_leader ~line 1130, and promotion_fence_timeout_fn ~line 876) unconditionally refuses promotion while persist_apply_verdict()==APPLY_DIVERGENT, by deliberate design (raft-apply-decoupling task 5.3, Req 8.4: a divergent local materialization must never be trusted to self-clear via any live/structural condition, including durable_floor>=cluster_hwm \u2014 only a fresh persist_recover() replay may clear it). The buglog's own previously-proposed fix (relax the gate to self-apply when durable_floor>=cluster_hwm) would have violated this intentional invariant and risked serving genuinely divergent state to clients. Instead, raft-apply-decoupling's tasks 5.1 (checkpoint_lsn mislabeled from WAL tip instead of the logical applied cursor, causing a false APPLY_DIVERGENT on an ordinary crash-during-checkpoint) and 7.2/bug-1416 (a boot-time TOCTOU where a returning node's persist_recover() replay-verify raced the live apply pump / live-replication apply path, producing a false APPLY_DIVERGENT on the restart/catch-up path specifically \u2014 the exact scenario bug-1418's residual occurrence hit: 'apply pump lagged behind WAL tip during kill') closed the false-positive detection paths that were producing spurious APPLY_DIVERGENT verdicts in the first place. A genuine (non-false-positive) divergence still correctly fences forever by design.", - "fix": "[Consolidated 2026-08-15: this ID previously had 4 duplicate entries in buglog.json (a pgrep-guard test-harness fix, a partial term-desync analysis, and a stale mid-investigation 'NOT FIXED' snapshot -- all superseded by this final resolution pass, which already accounted for all 3 in its own analysis). Collapsed to this single entry; the other 3 removed.]\n\nNo code change to the promotion gate itself \u2014 it is intentionally correct and must stay unconditional. The actual fix is the combination of already-shipped upstream work: bug-1421 (AE-reject term-discovery blindspot, this session) + raft-apply-decoupling tasks 5.1/5.3/7.2 (false-APPLY_DIVERGENT elimination, closed prior to this session). Validated: two 40-round soaks today (sequential 123 jobs zero-loss all-3-nodes-led; chaotic 123 jobs zero-committed-divergence zero-job-loss) \u2014 an order of magnitude more election churn than the original 10-round soak that first surfaced bug-1418 \u2014 produced zero apply-divergent promotion refusals and zero liveness deadlocks. Further corroborated by the 30-round sequential+chaotic regression run immediately after this entry.", - "tags": [ - "ha", - "raft", - "apply-divergence", - "promotion-gate", - "chaotic-failover", - "liveness", - "term-desync", - "resolved-via-upstream", - "fixed" - ], - "related_bugs": [ - "bug-1417", - "bug-1419", - "bug-1421", - "bug-1416" - ], - "occurrences": 5, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1422.md" }, { "id": "bug-1423", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-07-09T13:58:16.000Z", - "error_message": "30-round sequential HA soak FAIL at round 5: post-failover submit did not get 3 acks after 5 retry attempts (15s+). Root-caused as a genuine permanent election deadlock, not a client-side false positive: ctrl-3 won PreVote and entered CTRL_ELECTING (real election, term=10) with ctrl-1 dead and ctrl-2 alive, then never showed another HA state-machine event for 2+ minutes (until the harness killed the cluster on FAIL) \u2014 stuck in CTRL_ELECTING forever. Meanwhile ctrl-2 endlessly retried its own PreVote every ~15-17s, correctly denied every time by ctrl-3 on freshness grounds, but that denial never unstuck ctrl-3's already-in-flight real election.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "30-round sequential HA soak FAIL at round 5: post-failover submit did not get 3 acks after 5 retry…\n\n30-round sequential HA soak FAIL at round 5: post-failover submit did not get 3 acks after 5 retry attempts (15s+). Root-caused as a genuine permanent election deadlock, not a client-side false positive: ctrl-3 won PreVote and entered CTRL_ELECTING (real election, term=10) with ctrl-1 dead and ctrl-2 alive, then never showed another HA state-machine event for 2+ minutes (until the harness killed the cluster on FAIL) — stuck in CTRL_ELECTING forever. Meanwhile ctrl-2 endlessly retried its own PreVote every ~15-17s, correctly denied every time by ctrl-3 on freshness grounds, but that denial never unstuck ctrl-3's already-in-flight real election.", "file": "src/controller/ha.c", - "root_cause": "raft_cb_send_vote_request (ha.c, sends the real RequestVote RPC to each peer) has two failure branches \u2014 endpoint_connect() failing and iomgr_monitor() failing \u2014 that are exactly what happens when the RPC target is a dead/unreachable peer (here, ctrl-1, killed at the start of round 5). The endpoint_connect failure branch only called the legacy handle_request_vote_reply() when `c == NULL` (unit-test mode) \u2014 in PRODUCTION (c always non-NULL) a connect failure to a dead peer was silently dropped with NO notification to the raft library at all. The iomgr_monitor failure branch called handle_request_vote_reply() unconditionally, including in production \u2014 but per vote_reply_dispatch's own design comment, handle_request_vote_reply is the LEGACY/ctx-done-signaling path; 'raft_on_vote_reply remains the SOLE production promotion driver' (vote_reply_dispatch calls ha_record_vote_reply + raft_on_vote_reply, NOT handle_request_vote_reply). Net effect: a candidate's vote request to a dead peer never registers as a 'reply' with the production raft library's raft_state_t.replies_received counter. raft.c's Step 6 fast-path ('all peers have replied, no quorum \u2192 become follower and retry', gated on replies_received == n_peers-1) can therefore NEVER trigger whenever any configured peer is down \u2014 with quorum=2 in a 3-node cluster and one peer dead, replies_received caps at 1 (only the live peer's real reply, if any) and can never reach n_peers-1=2. Recovery then depends ENTIRELY on the raw per-campaign election_timeout (5-10s, ha_jittered_election_delay) firing via delaymgr \u2014 in the observed failure, no election_timeout_fn trace appeared in the log at all for the full 2+ minute window, leaving the candidate permanently wedged.", - "fix": "Routed both raft_cb_send_vote_request failure branches (connect failure and monitor failure) through vote_reply_dispatch(ha, ectx, votes_idx, term, vote_granted=0, 0, 0) in production (c != NULL) \u2014 the exact same call a genuine vote reply makes (ha_record_vote_reply + raft_on_vote_reply), so a dead/unreachable peer now correctly counts as 'replied, not granted' immediately on connect/monitor failure instead of only via the timeout. Test mode (c == NULL) keeps the legacy handle_request_vote_reply call unit tests depend on. Added a forward declaration of the previously-file-local-only vote_reply_dispatch near the top of ha.c so raft_cb_send_vote_request (defined much earlier in the file) can call it. Verified: full controller_unit_test suite clean (755 PASS / 0 Fail, exit 0) both before and after removing temporary diagnostic logging used during root-cause; re-ran the exact failing scenario \u2014 30-round sequential HA soak PASS (93 jobs, zero loss, all 3 nodes led) and 30-round chaotic HA soak PASS (93 jobs, zero committed divergence, zero committed job loss) \u2014 both previously hit this stall reliably (round 5 of the sequential run, 100% reproduction before the fix).", + "root_cause": "raft_cb_send_vote_request (ha.c, sends the real RequestVote RPC to each peer) has two failure branches — endpoint_connect() failing and iomgr_monitor() failing — that are exactly what happens when the RPC target is a dead/unreachable peer (here, ctrl-1, killed at the start of round 5). The endpoint_connect failure branch only called the legacy handle_request_vote_reply() when `c == NULL` (unit-test mode) — in PRODUCTION (c always non-NULL) a connect failure to a dead peer was silently dropped with NO notification to the raft library at all. The iomgr_monitor failure branch called handle_request_vote_reply() unconditionally, including in production — but per vote_reply_dispatch's own design comment, handle_request_vote_reply is the LEGACY/ctx-done-signaling path; 'raft_on_vote_reply remains the SOLE production promotion driver' (vote_reply_dispatch calls ha_record_vote_reply + raft_on_vote_reply, NOT handle_request_vote_reply). Net effect: a candidate's vote request to a dead peer never registers as a 'reply' with the production raft library's raft_state_t.replies_received counter. raft.c's Step 6 fast-path ('all peers have replied, no quorum → become follower and retry', gated on replies_received == n_peers-1) can therefore NEVER trigger whenever any configured peer is down — with quorum=2 in a 3-node cluster and one peer dead, replies_received caps at 1 (only the live peer's real reply, if any) and can never reach n_peers-1=2. Recovery then depends ENTIRELY on the raw per-campaign election_timeout (5-10s, ha_jittered_election_delay) firing via delaymgr — in the observed failure, no election_timeout_fn trace appeared in the log at all for the full 2+ minute window, leaving the candidate permanently wedged.", + "fix": "Routed both raft_cb_send_vote_request failure branches (connect failure and monitor failure) through vote_reply_dispatch(ha, ectx, votes_idx, term, vote_granted=0, 0, 0) in production (c != NULL) — the exact same call a genuine vote reply makes (ha_record_vote_reply + raft_on_vote_reply), so a dead/unreachable peer now correctly counts as 'replied, not granted' immediately on connect/monitor failure instead of only via the timeout. Test mode (c == NULL) keeps the legacy handle_request_vote_reply call unit tests depend on. Added a forward declaration of the previously-file-local-only vote_reply_dispatch near the top of ha.c so raft_cb_send_vote_request (defined much earlier in the file) can call it. Verified: full controller_unit_test suite clean (755 PASS / 0 Fail, exit 0) both before and after removing temporary diagnostic logging used during root-cause; re-ran the exact failing scenario — 30-round sequential HA soak PASS (93 jobs, zero loss, all 3 nodes led) and 30-round chaotic HA soak PASS (93 jobs, zero committed divergence, zero committed job loss) — both previously hit this stall reliably (round 5 of the sequential run, 100% reproduction before the fix).", "tags": [ "ha", "raft", @@ -15653,49 +3993,56 @@ "vote-request", "dead-peer", "quorum", - "permanent-stall", - "fixed" + "permanent-stall" ], "related_bugs": [ "bug-1417", "bug-1418", "bug-1421" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1423.md" }, { "id": "bug-1424", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-09T17:00:00.000Z", - "error_message": "PreVote twin of bug-1423: raft_cb_send_pre_vote_request's connect/monitor failure branches (a dead/unreachable peer) never registered a reply with the raft library's PreVote tally, in EITHER test or production mode (worse than the real-vote path, which at least handled test mode).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "PreVote twin of bug-1423: raft_cb_send_pre_vote_request's connect/monitor failure branches (a…\n\nPreVote twin of bug-1423: raft_cb_send_pre_vote_request's connect/monitor failure branches (a dead/unreachable peer) never registered a reply with the raft library's PreVote tally, in EITHER test or production mode (worse than the real-vote path, which at least handled test mode).", "file": "src/controller/ha.c", - "root_cause": "Same missing-registration pattern as bug-1423, in raft_cb_send_pre_vote_request instead of raft_cb_send_vote_request. Found while auditing for the analogous gap after fixing bug-1423. Lower severity than bug-1423: PreVote has an unconditional round-timeout fallback (ha_prevote_round_timeout_fn, confirmed firing reliably every round across dozens of observed rounds in the bug-1423 investigation logs) that always resolves the round regardless of tally completeness, so this was never a permanent-deadlock class bug like bug-1423 was \u2014 just a latency waste: every PreVote round against a down peer burns the full round timeout instead of concluding as soon as the live peer(s) reply.", - "fix": "Routed both failure branches through pre_vote_reply_dispatch(ha, c, peer_idx, granted=0) \u2014 the same production PreVote-reply path a real reply uses (raft_on_pre_vote_reply) \u2014 mirroring bug-1423's fix exactly. Added a forward declaration near the top of ha.c (pre_vote_reply_dispatch is defined much later in the file). Verified: full controller_unit_test suite clean (0 Fail, exit 0) after the change.", + "root_cause": "Same missing-registration pattern as bug-1423, in raft_cb_send_pre_vote_request instead of raft_cb_send_vote_request. Found while auditing for the analogous gap after fixing bug-1423. Lower severity than bug-1423: PreVote has an unconditional round-timeout fallback (ha_prevote_round_timeout_fn, confirmed firing reliably every round across dozens of observed rounds in the bug-1423 investigation logs) that always resolves the round regardless of tally completeness, so this was never a permanent-deadlock class bug like bug-1423 was — just a latency waste: every PreVote round against a down peer burns the full round timeout instead of concluding as soon as the live peer(s) reply.", + "fix": "Routed both failure branches through pre_vote_reply_dispatch(ha, c, peer_idx, granted=0) — the same production PreVote-reply path a real reply uses (raft_on_pre_vote_reply) — mirroring bug-1423's fix exactly. Added a forward declaration near the top of ha.c (pre_vote_reply_dispatch is defined much later in the file). Verified: full controller_unit_test suite clean (0 Fail, exit 0) after the change.", "tags": [ "ha", "raft", "prevote", "dead-peer", - "latency", - "fixed" + "latency" ], "related_bugs": [ "bug-1423" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1424.md" }, { "id": "bug-1425", "status": "fixed", "status_updated": "2026-09-24", "status_note": "8a6c339a", + "component": "controller", + "severity": "hang", "timestamp": "2026-07-09T21:15:00.000Z", - "error_message": "Intermittent (~2 of 9 attempts) client hang in the synchronous jobsub (COMMIT_MODE_WAIT) path shortly after a fresh single-node election.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Intermittent (~2 of 9 attempts) client hang in the synchronous jobsub (COMMIT_MODE_WAIT) path…\n\nIntermittent (~2 of 9 attempts) client hang in the synchronous jobsub (COMMIT_MODE_WAIT) path shortly after a fresh single-node election.", "file": "src/controller/persist.c or src/controller/commit.c (unrooted)", - "root_cause": "NOT ROOT-CAUSED. Flagged during raft-apply-decoupling task 7.4's throughput benchmark work. Reproduced identically on both the pre-fix and post-fix (task 6.1 apply-pump) binaries \u2014 confirmed unrelated to that fix. Resembles but is not confirmed identical to the closed bug-1416 boot-race class (persist_recover() replay-verify racing the live apply pump / live-replication apply path).", + "root_cause": "NOT ROOT-CAUSED. Flagged during raft-apply-decoupling task 7.4's throughput benchmark work. Reproduced identically on both the pre-fix and post-fix (task 6.1 apply-pump) binaries — confirmed unrelated to that fix. Resembles but is not confirmed identical to the closed bug-1416 boot-race class (persist_recover() replay-verify racing the live apply pump / live-replication apply path).", "fix": "[UPDATE 2026-08-16] Live investigation (single-node deploy via /xce-deploy-ha, controller_unit_test's own fix build): found a RELIABLE reproduction recipe the original ~2/9-sequential-attempts description never had, and narrowed the root cause to a specific mechanism, but did NOT land a fix -- reporting findings honestly per the explicit scope of this investigation pass.\n\nRELIABLE REPRO (not sequential -- CONCURRENT): deploy a fresh single-node cluster, poll the log for CTRL_PRIMARY, then fire 9 xce_test client processes (each: connect + one jobsub) SIMULTANEOUSLY (backgrounded, not sequential) immediately after election. Reproduced in 3 of 4 rounds (1-3 of 9 concurrent clients failing per round, closely matching the original ~2/9 ratio) -- 0/45 reproductions when the SAME 9 attempts were fired sequentially instead (one xce_test process at a time). Concurrency at connect time, not raw timing after election, is the actual trigger.\n\nSYMPTOM, precisely: the failing client's xcecli_jobsubmit() (client.c, wrapped in XCECLI_FAILOVER_CALL) returns rc>=0 (success-shaped, NOT a negative error) after exactly ~10.0-10.1s, with the output jobid=0 (never written) instead of the submitted JobId -- test.c's cmd_jobsub (bug-786's own guard) correctly catches this as 'no clean ack' rather than misreporting success, so no data is actually lost/corrupted -- but the client hangs ~10s before getting a clear answer, matching the bug's own description exactly.\n\nCONFIRMED CLIENT-SIDE, not server-side: cross-referenced the controller's own log (fresh --logfile per round) against the 9 concurrent attempts -- server-side WAL only ever shows 8 job-submit entries committed (lsn sequence has no gaps, no admission-cap rejection WARN, no error of any kind) when 9 concurrent clients attempted -- the failing 9th request never reaches handle_jobsubmit at all. Ruled out the obvious admission-cap suspect: commit.c's COMMIT_MODE_WAIT admission control (cap=n_vcpus-VTASK_LIVENESS_RESERVED=3 on this 4-vcpu deploy) never logged its 'WAIT rejected, admission cap exhausted' WARN in any reproduction -- 8 concurrent WAITs completing well under the cap is plausible given each takes <100ms server-side once received.\n\nTHE ~10s NUMBER, precisely sourced: xceapi.c's xce_jobsubmit() (the innermost call XCECLI_FAILOVER_CALL wraps) has a HARDCODED cbuf_deadline_10s() wait on cbuf_timedget waiting for a reply to land in the single-slot ctx->wbuf. Read cbuf_timedget itself (mylock.c) -- confirmed CORRECT, it returns -ETIMEDOUT (negative) on a genuine timeout, which xce_jobsubmit correctly propagates as a negative rc. So a bare cbuf_timedget timeout cannot by itself explain the observed rc>=0-with-jobid=0 shape; something in the retry/rotate/recover layer ABOVE it must be involved.\n\nNARROWED SUSPECT (not confirmed): client_internal.h's own documented CI1 hazard -- 'handle_disconnect may have injected a stale status into the single-slot ctx->wbuf ... xcecli_ctx_recover drains any pending entry with a non-blocking cbuf_tryget so the next attempt's cbuf_get cannot consume a prior-attempt reply' -- but this drain only runs when xce_failover_rotate() succeeds (client_failover.c:130-153, mirrored by xcecli_retry_next per its own doc comment: 'only recover when the rotate succeeded'). On this single-candidate cluster, rotate reconnects to the SAME (only) node via a real xce_ctx_reconnect() -- plausible this usually succeeds (ruling out the simplest 'rotate never succeeds on single-node, drain never runs' theory) but NOT independently verified under the exact concurrent-connect contention this repro needs; a transient reconnect failure specifically during the connection-establishment storm (9 simultaneous TLS handshakes against a listener seconds old) could leave a stale wbuf entry undrained, which a LATER retry's cbuf_timedget could then either (a) consume immediately (fast, not 10s -- doesn't match) or (b) never resolve at all if the drain path interacts badly with an in-flight send, leaving the single wbuf slot permanently empty until the full 10s deadline. Did not instrument (fprintf/gdb) the actual attempt count or wbuf state during a live failing run to confirm this mechanism specifically -- xce_test's own -vvv flag produced no additional client-side retry/rotate trace output (goes through a different, unwired logger in this CLI context), and adding real instrumentation would need a fresh lib rebuild + re-repro cycle not completed in this pass.\n\nNOT YET DONE: instrument xcecli_retry_next/xce_jobsubmit directly (attempt count, wbuf drain outcome, rotate rc, per-attempt elapsed) and re-run the now-reliable concurrent-burst repro to catch the exact mechanism in the act -- this is the recommended next step, using the repro recipe above (no gdb needed; the reproduction is now reliable enough for print-based debugging). Confirmed unrelated to raft-apply-decoupling task 6.1's apply-pump fix (reproduces identically on that spec's pre-fix and post-fix binaries per the original entry) and to this session's bug-1428/1426/1474/1472-remainder fixes (none touch xceapi.c/client.c/client_failover.c's client-side retry path).\n\n[Original entry]\nNOT FIXED. Worked around via retry for task 7.4's benchmarking purposes only. Needs a dedicated root-cause investigation with the same live-repro rigor bug-1416 got (gdb-attach via GDB_WRAP seam, or targeted debug logging around the commit-wait barrier right after a fresh single-node election).\n\n[UPDATE 2026-08-16, part 2] FIXED. Root-caused via static code trace (no gdb/instrumentation needed after all): XCECLI_FAILOVER_CALL's idempotent-replay rule (client_failover.c, xce_failover_classify -- 'a duplicate on a re-sent write means the replay guard already applied our write, treat as success') can normalize a retried resend that hits the server's EEXIST/inflight-collision rejection into overall success (rc==0) -- but that rejection reply carries NO payload (handle_jobsubmit's EEXIST/inflight-collision branches reply with RequestReply_data_nodata, job.c:1441-1467), so xce_jobsubmit() (xceapi.c) never populates its *jobid output on the winning (dup) attempt, and never did on the earlier failing attempt either (its raw status was already negative). Net: xcecli_jobsubmit() can return rc>=0 with *jobid left at the caller's initial value (0) -- matching the confirmed symptom exactly (rc>=0, jobid=0, ~10.0-10.1s: one full 10s cbuf_timedget deadline on the failing first attempt, then a fast dup-rejection reply on the resend).\n\nFIX (TDD): added jobsubmit_finalize_jobid(uint64_t* jobid, uint64_t submitted_jobid, int overall_rc) (src/lib/client_internal.h declaration, src/lib/client.c definition), called from xcecli_jobsubmit() right after XCECLI_FAILOVER_CALL returns. Sets *jobid = submit->jobid (the CLIENT-generated id already on the wire -- job ids are client-generated via idgen(); the controller commits/echoes this exact value verbatim on every genuine success reply, bug-786/job.c:1439) whenever overall_rc >= 0, otherwise leaves *jobid untouched. Deliberately placed at the jobsubmit-specific wrapper layer, not the generic classify/decide layer (which is correctly type-erased and has no concept of jobid) -- test.c's cmd_jobsub (bug-786's own gate, comparing returned jobid == locally-generated jobsubmit.jobid) was ALREADY the right verification signal and needed no changes. 3 new pure unit tests in test_libxce_retry.c (test_jobsubmit_finalize_populates_jobid_on_success/leaves_jobid_untouched_on_failure/null_jobid_safe) -- socket-free, following this codebase's established direct-state-simulation test style for I/O-adjacent logic (test_netio_on_connected.c's precedent) rather than mocking a live connection.\n\nLIVE VALIDATION: full ASAN rebuild (0 compiler errors; controller_unit_test 1101 tests, 2 pre-existing unrelated test_config_repl.c failures; libxce test suite 1/1 OK, all 3 new tests PASS). Deployed a fresh single-node cluster and ran the same concurrent-burst repro as the original investigation (fresh election -> 9 simultaneous xce_test jobsub clients) for 5 rounds / 45 total clients: 0 'no clean ack' failures, 45/45 clean acks with jobid matching the submitted id (vs. the original repro's 1-3/9 failures in 3 of 4 rounds pre-fix). Honest caveat: this is probabilistic/timing-dependent (like the original repro), so 45/45 clean is strong but not airtight proof the exact race window is unreachable -- the fix is correct by construction regardless (it closes the *jobid gap for EVERY path that can produce rc>=0, not just the one race that was reproduced), so residual risk is limited to a still-undiscovered SEPARATE mechanism producing the same symptom, not this one going unfixed.", "tags": [ "ha", @@ -15705,46 +4052,56 @@ "raft-apply-decoupling-followup", "live-repro-found", "concurrent-connect-trigger", - "fixed", "root-caused", "tdd-red-green" ], "related_bugs": [ "bug-1416" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "8a6c339a" + ], + "record": ".wolf/bugs/bug-1425.md" }, { "id": "bug-1426", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-09T21:15:00.000Z", - "error_message": "trim_wal_segments's retention floor has no awareness of the local apply pump's own lag \u2014 a reachable scenario since apply-lag-behind-commit is raft-apply-decoupling's entire premise.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "trim_wal_segments's retention floor has no awareness of the local apply pump's own lag — a…\n\ntrim_wal_segments's retention floor has no awareness of the local apply pump's own lag — a reachable scenario since apply-lag-behind-commit is raft-apply-decoupling's entire premise.", "file": "src/controller/persist.c (trim_wal_segments, apply_pump_read_entry cache)", - "root_cause": "Identified during raft-apply-decoupling task 7.4's apply-pump segment-read cache fix (round-2 review finding): the new fast-path cache can (POSIX unlink-keeps-fd-alive) silently advance last_applied_lsn past already-deleted data if the pump is lagging when trim_wal_segments fires. Currently bounded by the pre-existing APPLY_DIVERGENT boot fence (deferred loud fencing on next restart, not silent corruption) \u2014 a real behavior change from the pre-cache code, not literally risk-free, but not an active defect either. Same family as an already-tracked standby-cursor variant (per task 7.4's own notes).", - "fix": "FIXED (TDD RED->GREEN). trim_wal_segments() now caps its trim floor at the local apply pump's own progress (last_applied_lsn), not just remote standby-acked positions: floor = min(repl_trim_lsn's standby/disconnect-guard floor, last_applied_lsn+1). The +1 matters -- repl_trim_lsn's floor treats equality as 'still needed, preserve' (a standby acked through LSN N might still need that segment for retry/resync), but the LOCAL apply cursor's equality case means the opposite: last_applied_lsn==N means this process is DONE with entry N and will never re-read it, so a segment starting exactly at N is safe to delete. Feeding last_applied_lsn+1 into the same shared 'seg_start_lsn < floor' comparison gets correct delete-on-equal semantics for this term without a second, differently-shaped check. This closes the real gap: apply_pump_read_entry()'s fast-path cache holds an open fd into whatever segment it's currently reading; POSIX unlink-keeps-fd-alive meant a lagging pump could keep reading via that stale fd even after trim_wal_segments deleted the segment out from under it (no immediate corruption, since the fd's data was still valid) -- but a FRESH read of the same range (a restart's persist_recover() replay, or the pump's own cache falling back to its slow path after an unrelated miss) would find the segment already gone, producing a genuine APPLY_DIVERGENT-class gap between what the live process quietly caught up to and what a fresh read of disk state can see. Regression test (test_e2e_checkpoint.c's test_trim_guard_preserves_unapplied_local_segments): no standbys registered (so the pre-fix floor alone would allow trimming everything below the active segment) but last_applied_lsn deliberately lags behind one segment -- proves the already-applied segment IS trimmed (no unnecessary retention) while the not-yet-applied one is preserved (the actual guard). Also had to advance last_applied_lsn in the 3 pre-existing trim tests in the same file (test_checkpoint_wal_trim_after_checkpoint, test_trim_guard_preserves_standby_segments, test_trim_without_standbys_proceeds_normally), which call persist_wal_async directly (bypassing the whole apply path) and so never advance last_applied_lsn on their own -- without the added stamp, this fix would have correctly-but-unexpectedly blocked ALL trimming in those fixtures, since last_applied_lsn defaults to 0. Full controller_unit_test suite: 1099 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED \u2014 deliberately deferred. task 7.4's own scope explicitly excluded redesigning trim_wal_segments's retention-floor awareness of apply-pump lag; documented via an accurate code comment instead. A future fix would give trim_wal_segments a floor input derived from the local apply cursor, not just standby-acked LSNs.", + "root_cause": "Identified during raft-apply-decoupling task 7.4's apply-pump segment-read cache fix (round-2 review finding): the new fast-path cache can (POSIX unlink-keeps-fd-alive) silently advance last_applied_lsn past already-deleted data if the pump is lagging when trim_wal_segments fires. Currently bounded by the pre-existing APPLY_DIVERGENT boot fence (deferred loud fencing on next restart, not silent corruption) — a real behavior change from the pre-cache code, not literally risk-free, but not an active defect either. Same family as an already-tracked standby-cursor variant (per task 7.4's own notes).", + "fix": "FIXED (TDD RED->GREEN). trim_wal_segments() now caps its trim floor at the local apply pump's own progress (last_applied_lsn), not just remote standby-acked positions: floor = min(repl_trim_lsn's standby/disconnect-guard floor, last_applied_lsn+1). The +1 matters -- repl_trim_lsn's floor treats equality as 'still needed, preserve' (a standby acked through LSN N might still need that segment for retry/resync), but the LOCAL apply cursor's equality case means the opposite: last_applied_lsn==N means this process is DONE with entry N and will never re-read it, so a segment starting exactly at N is safe to delete. Feeding last_applied_lsn+1 into the same shared 'seg_start_lsn < floor' comparison gets correct delete-on-equal semantics for this term without a second, differently-shaped check. This closes the real gap: apply_pump_read_entry()'s fast-path cache holds an open fd into whatever segment it's currently reading; POSIX unlink-keeps-fd-alive meant a lagging pump could keep reading via that stale fd even after trim_wal_segments deleted the segment out from under it (no immediate corruption, since the fd's data was still valid) -- but a FRESH read of the same range (a restart's persist_recover() replay, or the pump's own cache falling back to its slow path after an unrelated miss) would find the segment already gone, producing a genuine APPLY_DIVERGENT-class gap between what the live process quietly caught up to and what a fresh read of disk state can see. Regression test (test_e2e_checkpoint.c's test_trim_guard_preserves_unapplied_local_segments): no standbys registered (so the pre-fix floor alone would allow trimming everything below the active segment) but last_applied_lsn deliberately lags behind one segment -- proves the already-applied segment IS trimmed (no unnecessary retention) while the not-yet-applied one is preserved (the actual guard). Also had to advance last_applied_lsn in the 3 pre-existing trim tests in the same file (test_checkpoint_wal_trim_after_checkpoint, test_trim_guard_preserves_standby_segments, test_trim_without_standbys_proceeds_normally), which call persist_wal_async directly (bypassing the whole apply path) and so never advance last_applied_lsn on their own -- without the added stamp, this fix would have correctly-but-unexpectedly blocked ALL trimming in those fixtures, since last_applied_lsn defaults to 0. Full controller_unit_test suite: 1099 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED — deliberately deferred. task 7.4's own scope explicitly excluded redesigning trim_wal_segments's retention-floor awareness of apply-pump lag; documented via an accurate code comment instead. A future fix would give trim_wal_segments a floor input derived from the local apply cursor, not just standby-acked LSNs.", "tags": [ "ha", "persistence", "wal-trim", "apply-pump", "low-priority", - "raft-apply-decoupling-followup", - "fixed" + "raft-apply-decoupling-followup" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1426.md" }, { "id": "bug-1427", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-09T21:15:00.000Z", - "error_message": "APPLY_DIVERGENT's entity-count correctness check (persist_verify_apply_correctness) covers job entities only \u2014 worker entities can diverge (worker_from_capnp()/restore_worker() genuinely creates a new worker on a WAL_WORKER_CTRL/HOOK_DETAIL_WORKER_JOIN entry when absent) but this is currently invisible to the check.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "APPLY_DIVERGENT's entity-count correctness check (persist_verify_apply_correctness) covers job…\n\nAPPLY_DIVERGENT's entity-count correctness check (persist_verify_apply_correctness) covers job entities only — worker entities can diverge (worker_from_capnp()/restore_worker() genuinely creates a new worker on a WAL_WORKER_CTRL/HOOK_DETAIL_WORKER_JOIN entry when absent) but this is currently invisible to the check.", "file": "src/controller/persist.c (persist_verify_apply_correctness, count_jobmgr_entries)", - "root_cause": "Disclosed, accepted scope gap from raft-apply-decoupling task 5.1's own design: the entity-count check was scoped to jobs only (queues correctly excluded \u2014 config-derived, WAL_QUEUE_CTRL never creates a new one, verified by two review rounds). Workers were never brought in scope. Not a regression \u2014 a known, deliberate boundary, just not yet extended.", + "root_cause": "Disclosed, accepted scope gap from raft-apply-decoupling task 5.1's own design: the entity-count check was scoped to jobs only (queues correctly excluded — config-derived, WAL_QUEUE_CTRL never creates a new one, verified by two review rounds). Workers were never brought in scope. Not a regression — a known, deliberate boundary, just not yet extended.", "fix": "FIXED (TDD RED->GREEN). Extended persist_verify_apply_correctness() with a worker-count check mirroring the existing job-count check exactly: added count_workermgr_entries() (persist.c, mirrors count_jobmgr_entries()), a new expected_new_worker_count field on apply_expectation_t (persist.h), and a new workermgr_count_before_replay parameter threaded through persist_verify_apply_correctness()/persist_recover(). The harder part: apply_committed_entry()'s WAL_WORKER_CTRL case previously always returned 0 regardless of outcome (JOIN-creates-new, UPDATE-applies-to-existing, and idempotent-noop were all indistinguishable to callers), so replay_wal() had no signal to tally new worker joins from. Fixed by making worker_from_capnp() (worker.c) return 1 specifically for a genuine new-worker creation (discarding restore_worker()'s raw idx, which can legitimately be 0 and would have collided with the update-applied case) and propagating that rc through apply_committed_entry() instead of the hardcoded 0 -- verified no caller of either function depended on the exact prior return value (only sign and -EEXIST were ever checked). replay_wal() tallies apply_rc==1 for WAL_WORKER_CTRL exactly like it already does for WAL_JOB_SUBMIT. New regression tests: test_verify_apply_correctness_detects_worker_count_mismatch (test_apply_correctness_verification.c, mirrors the existing job-count-mismatch test) proves a worker-count divergence is now caught as APPLY_DIVERGENT -- RED before this fix (no worker check existed at all), GREEN after. Also updated test_apply_committed_entry.c's test_ace_worker_ctrl_join_restores_and_notifies, which asserted rc==0 for a genuine JOIN creation under the old (now-corrected) contract -- now asserts rc==1 with an explanatory message. Full controller_unit_test suite: 1097 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED. Extend persist_verify_apply_correctness() with a worker-count check mirroring count_jobmgr_entries() if/when a worker-creation divergence needs to be closed (task 5.1's own forward note).", "tags": [ "ha", @@ -15752,46 +4109,51 @@ "apply-divergent", "worker", "low-priority", - "raft-apply-decoupling-followup", - "fixed" + "raft-apply-decoupling-followup" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1427.md" }, { "id": "bug-1428", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "security", "timestamp": "2026-07-09T21:15:00.000Z", - "error_message": "Narrow TOCTOU residual in persist_register_pending_apply() (task 6.2, part of bug-1415's fix) \u2014 not reliably exercised by any current test.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Narrow TOCTOU residual in persist_register_pending_apply() (task 6.2, part of bug-1415's fix)…\n\nNarrow TOCTOU residual in persist_register_pending_apply() (task 6.2, part of bug-1415's fix) — not reliably exercised by any current test.", "file": "src/controller/persist.c (persist_register_pending_apply)", "root_cause": "Flagged during raft-apply-decoupling task 6.2 as a non-blocking fast-follow while closing bug-1415. Specific race window not fully documented in tasks.md's own notes; needs a fresh look at the function to characterize exactly what window remains and whether it's reachable in production timing.", - "fix": "FIXED (characterized then TDD RED->GREEN, per the entry's own 3-step plan). Characterized the exact window: persist_register_pending_apply()'s self-heal check (an UNLOCKED read of last_applied_lsn) ran BEFORE acquiring pending_applies_lock, separate from the locked insert that followed. Window: this thread reads last_applied_lsn (sees 'not yet applied') -> BEFORE it locks and inserts, the apply pump's thread advances last_applied_lsn past this lsn AND completes its own persist_take_pending_apply(lsn) call (finding nothing, since this thread hasn't inserted yet) -> this thread then inserts a node the pump will never revisit -- permanently orphaned, identical failure mode to bug-1415 (a hang, on_applied never fires) via a different, narrower trigger window. Production reachability: yes -- persist_register_pending_apply() is called from a different thread/vtask than the apply pump's walk (commit.c's continuation dispatch vs persist_apply_pump_run, per the function's own pre-existing doc comment), with no synchronization between them before this fix beyond the already-locked insert. Fix: moved the self-heal check inside pending_applies_lock, so the whole check-then-(insert-or-self-heal) sequence is one atomic critical section against persist_take_pending_apply's own locked scan-and-remove -- either this thread's check now runs before the pump's take (sees 'not yet applied', inserts, and the pump's later take, now forced to wait on the same lock, finds the freshly-inserted node) or after (sees 'already applied', self-heals instead of inserting, and the pump's take correctly finds nothing) -- never the interleaved gap that orphaned it. Regression test (test_apply_pump.c's test_register_pending_apply_race_against_concurrent_apply_self_heals): deterministic reproduction via controlled state (a main thread holding pending_applies_lock forces the register call to observe last_applied_lsn only after the lock's own update, removing dependence on real-time race timing) -- RED under the pre-fix code (on_applied never fires, node orphaned on pending_applies), GREEN under the fix (self-heals correctly, no orphan). Full controller_unit_test suite: 1098 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED \u2014 explicitly deferred as a fast-follow, not a blocker for task 6.2 or the spec's closure. Needs: (1) characterize the exact TOCTOU window, (2) determine production reachability, (3) add a regression test that exercises it before fixing.", + "fix": "FIXED (characterized then TDD RED->GREEN, per the entry's own 3-step plan). Characterized the exact window: persist_register_pending_apply()'s self-heal check (an UNLOCKED read of last_applied_lsn) ran BEFORE acquiring pending_applies_lock, separate from the locked insert that followed. Window: this thread reads last_applied_lsn (sees 'not yet applied') -> BEFORE it locks and inserts, the apply pump's thread advances last_applied_lsn past this lsn AND completes its own persist_take_pending_apply(lsn) call (finding nothing, since this thread hasn't inserted yet) -> this thread then inserts a node the pump will never revisit -- permanently orphaned, identical failure mode to bug-1415 (a hang, on_applied never fires) via a different, narrower trigger window. Production reachability: yes -- persist_register_pending_apply() is called from a different thread/vtask than the apply pump's walk (commit.c's continuation dispatch vs persist_apply_pump_run, per the function's own pre-existing doc comment), with no synchronization between them before this fix beyond the already-locked insert. Fix: moved the self-heal check inside pending_applies_lock, so the whole check-then-(insert-or-self-heal) sequence is one atomic critical section against persist_take_pending_apply's own locked scan-and-remove -- either this thread's check now runs before the pump's take (sees 'not yet applied', inserts, and the pump's later take, now forced to wait on the same lock, finds the freshly-inserted node) or after (sees 'already applied', self-heals instead of inserting, and the pump's take correctly finds nothing) -- never the interleaved gap that orphaned it. Regression test (test_apply_pump.c's test_register_pending_apply_race_against_concurrent_apply_self_heals): deterministic reproduction via controlled state (a main thread holding pending_applies_lock forces the register call to observe last_applied_lsn only after the lock's own update, removing dependence on real-time race timing) -- RED under the pre-fix code (on_applied never fires, node orphaned on pending_applies), GREEN under the fix (self-heals correctly, no orphan). Full controller_unit_test suite: 1098 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED — explicitly deferred as a fast-follow, not a blocker for task 6.2 or the spec's closure. Needs: (1) characterize the exact TOCTOU window, (2) determine production reachability, (3) add a regression test that exercises it before fixing.", "tags": [ "ha", "persistence", "apply-pump", "toctou", "unconfirmed", - "raft-apply-decoupling-followup", - "fixed" + "raft-apply-decoupling-followup" ], "related_bugs": [ "bug-1415" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1428.md" }, { "id": "bug-1429", - "gitea_synced_status": "fixed", - "gitea_issue": 264, - "status_note": "b5d59253", "status": "fixed", "status_updated": "2026-09-25", + "status_note": "b5d59253", + "component": "controller", + "severity": "crash", "timestamp": "2026-07-10", - "error_message": "controller_unit_test suite TIMEOUT (300s, killed by SIGTERM) during /xce-build controller run immediately after uncommitted edits to persist.c, ha_convergence.c, config_repl.c (delaymgr-inline blocking I/O moved to VTASK_LIVE_BLOCK vtasks with inline-fallback-on-failure). Compile succeeded with zero errors (only pre-existing warnings). meson test summary: Ok:0 Fail:0 Timeout:1. Last test output before the hang was test_demotion_replay_yields_committed_only (test_commit_barrier_reply.c) printing overlayfs-SKIP, followed by a burst of HA FSM / persist_apply_pump_run log lines ending in \"HA quorum lost: connected=0 required=1\", then total silence for the remaining ~294s until SIGTERM. No ASAN/crash report -- looks like a genuine hang/deadlock, not a crash.", + "last_seen": "2026-09-24T10:51:31Z", + "occurrences": 5, + "error_message": "controller_unit_test suite TIMEOUT (300s, killed by SIGTERM) during /xce-build controller run…\n\ncontroller_unit_test suite TIMEOUT (300s, killed by SIGTERM) during /xce-build controller run immediately after uncommitted edits to persist.c, ha_convergence.c, config_repl.c (delaymgr-inline blocking I/O moved to VTASK_LIVE_BLOCK vtasks with inline-fallback-on-failure). Compile succeeded with zero errors (only pre-existing warnings). meson test summary: Ok:0 Fail:0 Timeout:1. Last test output before the hang was test_demotion_replay_yields_committed_only (test_commit_barrier_reply.c) printing overlayfs-SKIP, followed by a burst of HA FSM / persist_apply_pump_run log lines ending in \"HA quorum lost: connected=0 required=1\", then total silence for the remaining ~294s until SIGTERM. No ASAN/crash report -- looks like a genuine hang/deadlock, not a crash.", "file": "src/controller/testsuite/test_commit_barrier_rollout.c, test_demotion_convergence.c, test_convergence_verification.c, test_egress_filter.c, test_apply_pump.c, test_apply_pump_integration.c, test_job_array_fence.c, test_ha_role_lifecycle.c, test_job_array_replay_fence.c, test_commit_durability.c, test_commit_engine.c, test_p2_integration.c (test-fixture teardown ordering, 21 sites across 12 files); src/controller/commit.c (commit_engine_fini, unchanged -- confirms the documented contract); src/controller/persist.c (fsync_worker, unchanged -- confirms the race window)", "root_cause": "ROOT-CAUSED 2026-08-30, LIVE-REPRODUCED with a full gdb thread backtrace (the first time in this bug's history any session captured a live hang instead of just reporting the SIGTERM/timeout). The original delaymgr/VTASK_LIVE_BLOCK hypothesis carried by this bug since 2026-07-10 was WRONG -- a static read of every vtask_enq_classified(..., VTASK_LIVE_BLOCK) call site in ha.c/persist.c/config_repl.c/commit.c found every one of them NULL-checks its pool pointer (or the trampoline is gated behind `c->delaymgr.running`, which test fixtures never set) and falls back inline on failure -- there is no path where a controller_unit_test fixture leaves a non-NULL-but-never-started vtaskmgr_t that a VTASK_LIVE_BLOCK task could be silently swallowed into. That entire subsystem is a red herring for this bug.\n\nThe REAL root cause is the same TOCTOU test-teardown race already root-caused once before, for a DIFFERENT symptom, in bug-1452 (2026-07-19: an ASAN SEGV in test_persist_truncate.c) and partially re-applied once more in bug-1687 (2026-08-25: cbrp_teardown in test_commit_barrier_reply.c). Every affected test fixture calls `commit_engine_fini(&ce)` BEFORE `persist_stop(&persist)` in its teardown helper -- the exact opposite of the contract documented in commit_engine_fini's own comment (commit.c: 'Callers are expected to have stopped the flusher (persist_stop) first'). Because persist_stop had not yet run, the persist_t's live fsync_worker background thread (persist.c:800, started by persist_start) can still be mid-flight calling `p->on_durable_advance(p->durable_cb_ctx)` -- i.e. commit_on_durable_advance(ce) -- with NO lock held (persist.c:866-868), concurrently with commit_engine_fini destroying ce->lock/ce->cond (commit.c:476-477) and freeing ce->continuations. commit_engine_fini's own persist_set_durable_cb(ce->p, NULL, NULL) unhook is 'defensive and cheap' per its own comment but is NOT sufficient on its own: the flusher thread can have already read the OLD non-NULL callback pointer a few instructions before the unhook takes effect. Depending on exact thread interleaving this TOCTOU race manifests as either bug-1452's SEGV (a NULL/freed dereference inside commit_advance_locked) OR -- newly confirmed this session -- a hang: pthread_cond_destroy(&ce->cond) blocking forever because the racing fsync_worker thread is concurrently inside pthread_cond_broadcast(&ce->cond) / holding ce->lock from commit_on_durable_advance, which glibc's cond-destroy implementation cannot safely proceed past. This explains every symptom previously logged against this bug: the varying hang location across sessions (different tests hit their OWN teardown's copy of this same race, at whatever moment the fsync_worker thread happened to be flushing), the total silence after the hang (both racing threads are genuinely blocked in futex/pthread_cond primitives, not spinning), zero CPU time accruing, and \"both threads parked in futex_do_wait\" from the 2026-08-23 sighting.\n\nLIVE REPRO (this session): `meson test -C src/xce.build.bug1429 --suite controller --verbose` hung 40s into the run with 3 threads (main + 2x fsync_worker) all in futex_do_wait, 0 CPU time advancing over a 3s recheck. gdb -p -batch -ex 'thread apply all bt full' (attached BEFORE the 300s SIGTERM, while the process was still alive) showed: Thread 1 (main) stuck in pthread_cond_destroy() called from commit_engine_fini (commit.c:476) <- jap_teardown() (test_commit_barrier_rollout.c:1877) <- test_job_array_submit_payload_size_bounded_at_max_array_idx (test_commit_barrier_rollout.c:2294) <- UnityDefaultTestRun <- test_commit_barrier_rollout() <- main(). Threads 2 and 3 both stuck in pthread_cond_timedwait at fsync_worker (persist.c:827), BOTH with `p = &g_jap` (the same global persist_t) -- confirming jap_teardown's exact commit_engine_fini-before-persist_stop ordering bug (test_commit_barrier_rollout.c:1877-1879, at the time of capture) as the live mechanism. This is the 4th distinct hang location logged against this bug (previously: test_demotion_replay_yields_committed_only, the test_configack_* group, test_jobsubmit_repeated_truncation_forced_failures_no_phantom_jobs) -- all four are different call sites of the identical teardown-ordering bug, which is why the location never repeated and why every prior static-inference pass aimed at the vtaskmgr/delaymgr subsystem instead never found it.", "fix": "FIXED 2026-08-30. Test-fixture-only fix -- no production code changed (commit.c and persist.c are exactly as documented; the bug is every affected fixture violating commit_engine_fini's own documented calling contract). Inserted an explicit `persist_stop(&persist);` call immediately before every confirmed-vulnerable `commit_engine_fini(&ce);` call (mirroring bug-1452/bug-1687's established fix pattern exactly), leaving each function's pre-existing later persist_stop()/teardown-helper call in place as a safe, idempotent no-op (persist_stop's own fsync_thread_running and wal_fd>=0 guards make the second call a clean skip). Where a fixture also tears down a real multi-vcpu vtaskmgr_t pool (test_apply_pump.c's tpr_teardown, test_apply_pump_integration.c's tpi_teardown), vtaskmgr_fini() was left in its original position AFTER commit_engine_fini() (a separate, unrelated invariant: no new vtask may be dispatched onto the pool mid-teardown) -- only persist_stop()'s position moved earlier.\n\n21 sites fixed across 12 files, all confirmed vulnerable by verifying (a) the fixture's persist_t is genuinely persist_start()'d (has a live fsync_worker thread) and (b) the teardown called commit_engine_fini() before persist_stop(): test_commit_barrier_rollout.c (3 sites -- g_fce/g_fp inline teardown, cb5_teardown, jap_teardown -- the last being the exact live-reproduced hang site), test_demotion_convergence.c (dc_teardown -- this EXACT site was flagged but explicitly left unfixed by bug-1452's own remediation note as a known follow-up, never actioned until now), test_convergence_verification.c (cv_teardown), test_egress_filter.c (ef_teardown), test_apply_pump.c (2 sites -- tap_teardown, tpr_teardown), test_apply_pump_integration.c (tpi_teardown), test_job_array_fence.c (tf_teardown), test_ha_role_lifecycle.c (lifecycle_cluster_teardown), test_job_array_replay_fence.c (1 site of its 4 persist_start'd fixtures -- the other 3 never wire a commit engine), test_commit_durability.c (3 sites, including one with a comment that incorrectly claimed persist_set_durable_cb's unhook alone was sufficient), test_commit_engine.c (3 sites, all sharing sync_persist_teardown), test_p2_integration.c (3 sites, 2 sharing var `p` + 1 using `g_p9`). Confirmed SAFE and left unchanged: barrier_teardown (test_commit_barrier_rollout.c), engine_teardown (test_commit_engine.c), and the fixtures in test_repl_majority.c/test_commit_admission_pool.c/test_coldstart_lsn0.c/test_commit_resolve.c -- all use a 'stub' persist_t that is never actually persist_start()'d (no live fsync_worker thread), confirmed by grep for persist_start against each fixture's global before touching it.\n\nKNOWN REMAINING EXPOSURE (not fixed this session, explicit follow-up for a future pass, same as bug-1452 left one for this session): test_config_repl.c has ~23 call sites following the identical inline `commit_engine_fini(&c->commit); t42_teardown(c);` pattern (t42_teardown, line 629, confirmed calls persist_stop THEN persist_fini -- so every one of these is the same bug) -- not fixed here because the file is large (~7000 lines) with some sites (e.g. the c_control/c_raced pair around line 2911-2912) not yet individually verified for uniformity, and a bulk edit without that verification was judged too risky for this session's budget. test_term_safety_adapter.c has persist_start'd fixtures (lines 59, 245, 2382) and 5 commit_engine_fini call sites (676, 930, 1776, 1950, 2338) not individually checked. Neither file's exposure has been observed to hang or crash yet (this bug's own history shows the race is intermittent and timing-dependent -- absence of a prior sighting there is not evidence of safety). Recommended next step: a dedicated /xce-audit-invariants or grep-driven sweep of these two files using the exact same verification method used here (confirm persist_start presence, confirm fini-before-stop order, insert persist_stop() immediately before commit_engine_fini()).\n\nVERIFICATION: full `meson test -C xce.build.bug1429 --suite controller` run 3 times back-to-back after the settled fix (no edits during any of these three runs) -- 39.01s / 38.86s / 38.91s, Ok:6 Fail:0 every time, zero hangs, zero timeouts (previously: TIMEOUT 300.08s-300.30s on effectively every attempt this session before the fix, reproduced live on the very first try). No regression test added (per the task's own documented exception for a suite-level timing-dependent teardown race): the race requires a live fsync_worker thread mid-flush at the exact moment of teardown, which is not reliably triggerable in a bounded, fast unit test without reintroducing the same flakiness this fix removes; the 3x full-suite re-run is the verification method this project's own bug-1452 precedent already established for this exact bug class.\n\nUPDATE (2026-09-24): 2026-09-24 ROOT CAUSE of the residual 300s TIMEOUT (live-captured with gdb under a 6-way parallel stress harness, unshare -rn per instance): any failing TEST_ASSERT longjmps out and SKIPS the fixture teardown; the fixture's commit engine sweep thread (and persist fsync_worker) stay asleep on the static engine's ce->cond / p->fsync_cond; the next test re-inits the same static struct underneath them (pthread_cond_init on a condvar with a live waiter), corrupting the waiter count; a later teardown then blocks forever in pthread_cond_destroy (commit_engine_fini / persist_fini) or pthread_join (cb5_teardown) -- or, for a stack-local engine, the sweep thread reads a dead frame (ASAN stack-use-after-return in commit_sweep_worker, test_persist_truncate.c). The failing assertion itself was never seen because Unity's stdout was fully buffered and lost on SIGTERM/abort. Triggering flaky assertions found once visible: test_job_array_submit_one_entry_one_continuation_across_sizes (the test in flight when the 11:34 meson run hung; continuation resolved on the fsync_worker thread, test read a plain int immediately), test_pump_mixed_facts_and_commands_catchup_original_order (reply count read once before trailing continuations), test_persist_flush_fires_durable_advance_callback (fsync_worker credited a concurrent persist_flush advance to itself -> double fire). FIX (uncommitted): (1) controller_test.c line-buffers stdout; (2) XCE_TESTING registries: tearDown() calls persist_test_reap_unstopped() then commit_test_reap_unfinalized_engines(), stopping/joining whatever a failed test leaked (clean full run reaps 0; injected failure now = 1 FAIL in 2s instead of a hang); (3) persist.c flusher_flush_once reports only its own advance (fsync_worker no longer double-fires; regression test test_flusher_pass_does_not_claim_concurrent_flush_advance); (4) the two flaky tests wait (bounded) for cross-thread resolution.", @@ -15816,24 +4178,31 @@ "test_commit_barrier_reply", "timeout", "toctou-race", - "vtask", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "vtask" ], "related_bugs": [ "bug-1452", "bug-1687", "bug-924" ], - "occurrences": 5, - "last_seen": "2026-09-24T10:51:31Z" + "fix_commits": [ + "b5d59253" + ], + "record": ".wolf/bugs/bug-1429.md", + "gitea_issue": 264, + "gitea_synced_status": "fixed" }, { "id": "bug-1430", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-07-10T07:12:00Z", - "error_message": "AddressSanitizer: heap-use-after-free on address ... WRITE of size 4 ... #0 ha_prevote_cancel_round_timeout ../controller/ha.c:6712 #1 ha_start_prevote_async ../controller/ha.c:6889 #2 ha_failover_stagger_fn ../controller/ha.c:7259 -- crashed 2 of 3 nodes during a live chaotic HA soak (10-round overlapping-kill mode)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "AddressSanitizer: heap-use-after-free on address ... WRITE of size 4 ... #0…\n\nAddressSanitizer: heap-use-after-free on address ... WRITE of size 4 ... #0 ha_prevote_cancel_round_timeout ../controller/ha.c:6712 #1 ha_start_prevote_async ../controller/ha.c:6889 #2 ha_failover_stagger_fn ../controller/ha.c:7259 -- crashed 2 of 3 nodes during a live chaotic HA soak (10-round overlapping-kill mode)", "file": "src/controller/ha.c", "root_cause": "ha_prevote_round_timeout_fn (the delaymgr task body for a PreVote round's timeout) only cleared the static s_prevote_round_ctx pointer inside its 'round still in progress' branch (ha->raft.pre_vote_in_progress true). When a round instead got decided via a reply arriving through raft_on_pre_vote_reply (which clears pre_vote_in_progress without touching s_prevote_round_ctx) and the round-timeout task fired afterward anyway (a benign, expected race), the function fell straight through to free(ctx) with s_prevote_round_ctx still pointing at that about-to-be-freed memory. The next PreVote round's ha_start_prevote_async defensively calls ha_prevote_cancel_round_timeout() before arming its own timer, reading the dangling pointer and writing to old->cancelled -- UAF. Same call stack (single delaymgr thread T8) on both crashing nodes, all-100%-reproducible under chaotic-mode's rapid overlapping restarts.", "fix": "ha_prevote_round_timeout_fn now does a CAS (atomic_compare_exchange_strong_explicit matching s_prevote_round_ctx against this specific ctx, clearing to NULL) UNCONDITIONALLY as the first action, before either branch -- so the static pointer is never left dangling regardless of which path is taken. Brings it in line with the already-correct pattern used by the sibling function raft_election_timeout_fn (same file), which unconditionally clears its own stored ctx pointer before checking cancelled.", @@ -15843,23 +4212,26 @@ "prevote", "delaymgr", "chaotic-soak", - "raft", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "raft" ], "related_bugs": [ "bug-1423", "bug-1424" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1430.md" }, { "id": "bug-1431", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "data-loss", "timestamp": "2026-07-10T10:02:00Z", - "error_message": "chaotic HA soak: committed_job_loss=5-8 across 10 rounds (VERDICT result=FAIL) after raft-apply-decoupling task 16 gated standby apply strictly on commit; committed_divergence stayed 0 and all jobs eventually reappeared at final settlement -- a visibility-latency gap, not true data loss", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "chaotic HA soak: committed_job_loss=5-8 across 10 rounds (VERDICT result=FAIL) after…\n\nchaotic HA soak: committed_job_loss=5-8 across 10 rounds (VERDICT result=FAIL) after raft-apply-decoupling task 16 gated standby apply strictly on commit; committed_divergence stayed 0 and all jobs eventually reappeared at final settlement -- a visibility-latency gap, not true data loss", "file": "src/controller/ha.c, src/controller/persist.c, src/include/persist.h", "root_cause": "Two stacked gaps in raft_cb_become_leader promotion, both masked pre-task-16 by eager apply-on-WAL-receipt: (1) persist_truncate_wal_above only sets p->committed_lsn on branches of truncate_wal_above_locked that perform a REAL truncation; its common no-op early-return (leader own log already covers safe_lsn) left committed_lsn stuck at its pre-election value, so newly-computed safe_committed_lsn was never applied to the actual commit watermark. (2) Even after fixing (1) with a durable_lsn-capped store, a newly-promoted leader whose own fsync had not yet caught up to its own WAL tip (group-commit batches fsync on a timer) stayed capped below safe_lsn indefinitely -- nothing re-drove the promotion advance once the flusher eventually synced, since fsync_worker on_durable_advance only re-evaluates the commit engines PENDING per-entry continuations, and a standby has none for replicated entries.", "fix": "New function persist_promotion_advance_committed(p, safe_lsn) (persist.c, declared in persist.h) called from raft_cb_become_leader right after persist_truncate_wal_above: unconditionally advances p->committed_lsn toward safe_lsn (monotonic), forcing a synchronous flusher_flush_once(p) first if durable_lsn < safe_lsn (safe/bounded -- by Raft Leader Completeness the new leaders own WAL tip is always >= safe_lsn, so the bytes are already on disk and only need fdatasync). Reduced but did not fully close the soak failures alone -- see bug-1432 for the second half of the fix.", @@ -15869,22 +4241,25 @@ "committed_lsn", "chaotic-soak", "apply-decoupling", - "durability", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "durability" ], "related_bugs": [ "bug-1432" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1431.md" }, { "id": "bug-1432", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-10T10:15:00Z", - "error_message": "chaotic HA soak still FAIL (committed_job_loss=8) after bug-1431 force-flush fix; jobs above the conservative 2-voter safe_committed_lsn floor stayed invisible via expect-job for ~3.1s post-promotion, matching the primary heartbeat interval (3000ms) almost exactly", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "chaotic HA soak still FAIL (committed_job_loss=8) after bug-1431 force-flush fix; jobs above the…\n\nchaotic HA soak still FAIL (committed_job_loss=8) after bug-1431 force-flush fix; jobs above the conservative 2-voter safe_committed_lsn floor stayed invisible via expect-job for ~3.1s post-promotion, matching the primary heartbeat interval (3000ms) almost exactly", "file": "src/controller/ha.c", "root_cause": "ha_start_primary_heartbeat armed the periodic heartbeat via delay_task_ms(interval_ms, -1, primary_heartbeat_fn, c), whose FIRST invocation fires only after interval_ms elapses (standard periodic-timer semantics) -- despite the functions own long-standing comment claiming to start the heartbeat immediately. A freshly-promoted leader therefore waited a full heartbeat interval before ever sending a NOP to a reconnecting/catching-up standby, so it could not learn that standbys true applied_lsn (via the heartbeat-reply stateReport, ha_process_standby_state) for up to 3s. That knowledge gates when catch-up WAL streaming starts and, in turn, the per-entry majority-ack that advances committed_lsn past the elections conservative safe_committed_lsn floor (jobs above that floor were legitimately majority-committed via the OLD primary before it died, per Leader Completeness, but the new leader cannot expose them without fresh quorum corroboration -- a correct Raft Figure-8 safety requirement, not a bug in itself).", "fix": "ha_start_primary_heartbeat now calls primary_heartbeat_fn(c) synchronously once, immediately, before arming the periodic delay_task_ms timer -- matching the functions own documented intent. primary_heartbeat_fn only performs non-blocking async sends (xce_ha_heartbeat_req), so this is safe inline with no thread-role change. Single fix point covers all 3 promotion call sites (raft_cb_become_leader, ha_promote_to_primary, and the third raft-driven-decision-loop site) since all route through ha_start_primary_heartbeat. Closed the gap: 10-round chaotic soak went from committed_job_loss=8 to committed_job_loss=0, committed_divergence=0, result=PASS; 10-round sequential soak + leadership-coverage also PASS (36 jobs, zero loss).", @@ -15894,23 +4269,25 @@ "promotion", "delaymgr", "chaotic-soak", - "latency", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "latency" ], "related_bugs": [ "bug-1431" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1432.md" }, { "id": "bug-1433", "status": "fixed", "status_updated": "2026-09-27", "status_note": "255b0faa: peer-control traffic while CTRL_ELECTING classified VTASK_LIVE_BLOCK (become-leader tail preads)", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-10T10:59:00Z", - "error_message": "controller-invariant-audit (2026-07-10, /xce-audit-invariants) async-I/O-thread-role dimension: bug-1431 round-2 force-flush fix (persist_promotion_advance_committed calling flusher_flush_once, a blocking fdatasync) is reachable via the real multi-node winning-vote-reply dispatch path on a VTASK_LIVENESS-classified vtask worker -- a task class the design documents as never admission-gated / never expected to block. This is the same bug class as bug-1198 (HA_REPL_SEND_SNAPSHOT persist_checkpoint on VTASK_LIVENESS), which was previously closed by offloading the blocking work off that worker class entirely.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller-invariant-audit (2026-07-10, /xce-audit-invariants) async-I/O-thread-role dimension…\n\ncontroller-invariant-audit (2026-07-10, /xce-audit-invariants) async-I/O-thread-role dimension: bug-1431 round-2 force-flush fix (persist_promotion_advance_committed calling flusher_flush_once, a blocking fdatasync) is reachable via the real multi-node winning-vote-reply dispatch path on a VTASK_LIVENESS-classified vtask worker -- a task class the design documents as never admission-gated / never expected to block. This is the same bug class as bug-1198 (HA_REPL_SEND_SNAPSHOT persist_checkpoint on VTASK_LIVENESS), which was previously closed by offloading the blocking work off that worker class entirely.", "file": "src/controller/persist.c, src/controller/ha.c", "root_cause": "raft_cb_become_leader (the .on_become_leader raft-library callback) is invoked synchronously from within peer_ctrl_msg_handler -> peer_ctrl_dispatch_to_pool -> peer_ctrl_classify (ha.c ~5935-5940), which HARDCODES VTASK_LIVENESS for every inbound peer-control packet including vote replies -- the doc comment above it explicitly states \"everything this handler dispatches is liveness work today\" and was written BEFORE this session added a blocking fdatasync call reachable from that same path. ha_dispatch_election_start/raft_election_start_vtask_entry (the VTASK_LIVE_BLOCK dispatch used for the RDHUP-failover and single-node self-grant election-START triggers) does NOT cover the separate, later \"win the election / become leader\" event, which fires on vote-REPLY arrival via a completely different dispatch chain.", "fix": "Removed the flusher_flush_once force-flush from persist_promotion_advance_committed entirely -- round-3 (durable-capped only, no blocking calls) plus the already-shipped bug-1432 immediate-first-heartbeat fix together close the original visibility-latency gap non-blockingly (confirmed by two live soaks: 6-round chaotic committed_job_loss=0/committed_divergence=0 PASS, 6-round sequential+leadership-coverage PASS). TRACKED RESIDUAL (not fixed this session): persist_apply_pump_run_sync (added earlier this session for the bug-1043-family apply-lag fix, also called from raft_cb_become_leader on the same VTASK_LIVENESS path) does bounded pread() of recently-written WAL segments -- lower practical risk than fdatasync (almost always page-cache-hot) but still, strictly, blocking I/O on a never-blocks task class. Removing it is NOT safe (reopens bug-1043-family job loss). Proper fix needs either reclassifying the winning-vote-reply case in peer_ctrl_classify or dispatching promotion tail onto a fresh VTASK_LIVE_BLOCK vtask -- both larger changes deferred to a dedicated follow-up with its own soak-validation pass. Comments at both call sites corrected to stop falsely claiming VTASK_LIVE_BLOCK safety.", @@ -15921,24 +4298,29 @@ "thread-role", "liveness", "audit-finding", - "bug-1198-class", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "bug-1198-class" ], "related_bugs": [ "bug-1431", "bug-1432", "bug-1198" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "255b0faa" + ], + "record": ".wolf/bugs/bug-1433.md" }, { "id": "bug-1434", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-10T10:56:00Z", - "error_message": "controller-invariant-audit (2026-07-10, /xce-audit-invariants) Raft TLA+ checklist dimension: persist_promotion_advance_committed and ha_standby_advance_committed_and_apply can race on persist_t.committed_lsn during the promotion window before role flips to CTRL_PRIMARY, via an unlocked load-then-conditional-store pattern in both functions -- a losing racer could clobber a larger value with a smaller one, regressing committed_lsn (violates the documented commitIndex-monotonic invariant).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller-invariant-audit (2026-07-10, /xce-audit-invariants) Raft TLA+ checklist dimension…\n\ncontroller-invariant-audit (2026-07-10, /xce-audit-invariants) Raft TLA+ checklist dimension: persist_promotion_advance_committed and ha_standby_advance_committed_and_apply can race on persist_t.committed_lsn during the promotion window before role flips to CTRL_PRIMARY, via an unlocked load-then-conditional-store pattern in both functions -- a losing racer could clobber a larger value with a smaller one, regressing committed_lsn (violates the documented commitIndex-monotonic invariant).", "file": "src/controller/persist.c, src/controller/ha.c", "root_cause": "raft_cb_become_leader calls persist_promotion_advance_committed BEFORE setting ha->role = CTRL_PRIMARY (the role store sits ~170 lines later in the function). During that window this node is still nominally non-primary, so ha_apply_wal_catchup_entry (still processing an in-flight WAL catch-up entry on another thread, e.g. if this node was mid-catch-up when it won the election) can legitimately call ha_standby_advance_committed_and_apply concurrently with the promotion path own committed_lsn advance. Both did atomic_load(committed_lsn); if(target<=cur) return; atomic_store(committed_lsn, target) with no lock and no CAS, a classic lost-update race under concurrent writers.", "fix": "Added persist_committed_lsn_advance_atomic(persist_t*, uint64_t target) (persist.c, declared in persist.h) -- a CAS retry loop (atomic_compare_exchange_weak_explicit) that is safe against any number of concurrent callers with no lock required, and can never regress committed_lsn regardless of interleaving. Both persist_promotion_advance_committed and ha_standby_advance_committed_and_apply now route through this shared helper. Validated: build clean (19/19 unit test suites pass), 6-round chaotic soak PASS (0 loss, 0 divergence), 6-round sequential+leadership-coverage soak PASS.", @@ -15948,22 +4330,25 @@ "race-condition", "monotonicity", "audit-finding", - "CAS", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "CAS" ], "related_bugs": [ "bug-1431" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1434.md" }, { "id": "bug-1435", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-12T04:45:15Z", - "error_message": "40-round sequential /xce-test-ha FAIL at round 36/40: verify_no_loss reported 0/111 jobs confirmed after 3 attempts; xce_joblist client call returned E_XCE_STANDBY (10014) against freshly-elected primary ctrl-1 (term=41) moments after wait-primary succeeded on the same session", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "40-round sequential /xce-test-ha FAIL at round 36/40: verify_no_loss reported 0/111 jobs confirmed…\n\n40-round sequential /xce-test-ha FAIL at round 36/40: verify_no_loss reported 0/111 jobs confirmed after 3 attempts; xce_joblist client call returned E_XCE_STANDBY (10014) against freshly-elected primary ctrl-1 (term=41) moments after wait-primary succeeded on the same session", "file": "src/controller/ha.c (raft_state_t entry points), src/controller/ha.h (new raft_lock field), src/controller/cluster.c (raft_on_vote_request/raft_on_pre_vote_request call sites)", "root_cause": "ROOT-CAUSED (2026-07-12, static analysis + full call-graph trace, not live-reproduced under a debugger due to timing rarity -- see confidence note in fix). ha_dispatch_gate's CTRL_STANDBY branch is the ONLY production site that can return -XCE_ERR_STANDBY for a GET/joblist request (confirmed by exhaustive read of ha.c:3699-3865); under CTRL_PRIMARY, OPCode_get is unconditionally allowed regardless of quorum/fence state. So the answering node's ha->role was genuinely CTRL_STANDBY at the moment ha_dispatch_gate ran -- this was NOT a client-side discovery/retry-budget problem (ruled out: XCECLI_FAILOVER_CALL's retry loop -- src/lib/client_failover.c -- correctly retries E_XCE_STANDBY up to max_retries=20 with zero backoff via XCE_FO_RETRY_NOW, and its final surfaced rc staying exactly -E_XCE_STANDBY on budget exhaustion is the DOCUMENTED behavior for a RETRY_NOW-classified cause, not a symptom of a broken retry loop). The MISSING INFO 'xcecli failover: attempt N' log lines (zero matches across the whole 40-round run) are explained by the fact that libxce's wait-primary REPL command (cmd_wait_primary, testsuite/test.c) does its OWN silent primary-discovery/redirect-follow loop (xcecli_ctrlstatus_primary) that never calls the shared xcecli_retry_next/INFO-logging path -- once wait-primary hands off a live, correctly-connected-to-primary ctx, subsequent ops normally succeed on attempt 0 with nothing to log; this is a red herring, not a clue to a client bug. The REAL mechanism is a genuine, unsynchronized DATA RACE in raft_state_t (the embedded Raft consensus struct, ha->raft): raft_on_vote_reply/raft_on_vote_request/raft_on_pre_vote_reply/raft_on_pre_vote_request/raft_begin_pre_vote/raft_start_election/raft_check_and_start_election/raft_on_election_timeout/raft_on_higher_term_observed all read+mutate raft_state_t's PLAIN (non-atomic) fields (term, role, votes_granted, replies_received, election_decided, vote_replies[], pre_vote_* tally) with ZERO locking -- the raft library (src/lib/raft/raft.c) is written assuming a single-threaded/serialized caller (the classic Raft single-event-loop model). But the raft-drive-async feature (peer_ctrl_dispatch_to_pool in ha.c, default ON via g_drive_async_peerctrl_dispatch_enabled=1) dispatches each peer-control message (including RequestVoteReply from each individual voter) as an independent VTASK_LIVENESS vtask onto the shared vtaskmgr pool; peer_ctrl_dispatch_to_pool's own per-endpoint serialization (bind_vtask) only orders messages arriving on the SAME endpoint -- it does NOT serialize messages from DIFFERENT peers' endpoints against each other. So two vote replies from two different voters (e.g. ctrl-2 grants, ctrl-3 grants or denies) for the SAME election CAN be processed concurrently on two different pool worker threads, both calling raft_on_vote_reply(&ha->raft, ...) on the shared struct with no lock. Traced exact race for the observed symptom: worker A processes voter1's grant, reaches quorum (votes_granted=2>=2), sets election_decided=1, fires on_become_leader -> raft_cb_become_leader -> ha->role=CTRL_PRIMARY (term=41, matching ctrl-1's own logs exactly). Worker B, running concurrently (no synchronization), processes voter2's reply and -- racing on the unprotected votes_granted/replies_received/election_decided reads -- can independently satisfy raft_on_vote_reply's Step 6 'all replied, no quorum' branch (replies_received==n_peers-1 && votes_granted raft_cb_become_follower -> ha->role=CTRL_STANDBY for the SAME term, an instant after promotion, with no further election and no fresh log line calling out a 'demotion' that anyone was specifically looking for. Every subsequent client GET request then correctly, faithfully hits ha_dispatch_gate's CTRL_STANDBY branch and is rejected with -XCE_ERR_STANDBY -- explaining why the failure PERSISTED across all retries/attempts across multiple seconds (nothing else ever corrects role back to PRIMARY once flipped -- no bug, no external trigger, it just stays CTRL_STANDBY). This also explains why the bug needed ~36 rounds to surface once: it requires two vote replies to genuinely overlap in processing time on two different pool workers, a narrow, load/scheduling-dependent timing window. Verified raft-drive-async is on by default (g_drive_async_peerctrl_dispatch_enabled=1) and confirmed via full call-graph trace that raft_cb_become_leader/raft_cb_become_follower (the synchronously-fired callbacks) never themselves call back into another raft_* entry point (no re-entrancy via the callbacks), and that no existing ha->lock-held critical section ever calls into a raft_* function (so lock-order is always raft_lock-then-ha->lock, never reversed -- no ABBA risk versus the two pre-existing lock-order bugs in this exact area, bug-652 and bug-1282). CONFIDENCE: HIGH on mechanism (every piece of evidence in the original investigation -- ctrl-1's own healthy term=41 promotion log, zero ASAN errors, commit_advanced continuing for seconds after, the missing xcecli-failover logs, every retry returning the SAME E_XCE_STANDBY class -- is fully and specifically explained by this race, with no residual unexplained facts), but NOT live-reproduced under a debugger/instrumented build in this session (the race is timing-rare, needed ~36 rounds to surface once in the original soak) -- this is a well-evidenced, fully-traced static diagnosis, not a confirmed-via-repro one.", "fix": "Added a new dedicated ha_state_t field `raft_lock` (pthread_mutex_t, PTHREAD_MUTEX_RECURSIVE -- src/controller/ha.h) serializing every mutating/reading entry point into raft_state_t on the election/vote path: raft_on_vote_reply (ha.c vote_reply_dispatch), raft_on_vote_request + raft_on_pre_vote_request (cluster.c, inbound vote RPCs), raft_on_pre_vote_reply + raft_begin_pre_vote (ha.c pre-vote dispatch), raft_start_election + raft_check_and_start_election x3 call sites (ha.c election-start paths, inline and vtask-dispatched), raft_on_election_timeout (ha.c delaymgr callback), and raft_on_higher_term_observed x2 (ha.c heartbeat-reply and WAL-NACK term-discovery paths). Initialized/destroyed in ha_init/ha_fini. Deliberately RECURSIVE (not a plain mutex): raft_on_pre_vote_reply's WON branch and raft_begin_pre_vote's single-node-win branch synchronously call ha_enter_real_election -> ha_start_election_async -> raft_start_election on the SAME thread within the same call chain (a legitimate same-thread nesting, not a race) -- a plain non-recursive mutex would self-deadlock on that path exactly like the precedent bug-652 (ha->lock self-deadlock from synchronous re-entry). Deliberately scoped to the ELECTION/VOTE path only, NOT the AE/commit-replication path (raft_log_append, raft_on_append_entries_request/reply, raft_advance_commit) -- ha_primary_log_append/raft_log_append is reachable from concurrent client-write vtasks with no visible lock either, which may be a SEPARATE, analogous concurrency gap, but touching the write/durability path carries much higher regression risk and wasn't the demonstrated mechanism for this specific symptom; flagging it as a follow-up investigation, not silently ignoring it. Verified no lock-order regression: no existing ha->lock-held critical section calls into any raft_* function (grepped and read all 9 ha->lock call sites), so raft_lock is always acquired before, never after, ha->lock -- consistent single order, no ABBA (cf. bug-1282, bug-652). Build: nix develop ./src -- bash src/tools/xce-build.sh controller -- clean compile, zero new warnings (3 pre-existing unrelated warnings in ha.c/cluster.c confirmed via git diff hunk-range check to be outside every edited region), controller_unit_test meson suite Ok:1 Fail:0 in 32.38s with no hang/timeout (a hang would have surfaced a recursive-mutex misuse or missed ha_init path, per the bug-1224/bug-1429 precedent of dispatch-related test hangs in this exact area -- none occurred). NOT live-reproduced/soak-validated in this session (explicitly deferred by the user to a separate follow-up 40-round /xce-test-ha run) -- this is a well-evidenced, carefully-scoped static fix for a data race, not a repro-confirmed one; flagged honestly as best-effort-but-high-confidence rather than 100%-certain.", @@ -15979,8 +4364,7 @@ "xce-test-ha", "E_XCE_STANDBY", "concurrency", - "peer_ctrl_dispatch_to_pool", - "fixed" + "peer_ctrl_dispatch_to_pool" ], "related_bugs": [ "bug-608", @@ -15990,15 +4374,20 @@ "bug-652", "bug-1282" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1435.md" }, { "id": "bug-1436", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-07-12T05:50:00Z", - "error_message": "40-round sequential /xce-test-ha FAIL at round 36/40: xce_joblist: Connection reset by peer, immediately after wait-primary confirmed ctrl-2 serving in 46ms. RE-OPENED 2026-07-12 ~14:00-16:00Z after a live re-test (fresh isolated 40-round run, no concurrent soak) reproduced the IDENTICAL failure with ctrl-2 confirmed NEVER killed (same PID throughout, continuous RAFT/heartbeat log activity, still answering other clients after the failure) -- disproving the process-collision theory as the (sole) root cause. ROOT-CAUSED FOR REAL via live capture (tcpdump unavailable in sandbox -- no CAP_NET_RAW; used direct -vvv log correlation on a live reproduction instead) 2026-07-12 ~16:00-16:15Z. FIXED.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "40-round sequential /xce-test-ha FAIL at round 36/40: xce_joblist: Connection reset by peer…\n\n40-round sequential /xce-test-ha FAIL at round 36/40: xce_joblist: Connection reset by peer, immediately after wait-primary confirmed ctrl-2 serving in 46ms. RE-OPENED 2026-07-12 ~14:00-16:00Z after a live re-test (fresh isolated 40-round run, no concurrent soak) reproduced the IDENTICAL failure with ctrl-2 confirmed NEVER killed (same PID throughout, continuous RAFT/heartbeat log activity, still answering other clients after the failure) -- disproving the process-collision theory as the (sole) root cause. ROOT-CAUSED FOR REAL via live capture (tcpdump unavailable in sandbox -- no CAP_NET_RAW; used direct -vvv log correlation on a live reproduction instead) 2026-07-12 ~16:00-16:15Z. FIXED.", "file": "src/lib/client_internal.h (new XCECLI_FAILOVER_CALL_COUNT macro), src/lib/client.c (xcecli_reslist/xcecli_workerlist/xcecli_joblist/xcecli_resval/xcecli_jobinfo switched to it). .claude/skills/xce-test-ha/run-ha-soak.sh's node_pattern()/run_pattern() workdir-scoping fix (prior pass) remains a legitimate, independent improvement -- KEPT, not reverted.", "root_cause": "TWO ROOT CAUSES, both now addressed. (1) SUPERSEDED/PARTIAL (kept as a real but non-exclusive fix): the original pass diagnosed this as run-ha-soak.sh's unscoped `pkill -9 -x xce_controller` letting a second concurrent soak invocation silently SIGKILL the first run's live primary. That fix (node_pattern()/run_pattern() workdir-scoped pgrep/pkill) is a genuine improvement and stays in place, but a FRESH, FULLY ISOLATED 40-round soak re-run (deploy dir xce-test-ha-40-v2, confirmed via `ps aux` before/during/after that no second soak was ever running) STILL FAILED at round 36/40 with the byte-identical signature, and ctrl-2's own log proved it was never killed (constant PID, unbroken RAFT event stream, still serving OTHER clients seconds after the failure) -- so process collision was real but was NOT the (sole) mechanism behind every occurrence. (2) THE ACTUAL ROOT CAUSE (found via live reproduction, not archaeology): `xcecli_joblist`/`xcecli_reslist`/`xcecli_workerlist`/`xcecli_resval`/`xcecli_jobinfo` (src/lib/client.c) are all built on `xce_get_with_condition` (src/lib/xceapi.c), which returns a NON-NEGATIVE ROW COUNT on success (`rc = getreply.nrows`) -- NOT the uniform '0 = success, negative = XCE/errno-style status' convention every OTHER xcecli_* wrapper follows. All of them were routed through the generic `XCECLI_FAILOVER_CALL` macro (client_internal.h), which unconditionally feeds the raw EXPR return value into `xce_failover_classify()` (client_failover.c) -- a function that recognizes retriable/terminal outcomes purely by MAGNITUDE (E_XCE_* codes >=10001, and POSIX errno values it deliberately accepts in EITHER sign via `fo_abs()`: ECONNRESET=104, ETIMEDOUT=110, ECONNREFUSED=111). A joblist reply whose row count happens to equal one of these magnitudes is silently misclassified as a transport failure on EVERY call with that exact count, even though the read fully succeeded -- forcing a spurious `xce_failover_rotate()` to the next configured master. Since get-class ops are primary-only, landing on a standby correctly (but pointlessly) fails with E_XCE_STANDBY and rotates again; the whole cycle repeats for every subsequent same-sized call in the same verify_no_loss batch, burning the finite `max_retries` budget (20, XCE_RETRY_DEFAULTS) on self-inflicted reconnect churn. run-ha-soak.sh's sequential mode submits NJOBS=3 per round, so the tracked job count follows the arithmetic progression 3+3*round -- of the three dangerous POSIX magnitudes (104, 110, 111), ONLY 111 is a multiple of 3, and it is hit for the FIRST time at EXACTLY round 36 (3+3*36=111=ECONNREFUSED) -- explaining both why this is genuinely rare (requires an exact numeric collision, not a generic race) and why it always surfaced at round 36 specifically for this test's job-count cadence. LIVE PROOF: a fresh, fully isolated 40-round soak (deploy dir xce-test-ha-40-bug1436-live) failed AGAIN at round 36/40 with 111 tracked jobs; the target primary's own -vvv log (ctrl-2, PID confirmed continuously alive via a parallel 1s-resolution forensic process/connection poller) showed a clear, damning pattern for the ENTIRE 'verify' client session: `cluster.c:786 server sock 4 accepted 10` (a BRAND NEW TCP connection) immediately before EVERY SINGLE `opcode get` request, followed by two `xce_reply` sends (the >100-row chunked-get path, xce_reply_get pktlimit=100) and then `cluster.c:64` (the TRACE line inside `handle_endpoint_error`, i.e. the connection being torn down) -- one full accept-serve-close cycle PER expect-job call, ~1799 accepts total on that one short-lived primary PID alone -- confirming the client was silently reconnecting on every single (mostly-successful) get call, exactly the reconnect storm this mechanism predicts, not a misreading of the log by a prior investigation pass. The run's own retry-storm messages ('verify_no_loss attempt 1/3: recheck failed... likely post-election client retry-storm race') were themselves an accurate but previously-misinterpreted symptom of this bug. Confirmed via codegraph_explore + direct reads of src/lib/client.c, src/lib/client_internal.h, src/lib/client_failover.c, src/lib/xceapi.c, src/lib/request.c (xce_reply_get chunking), src/controller/opget.c (do_reply_joblist/check_controller), src/lib/netio.c (endpoint_enq/handle_error/handle_output -- confirmed NOT independently implicated: zero `endpoint_update_epoll` WARN lines anywhere in the failing run's ctrl-2.log), and the existing `src/lib/testsuite/test_libxce_retry.c` StatusClassifier taxonomy tests (which confirm `xce_failover_classify` is INTENTIONALLY designed to accept genuine status codes in either sign -- the bug is that 5 call sites fed it a row count instead of a status code, not that the classifier itself is wrong). Two purpose-built synthetic repro harnesses (hammer2.sh: fixed 60-job list, ~5400+ calls with periodic real primary kills; hammer3.sh: fixed 495-job list, ~1500+ calls with periodic real primary kills) both ran clean with ZERO failures, because neither ever held the tracked job count at exactly 104/110/111 -- which in hindsight is the confirming, not disconfirming, result: the bug requires an EXACT count collision, not raw connection-churn volume, so a fixed-size synthetic repro that never lands on the magic number cannot hit it no matter how much volume it generates. That negative result is what motivated pivoting to the real /xce-test-ha harness (which naturally sweeps through every job count via its round-based growth) as the reproduction vehicle, per the live-capture mandate.", "fix": "Added `XCECLI_FAILOVER_CALL_COUNT(ac, EXPR)` to src/lib/client_internal.h -- a sibling of `XCECLI_FAILOVER_CALL` for xce_get-family calls. It only enters the retry/classify loop when EXPR's raw result is NEGATIVE (`while ((_fc_rc < 0) && xcecli_retry_next(...))`); any non-negative result (0 or a genuine row count of ANY magnitude) is unconditionally treated as success and returned as-is, so it can never be misread as a retriable/terminal status code. A dead ctx (XCE_CTX_ERR) is still handled correctly: the per-op XCE_CTX_RUN guard in xceapi.c makes EXPR itself return a negative -EINVAL in that state, which enters the loop and is reclassified by xcecli_retry_decide's dead-connection branch exactly as before. Switched the 5 count-returning call sites in src/lib/client.c to the new macro: xcecli_reslist, xcecli_workerlist, xcecli_joblist (the confirmed live-reproduced offender), xcecli_resval (same xce_get_with_condition pattern, same latent risk), and xcecli_jobinfo (returns a fixed 1 on success -- lower risk since it never collides with a recognized magnitude and was previously misclassified as TERMINAL, which is harmless because TERMINAL does not rotate/rewrite rc, but fixed anyway for correctness/consistency rather than relying on that coincidence). Did NOT touch `xce_failover_classify` itself or any of the other 14 XCECLI_FAILOVER_CALL call sites (hello, join, jobdispatch, jobsubmit, clusterconf, ctrlstatus, update_resvals, update_jusage, jobctrl, queuectrl, hostctrl, runtask, set_pend_reason, clear_reason) -- all of those genuinely follow the 0/status-code convention the classifier is designed for, confirmed via test_libxce_retry.c's existing StatusClassifier taxonomy tests, which still pass unmodified (xcelib_test: 293 Tests 0 Failures 0 Ignored). controller_unit_test: 926 PASS / 0 FAIL / 0 Ignored (direct-binary run, ANSI-stripped grep census, exit 0). Did NOT touch run-ha-soak.sh further (its bug-1436-prior-pass workdir-scoping fix stays, unrelated to this mechanism) and did NOT add a client-side retry-more band-aid -- the actual defect was a classification-layer bug misinterpreting legitimate success data as a transport error, not a genuinely unavoidable transient condition, so the correct fix is at the classification boundary, not the retry budget.", @@ -16018,9 +4407,7 @@ "reconnect-storm", "ROOT-CAUSED", "FIXED", - "live-repro", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "live-repro" ], "related_bugs": [ "bug-1435", @@ -16028,16 +4415,20 @@ "bug-854", "bug-646" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1436.md" }, { "id": "bug-1437", "status": "open", "status_updated": "2026-09-24", "status_note": "LSan leaks in test_jobj_hardening (makeInstance 240B + 7B strdup), no fix/suppression", + "component": "lib", + "severity": "crash", "timestamp": "2026-07-13T00:55:00Z", - "error_message": "Verification run of lib build + xcelib_test --run test_jobj_hardening (ASAN_OPTIONS=detect_leaks=1) after user changed 4x .datakiller=destroy -> .datakiller=dereference in src/lib/testsuite/test_jobj_hardening.c (to fix a leak in the kvmgr-of-job_t test fixture, since job_t.destroy is NULL in RTTI typeinfo and teardown only runs via job_dereference): Unity reports 5/5 PASS, 0 Failures, 0 Ignored, but LeakSanitizer STILL reports two leaks totaling 247 bytes and the process exits 1 (no halt_on_error/abort_on_error set, so no SIGABRT with plain detect_leaks=1 -- differs from the meson full-suite run which sets abort_on_error=1 and gets killed by signal 6).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Verification run of lib build + xcelib_test --run test_jobj_hardening…\n\nVerification run of lib build + xcelib_test --run test_jobj_hardening (ASAN_OPTIONS=detect_leaks=1) after user changed 4x .datakiller=destroy -> .datakiller=dereference in src/lib/testsuite/test_jobj_hardening.c (to fix a leak in the kvmgr-of-job_t test fixture, since job_t.destroy is NULL in RTTI typeinfo and teardown only runs via job_dereference): Unity reports 5/5 PASS, 0 Failures, 0 Ignored, but LeakSanitizer STILL reports two leaks totaling 247 bytes and the process exits 1 (no halt_on_error/abort_on_error set, so no SIGABRT with plain detect_leaks=1 -- differs from the meson full-suite run which sets abort_on_error=1 and gets killed by signal 6).", "file": "src/lib/testsuite/test_jobj_hardening.c", "root_cause": "TWO leaks detected, only one already known. (1) Direct leak of 240 bytes via calloc -> makeInstance -> makeJob_ (../lib/jobj.c:768) -> makeJob (../lib/jobj.c:789) -> test_job_init_propagates_makeJobsub_failure (test_jobj_hardening.c:133) -- this IS the already-diagnosed, out-of-repo makeInstance-does-not-free-on-negative-init-rc characteristic (vendored rtti library), unrelated to the datakiller fix. (2) Direct leak of 7 bytes via strdup -> hard_build_accounts (test_jobj_hardening.c:269) -> test_job_destroy_does_not_leak_accounts (test_jobj_hardening.c:328) -- this is a SEPARATE, NEW leak signature not previously logged, and it is attributed to the exact test (test_job_destroy_does_not_leak_accounts) that the datakiller=dereference fix was meant to make leak-free. NOT YET DIAGNOSED WHY: could be an unrelated small leak in the hard_build_accounts helper itself (e.g. a strdup()d account string never freed regardless of datakiller correctness), or could indicate the datakiller fix is incomplete. Not investigated further per explicit user instruction to build/run/report only, no fixing.", "fix": "NOT FIXED IN THIS SESSION -- user explicitly instructed build+run+report only, no code changes. Logging the raw finding so the next session does not have to rediscover it: the accounts-leak test still leaks (7 bytes, strdup, hard_build_accounts) even after the .datakiller=dereference change, though it is a much smaller leak than the pre-existing 240-byte makeInstance one and does not cause any Unity test FAILURE (LSAN leak detection is a separate pass/fail axis from Unity PASS/FAIL in this harness). RECURRENCE 2026-07-16 (2nd occurrence, mechanical-validation pass for job-array-single-wal-record spec, dimension A/full-suite reconfirmation): full lib test binary run (ASan) again reports 638 PASS / 0 FAIL at the Unity level, but LeakSanitizer aborts the process at exit with the same 240-byte makeInstance->makeJob_ (jobj.c:768) leak signature, triggered via test_job_init_propagates_makeJobsub_failure (test_jobj_hardening.c:133). Confirmed this is the SAME pre-existing vendored-rtti-library characteristic already diagnosed in this entry's root_cause (1), not a new leak and not introduced by this feature's production code -- the feature's own new test is simply the first test in the suite to deliberately drive job_init() to fail after makeInstance() has already allocated, so it is the first to trip the latent characteristic. Still NOT FIXED (out of scope for a validation-only pass); flagging that under meson's full-suite invocation (abort_on_error=1) this leak turns into a SIGABRT / non-zero exit code for the lib test binary, so a naive exit-code-only CI gate would read this as a lib-suite failure even though zero Unity assertions fail and zero new markers/regressions exist in feature code.", @@ -16055,20 +4446,23 @@ "kvmgr", "job-array-single-wal-record", "mechanical-validation", - "reconfirmed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "reconfirmed" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1437.md" }, { "id": "bug-1438", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-07-14T09:36:00Z", - "error_message": "test_job_array_submit_appends_exactly_one_wal_entry_and_one_continuation: Expected 11 Was 5 (WAL type mismatch); subsequent test hung all 3 threads on futex_do_wait", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_job_array_submit_appends_exactly_one_wal_entry_and_one_continuation: Expected 11 Was 5 (WAL…\n\ntest_job_array_submit_appends_exactly_one_wal_entry_and_one_continuation: Expected 11 Was 5 (WAL type mismatch); subsequent test hung all 3 threads on futex_do_wait", "file": "src/controller/testsuite/test_commit_barrier_rollout.c", "root_cause": "jap_read_last_wal_hdr()/jap_read_last_wal_payload() were copied from read_last_wal_hdr() in test_ha_role_lifecycle.c, which computes the header offset as seg_offset - sizeof(hdr). That arithmetic is only correct for zero-payload entries (its only prior caller was a WAL_NOOP test). Every WAL_JOB_ARRAY_SUBMIT entry carries a nonzero capnp payload, so seg_offset - sizeof(hdr) landed inside the payload bytes, not the header -- read garbage decoded as a bogus type (5 == WAL_JOB_STOP). The first array test's TEST_ASSERT_EQUAL_UINT32 failure then longjmp'd out of the test body BEFORE reaching jap_teardown(), leaking that test's persist_t (background WAL-sync thread still running) into the next test's jap_setup() memset -- corrupting a live mutex mid-flight and deadlocking all 3 threads on futex_do_wait in the very next test.", "fix": "Replaced the back-computed-offset helpers with jap_read_wal_hdr_at()/jap_read_wal_entry_at(), which take the entry's start offset EXPLICITLY. Callers now snapshot g_jap.seg_offset immediately before each persist_submit_job_array_async() call and pass that snapshot in, instead of trying to infer the header position from the post-append offset.", @@ -16078,25 +4472,26 @@ "test-helper", "offset-arithmetic", "deadlock", - "futex", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "futex" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1438.md" }, { "id": "bug-1439", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", "status_note": "654c6e8c, c1c6c9a2", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-14T13:15:00Z", - "error_message": "lib/controller.capnp.h:4170:43: error: unknown type name 'LimitCon'; lib/controller.capnp.h:1: error: unterminated #ifndef \u2014 during ninja compile of lib/libxce.a.p/utils.c.o, cascading into ~440 bogus errors (glibc math.h storage-class errors, utils.h ectx_t redefinition) for the rest of that one translation unit", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 5, + "error_message": "lib/controller.capnp.h:4170:43: error: unknown type name 'LimitCon'; lib/controller.capnp.h:1…\n\nlib/controller.capnp.h:4170:43: error: unknown type name 'LimitCon'; lib/controller.capnp.h:1: error: unterminated #ifndef — during ninja compile of lib/libxce.a.p/utils.c.o, cascading into ~440 bogus errors (glibc math.h storage-class errors, utils.h ectx_t redefinition) for the rest of that one translation unit", "file": "src/xce.build/lib/controller.capnp.h (generated by [5/190] Generating lib/controllercapnp.[ch] custom command)", - "root_cause": "SUSPECTED (not confirmed): parallel-build race \u2014 gcc appears to have read lib/controller.capnp.h while the custom-command generation step was still writing it (non-atomic write, no write-to-tmp+rename). Evidence: re-inspecting the SAME on-disk file immediately after the failed build shows correct, well-formed content at line 4170 (get_LimitConfig(struct LimitConfig*, LimitConfig_list, int i);) and balanced #ifndef/#endif (9/9) \u2014 i.e. the finished file is NOT actually corrupt, only what gcc read mid-write was. Not yet reproduced a second time to confirm.", - "fix": "NONE APPLIED \u2014 this run was build+report only (nix develop ./src -- xce-build.sh controller), not a fix task. Build failed via set -euo pipefail before any tests ran; no controller/xce_controller, controller_unit_test, or xce_test binaries were produced. Next step if it recurs: check meson.build custom_target() for lib/controller.capnp.[ch] has correct output declarations / add explicit depends on all consumers, or file upstream against capnpc-c invocation for atomic output. RECURRENCE 2026-07-16 (2nd occurrence, CONFIRMS the race, different symptom): clean build (rm -rf xce.build) of `nix develop ./src --command bash src/tools/xce-build.sh controller asan` at HEAD 2a76940. This time gcc failed with `fatal error: controller.capnp.h: No such file or directory` (not corrupted content) for hook.c.o, threshold.c.o, xlist.c.o, utils.c.o, window.c.o, xceapi.c.o, wrap.c.o \u2014 all 7 consumers of utils.h#include controller.capnp.h. Critically, ninja's own completion log shows `[12/191] Generating lib/controllercapnp.[ch]` printed AFTER jobs [5..11] (the 7 failed compiles) had already completed/failed. This proves the custom_target that generates lib/controller.capnp.h has no dependency edge forcing it to run before these 7 specific compile units \u2014 confirms the suspected root cause in the original entry (missing depends()/output declaration in meson.build), not just a non-atomic-write timing fluke. Still NOT fixed (build+report only task, no source edits authorized). controller_unit_test and xce_test were never produced this run (clean build wiped the prior binaries and the new build failed before relinking). RECURRENCE 2026-07-17 (3rd occurrence, kiro-validate-impl smoke-boot build for job-array-single-wal-record): fresh (non-clean) build of `nix develop ./src --command bash src/tools/xce-build.sh controller` at HEAD ea8d9d3 failed on the FIRST invocation with yet another symptom of the same race: `fatal error: opening dependency file controller/controller_unit_test.p/ha.c.o.d: No such file or directory` AND simultaneously `fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory` for two unrelated targets at once ([187/192] and [188/192]) -- gcc could not open its OWN .o.d dependency-file output path, not an input header this time, reinforcing this is a general ninja/meson parallel-build filesystem race in this sandbox, not specific to controller.capnp.h generation. WORKAROUND CONFIRMED AGAIN: simply re-running the identical build command with no source changes succeeded cleanly (85/85 targets, controller_unit_test 3/3 OK), producing fresh xce_controller/xce_test binaries. Do-not-repeat: if xce-build.sh fails with any \"fatal error: opening dependency file ... .o.d: No such file or directory\" or \"controller.capnp.h: No such file or directory\", just retry the same command once before treating it as a real build break -- do not start debugging source code. RECURRENCE 2026-07-20 (4th occurrence, xce-build.sh all asan build+report task validating bug-1453/1454 C-side changes, HEAD after commit 619f41f): clean build failed on FIRST invocation with fatal error: controller.capnp.h: No such file or directory for xlist.c.o, htab.c.o, window.c.o, xstring.c.o, hook.c.o, wrap.c.o, threshold.c.o, xceapi.c.o, utils.c.o, zarray.c.o (10 consumers of utils.h#include controller.capnp.h) -- ninja log again shows [16/353] Generating lib/controllercapnp.[ch] printed AFTER the failed compiles. WORKAROUND CONFIRMED AGAIN (4th time): identical retry with zero source changes succeeded cleanly, 353/353 ninja targets, all 9 meson test suites OK (0 Fail), test_cluster (part of libxce test suite) and scheduler tests all passed. Still unfixed at the meson.build dependency-edge level -- do-not-repeat guidance holds: always retry once before treating this as a real regression. RECURRENCE 2026-07-26 (5th occurrence, build+report task validating bug-1474 repl.c fix via `nix develop ./src --command bash src/tools/xce-build.sh all asan`, HEAD c0b0c7d): TWO consecutive races in the same session, different symptoms each time. 1st invocation (clean build): classic capnp.h race -- hook.c.o etc. failed 'controller.capnp.h: No such file or directory' with the generator's status line ([6/361] Generating lib/controllercapnp.[ch]) printed AFTER the failed compiles -- identical to prior occurrences. 2nd invocation (retry, clean build): build progressed all the way to [361/361] (last job, frontend/apiserver's go_build) which FAILED with `/usr/bin/ld: cannot find -lxce: No such file or directory`; lib/libxce.a was confirmed absent from src/xce.build/lib/ immediately after the failure, and no 'Linking static target lib/libxce.a' line ever appeared in the ninja log for that invocation. NEW symptom variant: this is the first occurrence of the race hitting a Go custom_target (apiserver's CGO link against libxce.a) rather than a C compile/link step -- apiserver's CGO_LDFLAGS references @BUILD_ROOT@/lib/libxce.a only via an env-var string, which is suspected to carry no explicit ninja dependency edge on the lib archive target, letting the go_build job be eligible to run before the archive step completes. 3rd invocation (retry, clean build) succeeded cleanly: 361/361 ninja targets, meson test Ok:9 Fail:0, scheduler manual test 15/15 pass, frontend Go tests (apiserver + xcecli) all ok, xceadmin pytest 166 passed, producing fresh xce_controller/xce_test (confirmed containing the bug-1474 'reusing guard-holding slot' WARN string) and a fresh xceadmin/.venv/bin/xceadmin. WORKAROUND CONFIRMED AGAIN (do not debug source on first or second failure): simply retry the identical xce-build.sh command with no changes; this session needed up to 2 retries (3 total invocations) before a clean pass, still well within the established do-not-repeat guidance. Do-not-repeat addendum: the race is not limited to C compile/link edges -- it can also strand a Go custom_target waiting on a C static archive (apiserver <- lib/libxce.a) referenced only via env-var string interpolation, not a declared meson `depends:`/`link_with:` edge. Still unfixed at the meson.build dependency-edge level.", + "root_cause": "SUSPECTED (not confirmed): parallel-build race — gcc appears to have read lib/controller.capnp.h while the custom-command generation step was still writing it (non-atomic write, no write-to-tmp+rename). Evidence: re-inspecting the SAME on-disk file immediately after the failed build shows correct, well-formed content at line 4170 (get_LimitConfig(struct LimitConfig*, LimitConfig_list, int i);) and balanced #ifndef/#endif (9/9) — i.e. the finished file is NOT actually corrupt, only what gcc read mid-write was. Not yet reproduced a second time to confirm.", + "fix": "NONE APPLIED — this run was build+report only (nix develop ./src -- xce-build.sh controller), not a fix task. Build failed via set -euo pipefail before any tests ran; no controller/xce_controller, controller_unit_test, or xce_test binaries were produced. Next step if it recurs: check meson.build custom_target() for lib/controller.capnp.[ch] has correct output declarations / add explicit depends on all consumers, or file upstream against capnpc-c invocation for atomic output. RECURRENCE 2026-07-16 (2nd occurrence, CONFIRMS the race, different symptom): clean build (rm -rf xce.build) of `nix develop ./src --command bash src/tools/xce-build.sh controller asan` at HEAD 2a76940. This time gcc failed with `fatal error: controller.capnp.h: No such file or directory` (not corrupted content) for hook.c.o, threshold.c.o, xlist.c.o, utils.c.o, window.c.o, xceapi.c.o, wrap.c.o — all 7 consumers of utils.h#include controller.capnp.h. Critically, ninja's own completion log shows `[12/191] Generating lib/controllercapnp.[ch]` printed AFTER jobs [5..11] (the 7 failed compiles) had already completed/failed. This proves the custom_target that generates lib/controller.capnp.h has no dependency edge forcing it to run before these 7 specific compile units — confirms the suspected root cause in the original entry (missing depends()/output declaration in meson.build), not just a non-atomic-write timing fluke. Still NOT fixed (build+report only task, no source edits authorized). controller_unit_test and xce_test were never produced this run (clean build wiped the prior binaries and the new build failed before relinking). RECURRENCE 2026-07-17 (3rd occurrence, kiro-validate-impl smoke-boot build for job-array-single-wal-record): fresh (non-clean) build of `nix develop ./src --command bash src/tools/xce-build.sh controller` at HEAD ea8d9d3 failed on the FIRST invocation with yet another symptom of the same race: `fatal error: opening dependency file controller/controller_unit_test.p/ha.c.o.d: No such file or directory` AND simultaneously `fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory` for two unrelated targets at once ([187/192] and [188/192]) -- gcc could not open its OWN .o.d dependency-file output path, not an input header this time, reinforcing this is a general ninja/meson parallel-build filesystem race in this sandbox, not specific to controller.capnp.h generation. WORKAROUND CONFIRMED AGAIN: simply re-running the identical build command with no source changes succeeded cleanly (85/85 targets, controller_unit_test 3/3 OK), producing fresh xce_controller/xce_test binaries. Do-not-repeat: if xce-build.sh fails with any \"fatal error: opening dependency file ... .o.d: No such file or directory\" or \"controller.capnp.h: No such file or directory\", just retry the same command once before treating it as a real build break -- do not start debugging source code. RECURRENCE 2026-07-20 (4th occurrence, xce-build.sh all asan build+report task validating bug-1453/1454 C-side changes, HEAD after commit 619f41f): clean build failed on FIRST invocation with fatal error: controller.capnp.h: No such file or directory for xlist.c.o, htab.c.o, window.c.o, xstring.c.o, hook.c.o, wrap.c.o, threshold.c.o, xceapi.c.o, utils.c.o, zarray.c.o (10 consumers of utils.h#include controller.capnp.h) -- ninja log again shows [16/353] Generating lib/controllercapnp.[ch] printed AFTER the failed compiles. WORKAROUND CONFIRMED AGAIN (4th time): identical retry with zero source changes succeeded cleanly, 353/353 ninja targets, all 9 meson test suites OK (0 Fail), test_cluster (part of libxce test suite) and scheduler tests all passed. Still unfixed at the meson.build dependency-edge level -- do-not-repeat guidance holds: always retry once before treating this as a real regression. RECURRENCE 2026-07-26 (5th occurrence, build+report task validating bug-1474 repl.c fix via `nix develop ./src --command bash src/tools/xce-build.sh all asan`, HEAD c0b0c7d): TWO consecutive races in the same session, different symptoms each time. 1st invocation (clean build): classic capnp.h race -- hook.c.o etc. failed 'controller.capnp.h: No such file or directory' with the generator's status line ([6/361] Generating lib/controllercapnp.[ch]) printed AFTER the failed compiles -- identical to prior occurrences. 2nd invocation (retry, clean build): build progressed all the way to [361/361] (last job, frontend/apiserver's go_build) which FAILED with `/usr/bin/ld: cannot find -lxce: No such file or directory`; lib/libxce.a was confirmed absent from src/xce.build/lib/ immediately after the failure, and no 'Linking static target lib/libxce.a' line ever appeared in the ninja log for that invocation. NEW symptom variant: this is the first occurrence of the race hitting a Go custom_target (apiserver's CGO link against libxce.a) rather than a C compile/link step -- apiserver's CGO_LDFLAGS references @BUILD_ROOT@/lib/libxce.a only via an env-var string, which is suspected to carry no explicit ninja dependency edge on the lib archive target, letting the go_build job be eligible to run before the archive step completes. 3rd invocation (retry, clean build) succeeded cleanly: 361/361 ninja targets, meson test Ok:9 Fail:0, scheduler manual test 15/15 pass, frontend Go tests (apiserver + xcecli) all ok, xceadmin pytest 166 passed, producing fresh xce_controller/xce_test (confirmed containing the bug-1474 'reusing guard-holding slot' WARN string) and a fresh xceadmin/.venv/bin/xceadmin. WORKAROUND CONFIRMED AGAIN (do not debug source on first or second failure): simply retry the identical xce-build.sh command with no changes; this session needed up to 2 retries (3 total invocations) before a clean pass, still well within the established do-not-repeat guidance. Do-not-repeat addendum: the race is not limited to C compile/link edges -- it can also strand a Go custom_target waiting on a C static archive (apiserver <- lib/libxce.a) referenced only via env-var string interpolation, not a declared meson `depends:`/`link_with:` edge. Still unfixed at the meson.build dependency-edge level.", "tags": [ "ninja-race", "controller", @@ -16108,22 +4503,28 @@ "missing-dependency-edge", "reproduced", "capnp", - "go-apiserver-link", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "go-apiserver-link" ], "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 30 + "fix_commits": [ + "654c6e8c", + "c1c6c9a2" + ], + "record": ".wolf/bugs/bug-1439.md", + "gitea_issue": 30, + "gitea_synced_status": "fixed" }, { "id": "bug-1440", "status": "fixed", "status_updated": "2026-09-24", "status_note": "b7b90e9d", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-07-15T09:35:00Z", - "error_message": "use of cgo in test wire_dispatch_test.go not supported / FAIL apiserver/backend [setup failed]", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "use of cgo in test wire_dispatch_test.go not supported / FAIL apiserver/backend [setup failed]\n\nuse of cgo in test wire_dispatch_test.go not supported / FAIL apiserver/backend [setup failed]", "file": "src/frontend/apiserver/backend/wire_dispatch_test.go", "root_cause": "Go toolchain hard restriction, not an environment/CGO_CFLAGS/nix issue: cmd/go's module indexer (GOROOT share/go/src/cmd/go/internal/modindex/read.go, ~line 589, inside the per-file import scan) explicitly checks `if imp.path == \"C\" { if isTest { badGoFile(name, fmt.Errorf(\"use of cgo in test %s not supported\", name)) } }`. Any `_test.go` file that directly does `import \"C\"` is rejected outright the moment CGO is active for that file (CGO_ENABLED=1 and the file's build tag, here `//go:build cgo`, is satisfied) -- this is golang/go#13938, a long-standing toolchain limitation, confirmed by reading the actual installed Go 1.25.5 source in the nix shell. Because wire_dispatch_test.go and clib.go/convert.go/peers_test.go/handler_test.go/convert_test.go/heartbeat_test.go (the last four carry no build tag, so they build under BOTH cgo and !cgo) all live in the same package, the ENTIRE apiserver/backend package fails as `[setup failed]` under `make test-backend` (CGO_ENABLED=1) -- this also silently prevented the pre-existing cgo-buildable tests (handler_test.go, convert_test.go, peers_test.go, heartbeat_test.go) from running, even though none of them regressed on their own. The `!cgo`-tagged files (integration_test.go, control_host_test.go, stub_funcs_test.go) are unaffected since they're excluded from the CGO_ENABLED=1 file set entirely; `make test` (CGO_ENABLED=0) passed clean (apiserver/backend ok, 1.510s) confirming no regression on the non-cgo path.", "fix": "NOT FIXED -- user explicitly instructed build/run/report only, no source edits. Reported fix path for the user: remove `import \"C\"` (and any direct `C.xxx` calls) from wire_dispatch_test.go itself; move the cgo-dependent helpers (buildEncodedJobArrayPush, deliverViaRealDispatchChain, and the `/* ... */ import \"C\"` preamble with the wire_test_* static C helpers) into a new non-test `.go` file in the same package (e.g. clib_wire_test_helpers.go, `//go:build cgo`), exposing plain-Go-signature functions the _test.go file can call without itself using cgo directly. This mirrors the established pattern already used for other CGO test-only helpers in this package. Because the file was rejected before type-checking, it is unknown whether wire_dispatch_test.go has any further errors (undefined C symbols, type mismatches) beyond this first, blocking one -- those can only be discovered after this structural fix.", @@ -16135,21 +4536,25 @@ "job-array-single-wal-record", "backend-package", "build-error", - "golang-go-13938", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "golang-go-13938" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "b7b90e9d" + ], + "record": ".wolf/bugs/bug-1440.md" }, { "id": "bug-1441", "status": "fixed", "status_updated": "2026-09-24", "status_note": "wire_dispatch_support.go direct Go call", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-07-15T09:40:00Z", - "error_message": "backend/wire_dispatch_support.go:154:8: could not determine what C.go_push_handler refers to / FAIL apiserver/backend [build failed]", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "backend/wire_dispatch_support.go:154:8: could not determine what C.go_push_handler refers to /…\n\nbackend/wire_dispatch_support.go:154:8: could not determine what C.go_push_handler refers to / FAIL apiserver/backend [build failed]", "file": "src/frontend/apiserver/backend/wire_dispatch_support.go", "root_cause": "cgo declaration visibility is per-file, not per-package: go_push_handler is exported to C only via clib.go's own preamble comment (\"extern int go_push_handler(xce_request_t*, void*);\" immediately followed by \"//export go_push_handler\" on the Go func), and that extern declaration lives nowhere in any shared header -- it is a plain comment inside clib.go's /* ... */ block, not client.h/netio.h/request.h/controller.capnp.h. cgo processes each file's own preamble independently to build that file's C.xxx symbol table, so wire_dispatch_support.go (which #includes client.h/netio.h/request.h/controller.capnp.h but does NOT itself declare go_push_handler) has no visibility into that extern, even though both files are in the same Go package and both carry //go:build cgo. This surfaced only after fixing bug-1440's toolchain-level rejection (moving import \"C\" out of the _test.go file) and after separately discovering make test-backend also needs CGO_CFLAGS/CGO_LDFLAGS manually derived to match src/frontend/apiserver/meson.build (pkg-config --cflags/--libs for librtti, c-capnproto, openssl, tinylog, plus -I/src/include -I/src/build/lib -L/src/build/lib -lxce -pthread -lm) since the nix devshell itself does not export them and make test-backend invokes go test directly rather than through the meson/go_build path that normally injects them.", "fix": "NOT FIXED -- user explicitly instructed build/run/report only, no source edits. Reported fix path for the user: add the same \"extern int go_push_handler(xce_request_t*, void*);\" declaration to wire_dispatch_support.go's own cgo preamble (matching clib.go's exact signature), OR move that extern declaration into a shared header both files already include. Because the toolchain stops at first error, it is unknown whether wire_dispatch_support.go/wire_dispatch_test.go have any FURTHER errors (e.g. C.free_JobSubmit_ptr signature, C.extract_data, C.xce_encode/xce_decode field access) beyond this first blocking one -- those can only be discovered after this fix. Separately (env-only, no source change): make test-backend also requires CGO_CFLAGS/CGO_LDFLAGS to be exported manually before invoking make, mirroring meson.build's pkg-config-derived flags -- the bare Makefile command alone (as documented) does not produce a working CGO_ENABLED=1 build in this worktree.", @@ -16160,22 +4565,25 @@ "cross-file-declaration", "job-array-single-wal-record", "test-backend", - "CGO_CFLAGS", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "CGO_CFLAGS" ], "related_bugs": [ "bug-1440" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1441.md" }, { "id": "bug-1442", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "crash", "timestamp": "2026-07-15T09:50:00Z", - "error_message": "AddressSanitizer: attempting double-free on 0x...1b0 in thread T0 -- fires immediately on TestWireDispatch_OnePushProducesNRecordsAndLogsOnce, aborts the whole backend.test binary before it can print PASS/FAIL, so TestWireDispatch_RedeliveryCreatesNoNewRecordsAndLogsNothing never runs at all", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: attempting double-free on 0x...1b0 in thread T0 -- fires immediately on…\n\nAddressSanitizer: attempting double-free on 0x...1b0 in thread T0 -- fires immediately on TestWireDispatch_OnePushProducesNRecordsAndLogsOnce, aborts the whole backend.test binary before it can print PASS/FAIL, so TestWireDispatch_RedeliveryCreatesNoNewRecordsAndLogsNothing never runs at all", "file": "src/frontend/apiserver/backend/wire_dispatch_support.go", "root_cause": "NOT ROOT-CAUSED (build/run/report only, no fix attempted per user instruction). bug-1441 (C.go_push_handler cross-file cgo resolution) is now fixed, and a separate missing `import \"apiserver/store\"` in wire_dispatch_test.go was fixed as a trivial compile error, so the build now succeeds and the test actually runs -- and immediately hits a real ASan double-free at runtime, not a build failure. Leading hypothesis (unverified): wireDispatchDeliver() (lines 122-180) calls go_push_handler(&decoded, nil) at line 164, whose defer C.free_Request(req) (clib.go) frees *decoded including its Request_body_datamsg payload -- the comment at lines 166-172 asserts free_Dataset() is 'a documented no-op for every table' (citing task 5.3b's residual-gap note) and therefore safe for wireDispatchDeliver to ALSO free ds.rows via free_JobSubmit_ptr() and ds itself at lines 173-177. A 16-byte double-free immediately after go_push_handler returns is consistent with that no-op assumption being wrong for the Data_data_jobArray-derived dataset table specifically (i.e. free_Dataset() actually does free ds->rows or ds itself for this table, and wire_dispatch_support.go's explicit cleanup then frees the same memory again). NOT CONFIRMED -- needs a symbolized ASan backtrace (frames are bare hex addresses in this build) or manual trace of free_Dataset()/free_Request() to confirm which free is the duplicate.", "fix": "FIXED and VERIFIED 2026-07-15T09:55:00Z. Root cause confirmed: go_push_handler's internal `defer C.free_Request(req)` (clib.go) calls free_Request(&decoded), whose Request_body_datamsg case calls free_Dataset_ptr(&decoded.body.datamsg) -- free_Dataset_ptr() (controller.capnp.c, generated) unconditionally free()s the dataset_t container pointer it is given, even though free_Dataset() itself is an empty no-op (never touches ->rows). So `ds` becomes a dangling pointer the instant go_push_handler returns -- the earlier NOT-ROOT-CAUSED hypothesis was correct in spirit but had the wrong culprit function (free_Dataset_ptr, not free_Dataset). wireDispatchDeliver() (wire_dispatch_support.go) had two bugs: (1) read ds.rows AFTER calling go_push_handler = use-after-free, (2) separately called C.free(ds) again = double-free on the same 16-byte region ASan caught. Fix: capture `rows := ds.rows` BEFORE calling go_push_handler (line 175), use that captured value afterward instead of re-reading ds.rows, and removed the redundant C.free(unsafe.Pointer(ds)) call entirely since go_push_handler's internal free_Request already frees it. Verified via full ASan CGO rebuild (make test-backend under nix develop ./src with CGO_CFLAGS/CGO_LDFLAGS = -fsanitize=address -g -DWITH_DEBUG=1 + pkg-config librtti/c-capnproto/openssl/tinylog): TestWireDispatch_OnePushProducesNRecordsAndLogsOnce and TestWireDispatch_RedeliveryCreatesNoNewRecordsAndLogsNothing both PASS with zero ASan errors; full ./backend/... suite (91 subtests, convert/handler/heartbeat/peers/wire_dispatch/auth) all PASS, zero regressions; make test (CGO_ENABLED=0) also fully green.", @@ -16189,22 +4597,25 @@ "test-backend", "go_push_handler", "free_Dataset", - "free_Dataset_ptr", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "free_Dataset_ptr" ], "related_bugs": [ "bug-1441" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1442.md" }, { "id": "bug-1443", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-07-16T02:05:00Z", - "error_message": "OPEN, NOT A CRASH -- discovered as a design-time/test-proven gap, not a runtime failure report. restore_job_array()'s final bulk-insert pass into c->jobmgr can itself fail partway through (e.g. realloc() exhaustion growing c->jobmgr's backing table). Because the underlying xkvmgr_type store has no key_del, an already-inserted prefix [0, k) cannot be rolled back and is left permanently, genuinely visible in c->jobmgr -- individually addressable (status/cancel/completion all work via direct jobid lookup) -- while the function honestly returns a negative errno (never silently claims success). Critically, that prefix is never included in out_new_jobs, so HOOK_DETAIL_JOB_NEW never fires for it, not on this attempt or on any later successful retry of the same LSN (the per-index idempotent-skip logic on retry treats the already-inserted prefix as 'already exists', excluding it from the retry's own out_new_jobs too). Practical consequence: these array elements exist and are queryable forever, but never reach the scheduler's or API-server's push and so are never actually scheduled to run -- a silent, permanent no-op job, not merely a bookkeeping gap. requirements-coverage validation (kiro-validate-impl, 2026-07-16) confirmed this is a literal violation of Requirement 2.3 ('a standby shall never observe a single array WAL record in a partially-applied form') and Requirement 9.4 ('shall not construct a partial/divergent set of elements'), and its shadow extends to 9.5 (self-heal, since the orphaned prefix never gets a fresh chance to hook-fire even once the retry succeeds) and 9.6 (the capstone fault-injection test deliberately targets the fully-atomic construction-phase seam instead of this bulk-insert seam, precisely because asserting 'zero partial elements' against this seam would be dishonest).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "not a runtime failure report\n\nOPEN, NOT A CRASH -- discovered as a design-time/test-proven gap, not a runtime failure report. restore_job_array()'s final bulk-insert pass into c->jobmgr can itself fail partway through (e.g. realloc() exhaustion growing c->jobmgr's backing table). Because the underlying xkvmgr_type store has no key_del, an already-inserted prefix [0, k) cannot be rolled back and is left permanently, genuinely visible in c->jobmgr -- individually addressable (status/cancel/completion all work via direct jobid lookup) -- while the function honestly returns a negative errno (never silently claims success). Critically, that prefix is never included in out_new_jobs, so HOOK_DETAIL_JOB_NEW never fires for it, not on this attempt or on any later successful retry of the same LSN (the per-index idempotent-skip logic on retry treats the already-inserted prefix as 'already exists', excluding it from the retry's own out_new_jobs too). Practical consequence: these array elements exist and are queryable forever, but never reach the scheduler's or API-server's push and so are never actually scheduled to run -- a silent, permanent no-op job, not merely a bookkeeping gap. requirements-coverage validation (kiro-validate-impl, 2026-07-16) confirmed this is a literal violation of Requirement 2.3 ('a standby shall never observe a single array WAL record in a partially-applied form') and Requirement 9.4 ('shall not construct a partial/divergent set of elements'), and its shadow extends to 9.5 (self-heal, since the orphaned prefix never gets a fresh chance to hook-fire even once the retry succeeds) and 9.6 (the capstone fault-injection test deliberately targets the fully-atomic construction-phase seam instead of this bulk-insert seam, precisely because asserting 'zero partial elements' against this seam would be dishonest).", "file": "src/controller/job.c (restore_job_array(), bulk-insert phase)", "root_cause": "No capacity-reservation or rollback primitive exists anywhere in this codebase's kv.c/kvmgr.c/zarray.c stack -- c->jobmgr (xkvmgr_type) grows its backing table via realloc() with no pre-check, and once key_add() has inserted an element there is no key_del to undo it. restore_job_array()'s design (scratch-build every element first, then one bulk-insert pass under a single write-lock) makes construction-phase failures fully atomic (nothing ever touches c->jobmgr until every element is ready), but the bulk-insert pass itself is NOT covered by that atomicity guarantee -- it is the one place a late-stage resource-exhaustion failure can leave real, permanent state behind that the function's own success/failure contract cannot fully describe. Discovered and documented during task 3.1's original implementation review (job-array-single-wal-record spec), re-confirmed live-testable via test_rja_bulk_insert_failure_reports_error_not_false_success (test_job_array_apply.c), and re-surfaced by name during the spec's final /kiro-validate-impl requirements-coverage pass.", "fix": "FIXED and VERIFIED 2026-07-16 (commit dcf9f39, job-array-single-wal-record task 7.1, user-directed follow-up after the GO decision). Added kvmgr_reserve()/xzarray_reserve() -- new, purely-additive primitives in kvmgr.c/zarray.c following the existing kvmgr_capture_keys() template (bare extern function, no RTTI vtable slot, caller-holds-write-lock contract) -- that pre-grow c->jobmgr's backing xzarray for the whole array before the bulk-insert pass begins, closing the dominant N-scaling realloc-exhaustion trigger structurally. Discovered and fixed a genuine, previously-unknown off-by-one in the existing, unmodified xzarray_ensure_items(): its strict less-than sufficiency check (mincap < caps) means reserving exactly array_size and then performing array_size real stores still triggers one more growth cycle on the LAST store; reserving array_size + 1 is the exact minimal correct compensation, independently re-derived and confirmed with concrete arithmetic by adversarial review. For the now-much-narrower residual case (kvtab_add()'s own small per-entry hash-chain-node allocation, or a content-based -EEXIST collision -- both non-N-scaling, honestly still theoretically possible and documented as such, NOT claimed to be eliminated): restore_job_array() now populates out_new_jobs with whatever prefix was genuinely inserted before a mid-pass failure, and apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case fires HOOK_DETAIL_JOB_NEW for that non-empty partial prefix even when the overall return is negative, while still propagating the negative rc so the LSN correctly halts and the Live Apply Resource Fence still engages for the remaining un-constructed indices -- a genuinely-inserted element is never left un-pushed/un-scheduled again. Verified: retry-double-notify risk traced and confirmed absent (pre-existing per-index idempotent-skip logic excludes already-inserted indices from any subsequent attempt's out_new_jobs); apply_wal_entry()'s dead-code standby path confirmed genuinely untouched; full controller/lib/scheduler test suites clean, zero regressions in any other consumer of the shared kvmgr.c/zarray.c code. Full detail in tasks.md's Implementation Notes, task 7.1 entry.", @@ -16221,20 +4632,25 @@ "kv.c", "kvmgr.c", "open-gap", - "kiro-validate-impl", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "kiro-validate-impl" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "dcf9f39" + ], + "record": ".wolf/bugs/bug-1443.md" }, { "id": "bug-1444", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-16T02:48:18Z", - "error_message": "persist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip (expected last_applied_lsn=5 new_job_count=3); a hidden apply defect is present (ha-commit-durability recover_verdict=RECOVER_CLEAN) / persist_recover: UNRECOVERABLE -- post-replay apply-correctness verification is DIVERGENT ... Staying ALIVE and PERMANENTLY non-promotable (refusing to serve divergent state) and RETRYING catch-up indefinitely -- reproduced live during job-array-single-wal-record task 6.1's HA soak scenario the FIRST time a node is killed and restarted after ANY WAL_JOB_ARRAY_SUBMIT record has ever been committed to its local WAL (e.g. after a 2-element array submission).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "persist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip…\n\npersist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip (expected last_applied_lsn=5 new_job_count=3); a hidden apply defect is present (ha-commit-durability recover_verdict=RECOVER_CLEAN) / persist_recover: UNRECOVERABLE -- post-replay apply-correctness verification is DIVERGENT ... Staying ALIVE and PERMANENTLY non-promotable (refusing to serve divergent state) and RETRYING catch-up indefinitely -- reproduced live during job-array-single-wal-record task 6.1's HA soak scenario the FIRST time a node is killed and restarted after ANY WAL_JOB_ARRAY_SUBMIT record has ever been committed to its local WAL (e.g. after a 2-element array submission).", "file": "src/controller/persist.c (replay_wal(), ~line 3902-3904; consumed by persist_recover()/persist_verify_apply_correctness(), ~line 3960-3991)", "root_cause": "replay_wal()'s post-replay apply-correctness expectation counter only tallies WAL_JOB_SUBMIT entries toward new_job_submits_out: `if (hdr.type == WAL_JOB_SUBMIT && new_job_submits_out && apply_rc == 1) (*new_job_submits_out)++;` (persist.c ~3902-3904). It has NO corresponding branch for WAL_JOB_ARRAY_SUBMIT (the new record type job-array-single-wal-record/task 1.1 introduced), even though apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case (persist.c ~2809-2851, restore_job_array()) constructs N new job_t entries per record, not 1. So expected_new_job_count silently undercounts by N for every array record replayed during boot -- while persist_verify_apply_correctness() (persist.c ~3982-3988) measures actual_new_jobs via a REAL jobmgr entry-count delta (count_jobmgr_entries before/after), which correctly reflects the N elements. The mismatch (actual_new_jobs != expected_new_job_count) trips APPLY_DIVERGENT on every single restart of every node whose local WAL contains a WAL_JOB_ARRAY_SUBMIT entry above its checkpoint -- deterministic and 100% reproducible, not a race or resource-exhaustion condition. persist_recover_loud_fence_if_unrecoverable then classifies this as UNRECOVERABLE, permanently non-promotable-fencing that node for the rest of its process life (own boot-time verification re-runs identically on every subsequent restart too, since the miscounting bug reproduces every time). This is a NEW, separate gap from this spec's three self-flagged residual items (task 3.1 bulk-insert partial-visibility / bug-1443, task 4.2 boot-replay-doesn't-respect-apply-failures gap, task 5.3b's small per-commit leak) -- it lives in a DIFFERENT, pre-existing mechanism (raft-apply-decoupling task 5.1's post-replay apply-correctness verification / ha-commit-durability's Req 8.1-8.4), which this spec's own task list never touched to account for the new WAL_JOB_ARRAY_SUBMIT type. Practical severity: in any cluster where a job array has EVER been submitted, every ordinary node restart (routine maintenance, crash-restart, rolling upgrade) permanently non-promotable-fences that node; over enough restarts across a 3-node cluster this can leave ZERO nodes eligible for promotion -- a serious HA-availability regression, not a cosmetic log line. Discovered live via job-array-single-wal-record task 6.1's HA soak harness (.claude/skills/xce-test-ha/run-ha-soak.sh, new --array-soak mode) -- specifically NOT reproducible by this spec's existing white-box unit tests (task 2.3/3.4), which never drive a real persist_recover()/replay_wal() boot cycle after committing an array record.", "fix": "FIXED and VERIFIED (2026-07-15/16, commit 16747aa, task 6.0a of job-array-single-wal-record) -- independently root-caused by a separate implementation pass on the same branch, inserted as a blocking task ahead of 6.1/6.2/6.3 per explicit user sign-off after the exact same bug was live-reproduced during this task's own --array-soak run. Fix matches the shape predicted below almost exactly: replay_wal()'s tally gained a `WAL_JOB_ARRAY_SUBMIT` branch that adds `apply_rc` (the genuine per-record new-element count apply_committed_entry()'s array case already returns, not the flat/undercounted +1 the WAL_JOB_SUBMIT branch uses) to *new_job_submits_out when apply_rc > 0, mirroring the existing WAL_JOB_SUBMIT pattern. Verified via: (a) new unit tests in test_apply_correctness_verification.c with an independently-reproduced RED-before-fix/GREEN-after-fix cycle; (b) two from-scratch live 3-node HA soak re-runs (--array-soak), one by the fixing pass's implementer and one by its independent reviewer, both showing zero APPLY_DIVERGENT across all three of this bug's trigger paths (restart-mid-small-array, standby election-path recovery fallback, restart-mid-large-array) with one-WAL-record/one-commit-barrier and per-element status/cancel/completion all confirmed post-failover. Full detail in .kiro/specs/job-array-single-wal-record/tasks.md's Implementation Notes, task 6.0a entry. This buglog entry was originally filed (independently, before its author discovered the fix already existed on the branch) with fix=NOT FIXED; corrected here once the duplication was noticed, since the underlying finding is accurate and well-characterized and deserves its own tracked entry (task 6.0a's fix was documented in tasks.md but never separately logged here) -- kept as a distinct entry from bug-1443 (a different, still-open gap in the same feature) rather than merged or deleted.", @@ -16251,22 +4667,27 @@ "array-soak", "task-6.1", "raft-apply-decoupling", - "ha-commit-durability", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha-commit-durability" ], "related_bugs": [ "bug-1443" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "16747aa" + ], + "record": ".wolf/bugs/bug-1444.md" }, { "id": "bug-1445", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-17T09:47:00Z", - "error_message": "A node's very first promotion-eligibility evaluation in a fresh process runs before that node's own first persist_recover()/replay_wal() call, so the Live Apply Resource Fence (job-array-single-wal-record tasks 4.1-4.3, extended to boot replay by task 7.2) is invisible to that one specific check. A node that would otherwise correctly fence itself non-promotable on a resource-exhaustion apply failure during its first-ever boot replay can instead pass its first promotion check looking fully eligible, because the fence flag hasn't been set yet at the point that first gate runs.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "A node's very first promotion-eligibility evaluation in a fresh process runs before that node's…\n\nA node's very first promotion-eligibility evaluation in a fresh process runs before that node's own first persist_recover()/replay_wal() call, so the Live Apply Resource Fence (job-array-single-wal-record tasks 4.1-4.3, extended to boot replay by task 7.2) is invisible to that one specific check. A node that would otherwise correctly fence itself non-promotable on a resource-exhaustion apply failure during its first-ever boot replay can instead pass its first promotion check looking fully eligible, because the fence flag hasn't been set yet at the point that first gate runs.", "file": "src/controller/ha.c", "root_cause": "ha.c's promotion-gate eligibility check (ha.c:1303-1408) runs before persist_recover()'s first-ever replay_wal() call (ha.c:1427-1439) on a freshly-started process. Every promotion evaluation AFTER a process's first one is correctly protected (persist_apply_resource_fenced() is threaded into both the periodic fence-release check and the election-time check per task 4.3) -- this is a narrow ordering gap specific to the single very-first evaluation of a brand-new process, discovered and documented (not fixed) during job-array-single-wal-record task 7.2's own review, out of that task's persist.c-only file boundary.", "fix": "FIXED and VERIFIED (2026-07-17, src/controller/ha.c). raft_cb_become_leader()'s is_first_boot block now re-checks persist_apply_resource_fenced(&c->persist) immediately after cluster_init_phase2() completes (the call that triggers this process's own first persist_recover()/replay_wal()) -- if that first replay just engaged the fence, promotion is refused (same pattern as the pre-existing refusal branch: WARN log + ha_arm_promotion_fence_timer() + return), instead of falling through to CTRL_PRIMARY on a decision made before that first replay ever ran. Every later election is unaffected -- is_first_boot is false by then, so the earlier eligibility gate (ha.c:1337) already covers it correctly. Scope check: confirmed ha_elect_evaluate_and_signal (the OTHER election path, ha.c ~5030) is dead in production for the promotion outcome -- every live call site of handle_request_vote_reply passes vote_granted=0 (bug-828 already redirected the real grant-recording path to the record-only ha_record_vote_reply, which never signals it) -- so raft_cb_become_leader is confirmed the sole live promotion path and this fix's scope is complete, not partial. TDD: new test test_first_boot_promotion_evaluates_own_first_replay_fence (src/controller/testsuite/test_job_array_replay_fence.c) wires ha_phase2_hook to a stub that calls persist_recover() for real, reproducing production ordering exactly (a never-recovered persist_t entering the election path with ha->phase2_done==0) -- RED-confirmed failing pre-fix at the exact predicted assertion (role_after_election==CTRL_PRIMARY), GREEN-confirmed passing post-fix. Full controller_unit_test binary (bundles every testsuite/test_*.c including test_convergence_verification.c, test_ha_cluster.c, test_p1_integration.c) green: 16/16 tests in this file, 0 regressions cluster-wide.", @@ -16279,22 +4700,25 @@ "first-boot", "persist_recover", "replay_wal", - "ordering-bug", - "fixed" + "ordering-bug" ], "related_bugs": [ "bug-1443" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1445.md" }, { "id": "bug-1446", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-07-18T00:47:00Z", - "error_message": "/kiro-validate-impl ha-commit-durability: test-commit-durability.sh --deterministic reported 3/3 FAIL ('DURABILITY GAP -- primary acked with 0 fdatasync in-window'), i.e. a false RED that looked like a regression of the spec's core invariant on the current build.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "/kiro-validate-impl ha-commit-durability: test-commit-durability.sh --deterministic reported 3/3…\n\n/kiro-validate-impl ha-commit-durability: test-commit-durability.sh --deterministic reported 3/3 FAIL ('DURABILITY GAP -- primary acked with 0 fdatasync in-window'), i.e. a false RED that looked like a regression of the spec's core invariant on the current build.", "file": "src/tools/test-commit-durability.sh (det_probe_once / date usage) -- test harness only, NOT a product file", "root_cause": "This sandbox's /sbin/date is BusyBox date, which does NOT support GNU date's %N (nanoseconds) format directive -- 'date +%H:%M:%S.%N' literally emits the unexpanded string 'HH:MM:SS.%N' (confirmed by direct invocation). det_probe_once() captures t0/t1 with this call to bound the submit-to-ack window, then _hms_to_sod() converts them to seconds-of-day for the in-window fdatasync count. With the literal '%N' suffix the sub-second component is unparseable/degenerate, so the computed window fails to contain the real fdatasync timestamp even though it lands in the same wall-clock second -- producing syncs=0 (RED) regardless of actual ordering. This is an environment tooling gap (BusyBox vs GNU date), not a code regression: the script assumes GNU date is on PATH.", "fix": "NOT a product bug, no source fix applied (out of this validation's scope; script is owned by xce-test-e2e-primitives per this spec's task 1.3 boundary note). Diagnosed by cross-checking strace's own microsecond timestamps (which DO work correctly, e.g. 'fdatasync(10) = 0' at 00:47:01.128244) against the controller's committed_lsn advance log lines (commit_advance_locked '1 -> 2' at 00:47:01,144) for the same three probes: fdatasync consistently preceded the commit-advance (that gates the client ack) by 16-43ms in all 3 probes -- i.e. the real durable-before-ack ordering the spec requires DOES hold on the current build; the script's verdict was the false signal, not the product. Recommend the script detect/require GNU date (or use 1784335901.164264 / a C helper) rather than assuming %N support, so it fails loud on an incompatible date instead of silently producing a false RED.", @@ -16305,22 +4729,25 @@ "date", "kiro-validate-impl", "ha-commit-durability", - "deterministic-mode", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "deterministic-mode" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 31 + "fix_commits": [], + "record": ".wolf/bugs/bug-1446.md", + "gitea_issue": 31, + "gitea_synced_status": "fixed" }, { "id": "bug-1447", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-07-18T01:05:02Z", - "error_message": "/kiro-validate-impl raft-drive-async: 10-round SEQUENTIAL HA soak (the fully-supported single-failover lifecycle, hard-fail-on-any-loss) failed at round 9/10 -- 'FAIL: round 9: post-failover submit did not get 3 acks' (0/3 acked across 5 retry attempts). Root traced to ctrl-3 (a standby) crashing: AddressSanitizer DEADLYSIGNAL SEGV on address 0x40 (near-null, READ) in openssl's ssl3_read_bytes <- tls_get_message_header <- state_machine <- handle_ssl_handshake (src/lib/netio.c:548) <- io_handler (netio.c:588) <- io_worker (netio.c:633), thread T2 (an io_worker thread). With ctrl-1 already killed (the round's own kill target) and ctrl-3 now also dead, no 2-of-3 quorum remained for ctrl-2 (freshly elected PRIMARY, term=11) to commit writes -- explaining the observed 0/3-acks symptom as a downstream consequence, not the primary defect.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "/kiro-validate-impl raft-drive-async: 10-round SEQUENTIAL HA soak (the fully-supported…\n\n/kiro-validate-impl raft-drive-async: 10-round SEQUENTIAL HA soak (the fully-supported single-failover lifecycle, hard-fail-on-any-loss) failed at round 9/10 -- 'FAIL: round 9: post-failover submit did not get 3 acks' (0/3 acked across 5 retry attempts). Root traced to ctrl-3 (a standby) crashing: AddressSanitizer DEADLYSIGNAL SEGV on address 0x40 (near-null, READ) in openssl's ssl3_read_bytes <- tls_get_message_header <- state_machine <- handle_ssl_handshake (src/lib/netio.c:548) <- io_handler (netio.c:588) <- io_worker (netio.c:633), thread T2 (an io_worker thread). With ctrl-1 already killed (the round's own kill target) and ctrl-3 now also dead, no 2-of-3 quorum remained for ctrl-2 (freshly elected PRIMARY, term=11) to commit writes -- explaining the observed 0/3-acks symptom as a downstream consequence, not the primary defect.", "file": "src/lib/netio.c (handle_ssl_handshake / io_handler / io_worker -- crash site, shared iomgr epoll dispatch); src/controller/ha.c (ha_connect_prep_peer/ha_connect_finish_peer/reconnect_lane_fn, raft-drive-async Task 2.1/2.2 reconnect-offload -- suspected trigger); src/controller/cluster.c (handle_connection / endpoint_accept -- inbound accept path, alternate suspect)", "root_cause": "CONFIRMED (deterministic regression test + live soak, not speculative). The earlier 'unsynchronized concurrent SSL_do_handshake from two io_worker threads' hypothesis is WRONG and is retracted: iomgr_start() creates exactly ONE io_worker thread per iomgr_t (src/lib/netio.c:iomgr_start), and a single controller process has exactly one such iomgr/thread for all peer connections -- there is no second io_worker thread to race against.\n\nREAL mechanism: io_worker's epoll dispatch loop processes every ready fd from ONE epoll_wait() batch in a plain for-loop calling io_handler() per event (src/lib/netio.c, old io_worker ~line 612-640). A handler for an EARLIER event in that SAME batch can free a DIFFERENT endpoint that also has a ready event LATER in the SAME batch. In production this is exactly src/controller/ha.c:8257-8331 on_standby_connected(), the handshake-complete callback for a freshly-reconnected outbound standby channel (raft-drive-async Task 2.1/2.2's reconnect-offload path): when it detects a stale duplicate outbound channel to the same peer (ha.c:8286-8303, 'Different ep, same peer -> stale duplicate to evict'), it calls iomgr_unmonitor() (ha.c:8320-8321) then endpoint_unref() (ha.c:8327) on the STALE endpoint. iomgr_unmonitor()'s epoll_ctl(EPOLL_CTL_DEL) only stops a FUTURE epoll_wait() from returning events for that fd -- it does NOT invalidate an event for that SAME stale endpoint that is already sitting in the CURRENT batch's stack-local events[] array (very plausible during a reconnect burst, where several peer connections' fds become ready close together right after a new primary election). endpoint_unref() reaching refcnt 0 immediately SSL_free()s ep->ssl and free()s the endpoint_t. The loop's next iteration then calls io_handler() for that now-freed endpoint -- a heap-use-after-free that reads freed memory as 'ep', and (if it still looks like a plausible pointer chain) hands a freed/reused ep->ssl to SSL_do_handshake(), producing exactly the observed ASAN DEADLYSIGNAL SEGV deep inside OpenSSL's ssl3_read_bytes rather than an immediate top-of-function null-ssl crash.\n\nThis is a genuine gap in the bug-1282/1286/1298 hardening family: INVOKE_MSG_HANDLER (netio.h) already refs/unrefs an endpoint around its OWN msg_handler call (bug-1298 note in that macro's comment) so a handler cannot free the SAME ep it is currently dispatched on -- but that bracket never covered a DIFFERENT, already-batched endpoint torn down as a side effect (on_standby_connected's cross-endpoint eviction). io_worker's own epoll dispatch loop held no reference of any kind on the endpoints in its batch. Reproduced deterministically (no live soak needed to hit it) via a new unit test that drives io_worker's real batch-dispatch loop with a synthetic 2-event epoll batch where event 0's handler endpoint_unref()s event 1's endpoint -- confirmed RED pre-fix: ASAN heap-use-after-free at src/lib/netio.c:587 (io_handler reading ep->ssl_handshake_done), freed by src/lib/netio.c:843 (endpoint_unref) called from the injected handler, exactly mirroring the production on_standby_connected -> stale endpoint eviction -> later-batched io_handler() read chain.", "fix": "FIXED (2026-07-18). src/lib/netio.c: extracted io_worker's per-batch dispatch into a new static io_worker_dispatch_batch(struct epoll_event*, int) (~line 639-654), called from io_worker's while(1) loop in place of the old inline for-loop. It now takes an endpoint_ref() on every endpoint in the batch BEFORE dispatching any of them, dispatches all events via io_handler() as before, then endpoint_unref()s every endpoint only AFTER the whole batch has been processed -- mirroring the SAME endpoint_ref/endpoint_unref bracket idiom this codebase already uses for the vtask dispatch path (bug-1298) and INVOKE_MSG_HANDLER (netio.h), applied here to io_worker's own epoll dispatch loop, the one dispatcher that didn't yet have it. With this bracket, on_standby_connected's endpoint_unref() on a stale duplicate can only drop that endpoint's refcount from 2 (owner + io_worker's batch ref) to 1 during the batch -- never to 0 -- so the free is deferred until io_worker's own trailing unref runs after the ENTIRE batch (including that endpoint's own later event) has been safely dispatched. ref/unref are the existing lock-free _Atomic-int operations (netio.c endpoint_ref/endpoint_unref) with no I/O and no lock acquisition, so this adds zero blocking risk to the io_worker hot path (satisfies the async-io steering rule against blocking in vtask/event-loop handlers). Also added a small always-compiled test seam, io_worker_dispatch_batch_for_test() (netio.c + netio.h) -- not XCE_TESTING-gated because netio.c compiles once into libxce.a with no per-target defines (unlike controller/ha.c, which the controller test binary compiles directly with -DXCE_TESTING), so it follows this file's existing convention of unconditionally exporting its internals (endpoint_ref, endpoint_unref, endpoint_mark_tearing_down, etc. are all plain public functions already).\n\nRegression test: src/lib/testsuite/test_netio_batch_dispatch_uaf.c (test_batch_dispatch_survives_cross_endpoint_eviction), registered in src/lib/meson.build and src/lib/testsuite/test.c. Drives io_worker_dispatch_batch_for_test() with a synthetic 2-event batch: event 0 -> an endpoint whose conn_handler calls endpoint_unref() on event 1's endpoint (simulating on_standby_connected's stale-duplicate eviction); event 1 -> an endpoint with refcnt==1 (its only ref is the one event 0's handler drops) whose conn_handler just records that it ran. TDD verified: RED confirmed pre-fix (ASAN heap-use-after-free at netio.c:587 in io_handler, object freed at netio.c:843 endpoint_unref from the injected handler -- captured verbatim before the fix was restored); GREEN confirmed post-fix (both handlers run safely, exit 0, re-confirmed again standalone after both live soaks completed). Full regression: libxce lib suite 294/0/0 (xcelib_test, ASAN, ~/xce.build), controller_unit_test 972/0/0 with zero AddressSanitizer/heap-use-after-free/heap-buffer-overflow/double-free/SEGV hits in the full test log. Live verification beyond the unit test: two independent fresh sequential 10-round /xce-test-ha HA soaks on real ext4 (/home/claude/tmp/xce-bug1447-verify-1: 10/10 rounds, 33 jobs, zero job loss, all 3 nodes led; /home/claude/tmp/xce-bug1447-verify-2: 10/10 rounds, 36 jobs, zero job loss, all 3 nodes led including a forced-election leadership-coverage check on ctrl-3) -- both fully clean, zero crashes, and an explicit grep of every node's logs in both runs for the bug-1447 signature (AddressSanitizer/DEADLYSIGNAL/SEGV/ssl3_read_bytes/handle_ssl_handshake) found zero matches.", @@ -16341,26 +4768,29 @@ "round-9", "same-batch-uaf", "sequential-soak", - "ssl3_read_bytes", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ssl3_read_bytes" ], "related_bugs": [ "bug-1282", "bug-1286", "bug-1298" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 32 + "fix_commits": [], + "record": ".wolf/bugs/bug-1447.md", + "gitea_issue": 32, + "gitea_synced_status": "fixed" }, { "id": "bug-1448", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-07-18T01:54:00Z", - "error_message": "/kiro-validate-impl ha-heartbeat-committed-propagation: the xce-build skill's own documented pattern (\"dispatch via a plain Agent-tool call, run_in_background: false, wait for it\") produced an INCOMPLETE build -- ninja log ends with 'ninja: build stopped: interrupted by user.' at 85/192 targets, roughly 14 minutes after launch with zero nix/meson/ninja processes still running and no error in the log. The synchronous Agent call returned to the caller almost immediately (~38s) with text claiming 'The build is running in the background... I'll wait for it to complete' -- i.e. the subagent itself spawned an internal Bash run_in_background job and then ended its own turn, rather than blocking until the command finished.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "/kiro-validate-impl ha-heartbeat-committed-propagation: the xce-build skill's own documented…\n\n/kiro-validate-impl ha-heartbeat-committed-propagation: the xce-build skill's own documented pattern (\"dispatch via a plain Agent-tool call, run_in_background: false, wait for it\") produced an INCOMPLETE build -- ninja log ends with 'ninja: build stopped: interrupted by user.' at 85/192 targets, roughly 14 minutes after launch with zero nix/meson/ninja processes still running and no error in the log. The synchronous Agent call returned to the caller almost immediately (~38s) with text claiming 'The build is running in the background... I'll wait for it to complete' -- i.e. the subagent itself spawned an internal Bash run_in_background job and then ended its own turn, rather than blocking until the command finished.", "file": "src/tools/xce-build.sh (skill invocation pattern, not the script itself); .claude/skills/xce-build/SKILL.md's 'Always dispatch the build via the Agent tool' instruction", "root_cause": "Same defect family as the already-documented xce-test-ha/run-ha-soak.sh 'context: fork' SIGHUP-teardown bug (fixed 2026-07-08), but manifesting through a DIFFERENT mechanism: xce-build's skill doc explicitly avoids 'context: fork' and claims a plain Agent-tool dispatch is safe -- but that safety claim implicitly assumes the subagent's own Bash tool call blocks synchronously for the whole build. In practice the subagent chose to run the long nix/meson/ninja command via its OWN internal run_in_background Bash call and then finished its turn, un-detached (no setsid/nohup/disown) -- so when that subagent's session/turn ended, the still-attached background job's process group got interrupted (ninja saw SIGINT/SIGHUP-equivalent, logged literally as 'interrupted by user'). A second SendMessage asking the same agent to 'wait for it now' resumed a session with nothing left alive to wait for.", "fix": "Relaunched the build via a fresh Agent dispatch whose ONLY instruction was to run `setsid nohup nix develop ./src --command bash src/tools/xce-build.sh controller asan LOG 2>&1 & disown` and return immediately (fire-and-forget), then polled LOG/binary-mtimes independently from the main session via the Monitor tool. That build completed cleanly in ~10 minutes (85/85 ninja targets, clean link, meson test suite started normally) with no interruption. General fix for any long shell command dispatched through the Agent tool that must outlive the dispatching subagent's own turn: always explicit setsid+nohup+disown, never rely on the subagent's internal run_in_background alone or on run_in_background:false at the Agent-tool-call level to guarantee synchronous blocking all the way through.", @@ -16374,22 +4804,25 @@ "subagent-teardown", "kiro-validate-impl", "ha-heartbeat-committed-propagation", - "process-detach", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "process-detach" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 33 + "fix_commits": [], + "record": ".wolf/bugs/bug-1448.md", + "gitea_issue": 33, + "gitea_synced_status": "fixed" }, { "id": "bug-1449", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-07-18T09:56:00.000Z", - "error_message": "[xce-test-ha] FAIL: round 1: no new primary elected after kill (no serving primary at term>2) -- hit once during ha-elect-ctx-lifecycle kiro-validate-impl soak, did NOT reproduce on immediate retry (10/10 clean)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "[xce-test-ha] FAIL: round 1: no new primary elected after kill (no serving primary at term>2)…\n\n[xce-test-ha] FAIL: round 1: no new primary elected after kill (no serving primary at term>2) -- hit once during ha-elect-ctx-lifecycle kiro-validate-impl soak, did NOT reproduce on immediate retry (10/10 clean)", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh (sequential mode round-1 kill timing) / src/controller/ha.c ha_on_primary_ctrl_fd_rdhup (ha-config-bootstrap Req 2.8 voter-only guard, working as designed)", "root_cause": "Test-harness timing race, not a code defect. Root-caused from the controller logs (not assumed): cold-start election was unusually slow this run -- ctrl-1 only reached CTRL_PRIMARY (term=2) ~10.1s after process start (raft_cb_become_leader). Immediately on becoming primary it began its config-push handshake to both standbys (on_standby_connected: 'outbound standby connection to peer[N] established, ctrlhello sent', same ~10.1s timestamp). The driver's round-1 kill fired only ~1.7s later (its budget is 'submit 3 jobs + verify + walcompare' after primary is detected, no floor on how fast that completes) -- before either standby had received and applied the pushed config. Both standbys were still at config tag=0 when ctrl-1 died, so ha_on_primary_ctrl_fd_rdhup correctly refused to start PreVote/RDHUP re-election per ha-config-bootstrap's own Req 2.8 guard ('no local config (tag=0, n_peers=3) -- voter only, not starting PreVote/RDHUP re-election; awaiting configured primary'), leaving the cluster with no primary and no candidate. This is the guard working exactly as designed against an unconfigured standby self-electing -- the actual gap is that run-ha-soak.sh's sequential mode has no wait for initial 3-of-3 config convergence before round 1's first kill (unlike its wait for FULL reconvergence between later rounds). Sequential mode's round-1 timing has apparently never lined up this unluckily in any of the many other Group-1 kiro-validate-impl soaks run earlier the same session (ha-wal-replication, ha-heartbeat-committed-propagation, ha-prevote, raft-sim-conformance, libxce-failover, ha-entity-snapshots, ha-commit-durability, ha-term-safety, controller-status, job-array, job-array-single-wal-record) -- all of which use the same driver script unmodified and ran clean.", "fix": "FIXED (.claude/skills/xce-test-ha/run-ha-soak.sh). Added a round-1-only config-convergence gate that mirrors wait_for_full_convergence's exact polling style (same alive-count guard via pgrep -c -f \"$(run_pattern)\", same STABLE_TICKS=2-consecutive-converged-probes, same sleep 2 backoff, same ~90s/30-iteration bound, same log()/fail() convention) but gates on a NEW signal, cluster_config_converged (all 3 nodes report a non-zero configVersion via a new node_config_tag() single-node ctrlstatus client probe, mirroring node_probe's technique), instead of the existing cluster_committed_converged (committed-prefix catch-up) -- the two convergence signals are independent, and only the config-tag one covers the ha-config-bootstrap Req 2.8 voter-only guard. Added functions node_config_tag() / cluster_config_converged() / wait_for_initial_config_convergence() right after wait_for_full_convergence (~line 556). Call site inserted right after the chaotic-mode branch's closing `fi` and before the 'Steps 7-10 looped' round loop (~line 1897), so it applies only to the sequential/default path (chaotic mode intentionally tolerates this race by design -- 'no new primary' is a COUNTED outcome there, never a hard fail, so it was correctly left untouched). Diff: +77 lines, 0 lines removed -- no C source touched. VERIFIED: 3 back-to-back --sequential 6-round soaks on a genuine ext4 path (/home/claude/tmp, confirmed via `mount`/`df -T`, not overlayfs -- bug-924), all 3 PASS (6 rounds/24 jobs/zero loss/all 3 nodes led, identical verdict each run), and the new 'waiting for 3-of-3 initial config convergence before round 1' / 'confirmed' log lines appeared correctly ahead of every round 1. Cross-checked the actual cold-start timing from controller logs across all 3 runs to size the margin: election (raft_cb_become_leader -> CTRL_PRIMARY) took 10.3-11.7s after process start each run, and the config-push (handle_repl_config_push on both standbys) landed only 40-46ms after election every time -- i.e. the actual condition resolves in well under 100ms, while the new gate's minimum dwell is ~4s (2 ticks x 2s) and its ceiling is ~90s, giving roughly 2 orders of magnitude of margin over the observed timing, not just enough for these 3 runs' luck.", @@ -16403,23 +4836,25 @@ "timing-race", "config-push", "round-1", - "sequential-mode", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "sequential-mode" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 34 + "fix_commits": [], + "record": ".wolf/bugs/bug-1449.md", + "gitea_issue": 34, + "gitea_synced_status": "fixed" }, { "id": "bug-1450", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-27", "status_note": "7aac018c: demotion-convergence offload ON by default; inline fallback when slow lane not started", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-18", - "error_message": "xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_demotion_convergence_fn scheduled directly as a delaymgr task body with no vtask/slow-lane hop: it truncates the WAL (persist_truncate_wal_above, file I/O under p->lock) then calls ha_self_restart (tlog_exit synchronous log flush + execv) -- all inline on the delaymgr thread, the SAME single thread that drives heartbeats and election timeouts cluster-wide for the whole cluster.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_demotion_convergence_fn…\n\nxce-audit-invariants controller pass (dim: async I/O thread-role) found ha_demotion_convergence_fn scheduled directly as a delaymgr task body with no vtask/slow-lane hop: it truncates the WAL (persist_truncate_wal_above, file I/O under p->lock) then calls ha_self_restart (tlog_exit synchronous log flush + execv) -- all inline on the delaymgr thread, the SAME single thread that drives heartbeats and election timeouts cluster-wide for the whole cluster.", "file": "src/controller/ha_convergence.c, src/controller/ha.h", "root_cause": "ha_demotion_convergence_fn (ha_convergence.c) ran its full body -- truncate + self-restart -- synchronously on whatever thread invoked it (the delaymgr worker in production). A prior fix attempt (2026-07-10, see bug-1429) tried moving this to a VTASK_LIVE_BLOCK vtask via c->ha.pool and deadlocked controller_unit_test, because c->ha.pool is never initialized/started by this file's lightweight unit-test fixtures (dc_setup), so a vtask enqueued VTASK_LIVE_BLOCK blocked forever waiting for a worker that never existed; that attempt was reverted.", "fix": "Split ha_demotion_convergence_fn into a dispatcher (fast admin work: single-flight CAS on new ha_state_t field demotion_convergence_inflight, then slowlane_submit) and a new demotion_lane_fn slow-lane worker body running the actual truncate+self-restart (extracted into ha_demotion_convergence_do_run) -- the SAME established slow-lane pattern already used for snapshot send / config-push / peer connect in ha.c (ha_snapshot_send_dispatch, ha_config_push_offloadable, ha_connect_to_peer), not the vtask pool that deadlocked before. New feature flag g_drive_async_demotion_offload (ha_set_demotion_offload_for_test / ha_demotion_offload_enabled, ha.h/ha_convergence.c) gates the offload. IMPORTANT: defaulting this flag to ON (1) was tried first and reproduced a NEW real hang -- raft_cb_become_follower (which schedules this task) is exercised by a very broad set of pre-existing controller_unit_test fixtures across many files that never call slowlane_init, so the offloaded work was silently queued on an uninitialized c->slowlane (queued, never run -- no live workers) and downstream code waiting on the now-never-happening completion hung. Reverted the default to OFF (0), matching the ORIGINAL rollout discipline of the three reference offloads (all shipped OFF-by-default, flipped ON only in a later dedicated compatibility-audit pass -- raft-drive-async task 3.4). The mechanism is correct and covered by dedicated tests (test_demotion_offload_task_returns_promptly / test_demotion_inline_task_blocks_RED in test_demotion_convergence.c) that explicitly flip the flag ON against a fixture with a real slowlane_init'd lane. Flipping the cluster-wide default to ON is a follow-up compatibility audit across the full test suite, explicitly out of scope for this fix. Verified: standalone controller_unit_test binary run directly (bypassing meson) reports 976 Tests 0 Failures 0 Ignored, no hang/timeout.", @@ -16432,26 +4867,30 @@ "demotion", "raft-drive-async-pattern", "xce-audit-invariants", - "controller-tests", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "controller-tests" ], "related_bugs": [ "bug-1429", "bug-1451" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 25 + "fix_commits": [ + "7aac018c" + ], + "record": ".wolf/bugs/bug-1450.md", + "gitea_issue": 25, + "gitea_synced_status": "fixed" }, { "id": "bug-1451", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-27", "status_note": "7aac018c: catch-up offload ON by default; inline fallback when slow lane not started", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-18", - "error_message": "xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_catchup_continuation_fn scheduled directly as a delaymgr task body with no vtask/slow-lane hop: catchup_stream_range -> ha_stream_wal_to_standby does blocking WAL file reads + socket writes to a standby directly on the delaymgr thread. Flagged as the MORE URGENT of the two violations -- catch-up can stream a variable, potentially large amount of WAL data, so it can block delaymgr (and therefore heartbeats to every OTHER peer) for an unbounded, unpredictable time, a real spurious-election risk.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_catchup_continuation_fn…\n\nxce-audit-invariants controller pass (dim: async I/O thread-role) found ha_catchup_continuation_fn scheduled directly as a delaymgr task body with no vtask/slow-lane hop: catchup_stream_range -> ha_stream_wal_to_standby does blocking WAL file reads + socket writes to a standby directly on the delaymgr thread. Flagged as the MORE URGENT of the two violations -- catch-up can stream a variable, potentially large amount of WAL data, so it can block delaymgr (and therefore heartbeats to every OTHER peer) for an unbounded, unpredictable time, a real spurious-election risk.", "file": "src/controller/config_repl.c, src/controller/ha.h", "root_cause": "ha_catchup_continuation_fn (config_repl.c) ran its full body -- re-derive target, then catchup_stream_range's blocking WAL reads/socket writes -- synchronously on whatever thread invoked it (the delaymgr worker in production). Source comment at config_repl.c confirmed a prior VTASK_LIVE_BLOCK vtask-pool attempt here (2026-07-10) also deadlocked controller_unit_test and was reverted (bug-1429), tracked as a known open violation.", "fix": "Split ha_catchup_continuation_fn into a dispatcher (fast admin work: slowlane_submit keyed by peer_idx+1) and a new catchup_lane_fn slow-lane worker body running the original streaming logic unchanged -- the SAME established slow-lane pattern used for snapshot/config-push/reconnect in ha.c, not the vtask pool. c->ha.catchup_inflight[peer_idx] (set by ha_walack_on_ack before scheduling) remains the authoritative per-standby single-flight guard; the lane's own per-key dedup is a redundant backstop, mirroring ha_connect_to_peer's reconnect offload. New feature flag g_drive_async_catchup_offload (ha_set_catchup_offload_for_test / ha_catchup_offload_enabled, ha.h/config_repl.c) gates the offload. Defaulting this flag to ON (1) was tried first and reproduced a real hang: test_convergence_verification.c's pre-existing ACK-driven catch-up fixtures (which directly exercise this exact code path via ha_walack_continuation_for_test) never call slowlane_init, so offloaded work was silently queued and never ran, hanging controller_unit_test at ~90s+ (confirmed reproducible both via meson and a direct standalone binary run with SIGQUIT timeout). Reverted the default to OFF (0), matching the same rollout discipline as bug-1450's demotion-convergence fix and the three original raft-drive-async reference offloads. Mechanism covered by dedicated tests (test_catchup_offload_probe_runs_promptly / test_catchup_inline_probe_blocks_RED in test_config_repl.c) using a REAL, started, single-worker delaymgr (delaymgr_init+delaymgr_start, not just an un-started fixture) plus a probe task queued behind the continuation to directly prove the delaymgr thread is/isn't blocked. Flipping the cluster-wide default to ON is a follow-up compatibility audit, out of scope for this fix. Verified: standalone controller_unit_test binary run directly reports 976 Tests 0 Failures 0 Ignored, no hang/timeout.", @@ -16465,25 +4904,30 @@ "raft-drive-async-pattern", "xce-audit-invariants", "controller-tests", - "spurious-election-risk", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "spurious-election-risk" ], "related_bugs": [ "bug-1429", "bug-1450" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 35 + "fix_commits": [ + "7aac018c" + ], + "record": ".wolf/bugs/bug-1451.md", + "gitea_issue": 35, + "gitea_synced_status": "fixed" }, { "id": "bug-1452", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-07-18T00:00:00Z", - "error_message": "AddressSanitizer: SEGV on unknown address 0x000000000038 in commit_advance_locked (controller/commit.c:208), called from commit_on_durable_advance (commit.c:341) <- fsync_worker (persist.c:399), a background ASAN thread started by persist_start (persist.c:723) inside test_persist_truncate.c's make_fresh_persist / test_truncate_above_resolves_pending_continuations_as_failed. Reproduced by temporarily flipping BOTH g_drive_async_demotion_offload and g_drive_async_catchup_offload (bug-1450/1451) from their default 0 to 1 and running the standalone controller_unit_test binary directly (976-test suite) -- crashed immediately, before completing the suite. Confirmed absent with both flags at their committed default of 0: the same binary/build (just reverted) completed cleanly, 976 Tests 0 Failures 0 Ignored, twice.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: SEGV on unknown address 0x000000000038 in commit_advance_locked…\n\nAddressSanitizer: SEGV on unknown address 0x000000000038 in commit_advance_locked (controller/commit.c:208), called from commit_on_durable_advance (commit.c:341) <- fsync_worker (persist.c:399), a background ASAN thread started by persist_start (persist.c:723) inside test_persist_truncate.c's make_fresh_persist / test_truncate_above_resolves_pending_continuations_as_failed. Reproduced by temporarily flipping BOTH g_drive_async_demotion_offload and g_drive_async_catchup_offload (bug-1450/1451) from their default 0 to 1 and running the standalone controller_unit_test binary directly (976-test suite) -- crashed immediately, before completing the suite. Confirmed absent with both flags at their committed default of 0: the same binary/build (just reverted) completed cleanly, 976 Tests 0 Failures 0 Ignored, twice.", "file": "src/controller/commit.c (crash site, commit_advance_locked); src/controller/persist.c (fsync_worker background thread, apply_committed_entry/restore_job/makeJob chain also appears in a related non-crashing stack from the same investigation); src/controller/ha_convergence.c + config_repl.c (g_drive_async_demotion_offload / g_drive_async_catchup_offload -- the flags whose flip triggers this)", "root_cause": "ROOT-CAUSED 2026-07-19: the g_drive_async_demotion_offload/g_drive_async_catchup_offload flags were a RED HERRING -- this exact SEGV signature (NULL deref in the fsync_worker -> commit_on_durable_advance -> commit_advance_locked chain) was already independently observed on 2026-07-07, long before those flags existed (see memory: 'Pre-existing ASAN SEGV in test_persist_truncate Unrelated to Bug-1417 Work'). The flags only perturbed thread-scheduling timing enough to make a PRE-EXISTING, always-latent test-teardown race fire more reliably; they do not participate in the crash's causal chain at all. Actual root cause: in src/controller/testsuite/test_persist_truncate.c, the three continuation-related tests (test_truncate_above_resolves_pending_continuations_as_failed, test_truncate_noop_does_not_resolve_pending_continuations, test_truncate_concurrent_burst_resolves_above_cutoff_exactly_once) called `commit_engine_fini(&ce)` BEFORE `teardown_persist(p)` (which calls `persist_stop(p)`) -- the exact opposite of the contract documented in commit_engine_fini's own comment (commit.c:464-468: 'Callers are expected to have stopped the flusher (persist_stop) first'). Because persist_stop had not yet run, the fsync_worker background thread was still alive and could be mid-flight invoking the durable-advance callback (fsync_worker, persist.c:398-399, calls p->on_durable_advance(p->durable_cb_ctx) with NO lock held) concurrently with commit_engine_fini destroying ce->lock/ce->cond and, as its final step, setting ce->p = NULL. This produces a genuine TOCTOU data race with no synchronization between the two threads: commit_on_durable_advance's own guard (`if (ce == NULL || ce->p == NULL) return;`) can read ce->p as still non-NULL (racing ahead of or concurrent with commit_engine_fini's teardown), pass the guard, and then commit_advance_locked's SEPARATE, later re-read of ce->p (commit.c:208, `atomic_load(&ce->p->committed_lsn)`) observes the NOW-NULLED value written by the concurrently-running commit_engine_fini on the main thread -- producing a NULL-pointer dereference at exactly offset 0x38 (offsetof(persist_t, committed_lsn)), matching the crash address bit-for-bit. persist_set_durable_cb's unhook (called from commit_engine_fini) is 'defensive and cheap' per its own comment but is NOT sufficient on its own without persist_stop's pthread_join guaranteeing the flusher thread cannot be in-flight at all -- confirmed by the fact that persist_set_durable_cb existed and ran in this path the whole time and the crash still occurred.", "fix": "FIXED 2026-07-19 (commit pending): added an explicit `persist_stop(p);` call immediately before `commit_engine_fini(&ce);` in all three affected test functions in test_persist_truncate.c (previously commit_engine_fini ran first, then teardown_persist -> persist_stop ran second -- backwards from the documented contract). persist_stop joins the fsync_worker thread, so by the time commit_engine_fini tears down ce->lock/ce->cond/ce->p, no thread can be mid-callback referencing them. teardown_persist's own internal persist_stop(p) call becomes a safe idempotent no-op on the second invocation (fsync_thread_running is already 0 and wal_fd is already -1, so both of its internal branches skip). No production code changed -- this was purely a test-fixture ordering bug, not a bug in commit.c/persist.c itself. Verified: full ASAN controller_unit_test suite (976 tests) run 6 times total (1 via meson + 5 standalone direct binary runs per project convention of not trusting meson's aggregate alone) -- 976 Tests 0 Failures 0 Ignored on every run, zero ASAN reports (SEGV/UAF/etc.), all 8 test_persist_truncate.c tests PASS every run. NOTE for a future session: the identical commit_engine_fini-before-persist_stop ordering also exists in test_demotion_convergence.c (line 233 commit_engine_fini(&g_dc.commit) before line 235 persist_stop(&g_dc.persist), against a real persist_start'd fixture) -- same latent race, not yet observed to crash there (likely a narrower window against a static/global fixture rather than a per-test stack fixture), out of scope for this fix but worth a follow-up sweep of all commit_engine_fini call sites.", @@ -16503,28 +4947,31 @@ "ASAN", "SEGV", "FIXED", - "root-caused", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "root-caused" ], "related_bugs": [ "bug-1429", "bug-1450", "bug-1451" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 36 + "fix_commits": [], + "record": ".wolf/bugs/bug-1452.md", + "gitea_issue": 36, + "gitea_synced_status": "fixed" }, { "id": "bug-1453", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-20T00:00:00.000Z", - "error_message": "Unnecessary CA certificate plumbing across C daemons and Go CLI \u2014 ca_cert_path fields, --ca-cert flags, and CACert config are dead or misleading; TLS verification is SSL_VERIFY_NONE everywhere", - "file": "multiple \u2014 see root_cause", - "root_cause": "XCE design uses server-side TLS only (encryption without peer certificate verification). The CA cert plumbing was intended for future P3 mTLS hardening but was never implemented: (1) C-side: zero SSL_CTX_load_verify_locations calls in entire codebase \u2014 ca_cert_path in xce/cluster.h:35 is set by controller/worker/scheduler but never loaded into any SSL_CTX; do_connect() production path in cluster.c:73 is a NULL-returning stub; (2) Go-side: HTTPS is disabled by default; xcecli uses trust-on-first-use pinning of the server leaf cert (not a CA) stored via a misleadingly-named CACert field; falls back to InsecureSkipVerify:true. Design docs (persistence-and-ha-design.md:1700) explicitly tag peer verification as optional P3 hardening, confirming current encryption-only posture is intentional.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Unnecessary CA certificate plumbing across C daemons and Go CLI — ca_cert_path fields, --ca-cert…\n\nUnnecessary CA certificate plumbing across C daemons and Go CLI — ca_cert_path fields, --ca-cert flags, and CACert config are dead or misleading; TLS verification is SSL_VERIFY_NONE everywhere", + "file": "multiple — see root_cause", + "root_cause": "XCE design uses server-side TLS only (encryption without peer certificate verification). The CA cert plumbing was intended for future P3 mTLS hardening but was never implemented: (1) C-side: zero SSL_CTX_load_verify_locations calls in entire codebase — ca_cert_path in xce/cluster.h:35 is set by controller/worker/scheduler but never loaded into any SSL_CTX; do_connect() production path in cluster.c:73 is a NULL-returning stub; (2) Go-side: HTTPS is disabled by default; xcecli uses trust-on-first-use pinning of the server leaf cert (not a CA) stored via a misleadingly-named CACert field; falls back to InsecureSkipVerify:true. Design docs (persistence-and-ha-design.md:1700) explicitly tag peer verification as optional P3 hardening, confirming current encryption-only posture is intentional.", "fix": "DONE. C-side: removed ca_cert_path field from xce_cluster_config_t (src/include/xce/cluster.h); do_connect()/xce_connect_fn_t dropped the ca_cert param (src/lib/cluster.c); removed ca_cert_path non-empty check from xce_cluster_config_validate(); removed --ca-cert flag from worker (src/worker/src/main.c) and scheduler (src/scheduler/src/main.c); removed ca_cert_path population from src/controller/main.c:279-281; updated src/lib/testsuite/test_cluster.c mock_connect signature + fixture. Go-side: deleted xcecli/cmd/auth/cert/{cert.go,parser.go} (self-signed cert TOFU-pinning helper); removed CACert field from config.go APIServerInfo and models/logon.go LogonResponse; removed SetRootCAs/loadRootCAs from common/request/request.go and certPath param from Get/Post/DeleteRequestFromAPIServer; updated all ~18 xcecli cmd/* call sites; updated cmd/auth/logon.go to drop the https cert-fetch block; removed TestSetRootCAs_*/TestBuildTLSLogic from request_test.go. Verified: full C build (all components, ASAN) 353/353 targets, meson test suite 9/9 OK; xcecli go build/vet/test all PASS; grep -rn 'CACert|SetRootCAs|loadRootCAs|ca_cert' src/ clean (only historical design-doc mentions remain in src/docs/*.md, left as-is).", "tags": [ "dead-code", @@ -16535,72 +4982,85 @@ "worker", "scheduler", "xcecli", - "apiserver", - "fixed" + "apiserver" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 26 + "fix_commits": [], + "record": ".wolf/bugs/bug-1453.md", + "gitea_issue": 26, + "gitea_synced_status": "fixed" }, { "id": "bug-1454", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", "status_note": "94f50ce6", + "component": "scheduler", + "severity": "leak", "timestamp": "2026-07-20T00:00:00.000Z", - "error_message": "Scheduler --peers correctly wired for initial controller discovery; minor issues: nmasters=1 (no EndpointRotator) and leaked ctx on HA path", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Scheduler --peers correctly wired for initial controller discovery; minor issues: nmasters=1 (no…\n\nScheduler --peers correctly wired for initial controller discovery; minor issues: nmasters=1 (no EndpointRotator) and leaked ctx on HA path", "file": "src/scheduler/src/scheduler.c", - "root_cause": "VERIFICATION: --peers IS correctly parsed via xce_parse_peers() (main.c:129), stored via sched_set_cluster_cfg() (main.c:156), and iterated in sched_init() HA loop (scheduler.c:462-493) BEFORE sched_init \u2014 init ordering is correct. Narrowed findings: (1) nmasters=1 per attempt (line 468) \u2014 manual candidate loop instead of filling config->masters[] from all candidates so libxce EndpointRotator never engages. Functional but suboptimal. (2) Leaked apictx on HA path \u2014 xcecli_ctx_init at line 454 allocates sched->apictx, then HA loop at line 472 re-inits without xcecli_ctx_fini on the first allocation. On standby-failure branches (lines 480, 492) cleanup is correct. OUT OF SCOPE: runtime reconnect/rejoin after controller death belongs to .kiro/specs/component-failover-adoption/ (requirements + design approved, tasks 3.1/3.2 not yet approved).", + "root_cause": "VERIFICATION: --peers IS correctly parsed via xce_parse_peers() (main.c:129), stored via sched_set_cluster_cfg() (main.c:156), and iterated in sched_init() HA loop (scheduler.c:462-493) BEFORE sched_init — init ordering is correct. Narrowed findings: (1) nmasters=1 per attempt (line 468) — manual candidate loop instead of filling config->masters[] from all candidates so libxce EndpointRotator never engages. Functional but suboptimal. (2) Leaked apictx on HA path — xcecli_ctx_init at line 454 allocates sched->apictx, then HA loop at line 472 re-inits without xcecli_ctx_fini on the first allocation. On standby-failure branches (lines 480, 492) cleanup is correct. OUT OF SCOPE: runtime reconnect/rejoin after controller death belongs to .kiro/specs/component-failover-adoption/ (requirements + design approved, tasks 3.1/3.2 not yet approved).", "fix": "DONE (leak fix only). Fixed the confirmed defect: sched_init() called xcecli_ctx_init(&sched->apictx, config) unconditionally, then on the HA path re-called xcecli_ctx_init() inside the candidate loop without freeing the first ctx -> leaked one xcecli_ctx_t per sched_init() call. Fix: xcecli_ctx_fini(sched->apictx) added immediately before the HA loop starts (src/scheduler/src/scheduler.c), since the loop's first iteration always replaces it. Verified via full ASAN build: all 5 scheduler meson test suites + libxce suite pass, no regression. REJECTED the nmasters>1+EndpointRotator suggestion: memory obs 11825 (this session) confirms scheduler/worker deliberately use the candidates[]-loop path instead of filling config->masters[], so changing that would contradict the established design, not fix a bug. Runtime reconnect/rejoin gap remains out of scope, deferred to component-failover-adoption spec (unchanged from original triage).", "tags": [ "scheduler", "peers", "verified-working", "minor-ctx-leak", - "nmasters", - "fixed" + "nmasters" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 37 + "fix_commits": [ + "94f50ce6" + ], + "record": ".wolf/bugs/bug-1454.md", + "gitea_issue": 37, + "gitea_synced_status": "fixed" }, { "id": "bug-1455", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-07-20T00:00:00.000Z", - "error_message": "Scheduler --peers silently ignored: sched_init() memset(sched,0,...) wipes sched->cluster_cfg that main.c populated via sched_set_cluster_cfg() just before calling sched_init(), so the HA candidate loop's candidates_count check always reads 0 and falls back to the hardcoded localhost:9990 single-host path regardless of --peers.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Scheduler --peers silently ignored: sched_init() memset(sched,0,...) wipes sched->cluster_cfg that…\n\nScheduler --peers silently ignored: sched_init() memset(sched,0,...) wipes sched->cluster_cfg that main.c populated via sched_set_cluster_cfg() just before calling sched_init(), so the HA candidate loop's candidates_count check always reads 0 and falls back to the hardcoded localhost:9990 single-host path regardless of --peers.", "file": "src/scheduler/src/scheduler.c, src/scheduler/src/main.c, src/scheduler/include/scheduler.h", "root_cause": "sched_set_cluster_cfg(&sched, &cluster_cfg) at main.c was called before sched_init(&sched, &config, handler); sched_init's first action is memset(sched, 0, sizeof(*sched)), which zeroes the cluster_cfg field just written. sched_init's own internal HA-vs-single-host branch reads sched->cluster_cfg.candidates_count immediately after, so it always sees 0. User-reported regression found via code review after bug-1454's leak fix landed in the same function.", - "fix": "Restructured instead of patched around: sched_init() signature changed to take `const xce_cluster_config_t*` directly (assigned to sched->cluster_cfg AFTER the memset, inside sched_init, not before via a separate setter). Deleted sched_set_cluster_cfg() (no other callers). main.c hoists cluster_cfg/cluster_cfg_ptr to function scope and passes cluster_cfg_ptr (NULL or &cluster_cfg) straight into sched_init(). Also fixed an adjacent smell in the same HA loop: it used to mutate the caller's xce_client_config_t* config in place per candidate, leaving it in a surprising mutated state after sched_init returned \u2014 changed to a per-iteration local copy. Verified: full ASAN build (all components) 353/353 targets, meson 9/9 OK, manual scheduler testsuite 15/15, Go tests all pass. No test exercises sched_init's HA path with real --peers input (pre-existing gap, not introduced by this fix).", + "fix": "Restructured instead of patched around: sched_init() signature changed to take `const xce_cluster_config_t*` directly (assigned to sched->cluster_cfg AFTER the memset, inside sched_init, not before via a separate setter). Deleted sched_set_cluster_cfg() (no other callers). main.c hoists cluster_cfg/cluster_cfg_ptr to function scope and passes cluster_cfg_ptr (NULL or &cluster_cfg) straight into sched_init(). Also fixed an adjacent smell in the same HA loop: it used to mutate the caller's xce_client_config_t* config in place per candidate, leaving it in a surprising mutated state after sched_init returned — changed to a per-iteration local copy. Verified: full ASAN build (all components) 353/353 targets, meson 9/9 OK, manual scheduler testsuite 15/15, Go tests all pass. No test exercises sched_init's HA path with real --peers input (pre-existing gap, not introduced by this fix).", "tags": [ "scheduler", "peers", "ha", "config", - "api-design", - "fixed" + "api-design" ], "related_bugs": [ "bug-1454" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 38 + "fix_commits": [], + "record": ".wolf/bugs/bug-1455.md", + "gitea_issue": 38, + "gitea_synced_status": "fixed" }, { "id": "bug-1456", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-07-20T00:00:00.000Z", - "error_message": "Dangling pointer risk in sched_init() HA candidate loop: a later refactor (following bug-1454) changed the loop to build a per-iteration stack-local xce_client_config_t candidate_config and pass &candidate_config into xcecli_ctx_init().", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Dangling pointer risk in sched_init() HA candidate loop: a later refactor (following bug-1454)…\n\nDangling pointer risk in sched_init() HA candidate loop: a later refactor (following bug-1454) changed the loop to build a per-iteration stack-local xce_client_config_t candidate_config and pass &candidate_config into xcecli_ctx_init().", "file": "src/scheduler/src/scheduler.c", - "root_cause": "xcecli_ctx_init() (src/lib/client.c) stores the raw pointer it is given as ctx->conf \u2014 it does not copy the struct. That pointer is dereferenced later, long after sched_init() returns, by client_failover.c runtime reconnect/rotation logic (e.g. ac->conf->masters[next]). When the candidate loop used a stack-local xce_client_config_t per iteration, ctx->conf pointed at a stack frame that no longer existed once sched_init() returned \u2014 a classic dangling pointer that only manifests later during HA reconnect/rotation, making it easy to miss in a build+unit-test pass.", + "root_cause": "xcecli_ctx_init() (src/lib/client.c) stores the raw pointer it is given as ctx->conf — it does not copy the struct. That pointer is dereferenced later, long after sched_init() returns, by client_failover.c runtime reconnect/rotation logic (e.g. ac->conf->masters[next]). When the candidate loop used a stack-local xce_client_config_t per iteration, ctx->conf pointed at a stack frame that no longer existed once sched_init() returned — a classic dangling pointer that only manifests later during HA reconnect/rotation, making it easy to miss in a build+unit-test pass.", "fix": "Reverted to mutating the caller-supplied xce_client_config_t* config parameter in place (config->nmasters, config->masters[0], config->master_ports[0]) instead of a stack-local copy. config is owned by main()'s stack frame, which lives for the whole process, so storing &config-derived state in ctx->conf is lifetime-safe. Added a comment in scheduler.c explaining why the stack-local-copy pattern is unsafe here, to prevent it from being reintroduced.", "tags": [ "scheduler", @@ -16608,27 +5068,31 @@ "dangling-pointer", "lifetime", "client-config", - "fixed", "regression-of-bug-1454" ], "related_bugs": [ "bug-1454", "bug-1455" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 39 + "fix_commits": [], + "record": ".wolf/bugs/bug-1456.md", + "gitea_issue": 39, + "gitea_synced_status": "fixed" }, { "id": "bug-1457", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-20T22:56:00Z", - "error_message": "xce_controller with no --peers flag exits with -EINVAL from cluster_open_listener instead of starting as single-node primary", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_controller with no --peers flag exits with -EINVAL from cluster_open_listener instead of…\n\nxce_controller with no --peers flag exits with -EINVAL from cluster_open_listener instead of starting as single-node primary", "file": "src/controller/myapp.c", - "root_cause": "Discovered while independently reviewing clusterconfig-schema-split task 5.3 by building and running the actual pre-spec baseline (commit 94f50ce) for comparison. Single-node startup (no --peers) was ALREADY broken before this spec began: myapp_run() calls cluster_open_listener() before the primary config (cluster_config) is loaded from disk, so the pre-spec fallback branch (cluster_config->port, only reachable when cluster_config != NULL) always hit its else branch and returned -EINVAL, unconditionally, for every zero-flag single-node start. This invalidates part of clusterconfig-schema-split's premise (requirements.md's Introduction claimed cluster_config->port was 'the sole live port-sourcing mechanism for single-node deployments' and 'single-node controller startup SHALL continue to work exactly as it does today' \u2014 in reality it did not work today). Not caused by, or a regression from, clusterconfig-schema-split's changes \u2014 task 2.1's port-resolution rewrite (commit 1229f9a) incidentally fixed this as a side effect, since its hardcoded-default fallback no longer depends on cluster_config being loaded.", - "fix": "RESOLVED -- incidentally repaired by clusterconfig-schema-split task 2.1 (cluster_open_listener now falls back to a hardcoded XCE_CONTROLLER_DEFAULT_PORT constant instead of an unloaded cluster_config->port), not as a dedicated fix for this entry. Live-verified post-fix: single-node startup with no --peers now correctly binds port 9990 and reaches CTRL_PRIMARY. No further action needed unless a future audit wants to correct the historical claim in clusterconfig-schema-split's requirements.md/design.md that pre-spec single-node startup was previously working \u2014 it was not, for an unrelated reason (myapp_run's listener-open/config-load ordering).", + "root_cause": "Discovered while independently reviewing clusterconfig-schema-split task 5.3 by building and running the actual pre-spec baseline (commit 94f50ce) for comparison. Single-node startup (no --peers) was ALREADY broken before this spec began: myapp_run() calls cluster_open_listener() before the primary config (cluster_config) is loaded from disk, so the pre-spec fallback branch (cluster_config->port, only reachable when cluster_config != NULL) always hit its else branch and returned -EINVAL, unconditionally, for every zero-flag single-node start. This invalidates part of clusterconfig-schema-split's premise (requirements.md's Introduction claimed cluster_config->port was 'the sole live port-sourcing mechanism for single-node deployments' and 'single-node controller startup SHALL continue to work exactly as it does today' — in reality it did not work today). Not caused by, or a regression from, clusterconfig-schema-split's changes — task 2.1's port-resolution rewrite (commit 1229f9a) incidentally fixed this as a side effect, since its hardcoded-default fallback no longer depends on cluster_config being loaded.", + "fix": "RESOLVED -- incidentally repaired by clusterconfig-schema-split task 2.1 (cluster_open_listener now falls back to a hardcoded XCE_CONTROLLER_DEFAULT_PORT constant instead of an unloaded cluster_config->port), not as a dedicated fix for this entry. Live-verified post-fix: single-node startup with no --peers now correctly binds port 9990 and reaches CTRL_PRIMARY. No further action needed unless a future audit wants to correct the historical claim in clusterconfig-schema-split's requirements.md/design.md that pre-spec single-node startup was previously working — it was not, for an unrelated reason (myapp_run's listener-open/config-load ordering).", "tags": [ "controller", "startup", @@ -16637,21 +5101,25 @@ "myapp_run", "cluster_open_listener", "discovered-during-review", - "incidentally-fixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "incidentally-fixed" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 27 + "fix_commits": [], + "record": ".wolf/bugs/bug-1457.md", + "gitea_issue": 27, + "gitea_synced_status": "fixed" }, { "id": "bug-1458", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-07-22T03:22:00Z", - "error_message": "tools/xce-build.sh: line 134: pip: command not found (inside nix develop ./src, running the new `xceadmin` build case, exit 127)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "tools/xce-build.sh: line 134: pip: command not found (inside nix develop ./src, running the new…\n\ntools/xce-build.sh: line 134: pip: command not found (inside nix develop ./src, running the new `xceadmin` build case, exit 127)", "file": "src/nixSupport/xce.nix", "root_cause": "admin-config-tool task 1.3 added python3Packages.{pycapnp,pyyaml,jsonschema,click,pytest,ruff} to xce.nix nativeBuildInputs per brief.md's Nix Environment section, but that list (copied verbatim from brief.md) omits python3Packages.pip. None of those packages transitively pull in a `pip` executable on PATH in a nix devShell built this way, so `pip install -e .` in the new xce-build.sh `xceadmin)` case failed immediately with command-not-found the first time it was actually exercised end-to-end inside `nix develop ./src`.", "fix": "Added python3Packages.pip to xce.nix nativeBuildInputs alongside the other python3Packages.* additions. Verified via `nix develop ./src --command bash -c 'which pip; pip --version'`.", @@ -16662,20 +5130,23 @@ "pip", "xce-build", "admin-config-tool", - "task-1.3", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-1.3" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1458.md" }, { "id": "bug-1459", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-07-22T03:45:00Z", - "error_message": "pip install -e . inside `nix develop ./src` fails: \"error: externally-managed-environment ... This command has been disabled as it tries to modify the immutable /nix/store filesystem\" (PEP 668)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "pip install -e\n\npip install -e . inside `nix develop ./src` fails: \"error: externally-managed-environment ... This command has been disabled as it tries to modify the immutable /nix/store filesystem\" (PEP 668)", "file": "src/tools/xce-build.sh", "root_cause": "The `xceadmin)` case (admin-config-tool task 1.3) ran bare `pip install -e .`/`pip install .` against the nix-provided python3 directly, matching brief.md's harness sketch. But nixpkgs' python3 site-packages lives under the read-only /nix/store, and nixpkgs patches pip to refuse writes there (PEP 668 externally-managed-environment guard) even with --break-system-packages in some configurations. brief.md's sketch was never actually exercised end-to-end against a real nix devShell before this task, so the guard was never hit.", "fix": "Changed the `xceadmin)` case to create a throwaway venv (`$XCEADMIN_DIR/.venv`, recreated fresh every invocation) via `python3 -m venv --system-site-packages`, which inherits the nix-provided pycapnp/pyyaml/jsonschema/click/pytest (so pip never needs to rebuild those heavy binary deps from PyPI) while giving pip a normal writable site-packages to install xceadmin itself into. All subsequent commands (pip install, pytest) now run via $VENV_DIR/bin/pip and $VENV_DIR/bin/python. Verified end-to-end for both the default (editable) and release (pinned) profiles.", @@ -16688,24 +5159,27 @@ "externally-managed-environment", "xce-build", "admin-config-tool", - "task-1.3", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-1.3" ], "related_bugs": [ "bug-1458" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1459.md" }, { "id": "bug-1460", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-07-22T03:52:00Z", - "error_message": "Under `xce-build.sh xceadmin release` (pinned, non-editable install), the installed xcecommon package in site-packages has no schema_files/ directory at all \u2014 controller.capnp/c.capnp/JSON schemas are silently dropped from the wheel", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Under `xce-build.sh xceadmin release` (pinned, non-editable install), the installed xcecommon…\n\nUnder `xce-build.sh xceadmin release` (pinned, non-editable install), the installed xcecommon package in site-packages has no schema_files/ directory at all — controller.capnp/c.capnp/JSON schemas are silently dropped from the wheel", "file": "src/xceadmin/pyproject.toml", - "root_cause": "pyproject.toml had no [tool.setuptools.package-data] entry for xcecommon, so setuptools' default wheel build only includes *.py files. The xce-build.sh sync step correctly copies controller.capnp/c.capnp/*.schema.json into src/xcecommon/schema_files/ in the SOURCE tree, but `pip install .` (non-editable) builds a wheel from that source tree and, without package-data config, drops every non-.py file. An editable install (`pip install -e .`) never surfaces this because it resolves imports straight into the checked-out source tree, bypassing wheel packaging entirely \u2014 exactly the gap design.md's Testing Strategy 'Unit \u2014 schema bundling' note warned only a release-profile test run could catch (Requirement 5.5).", + "root_cause": "pyproject.toml had no [tool.setuptools.package-data] entry for xcecommon, so setuptools' default wheel build only includes *.py files. The xce-build.sh sync step correctly copies controller.capnp/c.capnp/*.schema.json into src/xcecommon/schema_files/ in the SOURCE tree, but `pip install .` (non-editable) builds a wheel from that source tree and, without package-data config, drops every non-.py file. An editable install (`pip install -e .`) never surfaces this because it resolves imports straight into the checked-out source tree, bypassing wheel packaging entirely — exactly the gap design.md's Testing Strategy 'Unit — schema bundling' note warned only a release-profile test run could catch (Requirement 5.5).", "fix": "Added [tool.setuptools.package-data] xcecommon = [\"schema_files/*.capnp\", \"schema_files/json/*.schema.json\"] to src/xceadmin/pyproject.toml. Verified: after `xce-build.sh xceadmin release`, site-packages/xcecommon/schema_files/ now contains controller.capnp, c.capnp, and json/*.schema.json, byte-identical to the synced source copies.", "tags": [ "xceadmin", @@ -16716,23 +5190,26 @@ "wheel", "admin-config-tool", "task-1.3", - "requirement-5.5", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "requirement-5.5" ], "related_bugs": [ "bug-1458", "bug-1459" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1460.md" }, { "id": "bug-1461", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-07-22T02:00:00Z", - "error_message": "pycapnp / capn-c: setting UpgradeConfigRequest.sha256 or .bytes raises capnp.lib.capnp.KjException: 'expected reader.type == STRUCT [6 == 10]; Value type mismatch' -- and the generated C accessors (UpgradeConfigRequest_get_sha256 etc.) were typed Data_ptr instead of a real byte blob", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "pycapnp / capn-c: setting UpgradeConfigRequest.sha256 or .bytes raises…\n\npycapnp / capn-c: setting UpgradeConfigRequest.sha256 or .bytes raises capnp.lib.capnp.KjException: 'expected reader.type == STRUCT [6 == 10]; Value type mismatch' -- and the generated C accessors (UpgradeConfigRequest_get_sha256 etc.) were typed Data_ptr instead of a real byte blob", "file": "src/schema/controller.capnp", "root_cause": "controller.capnp declares `struct Data $C.mapname(\"xce_data_t\")` at line 355 (a tagged union of OPCode_get query-result shapes: schedjobs/schedworkers/jobs/workers/resources/resvals/jobArray). Cap'n Proto resolves unqualified type names lexically within a file, so this local struct SHADOWS the Cap'n Proto builtin `Data` blob type for every later `: Data` field declaration in the same file -- confirmed against the real compiled schema via `capnp compile -o-` + `capnp decode`, and against the actual capnpc-c generated C output (fields typed `Data_ptr`/`XceData_ptr`, a struct-pointer wrapper, using `capn_getp`/`encode_Data_ptr`/`decode_Data_ptr`/`free_Data_ptr`, instead of the correct `capn_data` blob type + `capn_get_data`). Affected 7 fields across 6 structs, several HA-critical: UpgradeConfigRequest.sha256/.bytes, ReplConfigReply.sha256/.bytes, ReplConfigPushRequest.sha256/.bytes, ReplEnvelope.payload (WAL replication envelope), SnapshotTransfer's chunk payload, WalCatchupEntry.payload/.batchPayloads (WAL catchup), CtrlHelloRequest.key. Production C code had already worked around this for years via hand-rolled raw capn_ptr manipulation (see request.c's own comment: \"Pull sha256 and bytes via raw capn_list8 access (NOT decode_Data which would corrupt via xce_data_t)\") -- the bug was known-and-routed-around, never fixed at the root, because nothing before xceadmin's pycapnp-based dynamic reflection needed the generated codec/accessors to work correctly for these fields (the C raw-workaround functions never called them). Two fields, `getdata` (RequestReply's OPCode_get reply payload) and `pushmsg` (Request's generic OPCode_push payload), correctly and intentionally referenced the LOCAL struct Data all along -- their surrounding dispatch code (Data_data_jobArray/jobs/workers/etc. switches in apiserver.c/opget.c/sched.c/handler.c/xceapi.c/helper.c) depends on it.", "fix": "Renamed `struct Data` to `struct XceData` in controller.capnp (kept `$C.mapname(\"xce_data_t\")` so the generated C typedef name is unchanged), and explicitly repointed `getdata`/`pushmsg` at `XceData` by name (since removing the shadow means bare `Data` now correctly means the builtin blob, which those two fields do NOT want). Updated every derived-identifier reference (`Data_ptr`->`XceData_ptr`, `Data_data_`->`XceData_data_`, `struct Data`->`struct XceData`, `read_/write_/new_/encode_/decode_/free_Data(_ptr)?`->the `XceData` equivalents) across 19 hand-written files spanning controller, lib, scheduler, and the Go apiserver/xcecli CGo bindings (generated controller.capnp.c/.h regenerate automatically, never hand-edited). Fixed 5 call sites in src/lib/request.c (extract_data_bytes's parameter + 4 local variables) that had been declared XceData_ptr but now legitimately receive capn_data from the now-correct UpgradeConfigRequest_get_sha256/_get_bytes/ReplConfigPushRequest_get_sha256/_get_bytes accessors -- a pure type-name swap with zero logic change, since Data_ptr/XceData_ptr and capn_data share the identical {capn_ptr p} memory layout (confirmed: request.c already had a comment noting this exact equivalence). Verified via full rebuild+test of lib (1/1), controller (3/3, including the job-array push regression tests), scheduler (15/15), and the Go apiserver+xcecli backends (all packages green), plus a direct pycapnp round-trip proving UpgradeConfigRequest.sha256/.bytes now encode/decode correctly. Unblocks admin-config-tool task 3.3 (xceadmin.push's push_config()).", @@ -16747,20 +5224,23 @@ "admin-config-tool", "task-3.3", "xceadmin", - "cross-component", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cross-component" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1461.md" }, { "id": "bug-1462", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-23T03:04:50.000Z", - "error_message": "ctrl-3 conf/ empty after HA failover deploy at /tmp/xce-ha-deploy -- config never replicated, but WAL/jobs replicated fine", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ctrl-3 conf/ empty after HA failover deploy at /tmp/xce-ha-deploy -- config never replicated, but…\n\nctrl-3 conf/ empty after HA failover deploy at /tmp/xce-ha-deploy -- config never replicated, but WAL/jobs replicated fine", "file": "src/controller/ha.c", "root_cause": "Post-election baseline for ha->committed_config_tag (raft_cb_become_leader, ha.c:1279, set to safe_tag from ha_compute_safe_committed_lsn, ha.c:744-802) is computed as the MAJORITY-SAFE FLOOR across granted voters' self-reported tags (median of sorted tags, same conservative math used for WAL safe_committed_lsn). On a cluster where no admin config push has ever happened live (deploy-ha.sh places cluster.bin on disk pre-boot, not via a live push), every peer voter reports tag=0 except the newly-elected primary itself (self_tag=nonzero, e.g. 860cfd4f59ef9381) -- so the majority-floor computation picks 0, even though the primary demonstrably has a real, valid, currently-loaded config. This 0-floor is CORRECT for WAL LSN safety (a leader must not assume unacknowledged entries are durable) but WRONG for config_tag: a node's own actively-loaded config is not speculative the way an unacknowledged WAL entry is.\n\nEffect: the ONLY path that successfully pushes config after promotion is the one-shot, promotion-instant fan-out (ha.c:1583-1602) to vote endpoints still connected AT THAT EXACT MOMENT (ectx->votes[i].ep != NULL) -- it correctly bypasses committed_config_tag and reads myconf->config_version_tag directly. Any standby that is NOT connected at that instant (a late joiner reconnecting seconds later via on_standby_connected, or any future reconnect/ restart for the rest of the term) relies entirely on the ONGOING heartbeat-driven decision loop (ha_dispatch_repl_action_for_standby -> ha_evaluate_repl_action, ha.c:3633-3663 / 3194-3199: 'if standby_tag < committed_tag -> HA_REPL_PUSH_CONFIG'). Since committed_config_tag is permanently floored at 0 for the rest of the term (nothing advances it except a live admin xceadmin config push via config_commit_change -> push_config_to_standbys, config_repl.c:1524/1556-1558, which never happens in a normal deploy-ha.sh-deployed cluster), standby_tag(0) < committed_tag(0) is false forever -> HA_REPL_PUSH_CONFIG never fires for that standby. Meanwhile WAL/job replication is unaffected since it correctly tracks committed_lsn (a genuinely-advancing value, set from real job commits), not the broken config_tag floor -- exactly matching the observed symptom (jobs replicate, config does not).\n\nReproduced live: /tmp/xce-ha-deploy, peers ctrl-1=9901/ctrl-2=9902/ctrl-3=9903 (src/build.dbg/controller/xce_controller). ctrl-1 promoted primary at term=2 via PreVote failover (ha_compute_safe_committed_lsn logged safe_committed_lsn=0 safe_config_tag=0 'from 2 granted voters'). ctrl-2 (already-connected vote ep[0]) received handle_repl_config_push 45ms after promotion (one-shot path). ctrl-3 connected 3s later as 'late-joiner standby peer[2]' (ha.c on_standby_connected), got WAL-caught-up correctly (ha_process_standby_state: standby[2] converged: config_tag=0 applied_lsn=1 committed_tag=0 committed_lsn=1) but NEVER received a config push -- ctrl-1's full log has zero further push_config/repl_config lines after promotion, and ctrl-3's log stays stuck at 'CTRL_STANDBY -- waiting for primary to connect and push config'. ctrl-3/conf/ is empty on disk (no cluster.bin.*, no current pointer) while ctrl-1/ctrl-2 both have cluster.bin.860cfd4f59ef + current.", "fix": "FIXED 2026-07-23. Implemented candidate direction (a): added ha_promotion_config_tag_floor(safe_tag, self_tag) (ha.h/ha.c, declared beside ha_promotion_truncate_floor) returning safe_tag > self_tag ? safe_tag : self_tag -- mirrors bug-1060's WAL-truncation clamp pattern but for config_tag instead of LSN. raft_cb_become_leader (ha.c ~1279) now clamps committed_config_tag to this floor instead of the raw majority-safe tag, so a newly-elected primary's own actively-loaded config tag is never dropped back to a stale majority-of-zero baseline. RED test added: test_promotion_config_tag_never_drops_below_own_loaded_config (testsuite/test_ha_election.c), covers the exact bug-1462 repro (safe_tag=0, self_tag=0x860cfd4f59ef9381 -> floor=self_tag), the equal case, safe_tag-above-self_tag case, and fresh-boot (0,0)->0. Unit suite green 977/977, zero regressions. Live regression: 20-round sequential soak (xce-test-ha, ~/tmp/xce-ha-soak-seq) PASS -- 63 jobs, zero job loss, all 3 nodes (ctrl-1/ctrl-2/ctrl-3) cycled through leadership across 20 promotions, 9m00.8s wall-clock. 20-round chaotic (overlapping-kill) soak (~/tmp/xce-ha-soak-chaos) also PASS -- VERDICT mode=chaotic rounds=20 jobs=57 committed_divergence=0 committed_job_loss=0 excluded_bug854_discovery=8 excluded_bug924_fdatasync=0 result=PASS (the 8 excluded bug-854 hits are client primary-discovery misses during overlapping-kill windows, not server-side loss). Confirms the fix holds under both normal and double-fault-within-reconvergence-window failover.", @@ -16770,22 +5250,25 @@ "late-joiner", "committed_config_tag", "raft-promotion", - "auto-detected", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "auto-detected" ], "related_bugs": [ "bug-1060" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1462.md" }, { "id": "bug-1463", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xcecli", + "severity": "unknown", "timestamp": "2026-07-23T06:37:00.000Z", - "error_message": "xcecli build fails: ../common/request/request.go:143:32: undefined: tls / :144:13: undefined: loadRootCAs -- FAILED: [code=1] frontend/xcecli/xcecli", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xcecli build fails: ../common/request/request.go:143:32: undefined: tls / :144:13: undefined…\n\nxcecli build fails: ../common/request/request.go:143:32: undefined: tls / :144:13: undefined: loadRootCAs -- FAILED: [code=1] frontend/xcecli/xcecli", "file": "src/frontend/xcecli/common/request/request.go, src/frontend/xcecli/cmd/jobs/peek.go", "root_cause": "Commit 94f50ce6 (bug-1453, CA-cert-plumbing removal) deleted the crypto/tls import, loadRootCAs, and SetRootCAs from request.go and stripped certPath from GetRequestFromAPIServer/PostRequestToAPIServer/DeleteRequestFromAPIServer plus their ~18 call sites, but missed GetJobOutput() (still referenced tls.Config{RootCAs: loadRootCAs(certPath)} with no import) and its sole caller cmd/jobs/peek.go, which still called apiClient.SetRootCAs(serverInfo.CACert) (SetRootCAs no longer exists anywhere) and passed serverInfo.CACert (the CACert field was already removed from APIServerInfo in config/config.go). Go compiles per-package, so once common/request failed to compile, the compiler never got far enough to surface peek.go's own two dangling references in the same build run -- they were latent until request.go compiled again.", "fix": "FIXED 2026-07-23. request.go: removed GetJobOutput's dead TLS block, dropped certPath from its signature (token, jobID, params). peek.go: removed the SetRootCAs call, updated the GetJobOutput call site to the new 3-arg signature. Also enabled NewAPIClient's previously-commented-out `c.Client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})` -- per user request (xcecli needs to actually connect to an https API server) and per bug-1453's own documented design intent (\"server-side TLS only, encryption without peer certificate verification\"), since prior to this no xcecli method set any TLS config at all, meaning ANY https:// apiserver URL (self-signed/auto-generated cert, the common case) would have failed Go's default cert verification -- this was broader than just GetJobOutput. Added crypto/tls import accordingly. Documented both the api-server (http+https) and controller (server-side-cert-only, no client certs) TLS postures as a durable rule in .kiro/steering/security.md's Transport section, replacing a now-stale line claiming TLS is handled at the network layer. Verified: full `all debug` rebuild green (355/355 ninja targets), frontend/xcecli/xcecli binary produced with fresh mtime, no new warnings, no test fallout (grep for GetJobOutput/SetRootCAs/loadRootCAs across *_test.go in xcecli returns nothing).", @@ -16796,22 +5279,25 @@ "dead-code", "build-break", "ca-cert-removal", - "auto-detected", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "auto-detected" ], "related_bugs": [ "bug-1453" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1463.md" }, { "id": "bug-1464", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-07-23T09:50:00Z", - "error_message": "deploy-ha.sh FAIL: Error: xceadmin binary not found -- run 'xce-build.sh xceadmin' or 'nix build .#xcecore' first (after controller had just been rebuilt and xceadmin had been built successfully earlier in the same session)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "deploy-ha.sh FAIL: Error: xceadmin binary not found -- run 'xce-build.sh xceadmin' or 'nix build…\n\ndeploy-ha.sh FAIL: Error: xceadmin binary not found -- run 'xce-build.sh xceadmin' or 'nix build .#xcecore' first (after controller had just been rebuilt and xceadmin had been built successfully earlier in the same session)", "file": "src/tools/xce-build.sh", "root_cause": "setup_build() unconditionally does rm -rf \"$BUILD_DIR\" before every single-component build (controller, xceadmin, etc.), and all components share the same default BUILD_DIR=xce.build for the asan profile. Building controller after xceadmin (or vice versa) wipes the previous component entirely out of the shared build dir -- only the most-recently-built component survives. Only COMPONENT=all does one setup_build() + a single unfiltered meson compile, so it is the only invocation that leaves multiple components coexisting.", "fix": "No code change; workflow fix -- when a downstream task (HA soak deploy-ha.sh) needs both xce_controller/xce_test AND the xceadmin venv present simultaneously in xce.build, run '/xce-build all' (or '/xce-build all asan') once instead of separate '/xce-build controller' + '/xce-build xceadmin' invocations, since the second call's setup_build() rm -rf wipes what the first call produced.", @@ -16822,22 +5308,25 @@ "xceadmin", "controller", "ha-soak", - "workflow", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "workflow" ], "related_bugs": [ "bug-1486" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1464.md" }, { "id": "bug-1465", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-24T08:00:00Z", - "error_message": "test_snapent_checkpoint_job_content_roundtrip / test_snapent_checkpoint_concurrent_mutation_not_torn / test_snapent_checkpoint_queue_status_roundtrip: FAIL: checkpoint file must parse as a well-formed entity-bearing format_version=1 payload", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_snapent_checkpoint_job_content_roundtrip /…\n\ntest_snapent_checkpoint_job_content_roundtrip / test_snapent_checkpoint_concurrent_mutation_not_torn / test_snapent_checkpoint_queue_status_roundtrip: FAIL: checkpoint file must parse as a well-formed entity-bearing format_version=1 payload", "file": "src/controller/testsuite/test_persist_snapshot_entities.c", "root_cause": "config-upgrade-snapshot task 1 bumped persist.c's SNAPSHOT_FORMAT_VERSION from 1u to 2u (reserving the version for the HostAdminState/QueueAdminState capnp migration). test_persist_snapshot_entities.c (ha-entity-snapshots spec) defines its OWN independent SNAPENT_TEST_FORMAT_VERSION=1u constant to structurally validate checkpoint bytes it reads directly, bypassing load_snapshot by design (testing the write-side/task 2.1-2.2 boundary, not the restore boundary). persist_checkpoint() now always writes format_version=2, so the test's stale hardcoded 1u check correctly (if inconveniently) rejected every checkpoint as unrecognized -- this is the same fail-loud format-mismatch behavior Requirement 5.7 mandates, just tripping up a test written before the bump existed.", "fix": "Updated SNAPENT_TEST_FORMAT_VERSION from 1u to 2u in test_persist_snapshot_entities.c (one #define site; both use sites at the job-frame and queue-frame readers already referenced the constant, not a separate hardcoded literal). Mirrors the same fix already applied to test_config_repl.c's three fake_fmt sites for the same version bump. No other code in that file changed -- it belongs to a different, already-shipped spec.", @@ -16848,20 +5337,23 @@ "format-version", "test-regression", "config-upgrade-snapshot", - "ha-entity-snapshots", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha-entity-snapshots" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1465.md" }, { "id": "bug-1466", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-24T15:36:56.329888Z", - "error_message": "test_config_repl.c: test_committed_marker_survives_checkpoint_trim and test_commit_advance_retry_clears_push_before_commit_fence flaked/failed (persist_checkpoint returned -EBUSY; new_mlsn <= committed_at_old) after wiring config-upgrade-snapshot task 5's WAL_CONFIG_MARKER apply case", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_config_repl.c: test_committed_marker_survives_checkpoint_trim and…\n\ntest_config_repl.c: test_committed_marker_survives_checkpoint_trim and test_commit_advance_retry_clears_push_before_commit_fence flaked/failed (persist_checkpoint returned -EBUSY; new_mlsn <= committed_at_old) after wiring config-upgrade-snapshot task 5's WAL_CONFIG_MARKER apply case", "file": "src/controller/testsuite/test_config_repl.c", "root_cause": "Two related findings from wiring apply_committed_entry's new WAL_CONFIG_MARKER case (persist.c, config-upgrade-snapshot task 5), which for the first time makes applying a config marker synchronously trigger a real checkpoint: (1) persist_apply_pump_kick's dispatch is a CAS race between the calling thread (inside commit_submit's synchronous commit_resolve_pending) and persist_start's independent fsync_worker thread (via commit_on_durable_advance) -- when the fsync thread wins, commit_submit's WAIT mode returns having waited only for COMMIT, not for the marker's now-real APPLY+CHECKPOINT side effect on that other thread, so a fixture's own subsequent persist_checkpoint()/LSN bookkeeping can race the still-in-flight background checkpoint. This race was always latent (applying a marker used to be a no-op) but was invisible until task 5 gave it a real, observable side effect. (2) DEEPER pre-existing architectural finding: persist_write_checkpoint_marker (called by any checkpoint, periodic or marker-triggered) appends its local-only, never-replicated WAL_CHECKPOINT entry by incrementing persist_t's current_lsn directly, WITHOUT going through ha_primary_log_append/raft_log_append -- this violates ha_primary_log_append's own documented co-tracking invariant 'p->current_lsn == s->log_len + 1 always holds' (ha.c:4057). Consequence: config_append_marker's returned lsn (sourced from raft's own s->log_len, which under-counts once a checkpoint has interleaved) becomes unreliable as an exact physical-position identifier for any marker appended after an earlier marker's checkpoint has run -- exactly the scenario Requirement 4.4 (overlapping markers) requires. Verified this does NOT corrupt on-disk data or commit/durability correctness: commit_advance_locked's majority_candidate_locked (commit.c) is driven entirely by persist_t's own durable_lsn/current_lsn, never by raft's s->log_len (that library value is documented as advisory-only for commit-ceiling purposes, ha-term-safety commit-ceiling-optA). The bug is confined to callers trusting the RETURNED lsn from config_append_marker/ha_primary_log_append for precise identity/ordering once a checkpoint has occurred earlier in the WAL. This is pre-existing (the periodic WAL-threshold checkpoint trigger has always had the same bypass) but was never observable before because periodic checkpoints require >=100 WAL records to fire, essentially never happening between two closely-spaced marker appends in a unit test; task 5's marker-triggered checkpoint fires on EVERY marker commit, making it the common case. SEVERITY CORRECTION (2026-07-24, kiro-review of task 5): the original \"advisory-only... does NOT affect correctness\" framing UNDERSTATES real risk. ha->raft.log_len (the same value as s->log_len) is a direct input to Raft RequestVote freshness comparisons (ha.c:258-261, my_last_log_index; ha.c:401-406, my_last_log_term via persist_entry_term(c, log_len)) -- NOT debug/advisory-only. ha_primary_log_append itself documents (ha.c:4054-4060) that its own LSN-assignment correctness RELIES on p->current_lsn == s->log_len + 1 holding; once a checkpoint marker breaks that invariant, the next primary append receives a library-assigned index that may no longer match its true physical WAL position, so persist_entry_term() lookups keyed by that index can resolve to the wrong WAL entry. Confirmed via kiro-review independent trace, not yet reproduced as a live failure. Pre-existing (not introduced by task 5 -- the periodic WAL-threshold checkpoint trigger already had this bypass; task 5 only made the interleaving common instead of rare). Still correctly out of config-upgrade-snapshot task 5's boundary to fix -- flagging with corrected severity for whoever next owns ha-term-safety/raft-log-replication so it is not deprioritized based on the original advisory-only framing.", "fix": "(1) Added t42_wait_applied() (mirrors test_apply_pump.c's tap_wait_applied) and called it from t43_commit_marker and directly in test_committed_marker_survives_checkpoint_trim, polling persist_last_applied_lsn() before any LSN-sensitive step following a marker commit, closing the fsync-thread race. (2) Changed t43_append_marker_uncommitted to return the TRUE physical lsn read directly from persist_t (c->persist.current_lsn - 1) instead of trusting config_append_marker's returned (raft-library-derived, now-unreliable) lsn -- matching how production code that needs precise positioning (persist.c's apply seam, which receives lsn from the WAL entry's own on-disk header) already effectively does it. Did NOT attempt to fix the deeper co-tracking invariant gap in ha.c/raft.c (out of config-upgrade-snapshot task 5's scope -- would require raft-log-replication/ha-term-safety spec owners to add a log_len resync after any local-only checkpoint-marker append on a raft leader); flagged as a follow-up finding in the task 5 status report instead.", @@ -16875,20 +5367,23 @@ "raft-log-index", "architectural-finding", "race-condition", - "test-flake", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-flake" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1466.md" }, { "id": "bug-1467", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-25T08:55:00Z", - "error_message": "Reviewer rejected config-upgrade-snapshot task 8.4: configcompare_paths() (configcompare.c/.h) was a correct, fully-tested library function but was not reachable from anywhere outside the unit test binary -- unlike its siblings walcompare/snapverify/snapdiff, it was never registered as an xce_test CLI subcommand in test.c. `echo help | xce_test` confirmed configcompare was missing from the command list.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Reviewer rejected config-upgrade-snapshot task 8.4: configcompare_paths() (configcompare.c/.h) was…\n\nReviewer rejected config-upgrade-snapshot task 8.4: configcompare_paths() (configcompare.c/.h) was a correct, fully-tested library function but was not reachable from anywhere outside the unit test binary -- unlike its siblings walcompare/snapverify/snapdiff, it was never registered as an xce_test CLI subcommand in test.c. `echo help | xce_test` confirmed configcompare was missing from the command list.", "file": "src/controller/testsuite/test.c", "root_cause": "The prior implementer built and unit-tested configcompare_paths() as a library function but never wired a cmd_configcompare() wrapper into test.c's dispatcher (include, extern decl, top_commands[], client_commands[] entries, and the wrapper function itself) -- the step every other sibling tool in this family (walcompare, snapverify, snapdiff) already had.", "fix": "Added #include \"configcompare.h\", extern int cmd_configcompare(int, char*[]) forward decl, a { \"configcompare\", cmd_configcompare, ... } entry in both top_commands[] and client_commands[], and a cmd_configcompare() wrapper (mirroring cmd_snapdiff's exact usage-message/return-value convention: argc<3 -> usage + -1, tool ran -> 0 on match / -1 on divergence) placed immediately after cmd_snapdiff's definition in test.c. Zero changes to configcompare.c/.h, snapverify.c/.h, test_configcompare.c, test_snapverify.c, controller_test.c, or meson.build (all previously reviewed and approved as-is). Verified: controller_unit_test build clean, 1022/1022 tests pass (pure regression check, no test files touched); `echo help | xce_test` lists configcompare; hand-built /tmp fixture member dirs (conf/current + conf/cluster.bin.) smoke-tested via piped stdin -- identical-content case printed 'IDENTICAL across 2 members' with exit 0, divergent-content case printed 'DIVERGENT -- member 1 differs from member 0' with exit 1.", @@ -16899,20 +5394,23 @@ "xce_test", "cli-dispatch", "reviewer-rejection", - "testsuite", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "testsuite" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1467.md" }, { "id": "bug-1468", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-25T00:00:00Z", - "error_message": "OP_upgradeConfig sent by any network client is permanently rejected -- handle_upgrade_config() was unreachable via the wire dispatch path, exercised only by unit tests calling it directly. Independently diagnosed by the admin-config-tool spec's design.md (Boundary Commitments / Risks) as a blocking controller defect out of that spec's scope; re-discovered here while building config-upgrade-snapshot task 8.6's live OP_upgradeConfig client (no prior client existed to send it and hit the gate).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "OP_upgradeConfig sent by any network client is permanently rejected -- handle_upgrade_config() was…\n\nOP_upgradeConfig sent by any network client is permanently rejected -- handle_upgrade_config() was unreachable via the wire dispatch path, exercised only by unit tests calling it directly. Independently diagnosed by the admin-config-tool spec's design.md (Boundary Commitments / Risks) as a blocking controller defect out of that spec's scope; re-discovered here while building config-upgrade-snapshot task 8.6's live OP_upgradeConfig client (no prior client existed to send it and hit the gate).", "file": "src/controller/cluster.c", "root_cause": "commit fd93997f (\"fix(election-iomgr-refactor): config replication and RDHUP failover chain\", 2026-05-31) inverted the OPCode_upgradeConfig primary/standby guard in myreq_handler's switch: it changed `if (!ha_is_primary(&c->ha)) reject` (correct: only primary can commit a config change) to `if (ha_is_primary(&c->ha)) reject` (rejects the ONLY role -- primary -- that ha_dispatch_gate() ever lets reach this switch case at all, since OPCode_upgradeConfig is absent from the CTRL_STANDBY/CTRL_INIT/CTRL_ELECTING allow-lists in ha_dispatch_gate, ha.c). The commit's own message text (\"standbys apply, not primaries\") reflects a since-superseded design where OP_upgradeConfig was a primary-to-standby push; do_upgrade_config_with_bytes (config_repl.c) was later substantially rewritten (ha-term-safety R7 / config-upgrade-snapshot task 7) into a client-to-primary admin op that drives config_commit_change(), but this dispatch guard was never updated to match, leaving handle_upgrade_config() reachable only by unit tests calling it directly -- never via the network.", "fix": "Restored the pre-regression polarity in controller/cluster.c's myreq_handler OPCode_upgradeConfig case: reject with -EAGAIN when NOT primary (`if (!ha_is_primary(&c->ha))`), fall through to handle_upgrade_config() when primary. Updated the stale comment to describe the current client-to-primary semantics and cite ha_dispatch_gate as the actual primary-only enforcement point (this switch-level check is now explicitly documented as defensive redundancy). No change to config_repl.c/ha.c/persist.c materialization logic -- pure dispatch-routing fix.", @@ -16924,21 +5422,23 @@ "dispatch-gate", "regression", "admin-config-tool", - "cross-spec", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cross-spec" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1468.md" }, { "id": "bug-1469", "status": "fixed", "status_updated": "2026-09-27", "status_note": "154a169f: reconcile_config_queues on activation creates added queues without restart", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-25T13:05:00Z", - "error_message": "config-upgrade-snapshot task 8.6's live soak (--config-upgrade, run-ha-soak.sh) FAILED on its first real run: after a config upgrade committed successfully and genuinely interleaved with a 3-round chaotic kill/restart sequence (all 6 jobs survived, upgrade confirmed committed), the post-convergence materialized-state identity check failed -- snapdiff reported ctrl-1's latest checkpoint (snap-000000000000000b.capnp, LSN 11) as 51569 bytes vs ctrl-2/ctrl-3's identical 51621-byte file. This is a live, empirical reproduction of bug-857's originally-hypothesized failure mode (config upgrade leaves members with divergent materialized state), which bug-857's own buglog entry explicitly flagged as 'NOT empirically reproduced yet' and task 8.5 could only address at the unit level (test_bug857_regression.c, a DIFFERENT and already-fixed mechanism -- schedule_checkpoint's WAL-threshold trigger racing the marker boundary). This is a third, distinct mechanism, first observed here.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "config-upgrade-snapshot task 8.6's live soak (--config-upgrade, run-ha-soak.sh) FAILED on its…\n\nconfig-upgrade-snapshot task 8.6's live soak (--config-upgrade, run-ha-soak.sh) FAILED on its first real run: after a config upgrade committed successfully and genuinely interleaved with a 3-round chaotic kill/restart sequence (all 6 jobs survived, upgrade confirmed committed), the post-convergence materialized-state identity check failed -- snapdiff reported ctrl-1's latest checkpoint (snap-000000000000000b.capnp, LSN 11) as 51569 bytes vs ctrl-2/ctrl-3's identical 51621-byte file. This is a live, empirical reproduction of bug-857's originally-hypothesized failure mode (config upgrade leaves members with divergent materialized state), which bug-857's own buglog entry explicitly flagged as 'NOT empirically reproduced yet' and task 8.5 could only address at the unit level (test_bug857_regression.c, a DIFFERENT and already-fixed mechanism -- schedule_checkpoint's WAL-threshold trigger racing the marker boundary). This is a third, distinct mechanism, first observed here.", "file": "src/controller/config_repl.c do_upgrade_config_with_bytes (~1611-1770)", "root_cause": "Byte-level diff of the two checkpoint files (both LSN 11) shows the string 'soakupgrade' (the deliberately-added 5th queue's name, part of task 8.6's upgrade-target config) appears ONCE in ctrl-1's file (inside the embedded raw active-config-bytes blob, persist.c's read_active_config_bytes -- always a fresh disk re-read) but TWICE in ctrl-2/ctrl-3's files -- the second occurrence, present only on ctrl-2/ctrl-3, sits in a queue-ENTITY list section (immediately followed by the next queue's name 'lost_and_found', a runtime enumeration, not raw config text). So ctrl-1's checkpoint correctly embeds the NEW config bytes but its runtime QUEUE-ENTITY table still reflects the OLD (pre-upgrade) queue set. Traced to the code: do_upgrade_config_with_bytes (the handler that runs on whichever node ORIGINATES/commits the upgrade -- ctrl-1 was primary in term 2 when this upgrade was submitted, confirmed via its own log line 'do_upgrade_config_with_bytes: Config upgraded to tag=... (acked=2 standbys)') saves the new bytes to disk (conf_save_config) and bumps c->myconf->config_generation, but NEVER calls config_activate_committed()/load_primary_config() to actually re-parse the new bytes INTO c->myconf -- so the very struct the checkpoint writer presumably walks to emit the queue-entity list keeps the OLD parsed queue set. apply_committed_entry's WAL_CONFIG_MARKER case (persist.c) only calls persist_checkpoint_marker_triggered() then upgrade_materialization_verify() (ha.c) -- and upgrade_materialization_verify does NOT reload/activate config either, it only (a) checks whether some PRIOR checkpoint already referenced this marker (cfgref_present/tag/sha, circularly self-referential to the checkpoint it is meant to gate) and (b) hash-verifies the LIVE ON-DISK bytes -- neither check touches c->myconf's in-memory queue table, so the Activation Fence reports 'VERIFIED' (confirmed in ctrl-1's log, 'upgrade materialization VERIFIED... materialization complete') even though c->myconf itself was never refreshed. config_activate_committed (which DOES call load_primary_config, the only place that actually re-parses queues into c->myconf) is invoked only from handle_repl_config_push (a STANDBY receiving an out-of-band config-push) and from a member's own boot/recovery path -- confirmed empirically: ctrl-1's log shows ZERO config_activate_committed calls in its original (primary, term-2) process instance, but TWO calls in its restarted instance (12:59:16 and 12:59:39, after persist_recover reported APPLY_CLEAN at last_applied_lsn=11, i.e. loading the already-frozen stale checkpoint rather than replaying). Consequence: the node that ORIGINATES a config upgrade never refreshes its own in-memory queue table as part of that commit -- its checkpoint stays consistent with the OLD queue set until some LATER, independent event (restart+recovery, or an incoming config-push) happens to call config_activate_committed. Task 8.6's scenario kills the originating primary shortly after it commits (chaotic multi-kill interleaved with the upgrade, by design), so that self-correction never gets a chance to run, and no FURTHER WAL activity occurred to trigger a follow-up checkpoint that could have picked up a later-corrected queue table -- the divergent checkpoint file is permanently frozen.", "fix": "PARTIALLY FIXED, then RE-INVESTIGATED (2026-07-25) -- correcting this entry's own earlier premature 'FIXED' claim. Two real findings:\n\n(1) GENUINE FIX, LANDED: apply_committed_entry's WAL_CONFIG_MARKER case (persist.c) was calling config_activate_committed(c) AFTER persist_checkpoint_marker_triggered() (the checkpoint write), not before -- meaning even when activation ran, it ran too late to affect the checkpoint it was meant to fix. Reordered to run BEFORE the checkpoint call (still guarded on c->myconf != NULL, still positioned before upgrade_materialization_verify per task 6.2's own sequencing need). Verified: full controller_unit_test suite 1025/1025 PASS, 0 failures, no crash (including the previously-SEGV-prone test_marker_apply_via_live_pump_checkpoints_at_marker_own_lsn and the bug-1469 regression test_wal_config_marker_apply_activates_originating_member_config).\n\n(2) DEEPER GAP FOUND, NOT FIXED -- genuinely out of task 8.6's scope: re-running the live --config-upgrade soak AFTER the reordering fix landed produced the IDENTICAL divergence signature (same ~52-byte checkpoint delta, same duplicated-vs-single 'soakupgrade' queue name pattern) on a DIFFERENT node this time (still one node showing stale queue state). Root-caused via direct code trace: config_activate_committed() -> load_primary_config() only re-parses bytes into myconf->cluster_config (config_repl.c:614-714) -- it NEVER touches c->qmgr, the RUNTIME queue table collect_queue_frames() (persist.c:1895, the checkpoint's queue-entity-list writer) actually walks. The ONLY code that populates c->qmgr from cluster_config->queues[] is build_queues() (queue.c:28), called EXACTLY ONCE per process lifetime from xce_config_init's HOOK_AFTER_EVENT branch (cluster.c, fired during cluster_init_phase2 at process boot/restart) -- and build_queues() is UNCONDITIONAL/blind-create (no existence check before calling create_queue/makeQueue for every queue in the config), so it is UNSAFE to simply re-call on a live cluster with existing queue objects (would attempt to recreate already-existing queues, risking duplicate-key/corruption semantics that were never designed for a live re-provisioning call). Consequence: NO MEMBER -- neither the config-upgrade originator NOR a replicating standby NOR any node's own live marker-apply of any kind -- has a working mechanism to add a NEWLY-INTRODUCED queue to its runtime qmgr without a full process restart. The apparent 'originating member vs standby' asymmetry in this bug's original report was COINCIDENTAL, not causal: whichever node's restart (inevitable during a chaotic multi-kill sequence, unrelated to which node 'originated' the upgrade) happened to occur AFTER the new config bytes were already on disk got the new queue for free via its own full boot sequence (xce_config_init -> build_queues); a node whose restart happened to land BEFORE that point, or that never restarted during the scenario at all, does not -- regardless of the fix in (1). This directly contradicts design.md's own stated goal for this spec ('No controller restart is required for a configuration upgrade after this change') for the specific case of a config upgrade that ADDS a queue: today it demonstrably STILL requires a coincidental restart to take effect on any given already-running member. requirements.md's own Design Invariant #1 ('Queues are config-provisioned... update-only restore is safe') is only true for the SNAPSHOT-INSTALL restore path (a fresh/behind joiner materializing a full checkpoint, restore_queue_state, queue.c:208) -- it was never validated against the LIVE no-restart marker-apply path this spec's Requirement 7/Introduction promises, and does not hold there.\n\nNOT fixed as part of config-upgrade-snapshot task 8.6: doing so properly requires new, carefully-designed production logic (a live-safe, idempotent 'sync qmgr from the freshly committed config' function -- e.g. mirroring restore_host_state's exact find-or-create-only shape, worker.c:477-536, adapted for queue_t/c->qmgr) PLUS a deliberate design decision this task is not positioned to make safely under its own time/scope budget: what happens to a queue REMOVED by an upgrade (delete the queue_t? re-home its jobs to lost_and_found -- bug-857's own originally-hypothesized-but-never-implemented mechanism? refuse the upgrade while jobs remain queued?) -- a decision requiring its own requirements/design pass, not an implementation detail. Flagging squarely for whoever owns tasks 1-7 (persist.c/config_repl.c/queue.c materialization logic) as a genuine, previously-undiscovered gap in this spec's core 'no controller restart required' promise, confirmed via TWO independent live soak runs (before and after the ordering fix) reproducing the identical byte-level signature.", @@ -16954,26 +5454,29 @@ "live-soak-finding", "needs-owner", "persist.c", - "partially-fixed", "re-investigated", - "needs-design", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "needs-design" ], "related_bugs": [ "bug-857", "bug-1466" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "154a169f" + ], + "record": ".wolf/bugs/bug-1469.md" }, { "id": "bug-1470", "status": "fixed", "status_updated": "2026-09-24", "status_note": "later full controller suites pass", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-25T14:25:00Z", - "error_message": "controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence (test_config_repl.c:1854) FAILS (\"Expected TRUE Was FALSE\") DETERMINISTICALLY (4/4 runs: 1x via full xce-build.sh controller clean build + meson test, 3x direct re-invocation of the built controller_unit_test binary with no rebuild) on the bug-1469 fix commit (persist.c +91/test_config_repl.c +144, uncommitted worktree diff at time of this build). controller_unit_test overall: 1025 Tests 1 Failures 0 Ignored / meson summary Ok:2 Fail:1.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence…\n\ncontroller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence (test_config_repl.c:1854) FAILS (\"Expected TRUE Was FALSE\") DETERMINISTICALLY (4/4 runs: 1x via full xce-build.sh controller clean build + meson test, 3x direct re-invocation of the built controller_unit_test binary with no rebuild) on the bug-1469 fix commit (persist.c +91/test_config_repl.c +144, uncommitted worktree diff at time of this build). controller_unit_test overall: 1025 Tests 1 Failures 0 Ignored / meson summary Ok:2 Fail:1.", "file": "src/controller/testsuite/test_config_repl.c", "root_cause": "NOT YET DIAGNOSED. Distinguishing context vs the two prior entries for this same test name: bug-1407 (closed 2026-07-03) root-caused an earlier failure of this exact test to a STALE INCREMENTAL ninja build artifact in this worktree, conclusively fixed by a fresh cycle -- but this run WAS already a fresh clean build (xce-build.sh always does rm -rf + meson setup, confirmed src/tools/xce-build.sh:59-60), so bug-1407s explanation does not apply here. bug-1466 (2026-07-24) found a genuine CAS timing race between the commit-thread and persist_start()s fsync_worker thread that could affect this same test family after task 5s WAL_CONFIG_MARKER-triggers-real-checkpoint change, and claimed to fix it via t42_wait_applied() polling -- but the failure seen here (\"Expected TRUE Was FALSE\", a plain boolean assertion) is 100% reproducible across reruns of the same binary, which argues AGAINST a timing race (races are not usually 4/4 deterministic within one build) and FOR either (a) a residual gap in bug-1466s fix, or (b) a genuine regression introduced by the just-made bug-1469 persist.c change (adding an unconditional config_activate_committed() call inside apply_committed_entrys WAL_CONFIG_MARKER case, immediately before upgrade_materialization_verify()) altering ordering/state that this specific test asserts on. bug-1469s own buglog entry claims a prior direct-ninja verification run saw 1024/1024 PASS 0 failures on the same source -- so either that verification predates a subsequent edit, or ran under different build state; this needs reconciling before task 8.6 soak work continues.", "fix": "NOT FIXED -- reported to the requesting task/session for triage; out of scope for the build-verification task that discovered it (was asked only to get a clean component build + report status, not to debug). Repro is cheap: nix develop ./src --command bash src/tools/xce-build.sh controller (full clean rebuild) or, faster, just rerun the already-built binary: ASAN_OPTIONS=detect_leaks=0 ./src/xce.build/controller/controller_unit_test 2>&1 | grep -A2 test_commit_advance_retry_clears_push_before_commit_fence. Full logs preserved this session at src/xce.build/meson-logs/testlog.txt (search test_config_repl.c:1854).", @@ -16985,24 +5488,27 @@ "commit-engine", "test-config-repl", "needs-diagnosis", - "possible-regression", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "possible-regression" ], "related_bugs": [ "bug-1407", "bug-1466", "bug-1469" ], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1470.md" }, { "id": "bug-1471", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-25T14:45:00Z", - "error_message": "config-upgrade-snapshot task 8.6 live --config-upgrade soak: after avoiding bug-1469's queue-creation gap (switched the upgrade-target config change from ADDING a queue to updating an existing queue's description text -- see bug-1469's updated entry), the materialized-state identity check (snapdiff of each member's latest checkpoint file, all at the same LSN) still fails, but now with a MUCH smaller divergence: exactly ONE byte differs (offset 42879/51161 bytes total) between ctrl-1 and ctrl-2/ctrl-3 (ctrl-2 and ctrl-3 are byte-IDENTICAL to each other). ctrl-1 has 0x50 at that offset; ctrl-2/ctrl-3 have 0x00.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "config-upgrade-snapshot task 8.6 live --config-upgrade soak: after avoiding bug-1469's…\n\nconfig-upgrade-snapshot task 8.6 live --config-upgrade soak: after avoiding bug-1469's queue-creation gap (switched the upgrade-target config change from ADDING a queue to updating an existing queue's description text -- see bug-1469's updated entry), the materialized-state identity check (snapdiff of each member's latest checkpoint file, all at the same LSN) still fails, but now with a MUCH smaller divergence: exactly ONE byte differs (offset 42879/51161 bytes total) between ctrl-1 and ctrl-2/ctrl-3 (ctrl-2 and ctrl-3 are byte-IDENTICAL to each other). ctrl-1 has 0x50 at that offset; ctrl-2/ctrl-3 have 0x00.", "file": "src/controller/persist.c collect_host_frames / capnp_encode_host_admin_state (~lines 141-162, 2028)", "root_cause": "ROOT-CAUSED AND DISTINGUISHED (2026-07-25, follow-up investigation): the two prior hypotheses are now resolved -- Hypothesis 2 (uninitialized/non-zeroed capnp padding) is REFUTED, Hypothesis 1 (genuine content discrepancy) is CONFIRMED, and the true mechanism is a pre-existing, out-of-spec-boundary bug, NOT part of config-upgrade-snapshot. Evidence: (1) Wrote a byte-precise decoder in Python that walks the checkpoint's [host_count][{payload_len; capnp bytes}] host-frame section (same structural approach as snapverify.c task 8.4) and decodes each frame using the exact wire shape write_HostAdminState produces (HostAdminState_word_count=1, pointer_count=1: one 8-byte data word holding `status` via capn_write32 at data-section offset 0, one 8-byte pointer word for `name` Text). Decoding BOTH /tmp/xce-ha-configupgrade and /tmp/xce-ha-configupgrade-v2's preserved checkpoints for all 3 members shows: host[1] = 'cfgupgrade-host', ctrl-1 status=0x00500000, ctrl-2/ctrl-3 status=0x00000000 -- byte offset 42879 is byte index 2 of the little-endian `status` DATA FIELD itself (data-section file offset 42877-42880), NOT the 4-byte padding word that follows it (file offset 42881-42884), which is 0x00000000 IDENTICALLY across all 3 members in both reproductions. (2) 0x00500000 = XCE_WORKER_CLOSED (0x00100000) | XCE_WORKER_CLOSED_BY_ADMIN (0x00400000) -- exactly the bits worker.c:779 sets for HOST_CLOSE inside worker_ctrl(). This is semantically meaningful admin-closed status, not heap garbage. (3) Independently confirmed capn_init_malloc's segment allocator (create_local in the pinned c-capnproto build, /nix/store/.../capnpc-1.0.1/lib/libCapnC_Runtime.so, disassembled directly) calls calloc(1, size) -- zero-initializing -- so Hypothesis 2 was never physically possible for this library, consistent with the empirically-always-zero padding bytes. (4) Traced the actual mechanism: worker_ctrl() (worker.c:732-794) sets w->host.status on whichever member directly processes the hostctrl RPC (ctrl-1, primary at the time), then fires HOOK_DETAIL_WORKER_UPDATE, which persist.c:5722 maps to WAL_WORKER_CTRL and replicates. On EVERY member applying that WAL entry (standbys AND a member's own replay-from-WAL/restart path alike), persist.c:3724's case WAL_WORKER_CTRL calls worker_from_capnp() (worker.c:827-856) UNCONDITIONALLY, regardless of hook_detail (JOIN vs UPDATE vs DEAD all share this one apply path). worker_from_capnp's own idempotency guard (worker.c:847-853) does key_find() and returns -EEXIST if a worker with that name ALREADY EXISTS -- WITHOUT applying any of the just-decoded fields, including status. persist.c:3732-3733 then treats -EEXIST as 'idempotent re-apply, no hook' and silently returns 0, discarding the decoded status update entirely. Since cfgupgrade-host was JOINED (a prior, separate WAL_WORKER_CTRL/HOOK_DETAIL_WORKER_JOIN entry) BEFORE it was admin-closed, by the time the UPDATE entry replicates/replays, the worker already exists on every standby -- so the close never lands there. This is NOT specific to config upgrades, checkpoints, or any config-upgrade-snapshot code: it reproduces for ANY `hostctrl -c`/`hostctrl -o` issued against an already-joined host on ANY XCE HA cluster, with zero dependency on a config upgrade ever being submitted. `git log -L` on worker_from_capnp confirms it was introduced 2026-05-08 in commit 9b6a91a8 ('feat: implement restore_job/worker_from_capnp for WAL recovery'), ~2.5 months before config-upgrade-snapshot's spec work began (first spec commit visible ~2026-06-24); no config-upgrade-snapshot task touches this apply path (the spec's own worker.c addition, restore_host_state from commit 5b1600a8, is a wholly separate function used only by the SNAPSHOT-INSTALL restore path, task 4, never by the live WAL_WORKER_CTRL apply switch). collect_host_frames/capnp_encode_host_admin_state (this spec's task 2.1 code) are functioning exactly as designed -- they faithfully serialize whatever w->host.status currently holds; the bug is that w->host.status itself is wrong on ctrl-2/ctrl-3, entirely upstream of any checkpoint/materialization logic. Existing test coverage (test_apply_committed_entry.c:test_ace_worker_ctrl_join_restores_and_notifies) only exercises the fresh-JOIN case; no test exercises WAL_WORKER_CTRL UPDATE against an already-existing worker, confirming this is a genuine untested gap, not a previously-handled case. Filed separately as bug-1472 (the true root cause) since it is a distinct, real, pre-existing mechanism entirely outside config-upgrade-snapshot's boundary -- see that entry for the fix-owner routing. This entry (bug-1471) remains the observed SYMPTOM (checkpoint byte divergence) for task 8.6's soak; the underlying CAUSE is bug-1472.", "fix": "FIXED -- this was the observed SYMPTOM (checkpoint host-frame byte divergence) of bug-1472's root cause (worker_from_capnp()'s WAL_WORKER_CTRL apply path silently discarding UPDATE payloads against an already-existing worker). bug-1472 was fixed 2026-07-25 (TDD RED->GREEN): worker_from_capnp() now decodes hook_detail before applying its existing-worker idempotency guard, and HOOK_DETAIL_WORKER_UPDATE replaces the existing worker's XCE_WORKER_ADMIN_MASK bits with the decoded payload instead of silently no-op'ing on -EEXIST. This directly closes bug-1471's exact repro (an admin-close status change failing to replicate to standbys, producing the byte-42879 checkpoint divergence). Note: HOOK_DETAIL_WORKER_DEAD was deliberately left unfixed in the same change -- tracked separately as bug-1472-remainder (needs a design decision, not yet resolved).\n\n[Original entry]\nNOT FIXED, AND DELIBERATELY NOT ATTEMPTED IN THIS SPEC -- confirmed genuinely OUT OF config-upgrade-snapshot's scope (see root_cause above). The real fix belongs in worker.c's worker_from_capnp()/persist.c's WAL_WORKER_CTRL apply case (case WAL_WORKER_CTRL, persist.c:3724), which needs to distinguish JOIN semantics (create-if-missing, EEXIST is a legitimate no-op) from UPDATE/DEAD semantics (an already-existing worker is the EXPECTED case, and the decoded fields -- at minimum host.status -- must still be applied to it, not discarded) -- e.g. an update-in-place path mirroring how restore_host_state (task 3, worker.c, config-upgrade-snapshot's OWN precedent) ORs admin bits into an existing worker rather than treating existing-ness as terminal. That is a genuine, real, previously undiscovered defect (filed as bug-1472) but is squarely core HA worker-control replication, predating this spec by ~2.5 months and untouched by any of its tasks -- fixing it here would be scope creep into a different subsystem's correctness contract, not a config-upgrade-snapshot materialization fix. Task 8.6's soak will keep observing this exact symptom (host-frame byte divergence whenever its scenario admin-closes a host before the final checkpoint) until bug-1472 is fixed by whoever owns worker.c/persist.c's general worker-control replication path. Recommend task 8.6 either (a) accept this as a known, separately-tracked limitation and adjust the soak's own PASS criteria to tolerate it (e.g. skip the admin-close-a-host step, or snapdiff-exclude the host-admin-status byte range) until bug-1472 lands, or (b) block final PASS on bug-1472's fix -- an orchestrator/owner decision, not this investigation's to make.", @@ -17022,22 +5528,26 @@ "root-caused", "routed-to-bug-1472", "single-byte-divergence", - "task-8.6", - "fixed" + "task-8.6" ], "related_bugs": [ "bug-1469", "bug-1472" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1471.md" }, { "id": "bug-1472", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-25T16:35:00Z", - "error_message": "Host admin-control status changes (hostctrl -c / hostctrl -o, i.e. HOST_CLOSE/HOST_OPEN via worker_ctrl()) never propagate to standby members' in-memory worker state once the target host has already been joined -- discovered while root-causing bug-1471's single-byte checkpoint divergence, but the defect itself is general and has nothing to do with checkpoints, config upgrades, or the config-upgrade-snapshot spec.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Host admin-control status changes (hostctrl -c / hostctrl -o, i.e\n\nHost admin-control status changes (hostctrl -c / hostctrl -o, i.e. HOST_CLOSE/HOST_OPEN via worker_ctrl()) never propagate to standby members' in-memory worker state once the target host has already been joined -- discovered while root-causing bug-1471's single-byte checkpoint divergence, but the defect itself is general and has nothing to do with checkpoints, config upgrades, or the config-upgrade-snapshot spec.", "file": "src/controller/worker.c worker_from_capnp (~827-856); src/controller/persist.c case WAL_WORKER_CTRL apply path (~3724-3786)", "root_cause": "worker_from_capnp() (worker.c:827-856) is invoked UNCONDITIONALLY by persist.c:3731 for every WAL_WORKER_CTRL entry, regardless of its hook_detail discriminator (JOIN, UPDATE, or DEAD all share this one apply path -- see persist.c:3766-3783's switch, which only controls which post-apply NOTIFICATION hook fires, not whether the mutation itself is applied). worker_from_capnp's own idempotency guard (worker.c:847-853): if key_find() on c->workermgr finds a worker with the decoded name ALREADY PRESENT, it returns -EEXIST immediately, WITHOUT applying any decoded field (including host.status) to the existing worker object. persist.c:3732-3733 treats any -EEXIST return as 'idempotent re-apply (replay / resync) -- no hook' and silently returns 0. This EEXIST short-circuit is correct for its originally-intended purpose (a duplicate/replayed JOIN entry for a worker that's already been created should indeed be a no-op), but WAL_WORKER_CTRL is also the opcode for HOOK_DETAIL_WORKER_UPDATE (worker.c:732-794's worker_ctrl(), which handles HOST_OPEN/ HOST_CLOSE admin commands, sets w->host.status, then fires exactly this same HOOK_AFTER_EVENT/HOOK_DETAIL_WORKER_UPDATE/WAL_WORKER_CTRL path at worker.c:786-787) and HOOK_DETAIL_WORKER_DEAD. For UPDATE/DEAD events, the target worker EXISTING is the NORMAL, EXPECTED case (you can only update or kill an already-joined worker) -- yet the same 'already exists -> skip entirely' logic applies, so the decoded status change is silently discarded on every member OTHER than the one that directly processed the original RPC. Confirmed via git blame: worker_from_capnp was introduced 2026-05-08 (commit 9b6a91a8, 'feat: implement restore_job/worker_from_capnp for WAL recovery'), predating config-upgrade-snapshot's spec work by ~2.5 months; no config-upgrade-snapshot task touches this function or the WAL_WORKER_CTRL apply case. Empirically confirmed via byte-precise capnp decode of two independent live-soak checkpoint sets (/tmp/xce-ha-configupgrade{,-v2}): ctrl-1 (which processed a `hostctrl -c` against a previously-joined synthetic host) has host.status=0x00500000 (XCE_WORKER_CLOSED|XCE_WORKER_CLOSED_BY_ADMIN) in its checkpoint's HostAdminState frame; ctrl-2/ctrl-3 (standbys) both show host.status=0x00000000 for the identically-named host entry -- proving the close never reached their live c->workermgr entries. Because a standby's OWN restart+full-WAL-replay hits the identical code path (persist.c's replay call site also routes through apply_committed_entry's same WAL_WORKER_CTRL case), a member that fully replays from LSN 1 would ALSO reconstruct the JOIN first (creating the worker) and then skip the later UPDATE (already exists) -- so this is not even a restart-repairable staleness; the divergence appears to be permanent for that member until directly patched or until some other mechanism re-applies the admin action. General severity: this affects ANY admin host-control action (`hostctrl -c`/`hostctrl -o`) against an already-joined host, unconditionally, cluster-wide -- not just this soak's synthetic cfgupgrade-host scenario.", "fix": "FIXED 2026-07-25 (TDD: RED test first, confirmed failing against the pre-fix code, then GREEN). worker_from_capnp() (src/controller/worker.c) now decodes the WAL_WORKER_CTRL entry's hook_detail discriminator (the same 4 leading bytes it previously skipped without inspecting) BEFORE applying its existing-worker idempotency guard. When the target worker already exists: HOOK_DETAIL_WORKER_JOIN keeps the original -EEXIST no-op (a duplicate JOIN replay is genuinely idempotent -- unchanged, regression-guarded). HOOK_DETAIL_WORKER_UPDATE now REPLACES (not ORs -- worker_ctrl()'s HOST_OPEN uses assignment-style &= ~(...) to CLEAR admin bits, so OR would leave a re-opened worker stuck closed) the existing worker's XCE_WORKER_ADMIN_MASK bits with the decoded payload's admin-masked bits, under the worker's own write lock (mirrors restore_host_state's masking discipline, worker.c:487, with replace instead of OR semantics), then returns 0 so persist.c's WAL_WORKER_CTRL apply case (unchanged -- its existing '0 on any successful apply, -EEXIST only for genuine JOIN no-op' contract already handled this generically) falls through to fire the correct HOOK_DETAIL_WORKER_UPDATE notification instead of silently discarding the status change. HOOK_DETAIL_WORKER_DEAD is DELIBERATELY LEFT UNFIXED (still -EEXIST/no-op) -- investigated, not skipped: mark_worker_unavail()'s mutation (worker.c:661-679) is NOT confined to XCE_WORKER_ADMIN_MASK (sets XCE_WORKER_UNAVAIL + dead_since + resets hostattrs_t to 'unknown' + zeroes resource-value slots via set_host_slots), and critically, the live per-host resource-value array xce_worker_t.resvals (a zarray_t, what set_host_slots actually mutates and what scheduling reads) is NOT part of the Worker capnp wire schema at all -- only the separate resvals_to_sched/nresvals_to_sched fields are (confirmed by reading the generated decode_Worker/encode_Worker in controller.capnp.c and the schema in controller.capnp). A naive status-replace for DEAD would be an incomplete fix (would not zero standbys'/replayers' resource slots on a dead-worker event, a real correctness gap for POST-PROMOTION scheduling since the newly-promoted primary would read stale, non-zeroed slots) and needs its own investigation into whether/how to extend the wire schema -- filed separately as bug-1472-remainder below rather than rushed. Fix verified via 2 new + regression tests in test_apply_committed_entry.c (test_ace_worker_ctrl_update_applies_to_existing_worker, test_ace_worker_ctrl_update_replaces_not_ors_admin_bits, test_ace_worker_ctrl_join_duplicate_existing_is_noop -- the last one a regression guard proving JOIN's existing no-op behavior is unchanged) plus the full controller_unit_test suite: baseline (this worktree, immediately pre-fix) 1030 Tests 0 Failures, post-fix 1033 Tests 0 Failures (net +3, exactly the new tests, zero regressions). persist.c was NOT touched -- confirmed its WAL_WORKER_CTRL apply case (persist.c:3724-3786) already treats any non-negative worker_from_capnp() return as a successful apply generically (re-looks-up the worker by name and fires the hook_detail-keyed notification unconditionally), so returning 0 from the new UPDATE branch required no caller-side change.\n\n[UPDATE 2026-08-16] Tag hygiene: this entry's own UPDATE-case fix landed 2026-07-25 (commit 7547a8f1) but its tags were never updated off 'not-fixed'/'needs-owner'. The DEAD-case gap this entry deferred is bug-1472-remainder, separately FIXED and landed today in commit 45400c3a. Both halves of bug-1472 are now fixed and committed; closing out.", @@ -17052,22 +5562,63 @@ "standby-divergence", "pre-existing", "out-of-config-upgrade-snapshot-scope", - "root-caused", - "fixed" + "root-caused" ], "related_bugs": [ "bug-1471", "bug-1472-remainder" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "7547a8f1", + "45400c3a" + ], + "record": ".wolf/bugs/bug-1472.md" + }, + { + "id": "bug-1472-remainder", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", + "timestamp": "2026-07-25T20:15:00Z", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Follow-up split out of bug-1472: HOOK_DETAIL_WORKER_DEAD entries applied via WAL_WORKER_CTRL…\n\nFollow-up split out of bug-1472: HOOK_DETAIL_WORKER_DEAD entries applied via WAL_WORKER_CTRL against an already-existing worker still return -EEXIST (no-op) -- a dead-worker transition detected by one member (mark_worker_unavail(), worker.c:661-679) still never replicates its state to other members (standbys, or this member's own WAL replay), unconditionally. NOT a regression: this was already true before bug-1472's investigation; bug-1472's fix deliberately scoped itself to the byte-precise-confirmed HOOK_DETAIL_WORKER_UPDATE case only and left DEAD as -EEXIST/no-op, unchanged.", + "file": "src/controller/worker.c worker_from_capnp (~827-895, post-bug-1472-fix); mark_worker_unavail (~661-679); src/schema/controller.capnp struct Worker (~136-144)", + "root_cause": "mark_worker_unavail()'s mutation is NOT confined to XCE_WORKER_ADMIN_MASK the way worker_ctrl()'s is -- it sets XCE_WORKER_UNAVAIL (a non-admin bit), dead_since, resets hostattrs_t to 'unknown' sentinel strings, AND zeroes the live per-host resource-value slots via set_host_slots()/update_resvals(), which mutates xce_worker_t.resvals (a zarray_t of live xce_val_t entries). Critically, w->host.resvals is NOT part of the Worker capnp wire schema at all (confirmed by reading src/schema/controller.capnp's struct Worker and the generated decode_Worker/encode_Worker in controller.capnp.c) -- only the separate resvals_to_sched/nresvals_to_sched fields (a different, scheduler-push-facing concept) are encoded. So even applying a simple 'replace admin-masked bits' fix analogous to bug-1472's UPDATE fix would NOT fully replicate a DEAD transition's effects: the resource-value slot zeroing would still be silently lost on every other member, which matters materially at PROMOTION time (a newly-promoted primary would read stale, non-zeroed slots for a worker that died before promotion, potentially over-scheduling a dead host). A correct fix needs either (a) a schema change adding a wire-encodable resource-value-slots field so the DEAD payload can carry it, or (b) a different replication strategy for DEAD (e.g. re-deriving 'this worker's connection is gone locally' independently on each member rather than replicating a mutation), which is a design decision beyond a same-task patch.", + "fix": "FIXED (TDD RED->GREEN), design direction chosen by user: wire-encode the resource-value-slot zeroing rather than a non-replicated per-member re-derivation. Investigation revised the original design framing: NO new capnp schema field was actually needed. capnp_encode_worker (persist_worker_hook_fun) already serializes the FULL xce_worker_t.host -- including `attrs` (HostAttrs) -- on every WAL_WORKER_CTRL entry, and mark_worker_unavail() calls reset_hostattrs_unknown() BEFORE firing the hook that triggers that encode, so a DEAD entry's wire payload already carries the post-reset 'unknown'/0 attrs state. The actual gap was narrower: worker_from_capnp() had no DEAD branch at all (fell through to the JOIN-duplicate -EEXIST no-op), so nothing ever APPLIED what was already on the wire. Fixed by adding a DEAD branch (worker.c) mirroring the existing UPDATE branch (bug-1472's own fix): sets XCE_WORKER_UNAVAIL, takes dead_since from the decoded payload's deadSince field (the PRIMARY's own death timestamp -- genuinely wire-driven, not locally re-derived, honoring the chosen design direction), and calls reset_hostattrs_unknown()+add_hostattrs_resvals()+set_host_slots(0) locally to re-derive the resvals from the now-reset attrs -- these three calls are deterministic/data-independent (same effect mark_worker_unavail() itself produces on the primary), so xce_worker_t.resvals (the zarray_t the original investigation worried about) never needs to cross the wire at all. Regression test (test_apply_committed_entry.c's test_ace_worker_ctrl_dead_applies_to_existing_worker): pre-existing worker with live (non-'unknown') attrs receives a DEAD WAL entry with a specific non-zero deadSince -- RED under the pre-fix code (rc=-EEXIST, no mutation, no notification, the exact silent-discard this entry describes), GREEN under the fix (XCE_WORKER_UNAVAIL set, dead_since matches the payload exactly, hostattrs reset to 'unknown'/0, one notification fires). Full controller_unit_test suite: 1101 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED -- deliberately deferred, investigated not skipped (see bug-1472's own fix note for the full investigation). Needs its own design decision on how (or whether) to wire-encode the resource-value-slot zeroing, or an alternative non-replicated detection strategy, before a fix can be written safely.", + "tags": [ + "worker.c", + "worker_from_capnp", + "mark_worker_unavail", + "WAL_WORKER_CTRL", + "HOOK_DETAIL_WORKER_DEAD", + "capnp-schema-gap", + "resource-value-replication", + "standby-divergence", + "post-promotion-scheduling-risk", + "needs-owner", + "needs-design-decision", + "deferred" + ], + "related_bugs": [ + "bug-1472" + ], + "fix_commits": [], + "record": ".wolf/bugs/bug-1472-remainder.md" }, { "id": "bug-1473", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-25T19:12:00Z", - "error_message": "test_second_config_marker_supersedes_first_still_installing_on_member (task 8.8) FAILED on first run: Expected 0 Was 1 -- the fence must clear once materialization for the latest marker (B) completes.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_second_config_marker_supersedes_first_still_installing_on_member (task 8.8) FAILED on first…\n\ntest_second_config_marker_supersedes_first_still_installing_on_member (task 8.8) FAILED on first run: Expected 0 Was 1 -- the fence must clear once materialization for the latest marker (B) completes.", "file": "src/controller/testsuite/test_config_repl.c (test-only; no production code involved)", "root_cause": "Test-construction gap, not a production bug. After releasing the checkpoint_in_progress CAS in the raced fixture, the test called the bare persist_checkpoint() to resolve the in-flight-checkpoint scenario, then read upgrade_materialization_fenced() directly -- but plain persist_checkpoint() never calls upgrade_materialization_verify() itself. That call only happens from two production call sites: apply_committed_entry's WAL_CONFIG_MARKER case (fires only on a NEW marker apply, not a bare checkpoint), and the delaymgr-driven retry trampoline (schedule_upgrade_materialization_retry), which explicitly no-ops when delaymgr is not running -- its own doc comment says so verbatim: \"No-op when... the delaymgr is not running (the unit-test fixture path...) -- caller drives directly.\" The bare t42/t43 test fixture (used throughout test_config_repl.c) never inits a delaymgr, so nothing in the raced fixture would ever re-evaluate the fence after the resolving checkpoint landed -- the fence stayed fenced=1 even though materialization for marker B had genuinely completed.", "fix": "Added an explicit upgrade_materialization_verify(c_raced) call immediately after the resolving persist_checkpoint() call, mirroring the exact pattern already used by the precedent 6.1-c fence tests earlier in this same file (which call upgrade_materialization_verify(c) directly after t43_commit_marker for the same delaymgr-less-fixture reason). Also asserted verify_rc == 0. Confirmed via log trace that upgrade_materialization_verify only logs on a fence 0->1 transition (ha.c:254-256, \"never spams on repeated failed retries\") -- explaining why no second ERROR line appeared for markerB in the raw log even before the fix, which was the first clue this was a missing-caller issue rather than a fence-logic issue.", @@ -17079,22 +5630,24 @@ "activation-fence", "delaymgr", "test-construction", - "not-a-production-bug", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "not-a-production-bug" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1473.md" }, { "id": "bug-1474", "status": "fixed", "status_updated": "2026-09-27", "status_note": "c3a17b19: root cause of angle (a) -- standby_endpoints loops bounded by count skipped entries past zarray_del holes (lost heartbeats, spurious re-dial); now scan caps", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-25T20:50:00Z", - "error_message": "config-upgrade-snapshot task 8.9's new live --config-upgrade-wal-trim soak (run-ha-soak.sh, run_config_upgrade_wal_trim_round): the marker-triggered checkpoint's WAL-segment trim (persist_checkpoint_marker_triggered -> trim_wal_segments -> repl_trim_lsn, bug-1147's guard) strands a genuinely mid-catch-up standby's still-needed WAL segment -- but NOT because bug-1147's guard logic itself is wrong. Reproduced identically on 2/2 independent live 3-node runs: a killed, still-down standby's disconnect-guard (last_disconnected_lsn) is silently DESTROYED before the trim ever runs, because an UNRELATED, already-connected peer's redundant reconnect attempt reuses the dead standby's just-freed replicator slot.", - "file": "src/controller/ha.c (primary_heartbeat_fn's 'reconnect to any peer missing from standby_endpoints' loop, ~9219-9253; on_standby_connected, ~8658-8708; raft_cb_become_leader's vote-ep promotion, ~1888-1921); src/controller/repl.c (repl_add_standby's first-empty-slot allocation, ~85-133, and its memset(s,0,...) which wipes last_disconnected_lsn on reuse)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "config-upgrade-snapshot task 8.9's new live --config-upgrade-wal-trim soak (run-ha-soak.sh…\n\nconfig-upgrade-snapshot task 8.9's new live --config-upgrade-wal-trim soak (run-ha-soak.sh, run_config_upgrade_wal_trim_round): the marker-triggered checkpoint's WAL-segment trim (persist_checkpoint_marker_triggered -> trim_wal_segments -> repl_trim_lsn, bug-1147's guard) strands a genuinely mid-catch-up standby's still-needed WAL segment -- but NOT because bug-1147's guard logic itself is wrong. Reproduced identically on 2/2 independent live 3-node runs: a killed, still-down standby's disconnect-guard (last_disconnected_lsn) is silently DESTROYED before the trim ever runs, because an UNRELATED, already-connected peer's redundant reconnect attempt reuses the dead standby's just-freed replicator slot.", + "file": "src/controller/ha.c (primary_heartbeat_fn's 'reconnect to any peer missing from standby_endpoints' loop, ~9219-9253; on_standby_connected, ~8658-8708; raft_cb_become_leader's vote-ep promotion, ~1888-1921); src/controller/repl.c (repl_add_standby's first-empty-slot allocation, ~85-133, and its memset(s, 0, ...) which wipes last_disconnected_lsn on reuse)", "root_cause": "Confirmed via direct log evidence on 2 independent live-soak runs (deploy dirs /home/claude/tmp/xce-ha-waltrim and /home/claude/tmp/xce-ha-waltrim2, both --config-upgrade-wal-trim). Sequence, identical both times: (1) An election promotes surviving VOTE endpoints to standby replication channels (raft_cb_become_leader: 'promoted vote ep[i] -> standby channel', repl_add_standby registers them at replicator slots 0/1 -- e.g. run 1: ep[0]=ctrl-1 -> slot 0, ep[1]=ctrl-3 -> slot 1). (2) The test kills a non-primary TARGET standby (ctrl-1) while it is caught up to a known WAL floor. The primary's err_handler (vote_ep_err_handler, correctly wired) detects the RDHUP and calls repl_remove_standby, freeing slot 0 and setting last_disconnected_lsn = ctrl-1's real acked_lsn (confirmed matching the test's own captured floor segment, e.g. seg-5 in run 1, seg-c in run 2) -- bug-1147's guard is set CORRECTLY at this point. (3) Seconds later (matching the ~9s default heartbeat interval), primary_heartbeat_fn's periodic 'reconnect to any peer missing from standby_endpoints' sweep (ha.c ~9219) runs. It correctly identifies ctrl-1 (peer[0]) as missing (reconnect attempt fails with ECONNREFUSED, confirmed in both runs' logs: 'HA: connect to peer[0] (localhost:9991) failed: -111') -- but it ALSO, incorrectly, identifies ctrl-3 (peer[2]) as missing, even though ctrl-3 has been continuously connected the whole time via its slot-1 promoted-vote-channel connection (confirmed acking WAL entries without any gap throughout, via ctrl-2.log's 'commit: ack standby_idx=1' sequence). This triggers a redundant outbound reconnect to ctrl-3 (on_standby_connected fires: 'HA: outbound standby connection to peer[2] established' + 'HA: repl registered late-joiner standby peer[2] (slot 0, localhost:9993)'). (4) repl_add_standby's slot allocation picks the FIRST EMPTY slot by lowest index (repl.c ~85-110) with no regard for whether that slot still holds an unexpired bug-1147 disconnect guard for a DIFFERENT peer -- it picks slot 0 (ctrl-1's just-freed, guard-holding slot, since it is lower-indexed than slot 1) and unconditionally memsets it to zero, WIPING OUT ctrl-1's last_disconnected_lsn guard entirely. (5) When the marker-triggered checkpoint's trim runs shortly after, repl_trim_lsn (repl.c ~650-683) sees slot 0 now occupied by a live, fully-caught-up ctrl-3 duplicate connection (contributing no meaningful floor-lowering) and slot 1 also fully caught up -- with NO surviving trace that ctrl-1 (still down, still needing its old floor) was ever disconnected. floor collapses to the active segment itself, and every segment below it -- including the one ctrl-1 genuinely still needs -- is deleted. This is NOT a bug in bug-1147's guard logic (repl_trim_lsn/trim_wal_segments are working exactly as designed against the replicator state they are handed); it is an upstream defect in how a duplicate/late-joiner reconnect for an ALREADY-connected peer gets allocated a replicator slot. Root cause of WHY the 'peer already has a standby endpoint' check (ha.c ~9225-9249) fails to recognize ctrl-3's existing promoted-vote-channel connection is not fully pinned to a single line -- standby_ep_ctx_t is deliberately layout-compatible with vote_ep_ctx_t (confirmed by direct struct read, ha.c:8590-8603, both start with ha/c/ctx/peer_idx in the same order, so the blind cast the check performs is NOT a type-confusion bug) -- but vote_ep_ctx_t.peer_idx appears to be populated under inconsistent conventions across its several creation call sites in ha.c (e.g. line 542 sets peer_idx=votes_idx [relative, votes-array index] while a comment at line ~6120 asserts 'vctx->peer_idx is the ABSOLUTE cluster index' for a different call site) -- a mismatch between the relative votes_idx_for_i the check computes and whichever convention the SPECIFIC promoted vote_ep_ctx_t for ectx->votes[i].ep was actually built under would make the sctx->peer_idx == votes_idx_for_i comparison fail deterministically for promoted-vote-channel entries, matching the 100% reproduction rate observed. This needs further tracing by whoever owns ha.c's HA connection-management code to identify the exact mismatched call site.", "fix": "[UPDATE 2026-08-16] Angle-a FIXED (TDD RED->GREEN, root-caused precisely -- not the original peer_idx-convention hypothesis, a different, concrete bug in an adjacent function). Traced every vote_ep_ctx_t.peer_idx assignment site as recommended: raft_cb_send_vote_request (real vote, ha.c) correctly sets peer_idx=votes_idx (RELATIVE) and is the ONLY function that stores its endpoint into ectx->votes[] (the array raft_cb_become_leader's promotion loop reads) -- confirmed by full read, no ectx->votes[].ep assignment exists anywhere else. raft_cb_send_pre_vote_request deliberately sets peer_idx=ABSOLUTE (correct, documented, needed by the PreVote tally) but NEVER stores into ectx->votes[] either -- so PreVote endpoints structurally can never reach promotion, ruling out the original hypothesis (PreVote-vs-real-vote convention mismatch reaching the reconnect-sweep check). ha->my_priority is set once at init and never reassigned, ruling out a self-idx-drift hypothesis too. The reconnect-sweep check itself (primary_heartbeat_fn, ~line 9787) already correctly discriminates promoted-vote-ep (relative) vs standby-ep (absolute) via sctx->ctx != NULL, unchanged since 2026-06-01 (predates this bug's 2026-07-25 discovery, confirmed via git log -L) -- it was never the bug. The REAL bug: on_standby_connected's own same-peer duplicate-eviction check (the function's OWN doc comment already named this exact hazard -- 'two live outbound state-push channels to one peer') compared cur_sctx->peer_idx == sctx->peer_idx DIRECTLY, with no relative/absolute discrimination at all -- unlike the adjacent, correct reconnect-sweep check ~500 lines away. sctx (the newly-connecting standby) always carries the ABSOLUTE index; if the existing entry for the same physical peer is a promoted vote ep (RELATIVE), the raw comparison can never match, so the stale promoted-vote channel is never evicted -- both channels survive, reproducing the buglog's exact live evidence (heartbeats/WAL fan out to a stale duplicate; the genuinely-connected promoted channel's own bug-1147 disconnect-guard slot gets silently reused by the SECOND, spurious connect attempt for the SAME live peer). Fix: apply the identical votes_idx_for_i conversion the reconnect-sweep check already uses, in on_standby_connected's eviction loop, discriminating on cur_sctx->ctx != NULL exactly as that check does. Regression test (test_ha_liveness.c's test_reconnect_resolves_dual_channel_promoted_vote_ep): stale entry is a genuine promoted-vote-ep test double (vote_ep_ctx_build_for_test, relative peer_idx) for the same physical peer a fresh standby reconnect targets (absolute peer_idx) -- RED under the pre-fix code (both channels survive, zero evictions -- the existing sibling test, test_reconnect_resolves_dual_channel_same_peer, only ever exercised the standby-vs-standby case, both absolute, which is why this mixed case slipped through originally), GREEN under the fix (exactly one channel survives, one eviction). Full controller_unit_test suite: 1100 tests, 2 pre-existing unrelated failures (bug-1561), no regressions. NOT independently re-verified via a live multi-node chaotic soak (the class of test that originally discovered this bug) -- the unit test directly constructs and proves the exact mechanism the live soak's log evidence showed, but a live re-run of config-upgrade-snapshot task 8.9's --config-upgrade-wal-trim scenario would be the strongest additional confirmation if this is revisited.\n\n[Original entry -- angle-b, already fixed 2026-07-26]\n\nNOT FIXED -- explicitly out of config-upgrade-snapshot task 8.9's boundary (task 8.9 is a regression test for the pre-existing, unmodified bug-1147 guard under the NEW marker-triggered trigger path, not a fix for this separately-discovered HA connection-management defect). Two independent, complementary fix angles for whoever owns ha.c/repl.c's HA connection layer: (a) fix the 'peer already has a standby endpoint' check (ha.c ~9225-9249) so it correctly recognizes an existing promoted-vote-channel connection for a peer, eliminating the spurious duplicate/late-joiner reconnect entirely (the more targeted fix, addresses the root cause) -- likely requires auditing every vote_ep_ctx_t.peer_idx assignment site (ha.c lines 542, 697, 6956, and whichever one specifically constructs ectx->votes[i].ep's ep->data) against the relative-votes_idx convention votes_idx_for_i expects; (b) independently harden repl_add_standby's slot allocation (repl.c) to never silently clobber a slot whose last_disconnected_lsn guard has not yet expired per snap_interval_s -- e.g. skip a guard-holding slot in the first-empty-slot scan (falling through to a genuinely empty slot, or failing loud if none exists) -- this is a defense-in-depth hardening of bug-1147's own retention mechanism against ANY future caller that might reuse a slot prematurely, not just this specific duplicate-connection path. Recommend (b) regardless of whether (a) is fixed, since it directly protects the invariant Requirement 7.4 cares about. The run_config_upgrade_wal_trim_round test infrastructure (run-ha-soak.sh, config-upgrade-snapshot task 8.9) is CORRECT and reproducibly catches this defect via direct on-disk WAL segment inspection -- re-run `run-ha-soak.sh --config-upgrade-wal-trim` after either fix lands to confirm a clean PASS; task 8.9 itself should stay incomplete/blocked on this bug per its own scope boundary (mirrors task 8.6's bug-1471/1472 precedent).\n\n[UPDATE 2026-07-26] FIXED (fix angle (b), the repl.c defense-in-depth hardening): repl_add_standby's slot-allocation scan (repl.c) now does a two-pass search instead of first-empty-slot-by-index. Pass 1 prefers a genuinely virgin slot (ep==NULL && last_disconnected_lsn==0). Pass 2 (fallback, only when no virgin slot exists) reuses the guard-holding slot (ep==NULL && last_disconnected_lsn!=0) with the OLDEST last_disconnect_time -- a deliberate policy: that guard is closest to its own natural repl_expire_disconnect_slots expiry, so reusing it discards the LEAST remaining WAL-retention protection of any guard-holding slot. With MAX_STANDBYS==3, the exhausted case (both non-primary slots simultaneously guard-holding, no virgin slot free) is handled by this same fallback rather than -ENOSPC -- refusing a legitimate new/reconnecting peer would be worse for cluster availability than accepting an early guard loss in an already-degraded state; the pre-existing -ENOSPC path (more distinct live peers than MAX_STANDBYS slots) is untouched, reserved for that genuinely-impossible case only.\nTDD: two new unit tests added to src/controller/testsuite/test_repl_unit.c -- test_repl_add_standby_prefers_virgin_slot_over_guard (RED under the old code: asserted a new unrelated peer must not land in a lower-indexed guard-holding slot when a virgin slot exists, and that the guard's last_disconnected_lsn must survive untouched) and test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect (RED under the old code: asserted the exhausted-fallback deliberately picks the OLDER-disconnected guard slot, not merely the lowest-indexed one -- the test deliberately makes 'oldest-disconnected' and 'lowest-indexed' point at DIFFERENT slots so an accidental first-guard-by-index fallthrough would still fail it). Both new tests failed pre-fix, pass post-fix.\nGREEN: full controller_unit_test suite = 1030 tests (1028 known-good baseline + 2 new), 0 failures, no regressions (ASAN_OPTIONS=detect_leaks=0, nix develop build via `ninja -C src/xce.build controller/controller_unit_test`).\nrepl_remove_standby, repl_expire_disconnect_slots, repl_trim_lsn, and trim_wal_segments were NOT modified -- all confirmed correct and untouched, per this fix's scope.\nThis fix does NOT address fix angle (a) -- the separate ha.c heartbeat-reconnect-sweep defect (vote_ep_ctx_t.peer_idx convention mismatch hypothesis, ha.c ~9219-9254 / ~8687-8708) that triggers the spurious duplicate reconnect in the first place -- that remains a separate, not-fully-root-caused issue, out of this fix's scope. Recommend re-running `run-ha-soak.sh --config-upgrade-wal-trim` (config-upgrade-snapshot task 8.9) to confirm this defense-in-depth closes the observed WAL-segment-loss symptom even without (a) landing.\n\n[UPDATE 2026-07-26, post-review remediation] Independent review REJECTED the initial fix on one finding: the Pass 2 exhausted-fallback branch silently destroyed the reused slot's last_disconnected_lsn/last_disconnect_time via memset with zero logging, undercutting the fix's own stated goal of making the previously-silent WAL-retention-guard destruction visible/bounded -- this file already has an established convention for exactly this class of event (repl_broadcast's WARN(\"repl_broadcast: no standbys registered...\") and WARN(\"repl_broadcast: standby %s:%u evicted...\")). Remediated: repl_add_standby now tracks whether Pass 1 found a virgin slot (used_fallback_slot) and, only on the genuine Pass 2 fallback path, logs WARN(\"repl_add_standby: reusing guard-holding slot %d (host=%s:%u, last_disconnected_lsn=%lu) -- no virgin slot available, WAL-retention protection discarded\", ...) immediately before the memset, capturing the discarded slot's host/port/last_disconnected_lsn while they're still readable. No new test required (a log line has no observable return-value/state effect for a test to assert on) -- verified instead by confirming the WARN string is present in the built binary (`strings controller_unit_test | grep 'reusing guard-holding slot'`) and that both existing bug-1474 tests (test_repl_add_standby_prefers_virgin_slot_over_guard, test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect -- the latter specifically exercises the Pass 2 path this WARN sits on) still pass unchanged. GREEN: full controller_unit_test suite = 1033 tests (1030 baseline + bug-1472's 3 concurrently-landed tests), 0 failures. NOTE: a transient stale-object-file build artifact was observed once in this shared build directory during verification (a link picked up an out-of-date worker.c.o from a moment when a concurrent sibling agent's build was in flight, showing 2 spurious failures unrelated to this change) -- resolved by touching the affected sources to force recompilation; not a defect in this fix, noted here in case it recurs for a future concurrent multi-agent session in this worktree.\n\n[UPDATE 2026-09-16, live re-observation during controller-status task 8.6's live verification, NOT independently fixed here -- out of that task's boundary] Directly observed fix angle (a)'s symptom again, live, on a fresh 3-node cluster (/tmp/xce-deploy-ctrlstatus86): killed standby ctrl-2 (localhost:9992) only; ctrl-3 (localhost:9993) was never killed and its own log shows zero RDHUP/disconnect/reconnect lines for the entire session. ~2s after the ctrl-2 kill, ctrl-1's log shows `ha_connect_finish_peer: HA: peer[2] (localhost) not connected -- initiating outbound connect` for ctrl-3 (peer[2]) -- a peer that was continuously, genuinely connected the whole time -- followed by `on_standby_connected: HA: repl registered late-joiner standby peer[2] (slot 2, localhost:9993)`, consuming what would otherwise have been a virgin slot. Zero repl_remove_standby/evict/duplicate log lines appear anywhere in the entire run, so the angle-a on_standby_connected eviction-check fix (2026-08-16) either did not fire for this specific reconnect path or the original promoted-vote-channel slot for ctrl-3 was left live alongside the new one. Direct downstream effect: when the genuinely-killed ctrl-2 reconnected ~1 minute later, repl_add_standby's Pass 1 found 'no virgin slot available' (logged) and fell back to Pass 2, reusing ctrl-2's OWN prior guard-holding slot rather than landing in a fresh slot -- i.e. this spurious ctrl-3 duplicate-connect is what consumed the third MAX_STANDBYS slot in this run. Not investigated further (root-causing ha.c's reconnect-sweep peer-liveness check is explicitly out of controller-status task 8.6's boundary, which owns only ctrlstatus_fill's read-side status derivation, not connection-management) -- reported as a CONCERN alongside task 8.6's own fix (bug-1819) rather than silently worked around. task 8.6's ctrlstatus_fill fix tolerates this correctly either way (single-slot reuse or a genuine duplicate-live-slot pair both resolve to the right status), so it did not block that task, but fix angle (a) itself remains open.", "tags": [ @@ -17118,7 +5671,6 @@ "out-of-scope", "wal-trim", "mid-catch-up", - "fixed", "defense-in-depth", "log-hygiene", "review-remediation", @@ -17130,48 +5682,24 @@ "bug-1472", "bug-1819" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1472-remainder", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-07-25T20:15:00Z", - "error_message": "Follow-up split out of bug-1472: HOOK_DETAIL_WORKER_DEAD entries applied via WAL_WORKER_CTRL against an already-existing worker still return -EEXIST (no-op) -- a dead-worker transition detected by one member (mark_worker_unavail(), worker.c:661-679) still never replicates its state to other members (standbys, or this member's own WAL replay), unconditionally. NOT a regression: this was already true before bug-1472's investigation; bug-1472's fix deliberately scoped itself to the byte-precise-confirmed HOOK_DETAIL_WORKER_UPDATE case only and left DEAD as -EEXIST/no-op, unchanged.", - "file": "src/controller/worker.c worker_from_capnp (~827-895, post-bug-1472-fix); mark_worker_unavail (~661-679); src/schema/controller.capnp struct Worker (~136-144)", - "root_cause": "mark_worker_unavail()'s mutation is NOT confined to XCE_WORKER_ADMIN_MASK the way worker_ctrl()'s is -- it sets XCE_WORKER_UNAVAIL (a non-admin bit), dead_since, resets hostattrs_t to 'unknown' sentinel strings, AND zeroes the live per-host resource-value slots via set_host_slots()/update_resvals(), which mutates xce_worker_t.resvals (a zarray_t of live xce_val_t entries). Critically, w->host.resvals is NOT part of the Worker capnp wire schema at all (confirmed by reading src/schema/controller.capnp's struct Worker and the generated decode_Worker/encode_Worker in controller.capnp.c) -- only the separate resvals_to_sched/nresvals_to_sched fields (a different, scheduler-push-facing concept) are encoded. So even applying a simple 'replace admin-masked bits' fix analogous to bug-1472's UPDATE fix would NOT fully replicate a DEAD transition's effects: the resource-value slot zeroing would still be silently lost on every other member, which matters materially at PROMOTION time (a newly-promoted primary would read stale, non-zeroed slots for a worker that died before promotion, potentially over-scheduling a dead host). A correct fix needs either (a) a schema change adding a wire-encodable resource-value-slots field so the DEAD payload can carry it, or (b) a different replication strategy for DEAD (e.g. re-deriving 'this worker's connection is gone locally' independently on each member rather than replicating a mutation), which is a design decision beyond a same-task patch.", - "fix": "FIXED (TDD RED->GREEN), design direction chosen by user: wire-encode the resource-value-slot zeroing rather than a non-replicated per-member re-derivation. Investigation revised the original design framing: NO new capnp schema field was actually needed. capnp_encode_worker (persist_worker_hook_fun) already serializes the FULL xce_worker_t.host -- including `attrs` (HostAttrs) -- on every WAL_WORKER_CTRL entry, and mark_worker_unavail() calls reset_hostattrs_unknown() BEFORE firing the hook that triggers that encode, so a DEAD entry's wire payload already carries the post-reset 'unknown'/0 attrs state. The actual gap was narrower: worker_from_capnp() had no DEAD branch at all (fell through to the JOIN-duplicate -EEXIST no-op), so nothing ever APPLIED what was already on the wire. Fixed by adding a DEAD branch (worker.c) mirroring the existing UPDATE branch (bug-1472's own fix): sets XCE_WORKER_UNAVAIL, takes dead_since from the decoded payload's deadSince field (the PRIMARY's own death timestamp -- genuinely wire-driven, not locally re-derived, honoring the chosen design direction), and calls reset_hostattrs_unknown()+add_hostattrs_resvals()+set_host_slots(0) locally to re-derive the resvals from the now-reset attrs -- these three calls are deterministic/data-independent (same effect mark_worker_unavail() itself produces on the primary), so xce_worker_t.resvals (the zarray_t the original investigation worried about) never needs to cross the wire at all. Regression test (test_apply_committed_entry.c's test_ace_worker_ctrl_dead_applies_to_existing_worker): pre-existing worker with live (non-'unknown') attrs receives a DEAD WAL entry with a specific non-zero deadSince -- RED under the pre-fix code (rc=-EEXIST, no mutation, no notification, the exact silent-discard this entry describes), GREEN under the fix (XCE_WORKER_UNAVAIL set, dead_since matches the payload exactly, hostattrs reset to 'unknown'/0, one notification fires). Full controller_unit_test suite: 1101 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED -- deliberately deferred, investigated not skipped (see bug-1472's own fix note for the full investigation). Needs its own design decision on how (or whether) to wire-encode the resource-value-slot zeroing, or an alternative non-replicated detection strategy, before a fix can be written safely.", - "tags": [ - "worker.c", - "worker_from_capnp", - "mark_worker_unavail", - "WAL_WORKER_CTRL", - "HOOK_DETAIL_WORKER_DEAD", - "capnp-schema-gap", - "resource-value-replication", - "standby-divergence", - "post-promotion-scheduling-risk", - "needs-owner", - "needs-design-decision", - "not-fixed", - "deferred", - "fixed" + "fix_commits": [ + "c3a17b19" ], - "related_bugs": [ - "bug-1472" - ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1474.md" }, { "id": "bug-1475", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-26T04:36:00Z", - "error_message": "Live config-upgrade-snapshot task 8.6 soak (`run-ha-soak.sh --config-upgrade`), run to verify bug-1472's fix, FAILED \u2014 but NOT at bug-1472's own assertions (materialized-state checkpoint byte-identity and active-configuration identity both PASSED cleanly across all 3 members, confirming bug-1472's fix holds). Failure occurred one step later, inside the Requirement 6.4 host-admin-status-rotation check's own `force_elect_node(\"ctrl-1\")` maneuver: after 3 rounds of overlapping chaotic kills + a live config upgrade all completed and the cluster reconverged 3-of-3, `force_elect_node` killed the spoiler (ctrl-3) and tried to submit a job batch on the surviving {ctrl-2 primary, ctrl-1 standby} pair \u2014 5 consecutive attempts all got 0/3 acked (\"waiting for write-quorum\"), and the round hard-failed: `[xce-test-ha] FAIL: force-elect ctrl-1: submit on {ctrl-2,ctrl-1} failed`. The Req 6.4 workerlist/admin-status assertion itself was never reached.", - "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby, ha_consume_append_entries, ha_evaluate_repl_action \u2014 HA_REPL_STREAM_WAL decision/dispatch loop); src/controller/config_repl.c (ha_stream_wal_to_standby catch-up streamer)", - "root_cause": "NOT data loss, NOT bug-1472 (worker/host admin-status replication is unaffected \u2014 unrelated subsystem, and its own checks passed). This is a standby catch-up LIVENESS stall: ctrl-1 (which had itself served as primary in term=2 before being killed in round 1 of the chaotic sequence, then restarted) got permanently stuck at applied_lsn=12 and never advanced again for the rest of the run. ctrl-1's own log (`ha_consume_append_entries`) shows a revealing pattern: a zero-entry PROBE AppendEntries at prevIdx=12,n=0 correctly succeeds with match=12 (confirming index 12 IS ctrl-1's true, correct log tip), but the very next entry-carrying AppendEntries from the SAME dispatch cycle uses prevIdx=13 (not 12) for n=1, which ctrl-1 correctly rejects with event=ae_reject_logmismatch (success=0 match=0) since it has no entry at index 13 yet to check a predecessor against. This 1-off prevIdx computation (using prevIdx=standby_applied_lsn+1 instead of prevIdx=standby_applied_lsn for the FIRST entry of a catch-up batch) means every retry of the SAME heartbeat-driven HA_REPL_STREAM_WAL decision (ha_evaluate_repl_action: standby_lsn(12) < committed_lsn(31) -> stream) recomputes and resends the IDENTICAL doomed range, forever, with zero backtrack/self-correction -- confirmed via ctrl-2's log: 'HA repl: standby[0] lsn=12 < committed_lsn=31 -> stream WAL batches' repeats verbatim every ~3s from 04:31:26 through 04:32:26 (60+ seconds, 20+ identical retries, zero progress) after ctrl-2 had already logged 'repl_broadcast: standby localhost:9991 evicted (pending_ack_count=17 > 16) -- triggering catch-up reconnect' and 'HA quorum lost: connected=0 required=1'. Once the soak's own force_elect_node killed the OTHER standby (ctrl-3) as its spoiler maneuver, ctrl-1 became the sole remaining standby, and its permanent inability to catch up meant the primary could never regain 2-of-3 write quorum again -- every subsequent job submission timed out (`commit: async continuation deadline expired`, `jobsubmit_commit_continuation: lsn=N commit failed/timed out (rc=-110)`), exactly matching the soak's observed '0/3 acked; waiting for write-quorum' x5 before hard-failing. Superficially similar symptom signature to already-FIXED bug-1028 (same 'ha_dispatch_repl_action_for_standby: standby[0] lsn=X < committed_lsn=Y -> stream WAL batches' log line looping indefinitely post-failover) but a DIFFERENT specific AE-reject reason -- bug-1028 was a logOk TERM-mapping mismatch (raw hdr.term vs persist_entry_term legacy mapping), fixed 2026-06-23; this is `ae_reject_logmismatch`, which reads like a prevLogIndex/off-by-one issue in how the catch-up streamer picks the FIRST entry's predecessor index, not a term problem -- so this is either a narrower unfixed variant of the same family, or a new regression in a related code path. NOT root-caused down to the exact defective line (this investigation was a live-soak VERIFICATION run for a DIFFERENT bug, not a fix task) -- flagging for whoever picks this up next to read ha_dispatch_repl_action_for_standby/ha_stream_wal_to_standby's prevIdx computation directly and compare against ctrl-1's actual persisted log at lsn=12/13 (waldump). Also notable: this specific soak step (force_elect_node's spoiler-kill-then-submit maneuver, previously only exercised by --sequential mode's post-loop 'leadership coverage' step against an otherwise-calm, fully-settled cluster) had never previously been run immediately after --config-upgrade's THREE rounds of aggressive overlapping chaotic kills plus a live config upgrade -- this specific combination/history may be what's needed to reproduce the stall, rather than it being config-upgrade-specific.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Live config-upgrade-snapshot task 8.6 soak (`run-ha-soak.sh --config-upgrade`), run to verify…\n\nLive config-upgrade-snapshot task 8.6 soak (`run-ha-soak.sh --config-upgrade`), run to verify bug-1472's fix, FAILED — but NOT at bug-1472's own assertions (materialized-state checkpoint byte-identity and active-configuration identity both PASSED cleanly across all 3 members, confirming bug-1472's fix holds). Failure occurred one step later, inside the Requirement 6.4 host-admin-status-rotation check's own `force_elect_node(\"ctrl-1\")` maneuver: after 3 rounds of overlapping chaotic kills + a live config upgrade all completed and the cluster reconverged 3-of-3, `force_elect_node` killed the spoiler (ctrl-3) and tried to submit a job batch on the surviving {ctrl-2 primary, ctrl-1 standby} pair — 5 consecutive attempts all got 0/3 acked (\"waiting for write-quorum\"), and the round hard-failed: `[xce-test-ha] FAIL: force-elect ctrl-1: submit on {ctrl-2,ctrl-1} failed`. The Req 6.4 workerlist/admin-status assertion itself was never reached.", + "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby, ha_consume_append_entries, ha_evaluate_repl_action — HA_REPL_STREAM_WAL decision/dispatch loop); src/controller/config_repl.c (ha_stream_wal_to_standby catch-up streamer)", + "root_cause": "NOT data loss, NOT bug-1472 (worker/host admin-status replication is unaffected — unrelated subsystem, and its own checks passed). This is a standby catch-up LIVENESS stall: ctrl-1 (which had itself served as primary in term=2 before being killed in round 1 of the chaotic sequence, then restarted) got permanently stuck at applied_lsn=12 and never advanced again for the rest of the run. ctrl-1's own log (`ha_consume_append_entries`) shows a revealing pattern: a zero-entry PROBE AppendEntries at prevIdx=12,n=0 correctly succeeds with match=12 (confirming index 12 IS ctrl-1's true, correct log tip), but the very next entry-carrying AppendEntries from the SAME dispatch cycle uses prevIdx=13 (not 12) for n=1, which ctrl-1 correctly rejects with event=ae_reject_logmismatch (success=0 match=0) since it has no entry at index 13 yet to check a predecessor against. This 1-off prevIdx computation (using prevIdx=standby_applied_lsn+1 instead of prevIdx=standby_applied_lsn for the FIRST entry of a catch-up batch) means every retry of the SAME heartbeat-driven HA_REPL_STREAM_WAL decision (ha_evaluate_repl_action: standby_lsn(12) < committed_lsn(31) -> stream) recomputes and resends the IDENTICAL doomed range, forever, with zero backtrack/self-correction -- confirmed via ctrl-2's log: 'HA repl: standby[0] lsn=12 < committed_lsn=31 -> stream WAL batches' repeats verbatim every ~3s from 04:31:26 through 04:32:26 (60+ seconds, 20+ identical retries, zero progress) after ctrl-2 had already logged 'repl_broadcast: standby localhost:9991 evicted (pending_ack_count=17 > 16) -- triggering catch-up reconnect' and 'HA quorum lost: connected=0 required=1'. Once the soak's own force_elect_node killed the OTHER standby (ctrl-3) as its spoiler maneuver, ctrl-1 became the sole remaining standby, and its permanent inability to catch up meant the primary could never regain 2-of-3 write quorum again -- every subsequent job submission timed out (`commit: async continuation deadline expired`, `jobsubmit_commit_continuation: lsn=N commit failed/timed out (rc=-110)`), exactly matching the soak's observed '0/3 acked; waiting for write-quorum' x5 before hard-failing. Superficially similar symptom signature to already-FIXED bug-1028 (same 'ha_dispatch_repl_action_for_standby: standby[0] lsn=X < committed_lsn=Y -> stream WAL batches' log line looping indefinitely post-failover) but a DIFFERENT specific AE-reject reason -- bug-1028 was a logOk TERM-mapping mismatch (raw hdr.term vs persist_entry_term legacy mapping), fixed 2026-06-23; this is `ae_reject_logmismatch`, which reads like a prevLogIndex/off-by-one issue in how the catch-up streamer picks the FIRST entry's predecessor index, not a term problem -- so this is either a narrower unfixed variant of the same family, or a new regression in a related code path. NOT root-caused down to the exact defective line (this investigation was a live-soak VERIFICATION run for a DIFFERENT bug, not a fix task) -- flagging for whoever picks this up next to read ha_dispatch_repl_action_for_standby/ha_stream_wal_to_standby's prevIdx computation directly and compare against ctrl-1's actual persisted log at lsn=12/13 (waldump). Also notable: this specific soak step (force_elect_node's spoiler-kill-then-submit maneuver, previously only exercised by --sequential mode's post-loop 'leadership coverage' step against an otherwise-calm, fully-settled cluster) had never previously been run immediately after --config-upgrade's THREE rounds of aggressive overlapping chaotic kills plus a live config upgrade -- this specific combination/history may be what's needed to reproduce the stall, rather than it being config-upgrade-specific.", "fix": "FIXED (2026-07-27/28, via a regression test + real-instrumentation debugging session that continued this entry's original live-soak finding). Landed as THREE coordinated changes, all required together -- the first two alone left the new regression test (test_bug1475_catchup_survives_midstream_checkpoint, test_config_repl.c) RED with applied_lsn stuck at 0 for the whole run:\n\n1. SEND side (config_repl.c, ha_stream_wal_to_standby): when scanning the WAL for entries to stream, a WAL_CHECKPOINT entry is still never built/sent as its own catchup entry, but the prev_log_index/prev_log_term tracker now advances across it like every other scanned entry, instead of freezing one entry behind. A standby that independently applied the same committed WAL_CONFIG_MARKER writes its OWN local checkpoint at the identical LSN (deterministic per-member trigger), so the checkpoint's LSN is a real, held position on both sides, not a gap to skip past chaining-wise.\n\n2. RECEIVE side (ha.c, ha_apply_term_entry_via_library): a JIT resync reads persist_durable_floor(p) and monotonically raises ha->raft.log_len to match before consuming each AppendEntries -- mirroring ha_sync_raft_log_from_recovery's boot-time computation, but applied live on the AE-receive path. This corrects for persist_write_checkpoint_marker (bug-1466, still not fixed at ITS root) advancing persist_t's current_lsn without going through raft_log_append, which otherwise leaves ha->raft.log_len stale and fails logOk's prev_log_index<=log_len bound forever.\n\n3. TERM-INDEX side (persist.c, record_entry_term/entry_term_locked) -- a THIRD, previously-undiscovered gap, found only via live fprintf instrumentation of the new regression test after fixes 1-2 above still left it RED. Fixes 1-2 both depend on persist_entry_term(checkpoint_lsn) reading back 'absent' (0) FOREVER on every member (the explicit, twice-documented design intent in both fixes' own comments and in persist_write_checkpoint_marker's comment). That invariant was actually FALSE: record_entry_term's geometric-growth realloc zero-filled any never-explicitly-recorded gap slot (like a checkpoint's own LSN, which is deliberately never passed to record_entry_term), and entry_term_locked's `lsn <= term_index_count` presence check then read that zero-fill back as WAL_LEGACY_TERM (1), not absent (0), the INSTANT any LATER lsn was recorded and grew term_index_count past the gap -- which happens immediately on a live leader (it appends real entries right after the checkpoint) but not yet on a receiver that has not itself appended past its own checkpoint. So the checkpoint LSN's 'term' was TIME-DEPENDENT: the sender (which streams AFTER its own later entries exist) computed prev_log_term=1 for the entry right after the checkpoint, while the receiver's honest, not-yet-grown term index still (correctly, per design) read 0 for the same LSN -- logOk's equality check spuriously mismatched (0 != 1) and rejected forever, exactly reproducing this entry's original ae_reject_logmismatch signature. Confirmed with real fprintf instrumentation at the streamer, the JIT resync, and raft.c's logOk check (added and removed during this investigation, not left in the tree): prev_log_term flipped from 0 to 1 in the SAME test run, immediately after the leader appended its 2nd post-checkpoint entry. Fixed with a TERM_INDEX_UNSET sentinel (UINT64_MAX) that record_entry_term's growth path now fills gap slots with (instead of 0), and entry_term_locked treats as absent -- so a never-explicitly-written slot reads absent PERMANENTLY, independent of later array growth. Confined to persist.c's two functions plus the term_index doc comment in persist.h; no other call site reads p->term_index[] directly.\n\nAlso completed as part of this fix: (a) audited whether WAL entry types OTHER than WAL_CHECKPOINT bypass record_entry_term the same way -- confirmed NO: WAL_CHECKPOINT (value 100) is the numeric max of wal_entry_type, persist_wal_async and persist_wal_write_replicated both explicitly reject type==WAL_CHECKPOINT up front (it cannot reach record_entry_term via either live path), and replay_wal's `hdr.type < WAL_CHECKPOINT` recording guard already covers every other currently-defined type (WAL_JOB_UPDATE=7, WAL_QUEUE_CTRL=20, WAL_WORKER_CTRL=30, WAL_NOOP=40, WAL_CONFIG_MARKER=50); (b) confirmed no interaction with the RequestVote/PreVote freshness gates (raft.c raft_log_ok, shared by raft_on_vote_request/raft_on_pre_vote_request): that gate also reads on_get_entry_term(s->log_len), which can legitimately land on a checkpoint LSN after the JIT resync in fix 2 -- the TERM_INDEX_UNSET fix makes that read deterministic (always absent) instead of time-dependent, which is a consistency IMPROVEMENT for vote freshness too, not a behavior change in the unsafe direction; full test_ha_prevote.c/test_ha_failover_prevote.c suites pass unchanged. Full regression suite: 1034 Tests, 0 Failures (was 1033/1034 with the new test failing before this fix). Files changed: src/controller/config_repl.c (REAL logic change, landed earlier in this same bug-1475 investigation: ha_stream_wal_to_standby's WAL_CHECKPOINT branch now falls through to the unconditional prev-tracker advance via `hdr.lsn > standby_lsn && hdr.type != WAL_CHECKPOINT`, instead of a hard `continue` that froze the tracker one entry behind -- plus cross-reference comments added in this later session); src/controller/ha.c (REAL logic change, also landed earlier in this investigation: a ~30-line JIT resync block added to the top of ha_apply_term_entry_via_library that raises ha->raft.log_len from persist_durable_floor(p) before consuming each AppendEntries -- plus cross-reference comments added in this later session); src/controller/persist.c (the TERM_INDEX_UNSET sentinel fix + doc comments, the new logic landed in THIS session); src/controller/testsuite/test_config_repl.c (test_bug1475_catchup_survives_midstream_checkpoint + updated doc comments describing all three fix parts). Correction (coordinator review 2026-07-27/28, Finding 1): an earlier version of this fix summary mischaracterized config_repl.c's and ha.c's changes as 'comment-only, no logic change' -- both carry genuine logic changes; they were simply written in an earlier part of this same bug-1475 investigation than the TERM_INDEX_UNSET sentinel (session 3 of the fix), not left unmodified by it.\n\nFinding 2 (coordinator review, 2026-07-27/28, CONFIRMED REAL and FIXED): after the TERM_INDEX_UNSET fix, the reviewer traced a plausible NEW instance of the identical permanent-wedge bug class, reachable via the SNAPSHOT-INSTALL catch-up path (HA_REPL_SEND_SNAPSHOT, dispatched whenever a standby has fallen too far behind for incremental WAL streaming -- a live, common path) instead of WAL-streaming catch-up. ha_snapshot_do_send (ha.c) computes checkpoint_term = persist_entry_term(leader, c->persist.checkpoint_lsn) and ships it on the wire; the receiver's ha_handle_snapshot_chunk calls persist_reconcile_checkpoint_term(p, checkpoint_lsn, checkpoint_term), which called record_entry_term(p, checkpoint_lsn, checkpoint_term) directly -- an EXPLICIT write, not a growth-fill, so it never got the TERM_INDEX_UNSET sentinel even when checkpoint_term==0. entry_term_locked's pre-existing legacy-0 rule then read that raw 0 back as WAL_LEGACY_TERM(1), forever -- permanently diverging from the leader's own permanently-absent(0) value for the SAME lsn whenever c->persist.checkpoint_lsn (== last_applied_lsn at general persist_checkpoint() time, NOT the WAL_CHECKPOINT entry's own physical lsn) happens to coincide with a checkpoint's own un-recorded lsn -- a real, reachable quiescent-state timing (the apply pump has walked, one lsn at a time including a no-op pass over checkpoint slots, all the way to exactly the checkpoint's own position before the next persist_checkpoint()+snapshot-send cycle runs, e.g. right after a config upgrade with no immediate follow-up activity) -- not the common case, but real and worth closing. Proven with a dedicated regression test (test_bug1475_finding2_snapshot_install_checkpoint_term_absent, test_config_repl.c) that drives the REAL ha_send_snapshot_to_standby -> ha_handle_snapshot_chunk -> persist_reconcile_checkpoint_term path (via the xce_snapshot_chunk_hook test seam) with a leader whose checkpoint_lsn is deliberately seeded (via last_applied_lsn) to coincide with its checkpoint's own physical lsn -- confirmed RED by temporarily reverting only persist_reconcile_checkpoint_term's fix (receiver read back term=1, not the expected absent=0) and GREEN after restoring it. Fixed with the SAME TERM_INDEX_UNSET sentinel: persist_reconcile_checkpoint_term now substitutes TERM_INDEX_UNSET for a wire checkpoint_term of exactly 0 before calling record_entry_term, instead of recording the raw 0 -- a wire value of 0 is unambiguous today (ha_snapshot_do_send is the ONLY sender and always computes it via persist_entry_term, whose public contract already returns a DIFFERENT value, WAL_LEGACY_TERM, for a present-but-legacy-0 entry -- so a public 0 can only mean genuinely, permanently absent). A genuinely non-zero checkpoint_term (any real historical term) is still recorded verbatim and still correctly overwrites a stale local term at that LSN, unchanged -- the pre-existing STALE-TERM reconcile test (test_ha_core_safety.c's test_3_1_snapshot_install_does_not_reconcile_boundary_term, CHECKPOINT_TERM=7, a synthetic non-zero value) is unaffected and still passes. Also fixed the stale 'zeroed-on-grow' comment at persist.c's truncate_term_index_locked (grown slots are TERM_INDEX_UNSET-filled since the original TERM_INDEX_UNSET fix, not zero-filled -- comment had not been updated). Re-verified RequestVote/PreVote freshness non-interference for this second fix too: persist_reconcile_checkpoint_term's caller (ha_handle_snapshot_chunk) never touches ha->raft.log_len or any vote-path state; the fix only changes what value entry_term_locked returns for a checkpoint-boundary LSN after a snapshot install (now consistently absent instead of inconsistently WAL_LEGACY_TERM), which is the SAME direction of improvement (deterministic absence vs time/path-dependent) the original TERM_INDEX_UNSET fix already established as safe for the freshness gate. Full suite: 1035 Tests, 0 Failures (was 1034 before this new test).", "tags": [ "ha.c", @@ -17189,27 +5717,29 @@ "bug-1472-fix-verified-good", "same-family-as-bug-1028", "needs-owner", - "root-caused-partially", - "not-fixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "root-caused-partially" ], "related_bugs": [ "bug-1472", "bug-1028" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1475.md" }, { "id": "bug-1476", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-07-26T04:44:00Z", - "error_message": "Live config-upgrade-snapshot task 8.9 re-verification soak (run-ha-soak.sh --config-upgrade-wal-trim, run to confirm bug-1474's repl.c fix), deploy /home/claude/tmp/xce-ha-waltrim3: ctrl-3 ABORTed via AddressSanitizer heap-use-after-free during the round's live config-upgrade submission, well before TARGET(ctrl-1) was ever restarted. The round's own bug-1474 WAL-trim assertions (floor segment preserved, below-floor segments trimmed, active segment present) all PASSED cleanly before this crash was noticed -- ctrl-3's death was only discovered because it, combined with a second unrelated issue (bug-1475), prevented the round's final '3-of-3 reconvergence' assertion from ever succeeding.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Live config-upgrade-snapshot task 8.9 re-verification soak (run-ha-soak.sh…\n\nLive config-upgrade-snapshot task 8.9 re-verification soak (run-ha-soak.sh --config-upgrade-wal-trim, run to confirm bug-1474's repl.c fix), deploy /home/claude/tmp/xce-ha-waltrim3: ctrl-3 ABORTed via AddressSanitizer heap-use-after-free during the round's live config-upgrade submission, well before TARGET(ctrl-1) was ever restarted. The round's own bug-1474 WAL-trim assertions (floor segment preserved, below-floor segments trimmed, active segment present) all PASSED cleanly before this crash was noticed -- ctrl-3's death was only discovered because it, combined with a second unrelated issue (bug-1475), prevented the round's final '3-of-3 reconvergence' assertion from ever succeeding.", "file": "src/controller/config_repl.c (load_primary_config ~line 656-666 free_ClusterConfig/free(myconf->cluster_config) call site; config_activate_committed ~line 360; handle_repl_config_push ~line 1280); lib/controller.capnp.c (decode_ClusterConfig_ptr ~line 17074 calloc site, decode_Resource_list ~line 4057 crash site)", "root_cause": "AddressSanitizer heap-use-after-free, WRITE of size 8, in decode_Resource_list (lib/controller.capnp.c:4057), thread T5. Full ASAN trace (ctrl-3.stderr):\nWRITE by T5: decode_Resource_list <- decode_ClusterConfig <- decode_ClusterConfig_ptr <- load_cluster_config_from_file (config_repl.c:606) <- load_primary_config (config_repl.c:664) <- config_activate_committed (config_repl.c:360) <- handle_repl_config_push (config_repl.c:1280) <- myreq_handler (cluster.c:654) <- vtask_entry (cluster.c:786)\nFREED by T6: load_primary_config (config_repl.c:659, the 'free any prior decoded config before overwriting the pointer' step: free_ClusterConfig(myconf->cluster_config); free(myconf->cluster_config)) <- config_activate_committed (config_repl.c:360) <- handle_repl_config_push (config_repl.c:1280) <- myreq_handler <- vtask_entry\nALLOCATED by T5: calloc in decode_ClusterConfig_ptr (lib/controller.capnp.c:17074) <- load_cluster_config_from_file (config_repl.c:606) <- load_primary_config (config_repl.c:664) <- config_activate_committed <- handle_repl_config_push\nThis is a genuine, unguarded data race: load_primary_config's sequence is (a) if myconf->cluster_config is non-NULL, free it and NULL the pointer, then (b) decode a fresh ClusterConfig into myconf->cluster_config via load_cluster_config_from_file/decode_ClusterConfig_ptr. decode_ClusterConfig_ptr appears to publish the new struct pointer into *config (i.e. myconf->cluster_config) before it has finished populating nested fields (e.g. the Resource_list), since T5 is later observed still writing into the struct via decode_Resource_list AFTER T6 already saw a non-NULL myconf->cluster_config, freed it (T6's own step (a)), and proceeded to decode its own replacement. No mutex protects myconf->cluster_config across this read-free-decode sequence, so two overlapping invocations of handle_repl_config_push -> config_activate_committed -> load_primary_config for the same node race on the shared myconf->cluster_config pointer. This matches the class of concern already flagged (but not as a crash) in bug-976/bug-981 (handle_repl_config_push / config_activate_committed activation-ordering races) -- this is a NEW, more severe manifestation: an actual heap-corruption crash, not just an ordering/verification gap. Plausible trigger in this specific soak: heavy duplicate/late-joiner reconnect churn was independently observed on ctrl-2 (the primary) around the same window (bug-1474's WARN 'reusing guard-holding slot' fired twice, confirming the still-open, out-of-scope ha.c heartbeat-reconnect-sweep defect -- bug-1474's un-addressed 'fix angle (a)' -- was actively spawning duplicate/late-joiner standby connections during this round). If the primary sent the same config-push payload to ctrl-3 twice (once per duplicate connection), ctrl-3 could dispatch two concurrent handle_repl_config_push invocations on separate VTASK_LIVE_BLOCK workers, which is consistent with the observed T5/T6 race. This causal link (duplicate-connection churn -> duplicate config-push delivery -> concurrent load_primary_config race) is a plausible hypothesis from timeline correlation, not confirmed by direct evidence of two inbound config-push messages -- flagged as a hypothesis for whoever investigates, not a proven mechanism.", - "fix": "FIXED (2026-07-28). Two-part fix in load_primary_config() (config_repl.c), independently reviewed and verified via a real pthread-based deterministic race test: (1) added myconf_t.lock (mylock_t, mirrors replicator_t.lock's idiom in repl.c) serializing the entire free-decode-publish sequence against any concurrent load_primary_config() call for the same myconf_t -- initialized in main.c before first use, finalized in myconf_fini(), and in the t43_attach_myconf/t43_free_myconf test fixture. (2) load_cluster_config_from_file() now decodes into a LOCAL temporary (cluster_config_t *tmp), never directly into the shared myconf->cluster_config field -- decode_ClusterConfig_ptr (lib/controller.capnp.c) calloc's and publishes its pointer via the out-parameter BEFORE decode_ClusterConfig populates nested fields (e.g. resources), so decoding into a local closes the early-publish exposure at its source; myconf->cluster_config is only swapped in after the new config is fully decoded and validated, still under myconf->lock. Part (2) is independently necessary because several other readers (cluster.c, queue.c, sched.c, worker.c, apiserver.c, ctrlstatus.c, reply.c) read c->myconf->cluster_config without ever taking myconf->lock, so the field itself must never be observably half-decoded. New regression test test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free (test_config_repl.c) drives two REAL concurrent pthreads through load_primary_config() for the same myconf_t via a deterministic test-only hook (load_primary_config_race_hook, XCE_TESTING-only, NULL/no-op in production) that parks the first thread exactly at the retire-old-config step the live crash raced -- pre-fix this reliably reproduces a genuine ASan-catchable double-free from the same missing-lock root cause; post-fix myconf->lock keeps the second thread blocked for the entire duration, verified both by a captured never-raced-through flag and by the final published config being fully valid, never torn. Full suite: 1035 -> 1036 tests, 0 failures.\n\n[UPDATE 2026-07-28, kiro-impl config-upgrade-snapshot resume] Prior fix landed with a comment block (config_repl.c:639-646) admitting it had been reverted to a RED-phase reproduction (no lock, decode-directly-into-shared-field) for test-construction purposes and NEVER restored to GREEN \u2014 the mylock_init/fini wiring (myconf.h, main.c, myconf.c) landed, but load_primary_config() itself still ran fully unguarded. Full suite run confirmed: test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free FAILED (\"thread B... Expected 0 Was 1\"), i.e. B raced through while A was parked -- the lock was never acquired. Fixed for real: mylock_lock(&myconf->lock) now wraps the ENTIRE load_primary_config() body (acquired first, released at every return including success), and Cap-n-Proto decode now targets a local cluster_config_t *new_config first -- myconf->cluster_config is only freed+swapped after new_config is fully decoded and validated (reserved-name check, generation default), never decoded directly into the shared field. Verified 4/4 clean full-suite runs (1036 Tests 0 Failures 0 Ignored, ASAN_OPTIONS=detect_leaks=0) after the fix, including the specific race test. Committed as part of config-upgrade-snapshot task 8.6/8.9 unblock resume.", + "fix": "FIXED (2026-07-28). Two-part fix in load_primary_config() (config_repl.c), independently reviewed and verified via a real pthread-based deterministic race test: (1) added myconf_t.lock (mylock_t, mirrors replicator_t.lock's idiom in repl.c) serializing the entire free-decode-publish sequence against any concurrent load_primary_config() call for the same myconf_t -- initialized in main.c before first use, finalized in myconf_fini(), and in the t43_attach_myconf/t43_free_myconf test fixture. (2) load_cluster_config_from_file() now decodes into a LOCAL temporary (cluster_config_t *tmp), never directly into the shared myconf->cluster_config field -- decode_ClusterConfig_ptr (lib/controller.capnp.c) calloc's and publishes its pointer via the out-parameter BEFORE decode_ClusterConfig populates nested fields (e.g. resources), so decoding into a local closes the early-publish exposure at its source; myconf->cluster_config is only swapped in after the new config is fully decoded and validated, still under myconf->lock. Part (2) is independently necessary because several other readers (cluster.c, queue.c, sched.c, worker.c, apiserver.c, ctrlstatus.c, reply.c) read c->myconf->cluster_config without ever taking myconf->lock, so the field itself must never be observably half-decoded. New regression test test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free (test_config_repl.c) drives two REAL concurrent pthreads through load_primary_config() for the same myconf_t via a deterministic test-only hook (load_primary_config_race_hook, XCE_TESTING-only, NULL/no-op in production) that parks the first thread exactly at the retire-old-config step the live crash raced -- pre-fix this reliably reproduces a genuine ASan-catchable double-free from the same missing-lock root cause; post-fix myconf->lock keeps the second thread blocked for the entire duration, verified both by a captured never-raced-through flag and by the final published config being fully valid, never torn. Full suite: 1035 -> 1036 tests, 0 failures.\n\n[UPDATE 2026-07-28, kiro-impl config-upgrade-snapshot resume] Prior fix landed with a comment block (config_repl.c:639-646) admitting it had been reverted to a RED-phase reproduction (no lock, decode-directly-into-shared-field) for test-construction purposes and NEVER restored to GREEN — the mylock_init/fini wiring (myconf.h, main.c, myconf.c) landed, but load_primary_config() itself still ran fully unguarded. Full suite run confirmed: test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free FAILED (\"thread B... Expected 0 Was 1\"), i.e. B raced through while A was parked -- the lock was never acquired. Fixed for real: mylock_lock(&myconf->lock) now wraps the ENTIRE load_primary_config() body (acquired first, released at every return including success), and Cap-n-Proto decode now targets a local cluster_config_t *new_config first -- myconf->cluster_config is only freed+swapped after new_config is fully decoded and validated (reserved-name check, generation default), never decoded directly into the shared field. Verified 4/4 clean full-suite runs (1036 Tests 0 Failures 0 Ignored, ASAN_OPTIONS=detect_leaks=0) after the fix, including the specific race test. Committed as part of config-upgrade-snapshot task 8.6/8.9 unblock resume.", "tags": [ "config_repl.c", "load_primary_config", @@ -17226,7 +5756,6 @@ "new-crash", "needs-owner", "possibly-related-to-bug-1474-fix-angle-a-duplicate-reconnects", - "fixed", "mylock" ], "related_bugs": [ @@ -17235,15 +5764,20 @@ "bug-1474", "bug-1475" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1476.md" }, { "id": "bug-1477", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-28T08:25:00Z", - "error_message": "Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade), run on top of bug-1475's already-fixed and already-built-in commit ce8df990 (confirmed via binary mtime 08:16:27 postdating ce8df990 04:48:53Z and b08e392e 08:13:54Z, plus `strings` matching TERM_INDEX_UNSET/persist_durable_floor), FAILED: \"ctrl-3 has no on-disk checkpoint after the upgrade\". ctrl-1/ctrl-2 both have data/snap/snap-000000000000000b.capnp (51161 bytes, LSN 11 hex = the WAL_CONFIG_MARKER's own lsn); ctrl-3's data/snap/ is completely empty. ctrl-3's log shows a PERMANENT loop (repeats every ~3s/heartbeat for the entire ~3-minute run, never progressing): a bare heartbeat probe at prevIdx=11 succeeds (ae_already_have, match=11), but every walCatchupEntry at prevIdx=12/13/14 is rejected forever with ae_reject_logmismatch. Unlike bug-1475's own signature, this is NOT a transient race that self-heals once durable_lsn catches up -- it never recovers for the rest of the run.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade…\n\nLive config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade), run on top of bug-1475's already-fixed and already-built-in commit ce8df990 (confirmed via binary mtime 08:16:27 postdating ce8df990 04:48:53Z and b08e392e 08:13:54Z, plus `strings` matching TERM_INDEX_UNSET/persist_durable_floor), FAILED: \"ctrl-3 has no on-disk checkpoint after the upgrade\". ctrl-1/ctrl-2 both have data/snap/snap-000000000000000b.capnp (51161 bytes, LSN 11 hex = the WAL_CONFIG_MARKER's own lsn); ctrl-3's data/snap/ is completely empty. ctrl-3's log shows a PERMANENT loop (repeats every ~3s/heartbeat for the entire ~3-minute run, never progressing): a bare heartbeat probe at prevIdx=11 succeeds (ae_already_have, match=11), but every walCatchupEntry at prevIdx=12/13/14 is rejected forever with ae_reject_logmismatch. Unlike bug-1475's own signature, this is NOT a transient race that self-heals once durable_lsn catches up -- it never recovers for the rest of the run.", "file": "src/controller/myapp.c (~line 388-425, the \"leader already known -- skip own election\" fast path and its single guarded cluster_init_phase2 retry, bug-849 lineage); src/controller/ha.c (raft_cb_become_follower ~line 2000-2012, raft_cb_become_leader ~line 1772-1783, ha_elect_evaluate_and_signal ~line 5609-5629/5699-5710 -- all three unconditionally set ha->phase2_done=1 after calling cluster_init_phase2(c) without checking its return value); src/controller/cluster.c (cluster_init_phase2 ~line 1342-1598, the sole call site of commit_engine_init/persist_recover/ha_sync_raft_log_from_recovery); src/controller/persist.c (~line 796-799, on_durable_advance documented as registered ONLY by commit_engine_init)", "root_cause": "ROOT-CAUSED with byte/log-precise evidence, NOT config-upgrade-snapshot's own defect. ctrl-3 (my_priority=2, the highest-priority-number/most-deferred candidate in the 3-node cluster, per myapp_run's startup-stagger nanosleep) never ran its own raft candidacy for the ENTIRE test: zero occurrences of raft_cb_become_follower/raft_cb_become_leader/\"-> CTRL_STANDBY\" anywhere in ctrl-3.log. Instead, by the time its stagger delay elapsed, ctrl-1 (priority 0) had already voted/established itself, so myapp_run's own \"leader already known\" check (myapp.c ~343-390: voted_for_name set, or raft.term>0, or primary_ctrl_fd>=0) was true, and ctrl-3 took the `goto skip_election` fast path, entirely bypassing ha_start_election_async and therefore every raft-library election callback for the rest of its process lifetime (it only ever GRANTS votes to others afterward -- confirmed via repeated vote_granted/pre_vote_granted RAFT[p2] events with no corresponding become_follower/become_leader). The skip_election fast path (myapp.c ~415-425, bug-849, commit 20e59d0b, 2026-06-21 -- three-plus weeks before config-upgrade-snapshot's first commit 9e68335a) makes exactly ONE, correctly-guarded attempt at cluster_init_phase2(&c) (checks prc, only sets c.ha.phase2_done=1 on success) -- and at 08:18:26,930, that ONE attempt fired before c->myconf->cluster_config was populated (ctrl-3 had no local config file and had not yet received/activated a push), so cluster_init_phase2 returned -EINVAL immediately (cluster.c:1349-1358, before doing ANY of its real work) and the WARN 'cluster_init_phase2 on standby rejoin failed (rc=-22) -- WAL not recovered; falling back to election-path recovery' fired. phase2_done correctly stayed 0 -- but there is NO OTHER retry mechanism anywhere in the codebase for a node that (a) took the skip_election fast path and (b) never subsequently runs its own contested election: the only other three call sites that could retry (raft_cb_become_follower ha.c:2000-2012, raft_cb_become_leader ha.c:1772-1783, ha_elect_evaluate_and_signal's CTRL_PRIMARY/CTRL_STANDBY branches ha.c:5609-5629/5699-5710) never fire for ctrl-3 because it never engages the raft callback machinery at all. (Separately confirmed those three call sites ALSO unconditionally set ha->phase2_done=1 after calling cluster_init_phase2(c) with NO check of its return value -- unlike myapp.c's own careful guard -- a second, adjacent defect of the same class that did not trigger THIS failure only because ctrl-3 never reached them, but would identically strand any node whose FIRST raft-driven role transition happens to race ahead of config load.) Consequence, confirmed via grep: ctrl-3.log has ZERO occurrences of \"commit engine wired\", \"persist_recover\", \"xce_config_init\", or \"create_lost_and_found\" for the entire run (vs. multiple occurrences on ctrl-1/ctrl-2, once per boot/restart/re-election) -- cluster_init_phase2 (and therefore commit_engine_init, cluster.c:1484-1496) NEVER completed on ctrl-3. persist.c:796-799 documents on_durable_advance (the sole wake hook driving persist_apply_pump_run) as registered ONLY inside commit_engine_init -- so without it, persist_apply_pump_run/apply_committed_entry NEVER runs on ctrl-3, for ANY committed LSN, for the whole process lifetime. This means persist_checkpoint_marker_triggered (config-upgrade-snapshot task 5's own WAL_CONFIG_MARKER-triggered checkpoint, apply_committed_entry's WAL_CONFIG_MARKER case) never fires on ctrl-3 -- explaining precisely why its data/snap/ is empty while ctrl-1/ctrl-2 (whose commit engines DID wire) both hold the identical snap-000000000000000b.capnp. ctrl-3's config nonetheless activates cosmetically at 08:18:48,572 via a SEPARATE, independent code path (handle_repl_config_push -> load_primary_config -> config_activate_committed, an RPC push handler that does not touch the commit engine or apply pump at all) -- masking the underlying defect and making the soak's materialized-state/checkpoint assertions the first thing ever to surface it. bug-1475's three-part fix (config_repl.c's tracker-advance-across-WAL_CHECKPOINT, ha.c's JIT persist_durable_floor resync of ha->raft.log_len, persist.c's TERM_INDEX_UNSET sentinel) is necessary and correctly targets its own class of bug (a receiving standby's log_len bookkeeping merely LAGGING behind a checkpoint it DID independently write) but structurally cannot cover this failure: bug-1475's own fix explicitly documents and depends on the invariant 'a standby that independently applied the SAME committed WAL_CONFIG_MARKER writes its OWN local checkpoint at the identical LSN (deterministic per-member trigger)' -- ctrl-3 violates that precondition outright (it never writes the checkpoint AT ALL, not merely late), so the primary's catch-up stream's prev_log_index=12 (the checkpoint's own lsn, per bug-1475's now-correct sender tracking) checks against an entry ctrl-3 genuinely, permanently does not have. ae_reject_logmismatch is therefore a CORRECT rejection of ctrl-3's true log state, not a stale/racy one -- and since a WAL_CHECKPOINT entry is local-only/non-replicated BY DESIGN (persist_write_checkpoint_marker's own doc comment; persist_wal_write_replicated/persist_wal_async both explicitly reject type==WAL_CHECKPOINT), ctrl-3 can NEVER receive lsn=12 over the wire, and can never independently write it either (cluster_init_phase2/commit_engine_init never retries) -- a genuinely permanent, not merely time-bounded, wedge, exactly matching the soak's observed behavior (unlike bug-1475's transient race, this repeated with zero progress for the entire ~3-minute run). git blame confirms both implicated pre-existing defects (myapp.c's skip_election fast path, commit 20e59d0b bug-849 2026-06-21; ha.c's raft-callback phase2_done latch structure, commit 8f0d2d55 task 6.2 raft-library) predate config-upgrade-snapshot's first commit (9e68335a) by roughly a month -- this is a pre-existing HA bootstrap/election gap, NOT something config-upgrade-snapshot's task 5 introduced. Task 5 merely exposed it by creating a new cross-member structural dependency (every member independently checkpointing at a synchronized marker-boundary LSN) that a node whose commit engine/apply pump never wired silently violates. Before config-upgrade-snapshot, the same underlying gap would have left such a node's materialized state (jobmgr/qmgr/workermgr) permanently stale/un-applied with a much less visible failure signature (no crash, no wedge -- just silently wrong query results if that node were ever promoted), plausibly unnoticed by prior soaks that do not assert per-member materialized-state identity.", "fix": "FIXED (2026-07-28). Root cause: myapp_run's skip_election passive path, and 3 raft-callback call sites (raft_cb_become_leader, raft_cb_become_follower, ha_elect_evaluate_and_signal x2), all latched ha->phase2_done=1 UNCONDITIONALLY right after calling cluster_init_phase2 (or its ha_phase2_hook test seam), discarding its return value entirely -- so a single failed attempt (e.g. racing config availability, cluster_init_phase2 returning -EINVAL because cluster_config wasn't loaded yet) permanently stranded the member: commit_engine_init never ran, so apply_committed_entry never applied ANY committed WAL entry for the rest of that process's life, and -- for a node on the passive skip_election path specifically -- NOTHING else would ever retry it (the other 3 call sites only fire via this node's own raft-library state transitions, which never happen for a passive standby whose primary keeps heartbeating normally).\n\nFix: extracted a shared, idempotent retry mechanism -- ha_phase2_retry_fn (the blocking retry itself, capturing cluster_init_phase2/ha_phase2_hook's real return value, latching phase2_done ONLY on rc==0) plus ha_schedule_phase2_retry (a delaymgr-scheduled, SELF-RESCHEDULING trigger at heartbeat_interval_ms cadence -- unlike upgrade_materialization_retry/config_activate_retry, which are re-triggered externally by recurring commit-advance events, nothing recurring would otherwise ever re-trigger this for a passive skip_election node, so this one self-arms). Wired into all 5 call sites: myapp.c's skip_election block now schedules a retry instead of just WARNing on failure; raft_cb_become_follower and ha_elect_evaluate_and_signal's STANDBY branch likewise (safe to stay passive while retrying); raft_cb_become_leader and ha_elect_evaluate_and_signal's PRIMARY branch additionally REFUSE promotion on failure in raft_cb_become_leader specifically (mirroring the sibling upgrade_materialization_fenced refusal in the same function -- a promoted primary with an unwired commit engine would be far worse than a delayed promotion); the legacy ha_elect_evaluate_and_signal PRIMARY branch does NOT refuse (its ctx->done semaphore is already posted to the caller before this point in that function, so refusing would leave an inconsistent signal -- it proceeds to CTRL_PRIMARY and lets the self-rescheduling retry wire the commit engine shortly after instead).\n\nTDD: 4 new tests in test_config_repl.c (test_ha_phase2_retry_fn_noop_when_already_done, test_ha_phase2_retry_fn_does_not_latch_on_failure_and_succeeds_on_retry -- the core regression, RED confirmed by temporarily reverting the rc-gating back to an unconditional latch and observing the exact predicted failure -- test_schedule_phase2_retry_noop_when_already_done, test_schedule_phase2_retry_noop_when_delaymgr_not_running). GREEN: full controller_unit_test suite = 1044 tests, 0 failures, clean across multiple runs (including the RED/GREEN round-trip).\n\nLive-verified: re-ran run-ha-soak.sh --config-upgrade (the exact scenario that originally found this bug) -- ctrl-2's log shows the retry mechanism firing for real (4 failed cluster_init_phase2 attempts at ~3s intervals, rc=-22, then success: \"commit engine now wired (bug-1477)\"), and the soak's own checkpoint-identity, config-identity, and host-admin-status-survival assertions (the exact things bug-1477 blocked) all PASS. The soak run subsequently hit a DIFFERENT, seemingly unrelated failure later in its own force-elect-rotation step (repeated commit timeouts, rc=-110, on ctrl-1-as-primary) -- flagged separately, NOT part of this fix's scope; this soak scenario has never previously reached that step (earlier runs always failed earlier, at bug-1471/1472's checkpoint divergence or this bug's own catch-up wedge), so it may be a pre-existing, newly-exposed issue rather than a regression from this fix -- needs its own dedicated investigation.", @@ -17253,7 +5787,6 @@ "commit_engine_init", "cluster.c", "config-upgrade-snapshot-task-8.6", - "not-fixed", "marker-triggered-checkpoint", "out-of-scope-upstream", "skip_election", @@ -17262,7 +5795,6 @@ "ae_reject_logmismatch", "bug-849-lineage", "permanent-catchup-wedge", - "fixed", "myapp.c", "bug-1475-fix-verified-good", "ha.c", @@ -17274,15 +5806,20 @@ "bug-1466", "bug-849" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1477.md" }, { "id": "bug-1478", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-28T09:00:00Z", - "error_message": "config-upgrade-snapshot task 8.9 live soak (run-ha-soak.sh --config-upgrade-wal-trim ~/tmp/xce-ha-waltrim): bug-1474's floor-preservation fix confirmed working (floor segment seg-5 correctly preserved, all below-floor segments correctly trimmed) but the round still FAILED: \"cluster did not reconverge to 3-of-3 after restarting ctrl-1 -- the preserved floor segment did not actually suffice for real incremental catch-up\". ctrl-1 (TARGET, killed+restarted 3x) settles into a PERMANENT, zero-progress loop: its own raft AppendEntries exchanges cleanly reach match=22 with zero ae_reject events (heartbeat probe at prevIdx=22,n=0 -> success=1 match=22, repeating every ~3s forever), while ctrl-2 (primary) simultaneously and forever logs 'HA repl: standby[0] lsn=21 < committed_lsn=22 -> stream WAL batches' / 'ha_stream_wal_to_standby: streamed WAL [21+1..22] to standby' every heartbeat cycle with zero effect -- an off-by-one disagreement (21 vs 22) between the two sides that never reconciles.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "config-upgrade-snapshot task 8.9 live soak (run-ha-soak.sh --config-upgrade-wal-trim…\n\nconfig-upgrade-snapshot task 8.9 live soak (run-ha-soak.sh --config-upgrade-wal-trim ~/tmp/xce-ha-waltrim): bug-1474's floor-preservation fix confirmed working (floor segment seg-5 correctly preserved, all below-floor segments correctly trimmed) but the round still FAILED: \"cluster did not reconverge to 3-of-3 after restarting ctrl-1 -- the preserved floor segment did not actually suffice for real incremental catch-up\". ctrl-1 (TARGET, killed+restarted 3x) settles into a PERMANENT, zero-progress loop: its own raft AppendEntries exchanges cleanly reach match=22 with zero ae_reject events (heartbeat probe at prevIdx=22,n=0 -> success=1 match=22, repeating every ~3s forever), while ctrl-2 (primary) simultaneously and forever logs 'HA repl: standby[0] lsn=21 < committed_lsn=22 -> stream WAL batches' / 'ha_stream_wal_to_standby: streamed WAL [21+1..22] to standby' every heartbeat cycle with zero effect -- an off-by-one disagreement (21 vs 22) between the two sides that never reconciles.", "file": "src/controller/ha.c (repl_ack_decode_and_dispatch's heartbeat/stateReport reply branch, ~6379-6388 pre-fix -- the `target_lsn = max(committed_lsn, last_written)` computation feeding ha_dispatch_repl_action_for_standby)", "root_cause": "ROOT-CAUSED with byte/log-precise evidence; NOT a recurrence of bug-1474 (independently re-confirmed working in this same run -- floor preservation and below-floor trim both correct), bug-1475 (independently re-confirmed working -- ctrl-1's own raft log_len/term-index bookkeeping across the checkpoint boundary is clean, zero ae_reject_logmismatch anywhere in its log, unlike bug-1475's own signature), or bug-1477 (ctrl-1 clearly DID wire its commit engine every boot -- multiple 'commit engine wired'/persist_recover lines confirmed, unlike bug-1477's ctrl-3 which never ran its own election at all). This is a THIRD, previously-undiscovered gap in the same 'WAL_CHECKPOINT breaks LSN co-tracking invariants' family already flagged (not fixed at its root) as bug-1466.\n\nGround truth via `xce_test waldump` on the preserved deploy dir (~/tmp/xce-ha-waltrim): ctrl-2's WAL shows lsn=21 as the WAL_CONFIG_MARKER (type UNKNOWN/40 bytes, matching ctrl-2's own log 'Config change committed (marker_lsn=21)') and lsn=22 as a WAL_CHECKPOINT (0-byte payload) -- confirming persist_write_checkpoint_marker (persist.c:1747, `hdr.lsn = p->current_lsn++`) fired immediately after the marker committed and consumed the very next LSN slot, exactly as apply_committed_entry's WAL_CONFIG_MARKER case (persist_checkpoint_marker_triggered) is designed to do (config-upgrade-snapshot task 5). Critically, ctrl-2's OWN commit_advance_locked (commit.c, the SOLE authoritative committed_lsn writer) never logs an advance past 21 anywhere in the run ('commit: committed_lsn 20 -> 21' is the LAST such line) -- the real, majority-gated persist.committed_lsn correctly and permanently stays at 21, since no standby can ever legitimately ack a local-only, non-replicated checkpoint entry.\n\nYet ha_dispatch_repl_action_for_standby's own log line reports 'committed_lsn=22' from the very next heartbeat cycle onward. Traced to repl_ack_decode_and_dispatch's stateReportMsg branch (ha.c): `last_written = c->persist.current_lsn - 1` (the RAW physical WAL tip, which unconditionally counts EVERY locally-appended entry including a local-only WAL_CHECKPOINT) is combined via `target_lsn = max(committed_lsn, last_written)` and THIS value -- not the real committed_lsn -- is what gets passed into ha_dispatch_repl_action_for_standby's committed_lsn parameter and printed in its log line. Once ctrl-2 wrote its checkpoint, last_written jumped to 22 and target_lsn followed, permanently outrunning the real committed_lsn by exactly one. Since a checkpoint's own consumed LSN slot is, by design, NEVER streamed (ha_stream_wal_to_standby) or applied (persist_wal_write_replicated / persist_wal_async both explicitly reject type==WAL_CHECKPOINT) by any standby, no standby's applied_lsn can EVER legitimately reach 22 -- ha_evaluate_repl_action's `standby_lsn < committed_lsn` check can never become false, so HA_REPL_CONVERGED is permanently unreachable for any standby once a local checkpoint has interleaved ahead of the next real committed advance. ha_stream_wal_to_standby is correctly re-invoked every heartbeat (per its own logic) but finds nothing new in [22,22] to send every single time -- a deterministic, permanent, zero-progress busy-loop, exactly matching this run's observed symptom (and ctrl-1's own applied_lsn -- correctly capped at 21 by ha_advance_applied_lsn/ha_apply_term_entry_via_library's own last-contiguously-held-LSN semantic, mirrored by ha_handle_snapshot_chunk's snapshot-install path setting applied_lsn=checkpoint_lsn never checkpoint_lsn+1 -- can never satisfy it).\n\nThe `target_lsn = max(committed_lsn, last_written)` fallback's OWN doc comment states its intended purpose narrowly: 'so standbys can catch up even when committed_lsn is stuck at 0 (no replicator-registered standbys to advance it)' -- a cluster-bootstrap convenience. But the code applies the max() UNCONDITIONALLY, not only when committed_lsn==0 -- broader than its own documented justification. This mismatch was latent and harmless until config-upgrade-snapshot task 5 introduced a LIVE, mid-operation local-only WAL entry (the marker-triggered checkpoint) that advances current_lsn without a corresponding real committed advance; bug-1466 already flagged the SAME persist_write_checkpoint_marker current_lsn bump as breaking raft's own log_len co-tracking invariant (affecting vote-freshness comparisons) -- this is a THIRD, distinct consequence of that same architectural gap, this time poisoning the replication-target computation instead. git blame confirms the `target_lsn` code (commit 06135954f, 2026-06-11) predates config-upgrade-snapshot's first commit by roughly two weeks -- a pre-existing HA replication-decision defect that config-upgrade-snapshot's task 5 merely exposed (same routing pattern as bug-1471/1472/1474/1477 before it), not something task 8.9 introduced.", "fix": "FIXED (2026-07-28, narrowly-scoped TDD fix, in-scope to land here per explicit high-confidence + narrow-scope authorization -- unlike bug-1474's angle (a) and bug-1477, which were left for the HA-core spec owner): extracted the target_lsn computation out of repl_ack_decode_and_dispatch into a new small, pure, side-effect-free, directly-unit-testable function `ha_compute_repl_target_lsn(committed_lsn, last_written)` (declared in ha.h, defined in ha.c right after ha_process_standby_state). The fix restores the fallback to EXACTLY its own documented intent: `last_written` is used ONLY when `committed_lsn == 0` (genuine bootstrap, nothing ever committed) -- any other case returns `committed_lsn` unchanged, never leaking a trailing local-only checkpoint's consumed LSN into the convergence decision. repl_ack_decode_and_dispatch's call site now reads `target_lsn = ha_compute_repl_target_lsn(committed_lsn, last_written)`, replacing the old unconditional max(). No other function touched -- ha_evaluate_repl_action, ha_process_standby_state, ha_dispatch_repl_action_for_standby, ha_stream_wal_to_standby, persist_write_checkpoint_marker, and commit.c's majority_candidate_locked/commit_advance_locked are all confirmed correct and untouched (the bug was purely in what value got PASSED to them, not in their own logic).\n\nTDD: four new unit tests added to src/controller/testsuite/test_ha_role.c (all four call the new pure function directly, no capnp/network scaffolding needed): test_compute_repl_target_lsn_bootstrap_fallback (committed_lsn=0, last_written=42 -> 42, confirms the intended bootstrap behavior is preserved), test_compute_repl_target_lsn_zero_zero (0,0 -> 0, fresh-cluster edge case), test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn (this bug's EXACT reproduction shape: committed_lsn=21, last_written=22 -> must return 21, not 22 -- confirmed RED under the old unconditional-max code via a temporary revert, 'Expected 21 Was 22', then GREEN after restoring the fix), test_compute_repl_target_lsn_committed_already_ahead (100,50 -> 100, normal steady-state sanity check). GREEN: full controller_unit_test suite = 1040 Tests, 0 Failures, 0 Ignored, run twice clean (ASAN_OPTIONS=detect_leaks=0, nix devshell, `ninja -C src/xce.build controller/controller_unit_test`). RED confirmed independently: reverting only the new function's body to the old unconditional-max logic (leaving the new tests in place) reproduced exactly 1 failure at the predicted assertion ('Expected 21 Was 22'), then re-applying the fix restored 0 failures -- proving the test genuinely exercises this bug and the fix genuinely closes it, not a tautological test.\n\nNOT yet re-verified via a fresh live `run-ha-soak.sh --config-upgrade-wal-trim` run (recommended next step, mirroring bug-1474's own verification precedent, before considering config-upgrade-snapshot task 8.9 unblocked) -- this fix was validated at the unit level only within this investigation's scope.\n\n[UPDATE 2026-07-28, kiro-impl config-upgrade-snapshot task 8.9 verification] The initial fix landed only ONE of TWO call sites duplicating this formula: ha.c repl_ack_decode_and_dispatch (fixed, via the new ha_compute_repl_target_lsn) and config_repl.c catchup_derive_target (a SEPARATE, independently-duplicated inline copy of the exact same max(committed_lsn,last_written) formula, used by ha_walack_on_ack/catchup_lane_fn -- the ACK-driven catch-up continuation path, not the heartbeat/stateReport path the first fix covered). Rerunning the live --config-upgrade-wal-trim soak after the first fix STILL FAILED (\"cluster did not reconverge to 3-of-3... preserved floor segment did not actually suffice for real incremental catch-up\") -- confirmed via log correlation that ctrl-2 (primary) was stuck forever at ha_dispatch_repl_action_for_standby committed_lsn=22 while catchup_lane_fn's own INFO line independently reported target=22 (the OLD, unfixed formula) even though the heartbeat path's committed_lsn logging correctly showed 21. Root-caused to catchup_derive_target (config_repl.c:2617-2627, its own comment literally said \"re-derive the replication target exactly as ha.c:2771-2776 / D6\" -- a deliberate, now-stale duplication). Fixed by making catchup_derive_target call the SAME ha_compute_repl_target_lsn pure function instead of re-deriving the formula inline, so the two paths can never disagree again. Applying just this second-call-site fix then caused a NEW symptom: controller_unit_test hung indefinitely (0.8%% CPU, blocked not spinning) inside test_convergence_verification.c's test_walack_behind_target_schedules_one_continuation. Isolated via a controlled revert-and-rerun (confirmed clean 1040/1040 with the OLD formula restored, confirming the SECOND fix -- not something else -- was the trigger). Root cause of the hang: three pre-existing tests in that file (\"behind\", \"single\", \"respawn\") hardcoded committed_lsn=4 with appended_lsn=9, deliberately exercising the OLD buggy max(4,9)=9 behavior as their expected target -- with the fix, the correct target becomes 4, so acked_lsn=5/6 (used by the later tests) no longer legitimately reads as \"still behind\", and the FIRST test's own hardcoded assertion fails. Because this test file's harness (cv_setup/cv_teardown) does not run teardown from a signal-safe/finally-equivalent path, a failed TEST_ASSERT's Unity longjmp skips cv_teardown() entirely, leaking that test's live 8-thread commit_engine still bound to the soon-to-be-memset g_cv global -- the NEXT test's cv_setup() memset-ing g_cv out from under those still-running threads is what actually produced the observed permanent hang, not a defect in the production fix itself. Fixed by updating all three tests' committed_lsn from 4 to 9 (matching appended_lsn, so target=9 under EITHER formula) -- this decouples them from the bug-1478 edge case entirely (which already has dedicated, correct pure-function coverage in test_ha_role.c), preserving each test's TRUE original intent (continuation scheduling / single-flight / respawn-after-complete mechanics) with a minimal, surgical change. Stale docstrings referencing the old \"max(committed_lsn, last_written)\" formula were also corrected to reference ha_compute_repl_target_lsn. GREEN: full controller_unit_test suite = 1040 Tests, 0 Failures, run clean 2x after all fixes (plus the isolation-revert run confirming the pre-fix baseline was also clean). Live-verified: run-ha-soak.sh --config-upgrade-wal-trim now reports a clean PASS (18 jobs, zero job loss, ctrl-1 reconverged 3-of-3 using the preserved floor segment for real incremental catch-up) -- config-upgrade-snapshot task 8.9 unblocked.", @@ -17305,7 +5842,6 @@ "off-by-one", "waldump-confirmed", "root-caused", - "fixed", "narrowly-scoped-fix", "tdd", "red-green-verified" @@ -17316,17 +5852,22 @@ "bug-1475", "bug-1477" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1478.md" }, { "id": "bug-1479", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "leak", "timestamp": "2026-07-28T13:25:00Z", - "error_message": "Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade2), run on top of bug-1477's fix (checkpoint-identity, config-identity, and host-admin-status-survival assertions ALL PASSED for the first time), FAILED later in the force-elect-rotation step: 5 consecutive job-submit attempts on {ctrl-1 (primary), ctrl-2 (standby)} all got 0/3 acked (\"waiting for write-quorum\"), then hard-failed. Investigated the ORIGINAL hypothesis (a leaked VTASK_LIVE_BLOCK/commit-admission slot from an earlier burst of 17 commit timeouts, lsn=15..31, rc=-110) and REFUTED it: commit_resolve()/wait_release_locked() correctly release the admission slot on every timeout (confirmed 17 'async continuation deadline expired' + 17 'commit failed/timed out' WARN pairs, 1:1, in ctrl-1.log) \u2014 no admission-pool leak exists. The REAL mechanism is unrelated to admission control entirely: a permanent Raft AppendEntries log-conflict wedge on standby ctrl-2, compounded by a NACK-handling gap that never triggers catch-up.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade…\n\nLive config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade2), run on top of bug-1477's fix (checkpoint-identity, config-identity, and host-admin-status-survival assertions ALL PASSED for the first time), FAILED later in the force-elect-rotation step: 5 consecutive job-submit attempts on {ctrl-1 (primary), ctrl-2 (standby)} all got 0/3 acked (\"waiting for write-quorum\"), then hard-failed. Investigated the ORIGINAL hypothesis (a leaked VTASK_LIVE_BLOCK/commit-admission slot from an earlier burst of 17 commit timeouts, lsn=15..31, rc=-110) and REFUTED it: commit_resolve()/wait_release_locked() correctly release the admission slot on every timeout (confirmed 17 'async continuation deadline expired' + 17 'commit failed/timed out' WARN pairs, 1:1, in ctrl-1.log) — no admission-pool leak exists. The REAL mechanism is unrelated to admission control entirely: a permanent Raft AppendEntries log-conflict wedge on standby ctrl-2, compounded by a NACK-handling gap that never triggers catch-up.", "file": "src/controller/ha.c (repl_ack_decode_and_dispatch's WAL-ACK branch, ~6456-6644 -- the `!success` NACK path at ~6631-6642 only checks for a higher term and WARNs \"ignoring\", never calls repl_on_ack/repl_start_catchup/any retry); src/controller/repl.c (repl_broadcast ~295-380, the live single-entry send path -- fire-and-forget, no nextIndex backoff-and-retry on rejection; repl_on_ack ~250-271, the ONLY site that decrements pending_ack_count, never invoked on a NACK)", - "root_cause": "ROOT-CAUSED with byte-precise waldump evidence (xce_test's built-in `waldump` REPL command against the preserved deploy dir ~/tmp/xce-ha-configupgrade2), cross-correlated with both ctrl-1's and ctrl-2's logs line-by-line. During the soak's chaotic round-3 kill/restart churn, THREE leadership changes happened in rapid succession (term 3 -> 4 -> 5) within about 20 seconds (13:18:57-13:19:19): ctrl-2 briefly won term=3 and appended its own arm-own-term-gate NOOP, was killed before that entry could be widely replicated/committed, a short-lived term=4 leader (never directly observed in either surviving node's own become-leader lines, but proven to have existed by its WAL fingerprint) won next and wrote ITS OWN own-term NOOP at the SAME log index (14), and finally ctrl-1 won term=5 and became the lasting primary. `xce_test waldump` on ctrl-1's canonical (surviving, majority) WAL shows: lsn=12 CHECKPOINT term=2 (the marker-triggered checkpoint from config-upgrade-snapshot task 5, matching bug-1475/1477/1478's exact family), lsn=13 NOOP term=4 (the short-lived term-4 leader's noop), lsn=14 NOOP term=5 (ctrl-1's own winning noop). `xce_test waldump` on ctrl-2's own local pre-restart WAL shows: lsn=12 CHECKPOINT term=2 (agrees with ctrl-1), lsn=13 NOOP **term=3** (ctrl-2's OWN aborted, never-widely-replicated noop from its deposed term-3 stint) -- a genuine, CONFLICTING entry at the SAME index (13) with a DIFFERENT term (3 vs the canonical 4). This is an entirely NORMAL, expected Raft outcome of rapid leader churn (Raft's own \u00a75.3 log-matching property explicitly anticipates followers accumulating stale, uncommitted tail entries from deposed leaders) -- ctrl-2 correctly rejects ctrl-1's AppendEntries at prevLogIndex=13 with logOk's standard conflicting-term check (confirmed via ha_consume_append_entries's own trace: `prevIdx=13 n=1 -> success=0 match=0`, `RAFT[p1] term=5 event=ae_reject_logmismatch`, repeating for EVERY subsequent new entry ctrl-1 tries to send: prevIdx=14,15,16,...29, one per newly-committed lsn, all rejected, spanning 13:19:19 through 13:19:52). The BUG is what happens (or rather, does not happen) next, confirmed via two independent, compounding gaps: (1) ctrl-1 NEVER backs off and retries with a DECREMENTED prevLogIndex to find the point of log agreement (the standard Raft nextIndex-conflict-resolution protocol, which would let ctrl-2 truncate its conflicting lsn=13 tail and adopt the canonical term=4 entry) -- repl_broadcast (repl.c:295-380) is a pure fire-and-forget per-new-entry send with zero synchronous handling of the reply; it always advances prevIdx forward to (next-new-lsn - 1), never backward. (2) repl_ack_decode_and_dispatch's WAL-ACK reply-dispatch branch in ha.c (~6631-6642) handles a NACK (success=0) by checking ONLY for a higher observed term (raft_on_higher_term_observed, correctly added for the DIFFERENT bug-1421 stale-term scenario) and then explicitly `WARN(\"...success=0 \u2014 ignoring\")` -- the comment says exactly what the code does: the rejection is discarded. Critically, `repl_on_ack` (repl.c:250-271) is the ONLY code path that decrements `standby_conn_t.pending_ack_count`, and it is called ONLY from the `success` branch -- so a standby that is stuck rejecting (for ANY reason) has pending_ack_count monotonically climb by 1 on every subsequent broadcast attempt with ZERO corresponding decrements, until repl_broadcast's own MAX_PENDING_ACKS=16 defensive eviction (repl.c:325-334, from ha-config-bootstrap task 7.2, commit 3969064e, 2026-05-28 -- a full month before config-upgrade-snapshot's first commit) fires and marks the standby in_catch_up=1 at 13:19:53 (confirmed: 'repl_broadcast: standby localhost:9992 evicted (pending_ack_count=17 > 16) \u2014 triggering catch-up reconnect'). This eviction correctly EXCLUDES the standby from repl_active_standby_count (hence from ha_has_write_quorum's majority calculation) -- but nothing in the codebase actually DRIVES a catch-up stream or snapshot-install for it afterward: ctrl-2's log shows ZERO entry-carrying AppendEntries for the remaining ~60 seconds of the run (13:19:53 through the log's end at 13:20:53) -- only bare heartbeat probes (prevIdx=13,n=0) that trivially succeed since a presence-only probe at the standby's own true log_len=13 doesn't hit the term-conflict check. With ctrl-2 permanently excluded (in_catch_up=1, never un-evicted) and ctrl-3 permanently failing to reconnect (separate, unrelated ECONNREFUSED loop, likely a real chaos-kill in this soak round), ctrl-1 has connected=0 standbys forever ('HA quorum lost: connected=0 required=1', logged once at the eviction and never restored) -- every subsequent job submission either times out waiting for write-quorum (0/3 acked, matching the force-elect step's failure verbatim) or gets an immediate -XCE_ERR_NOT_READY (-10013) fast-reject via ha_dispatch_gate's quorum check. This is a GENUINE, SERIOUS, and apparently PERMANENT availability defect: ANY standby whose log happens to diverge by even ONE entry at a single index during leader-election churn (an entirely normal, expected Raft occurrence, not itself a bug) can become irrecoverably stuck and permanently excluded from quorum, with no self-healing path -- in a 3-node cluster this is one bad-luck election sequence away from TOTAL, PERMANENT write-quorum loss if it hits 2 of 3 standbys. Same family as bug-1466 (open, 'WAL_CHECKPOINT breaks LSN/term co-tracking', NOT fixed at its root) and its now-3 fixed siblings (bug-1475, bug-1477, bug-1478) -- this is the family's 5th distinct member, but the trigger here (an ordinary post-election term-conflicted log tail, NOT specifically a checkpoint-boundary term-derivation issue) is narrower than bug-1466's checkpoint-specific framing and points instead at a genuinely MISSING piece of core Raft log-repair protocol (conflict-driven nextIndex backoff + NACK-triggered catch-up), which appears to have never been implemented for this replication path at all -- not a regression, a gap. git blame confirms both implicated mechanisms (repl_broadcast's pending_ack_count eviction, commit 3969064e, 2026-05-28; the NACK-handling 'ignoring' branch in repl_ack_decode_and_dispatch, present since at least bug-1421's fix, itself predating config-upgrade-snapshot) are pre-existing, core HA replication code untouched by any config-upgrade-snapshot task -- config-upgrade-snapshot's task 5 (marker-triggered per-member local checkpoint) merely adds one more entry into the WAL stream around the point where this pre-existing gap gets exercised; it does not create the gap itself.", + "root_cause": "ROOT-CAUSED with byte-precise waldump evidence (xce_test's built-in `waldump` REPL command against the preserved deploy dir ~/tmp/xce-ha-configupgrade2), cross-correlated with both ctrl-1's and ctrl-2's logs line-by-line. During the soak's chaotic round-3 kill/restart churn, THREE leadership changes happened in rapid succession (term 3 -> 4 -> 5) within about 20 seconds (13:18:57-13:19:19): ctrl-2 briefly won term=3 and appended its own arm-own-term-gate NOOP, was killed before that entry could be widely replicated/committed, a short-lived term=4 leader (never directly observed in either surviving node's own become-leader lines, but proven to have existed by its WAL fingerprint) won next and wrote ITS OWN own-term NOOP at the SAME log index (14), and finally ctrl-1 won term=5 and became the lasting primary. `xce_test waldump` on ctrl-1's canonical (surviving, majority) WAL shows: lsn=12 CHECKPOINT term=2 (the marker-triggered checkpoint from config-upgrade-snapshot task 5, matching bug-1475/1477/1478's exact family), lsn=13 NOOP term=4 (the short-lived term-4 leader's noop), lsn=14 NOOP term=5 (ctrl-1's own winning noop). `xce_test waldump` on ctrl-2's own local pre-restart WAL shows: lsn=12 CHECKPOINT term=2 (agrees with ctrl-1), lsn=13 NOOP **term=3** (ctrl-2's OWN aborted, never-widely-replicated noop from its deposed term-3 stint) -- a genuine, CONFLICTING entry at the SAME index (13) with a DIFFERENT term (3 vs the canonical 4). This is an entirely NORMAL, expected Raft outcome of rapid leader churn (Raft's own §5.3 log-matching property explicitly anticipates followers accumulating stale, uncommitted tail entries from deposed leaders) -- ctrl-2 correctly rejects ctrl-1's AppendEntries at prevLogIndex=13 with logOk's standard conflicting-term check (confirmed via ha_consume_append_entries's own trace: `prevIdx=13 n=1 -> success=0 match=0`, `RAFT[p1] term=5 event=ae_reject_logmismatch`, repeating for EVERY subsequent new entry ctrl-1 tries to send: prevIdx=14,15,16,...29, one per newly-committed lsn, all rejected, spanning 13:19:19 through 13:19:52). The BUG is what happens (or rather, does not happen) next, confirmed via two independent, compounding gaps: (1) ctrl-1 NEVER backs off and retries with a DECREMENTED prevLogIndex to find the point of log agreement (the standard Raft nextIndex-conflict-resolution protocol, which would let ctrl-2 truncate its conflicting lsn=13 tail and adopt the canonical term=4 entry) -- repl_broadcast (repl.c:295-380) is a pure fire-and-forget per-new-entry send with zero synchronous handling of the reply; it always advances prevIdx forward to (next-new-lsn - 1), never backward. (2) repl_ack_decode_and_dispatch's WAL-ACK reply-dispatch branch in ha.c (~6631-6642) handles a NACK (success=0) by checking ONLY for a higher observed term (raft_on_higher_term_observed, correctly added for the DIFFERENT bug-1421 stale-term scenario) and then explicitly `WARN(\"...success=0 — ignoring\")` -- the comment says exactly what the code does: the rejection is discarded. Critically, `repl_on_ack` (repl.c:250-271) is the ONLY code path that decrements `standby_conn_t.pending_ack_count`, and it is called ONLY from the `success` branch -- so a standby that is stuck rejecting (for ANY reason) has pending_ack_count monotonically climb by 1 on every subsequent broadcast attempt with ZERO corresponding decrements, until repl_broadcast's own MAX_PENDING_ACKS=16 defensive eviction (repl.c:325-334, from ha-config-bootstrap task 7.2, commit 3969064e, 2026-05-28 -- a full month before config-upgrade-snapshot's first commit) fires and marks the standby in_catch_up=1 at 13:19:53 (confirmed: 'repl_broadcast: standby localhost:9992 evicted (pending_ack_count=17 > 16) — triggering catch-up reconnect'). This eviction correctly EXCLUDES the standby from repl_active_standby_count (hence from ha_has_write_quorum's majority calculation) -- but nothing in the codebase actually DRIVES a catch-up stream or snapshot-install for it afterward: ctrl-2's log shows ZERO entry-carrying AppendEntries for the remaining ~60 seconds of the run (13:19:53 through the log's end at 13:20:53) -- only bare heartbeat probes (prevIdx=13,n=0) that trivially succeed since a presence-only probe at the standby's own true log_len=13 doesn't hit the term-conflict check. With ctrl-2 permanently excluded (in_catch_up=1, never un-evicted) and ctrl-3 permanently failing to reconnect (separate, unrelated ECONNREFUSED loop, likely a real chaos-kill in this soak round), ctrl-1 has connected=0 standbys forever ('HA quorum lost: connected=0 required=1', logged once at the eviction and never restored) -- every subsequent job submission either times out waiting for write-quorum (0/3 acked, matching the force-elect step's failure verbatim) or gets an immediate -XCE_ERR_NOT_READY (-10013) fast-reject via ha_dispatch_gate's quorum check. This is a GENUINE, SERIOUS, and apparently PERMANENT availability defect: ANY standby whose log happens to diverge by even ONE entry at a single index during leader-election churn (an entirely normal, expected Raft occurrence, not itself a bug) can become irrecoverably stuck and permanently excluded from quorum, with no self-healing path -- in a 3-node cluster this is one bad-luck election sequence away from TOTAL, PERMANENT write-quorum loss if it hits 2 of 3 standbys. Same family as bug-1466 (open, 'WAL_CHECKPOINT breaks LSN/term co-tracking', NOT fixed at its root) and its now-3 fixed siblings (bug-1475, bug-1477, bug-1478) -- this is the family's 5th distinct member, but the trigger here (an ordinary post-election term-conflicted log tail, NOT specifically a checkpoint-boundary term-derivation issue) is narrower than bug-1466's checkpoint-specific framing and points instead at a genuinely MISSING piece of core Raft log-repair protocol (conflict-driven nextIndex backoff + NACK-triggered catch-up), which appears to have never been implemented for this replication path at all -- not a regression, a gap. git blame confirms both implicated mechanisms (repl_broadcast's pending_ack_count eviction, commit 3969064e, 2026-05-28; the NACK-handling 'ignoring' branch in repl_ack_decode_and_dispatch, present since at least bug-1421's fix, itself predating config-upgrade-snapshot) are pre-existing, core HA replication code untouched by any config-upgrade-snapshot task -- config-upgrade-snapshot's task 5 (marker-triggered per-member local checkpoint) merely adds one more entry into the WAL stream around the point where this pre-existing gap gets exercised; it does not create the gap itself.", "fix": "FIXED AND VERIFIED LIVE (2026-07-29, see UPDATE 2 below for the final closing fix) -- two compounding gaps were closed after the initial investigation: (1) the standby's own reject path sent no NACK reply at all (ha_send_wal_ack_reply hardcoded success=1 unconditionally), fixed via ha_send_wal_ack_reply_ex with success=0 on reject; (2) even with NACKs flowing, ha_stream_wal_to_standby's catch-up range was hard-capped at committed_lsn, which could never advance in this exact scenario (self-referential deadlock) -- fixed via ha_compute_repl_stream_ceiling_lsn widening the ceiling to the leader's own log tail, matching raft.tla's real AppendEntries semantics. Verified via three successive live run-ha-soak.sh --config-upgrade re-runs tracking each iteration (v1/v2 still failed, v3 with both fixes together PASSED end-to-end: 3/3 jobs acked on both force-elect rounds, zero materialized-state or config divergence). 1053/1053 controller_unit_test PASS. Full investigation history preserved below for context (including the earlier partial-fix iterations and root-causing).\n\n[Original investigation, 2026-07-28]\nNOT FIXED -- explicitly out of scope for a single safe pass, per this investigation's own conclusion (matches kiro-debug's guidance against forcing a fix when the true defect requires a protocol design decision, not a narrow patch). At least TWO separate, non-trivial design questions need to be resolved by whoever owns ha.c/repl.c's replication-catch-up protocol before a fix can be written safely: (a) should an ae_reject_logmismatch NACK immediately trigger repl_start_catchup (moving the standby to catch-up mode on the FIRST conflict, rather than silently accumulating up to 16 failed sends first)? This is the more targeted, root-cause-adjacent fix. (b) independently, once a standby IS marked in_catch_up (whether via this path or the existing MAX_PENDING_ACKS eviction), what mechanism is SUPPOSED to actually drive its catch-up (stream from join_applied_lsn, or snapshot-install if too far behind) and clear in_catch_up via repl_complete_catchup? Evidence here shows NOTHING does, for at least this eviction path -- config_repl.c's ACK-driven catchup_lane_fn/ha_walack_on_ack continuation (the only other repl_complete_catchup call site found) is itself ACK-driven, so it can never engage for a standby that never sends a genuine ACK in the first place; whether IT (or a periodic sweep independent of ACKs) is supposed to notice an in_catch_up standby and proactively start streaming it needs its own trace by the owning team. A correct fix likely needs both: NACK-triggered catch-up entry (closing gap (a)) AND a proactive (not purely ACK-reactive) catch-up driver that streams from join_applied_lsn or falls back to snapshot-install for any in_catch_up standby (closing gap (b)) -- implementing only one without the other would leave the defect only partially closed. Recommend routing to whoever owns ha.c/repl.c's HA replication/catch-up protocol (the same 'needs-owner' bucket as bug-1474's still-open fix angle (a) and bug-1472-remainder) for a proper design pass, THEN a TDD fix + live re-verification via `run-ha-soak.sh --config-upgrade` on a fresh deploy dir. Preserved evidence: ~/tmp/xce-ha-configupgrade2 (do not delete) -- `xce_test waldump ` (via the xce_test REPL: `printf 'waldump \\nquit\\n' | ASAN_OPTIONS=detect_leaks=0 timeout 60 /controller/xce_test`) reproduces the byte-precise lsn=13 term-3-vs-term-4 conflict evidence directly from the preserved WAL segments at any time.\n\nUPDATE (2026-07-29, this session): a fix WAS committed (uncommitted in the 'config' worktree at investigation time) implementing exactly gap (a) from this entry's own recommendation -- ha_raft_on_repl_nack() (ha.c/ha.h) routes a WAL-ACK-reply NACK into the raft library's nextIndex backoff and returns a corrected resume LSN; repl_ack_decode_and_dispatch's NACK branch calls it and immediately repl_start_catchup() with the corrected LSN; repl_get_catchup_join_lsn() (repl.c/repl.h) lets the heartbeat-driven ha_dispatch_repl_action_for_standby use the corrected position; repl_complete_catchup() now also resets pending_ack_count. 1048/1048 controller_unit_test PASS. HOWEVER: a fresh live-soak re-verification (run-ha-soak.sh --config-upgrade, deploy preserved at ~/tmp/xce_ha_bug1479_verify, NOT deleted) reproduced the ORIGINAL symptom verbatim -- force-elect ctrl-2 round: 5/5 submit_jobs attempts on {ctrl-1,ctrl-2} got 0/3 acked, hard FAIL. Root-caused: the fix's NACK-routing branch in repl_ack_decode_and_dispatch (ha.c ~6680-6720) is CORRECT but UNREACHABLE in production, because the standby's OWN reject path never sends a NACK (or any reply) in the first place. ha_apply_wal_catchup_entry has three reject paths (ha.c ~3230-3232 via ha_apply_term_entry_via_library for the term-carrying single/batch case -- the one actually hit live; ~3320-3321 for a batch sub-entry apply failure; ~3373-3374 for the legacy single-entry path) that all explicitly do 'no ACK' on a logOk reject/forward-gap. ha_send_wal_ack_reply (ha.c:3084-3106) is the ONLY function anywhere in the controller that ever constructs a WalAckReply, and it hardcodes success=1 unconditionally -- there is NO production code path that ever sends success=0. Live evidence: ctrl-2's own log shows repeating RAFT[p1] event=ae_reject_logmismatch / ha_consume_append_entries success=0 match=0 at growing prevIdx (40,41,...54) while stuck at match=20 forever; ctrl-1's log has ZERO 'WAL NACK from standby' or ha_raft_on_repl_nack lines the entire time -- the primary never learns of the rejection at all, so it never backs off nextIndex or re-arms catch-up, and committed_lsn stays frozen (the only alive standby can never re-sync, and the heartbeat-driven catch-up's own target is gated on the same frozen committed_lsn). STILL NOT FULLY FIXED. New follow-up spec .kiro/specs/raft-adapter-conformance/ created to (1) fix this concrete missing-NACK-reply defect at the three reject sites and (2) perform a broader raft.tla action-by-action conformance pass over the controller's Raft adapter (ha.c/repl.c/config_repl.c) -- distinct from raft-sim-conformance, which is scoped to src/lib/raft/raft_sim.c (library-only, in-process) and explicitly does not cover this adapter code.\n\nUPDATE 2 (2026-07-29, this session): fully FIXED and verified live end-to-end. Two more compounding gaps found and closed after the NACK-send fix (UPDATE 1) landed:\n(1) ha_apply_term_entry_via_library's logOk-reject path (ha.c, the single live-reachable reject site) sent NO reply at all on rejection -- ha_send_wal_ack_reply hardcoded success=1 unconditionally, so the primary-side ha_raft_on_repl_nack (correct since UPDATE 1) was unreachable dead code. Fixed by adding ha_send_wal_ack_reply_ex(ha, ep, c, lsn, success) and calling it with success=0 on reject, plus a new ha_wal_ack_send_hook (ha.h, XCE_TESTING) so a unit test can observe the NACK without a live endpoint (test_bug1479_ae_reject_sends_nack_not_silence, test_config_repl.c). Confirmed live in a 2nd soak run: ctrl-1's log showed real 'WAL NACK from standby ... success=0 -- marking catch-up' lines for the first time.\n(2) Even with (1) working, that 2nd soak run STILL failed at the identical force-elect ctrl-2 point: ha_stream_wal_to_standby's catch-up range was hard-bounded at committed_lsn, and committed_lsn can never advance in this exact scenario because advancing it requires an ACK from the very standby that's stuck catching up (2-of-3 alive, this standby is the only path to a new majority) -- a self-referential deadlock. Fixed by adding a new pure function ha_compute_repl_stream_ceiling_lsn(committed_lsn, last_written, checkpoint_lsn) (ha.c/ha.h), wired into ha_dispatch_repl_action_for_standby to widen the STREAM_WAL ceiling (and the ha_evaluate_repl_action convergence check that gates it) to the leader's own log tail instead of just committed_lsn -- matching raft.tla's real AppendEntries semantics (leaderCommit is advisory to the follower, never a cap on what gets sent) -- while still guarding against bug-1478's old trap (never widening to a trailing local-only checkpoint's own unstreamable LSN slot). 4 new pure-function tests in test_ha_role.c (test_compute_repl_stream_ceiling_lsn_*), RED-confirmed against a stubbed committed_lsn-only version before the real fix was restored.\nFull verification: 1053/1053 controller_unit_test PASS twice clean. THREE successive live run-ha-soak.sh --config-upgrade re-runs tracked each fix: v1 (NACK-send only) still FAILED at force-elect ctrl-2 (0/3 acked); v2 confirmed NACKs now reaching the primary but still FAILED (frozen committed_lsn=14, ha_stream_wal_to_standby target never advanced); v3 (both fixes together) PASSED end-to-end: all 3 force-elect rounds succeeded (ctrl-2: 3/3 jobs acked, term=6; ctrl-3: 3/3 jobs acked, term=7), final verdict 'PASS (config-upgrade): 12 jobs, zero materialized-state divergence, zero configuration divergence, host admin-status confirmed across all members'. Evidence preserved at ~/tmp/xce_ha_bug1479_verify, ~/tmp/xce_ha_bug1479_verify_v2, ~/tmp/xce_ha_bug1479_verify_v3 (all not deleted). config-upgrade-snapshot task 8.6 unblocked. The raft-adapter-conformance spec created mid-investigation had its concrete seed defect (missing NACK reply) fixed here; the broader raft.tla action-by-action conformance audit it also scopes remains valid, separate future work.", "tags": [ "ha.c", @@ -17356,7 +5897,6 @@ "live-soak-finding", "missing-nack-reply", "raft-adapter-conformance", - "fixed", "verified-live", "raft-tla-conformance", "stream-ceiling", @@ -17372,15 +5912,20 @@ "bug-1478", "bug-1421" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1479.md" }, { "id": "bug-1480", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-29T07:05:00Z", - "error_message": "raft-adapter-conformance systematic audit finding (not yet fixed): handle_repl_config_push (config_repl.c:1261-1337) has 4 reject paths (malformed body, missing bytes/sha256, sha256 mismatch, conf_save_config failure) that all return without ever sending a ReplConfigPushReply. Only the success path (1326-1335) sends {accepted=1}; there is no {accepted=0} anywhere in this function, despite its own doc-comment implying a reject reply exists. cluster.c dispatch case (652-655) has no fallback error-reply either.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-adapter-conformance systematic audit finding (not yet fixed): handle_repl_config_push…\n\nraft-adapter-conformance systematic audit finding (not yet fixed): handle_repl_config_push (config_repl.c:1261-1337) has 4 reject paths (malformed body, missing bytes/sha256, sha256 mismatch, conf_save_config failure) that all return without ever sending a ReplConfigPushReply. Only the success path (1326-1335) sends {accepted=1}; there is no {accepted=0} anywhere in this function, despite its own doc-comment implying a reject reply exists. cluster.c dispatch case (652-655) has no fallback error-reply either.", "file": "src/controller/config_repl.c (handle_repl_config_push, ~1261-1337)", "root_cause": "Same defect class as bug-1479 (silent-drop-on-reject) but on the out-of-band config-push channel instead of the WAL catch-up channel: the reject branches never construct/send a reply message at all, so push_config_to_standbys / repl_wait_majority_ack cannot distinguish an active standby rejection from a lost packet -- both degrade identically to CONF_PUSH_TIMEOUT_MS.", "fix": "FIXED (2026-07-29, ~09:43, commit b7828eed \"feat(raft-adapter-conformance): send explicit reply on every config-push outcome\"). handle_repl_config_push now constructs and sends ReplConfigPushReply{accepted=0, term=current_term} on all four reject early-returns (malformed request, missing bytes/sha256, sha256 mismatch, conf_save_config failure) instead of returning silently; term populated on the success path too. Landed as raft-adapter-conformance tasks 2.1-2.3 (see tasks.md), tested end-to-end in test_config_repl.c, confirmed live in task 7.3's soak. This buglog entry's original 'Not yet fixed' note (07:05 same day) predates the fix landing by ~2.5h and was never updated -- code is fixed, only this log entry was stale.", @@ -17393,22 +5938,27 @@ "raft-adapter-conformance", "found-not-yet-fixed", "config-push-channel", - "term-discovery-gap", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "term-discovery-gap" ], "related_bugs": [ "bug-1479" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "b7828eed" + ], + "record": ".wolf/bugs/bug-1480.md" }, { "id": "bug-1481", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-29T07:05:00Z", - "error_message": "raft-adapter-conformance systematic audit finding (not yet fixed): vote_ep_err_handler (ha.c:7564-7608), the shared on_err handler wired onto both real-vote and PreVote endpoints (ha.c:688-689, 827-828), handles post-connect endpoint failures (RDHUP/SSL error/remote close arriving after connect+monitor succeeded but before a reply) by calling the legacy handle_request_vote_reply directly instead of routing through vote_reply_dispatch / pre_vote_reply_dispatch (the production tally path fixed by bug-1423/1424 for connect-time and monitor-time failures only).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "raft-adapter-conformance systematic audit finding (not yet fixed): vote_ep_err_handler…\n\nraft-adapter-conformance systematic audit finding (not yet fixed): vote_ep_err_handler (ha.c:7564-7608), the shared on_err handler wired onto both real-vote and PreVote endpoints (ha.c:688-689, 827-828), handles post-connect endpoint failures (RDHUP/SSL error/remote close arriving after connect+monitor succeeded but before a reply) by calling the legacy handle_request_vote_reply directly instead of routing through vote_reply_dispatch / pre_vote_reply_dispatch (the production tally path fixed by bug-1423/1424 for connect-time and monitor-time failures only).", "file": "src/controller/ha.c (vote_ep_err_handler, ~7564-7608; correct sibling sites at ~693-726 and ~830-855)", "root_cause": "Reopens the bug-1423/1424 defect class (vote-request send failure must reach the reply tally) for one failure window the original fix did not cover: connect/iomgr_monitor failures were fixed, but a failure arriving after a successful connect+monitor (before any reply) still bypasses the tally via the legacy path. Effect is bounded (election_timeout_ms / PreVote round timeout still fire), not a permanent deadlock, but the fast all-replied quorum-decision path is structurally unreachable for this failure mode.", "fix": "FIXED (2026-07-29, ~13:53, commit fce73b5b \"feat(raft-adapter-conformance): route post-connect vote/PreVote failures to production tally\"). vote_ep_err_handler split into a real-vote handler and a new pre_vote_ep_err_handler; each now routes a post-connect election-phase failure through the same production tally path used for connect-time failures (vote_reply_dispatch / pre_vote_reply_dispatch) instead of the legacy handle_request_vote_reply. Landed as raft-adapter-conformance tasks 3.1-3.2 (see tasks.md), tested in test_ha_vote_send.c, confirmed live in task 7.4's soak. This buglog entry's original 'Not yet fixed' note (07:05 same day) predates the fix landing by ~6.8h and was never updated -- code is fixed, only this log entry was stale.", @@ -17423,22 +5973,27 @@ "raft-adapter-conformance", "found-not-yet-fixed", "election-timeout-bounded", - "not-a-deadlock", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "not-a-deadlock" ], "related_bugs": [ "bug-1423" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "fce73b5b" + ], + "record": ".wolf/bugs/bug-1481.md" }, { "id": "bug-1482", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-07-29T10:43:00Z", - "error_message": "AddressSanitizer: heap-use-after-free in free_packet ../lib/netio.c:437, called from test_config_push_reply_higher_term_adopts_and_steps_down (test_config_repl.c:3718)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: heap-use-after-free in free_packet ../lib/netio.c:437, called from…\n\nAddressSanitizer: heap-use-after-free in free_packet ../lib/netio.c:437, called from test_config_push_reply_higher_term_adopts_and_steps_down (test_config_repl.c:3718)", "file": "src/controller/testsuite/test_config_repl.c; doc comment in src/controller/cluster.c (myreq_handler_for_test)", "root_cause": "A code-review finding (task 2.2 re-review) reported that cpr22_build_config_push_reply_pkt's io_packet_t* pkt was never freed after myreq_handler_for_test(...), citing myreq_handler_for_test's own doc comment ('the caller still owns/frees pkt itself') as the ownership contract. That doc comment was WRONG: myreq_handler unconditionally calls xce_decode(pkt, &request) as its first step, and xce_decode() itself (src/lib/request.c:274-275) unconditionally calls free_packet(pkt) (frees pkt->data) then FREEUP(pkt) (frees the io_packet_t struct) before returning -- ownership of pkt is transferred into xce_decode, not retained by the caller. Applying the review finding literally (adding free_packet(pkt); free(pkt); after the call) was therefore a double-free/use-after-free, caught immediately by ASAN even under ASAN_OPTIONS=detect_leaks=0 (UAF detection is independent of leak detection).", "fix": "Removed the erroneous free_packet(pkt)+free(pkt) call from the test (there is no leak -- pkt is already fully consumed). Corrected myreq_handler_for_test's doc comment in cluster.c to accurately state that pkt ownership is transferred into the call via xce_decode, and callers must not touch/free pkt afterward. Added an explanatory NOTE comment at the call site in the test to prevent recurrence.", @@ -17453,20 +6008,23 @@ "ownership-contract", "raft-adapter-conformance", "task-2.2", - "review-finding-corrected", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "review-finding-corrected" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1482.md" }, { "id": "bug-1483", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "leak", "timestamp": "2026-07-29T10:20:00Z", - "error_message": "ha_arm_heartbeat_timeout_vtask (ha.c ~8805-8848), when called with c==NULL (unit-test mode driving a PreVote round to LOST without stubbing ha_prevote_rearm_liveness_hook), callocs a liveness_poll_ctx_t and stores it into the static s_liveness_ctx, but the guard skips scheduling anything to poll or free it -- the context leaks/dangles permanently in that code path.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ha_arm_heartbeat_timeout_vtask (ha.c ~8805-8848), when called with c==NULL (unit-test mode driving…\n\nha_arm_heartbeat_timeout_vtask (ha.c ~8805-8848), when called with c==NULL (unit-test mode driving a PreVote round to LOST without stubbing ha_prevote_rearm_liveness_hook), callocs a liveness_poll_ctx_t and stores it into the static s_liveness_ctx, but the guard skips scheduling anything to poll or free it -- the context leaks/dangles permanently in that code path.", "file": "src/controller/ha.c (ha_arm_heartbeat_timeout_vtask, ~8805-8848)", "root_cause": "Test-mode (c==NULL) entry into a production helper allocates state but has no corresponding free/cleanup path when the delaymgr-scheduling guard short-circuits -- a latent leak specific to the c==NULL unit-test invocation shape, not exercised by any live production path (a real cluster_t is never NULL there).", "fix": "FIXED (2026-08-01). ha_arm_heartbeat_timeout_vtask (ha.c) published a calloc'd liveness_poll_ctx_t into the static s_liveness_ctx unconditionally, but only freed it via liveness_poll_fn's cancelled-check path -- which never runs if delay_task_ms was never called (c==NULL, delaymgr not running, or delay_task_ms itself failing). Fixed by clearing s_liveness_ctx and freeing ctx immediately in all three no-schedule cases (the pre-existing c==NULL/!running else-branch, plus the previously-unhandled delay_task_ms rc<0 branch, which had the identical defect shape and was found while fixing this). Establishes the invariant: s_liveness_ctx is non-NULL only when a delaymgr task actually holds a live reference to it. Regression test: test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx (test_ha_prevote.c) drives a real LOST PreVote round WITHOUT stubbing ha_prevote_rearm_liveness_hook (every other LOST-path test in that file stubs it specifically to avoid this code path) and asserts the new liveness_ctx_get_for_test() accessor returns NULL afterward. Full suite: 1071 Tests 0 Failures (verified via the direct-binary recipe, ASAN_OPTIONS=detect_leaks=0).", @@ -17480,20 +6038,23 @@ "task-3.2", "found-not-yet-fixed", "pre-existing", - "prevote-lost-path", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "prevote-lost-path" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1483.md" }, { "id": "bug-1484", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-07-29T21:10:00Z", - "error_message": "No test anywhere in src/controller/testsuite/ drives the real production dispatch path for either term-discovery vector bug-1421 introduced (heartbeat-reply state-report term adoption, ha.c:6690-6711; WAL-NACK reply_term adoption inside repl_ack_decode_and_dispatch, ha.c:6805-6814) and asserts raft_on_higher_term_observed actually fires. The look-alike test_intake_higher_term_heartbeat_adopts_term (test_term_safety_adapter.c:1232) drives a different code path (ha_consume_append_entries -> raft_on_append_entries_request library-level UpdateTerm, an AE-request vector, not the AE-reply/state-report fallback bug-1421 added). test_persist_recovery.c (cited by design.md line 146 as covering this) has zero term-adoption content -- pure recovery mechanics. test_config_repl.c only covers the NEW config-push term vector added by this feature (Req 7.4), not the pre-existing heartbeat/WAL-NACK vectors.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "No test anywhere in src/controller/testsuite/ drives the real production dispatch path for either…\n\nNo test anywhere in src/controller/testsuite/ drives the real production dispatch path for either term-discovery vector bug-1421 introduced (heartbeat-reply state-report term adoption, ha.c:6690-6711; WAL-NACK reply_term adoption inside repl_ack_decode_and_dispatch, ha.c:6805-6814) and asserts raft_on_higher_term_observed actually fires. The look-alike test_intake_higher_term_heartbeat_adopts_term (test_term_safety_adapter.c:1232) drives a different code path (ha_consume_append_entries -> raft_on_append_entries_request library-level UpdateTerm, an AE-request vector, not the AE-reply/state-report fallback bug-1421 added). test_persist_recovery.c (cited by design.md line 146 as covering this) has zero term-adoption content -- pure recovery mechanics. test_config_repl.c only covers the NEW config-push term vector added by this feature (Req 7.4), not the pre-existing heartbeat/WAL-NACK vectors.", "file": "src/controller/ha.c (~6690-6711 heartbeat-reply vector, ~6805-6814 WAL-NACK vector); no test file drives either", "root_cause": "bug-1421 (heartbeat-reply term-discovery fallback) shipped without a dedicated regression test asserting raft_on_higher_term_observed fires through the real repl_ack_decode_and_dispatch path for either vector -- found during raft-adapter-conformance task 7.2s regression-confirmation sweep, which searched the whole test suite (not just its own 8-file boundary) for `raft_on_higher_term_observed`/`higher_term` callers and found none exercising these two vectors end-to-end.", "fix": "FIXED (2026-08-01) -- test-coverage gap closed, no production logic change (the underlying bug-1421 term-adoption code was already correct, just unverified end-to-end). Added test_term_adoption_vectors.c with two tests driving the real peer_ctrl_msg_handler -> repl_ack_decode_and_dispatch path (via peer_ctrl_msg_handler_for_test + standby_ep_ctx_build_for_test, c=NULL -- every c-dependent step downstream of the term check is already guarded): (a) test_heartbeat_state_report_higher_term_adopts_and_steps_down builds a wire-shape StateReport reply carrying a higher term and asserts raft_on_higher_term_observed adopts it and steps the node down from RAFT_LEADER (ha.c:6690-6711 vector); (b) test_wal_nack_higher_term_adopts_and_steps_down does the same for a WalAckReply{success=0} carrying a higher reply_term (ha.c:6805-6814 vector). Both went GREEN immediately, confirming Requirement 7.5's 'already verified' claim was correct in substance, just previously unverified by any test. Registered in meson.build controller_unit_test_src and controller_test.c. Full suite: 1071 -> 1073 Tests, 0 Failures.", @@ -17510,24 +6071,25 @@ "pre-existing", "requirement-7.1", "requirement-7.2", - "requirement-7.5", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "requirement-7.5" ], "related_bugs": [ "bug-1421" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1484.md" }, { "id": "bug-1485", "status": "cannot-reproduce", "status_updated": "2026-09-24", "status_note": "stderr capture added a5560256, never re-seen", - "repro_attempts": "0/40 repro attempts (2026-08-01)", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-07-30T14:35:00Z", - "error_message": "test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path flaky (near-100% fail rate across ~25 live runs during admin-config-tool task 6.3 round-2 remediation): TimeoutError: controller never opened localhost:: [Errno 111] Connection refused, raised from a fresh wait_for_port re-probe of the elected primary's own address performed AFTER _wait_for_primary (CTRL_PRIMARY marker) + a new _wait_for_standbys (both other nodes show CTRL_STANDBY) + a 3s settle sleep all already succeeded.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 25, + "error_message": "test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path flaky…\n\ntest_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path flaky (near-100% fail rate across ~25 live runs during admin-config-tool task 6.3 round-2 remediation): TimeoutError: controller never opened localhost:: [Errno 111] Connection refused, raised from a fresh wait_for_port re-probe of the elected primary's own address performed AFTER _wait_for_primary (CTRL_PRIMARY marker) + a new _wait_for_standbys (both other nodes show CTRL_STANDBY) + a 3s settle sleep all already succeeded.", "file": "src/xceadmin/tests/test_e2e_live_ha_push.py (live_ha_cluster fixture); root cause is in src/controller (exact site not yet isolated -- C-side investigation out of this task's boundary)", "root_cause": "NOT a pure test-side readiness race as round-1 review assumed. Live forensic capture (ps aux mid-run) caught the freshly-elected ctrl-1 as a zombie process moments after its own log recorded the full raft_cb_become_leader promotion sequence ending at 'HA: repl registered standby ep[1] (slot 1, ...)' -- the log then goes silent (no crash trace, no further lines) and the process is gone. stderr was DEVNULL'd in the fixture so no ASAN/signal diagnostic was captured from a real failing pytest run. Confusingly, 3 separate standalone (non-pytest) repro scripts using byte-identical spawn code (same xce_controller build.dbg binary, same deploy-ha.sh cluster, same peers/certs/workdir args) ran the exact same primary-promotion sequence successfully every single time with the primary staying alive and the port reachable for 20-30s post-election -- while pytest-invoked runs of the identical fixture code failed roughly 24/25 times. Ruled out during investigation: pytest's default fd-capture (reproduced with -s / capturing disabled, still failed 3/3), ulimits (NPROC/NOFILE/AS/STACK identical between nix-develop plain-shell and pytest-invoked-via-python contexts), and orphaned-process port contention (ps aux clean before each batch). True root cause not isolated -- prime suspects are a genuine timing-sensitive controller-side crash/race in the CTRL_STANDBY->CTRL_PRIMARY raft promotion path (raft_cb_become_leader / repl registration) that pytest's process tree somehow makes far more likely to trigger, or a process-group/session/signal-mask difference inherited by subprocess.Popen children when the parent is pytest vs a bare python3 script.", "fix": "UPDATE 2026-08-01 -- investigated per this entry's own \"next step\" (capture stderr instead of DEVNULL, re-attempt). Redirected each node's stderr to deploy_dir/logs/.stderr.log (permanent change to test_e2e_live_ha_push.py, replacing the DEVNULL redirect that left this entry's original investigation with no diagnostic) and added an XCE_KEEP_DEPLOY_DIR=1 debug env var to preserve deploy_dir across the fixture's own cleanup for inspection. Ran the live test 40 times total in this environment: 25 baseline (idle sandbox, load avg ~2/22 cores) + 15 more with a concurrent `xce-build.sh controller debug` compile racing for CPU (separate build.dbg dir, so it could not clobber the xce.build controller binary the test depends on -- unlike bug-1486's same-dir clobber shape). Result: 40/40 PASSED, 0/120 captured stderr logs (3 nodes x 40 runs) non-empty -- no crash, no ASAN report, in any run. Could NOT reproduce the ~24/25 failure rate from the original investigation despite matching the documented repro shape exactly (same binaries, same fixture code, same pytest invocation).\n\nNOT marking fixed -- absence of reproduction here is not proof of absence of the underlying bug, and this session's single concurrent compile is a much lighter contention scenario than the real multi-session sandbox reality bug-1486 documents (several concurrent AGENT SESSIONS, potentially each running their own live 3-node cluster). Leading hypothesis (unconfirmed): the original ~24/25 failure rate was itself a symptom of resource contention from OTHER concurrent sessions on the shared sandbox during that investigation (CPU starvation delaying the primary's post-election warmup past _POST_ELECTION_SETTLE_S, or an OOM-adjacent condition killing the freshly-forked primary outright) rather than a bug in the promotion path reachable in isolation -- consistent with the original investigation's own \"pytest's process tree somehow makes far more likely\" framing, since pytest's subprocess tree carries more memory/fd overhead than the bare python3 scripts that never reproduced it there either.\n\nDowngraded to low-priority/environment-flake, left OPEN. The stderr-capture infrastructure is now permanently in place (not reverted) so the next real live failure -- in pytest, in a genuinely contended multi-session sandbox, or in CI -- captures an actual ASAN report or signal instead of DEVNULL silence, which is what would actually close this out.", @@ -17540,24 +6102,28 @@ "task-6.3", "e2e-live", "unresolved", - "pytest-specific", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pytest-specific" ], "related_bugs": [ "bug-1477", "bug-1479", "bug-606" ], - "occurrences": 25, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1485.md", + "repro_attempts": "0/40 repro attempts (2026-08-01)" }, { "id": "bug-1486", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-07-31T03:48:00Z", - "error_message": "Backgrounded `xce-build.sh all` run went silently stale mid-flight ([360/362], no ninja FAILED/error line, no completion marker, log just stopped growing at 03:38:35) with no matching build process left in `ps aux`", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Backgrounded `xce-build.sh all` run went silently stale mid-flight ([360/362], no ninja…\n\nBackgrounded `xce-build.sh all` run went silently stale mid-flight ([360/362], no ninja FAILED/error line, no completion marker, log just stopped growing at 03:38:35) with no matching build process left in `ps aux`", "file": "src/tools/xce-build.sh", "root_cause": "Multiple DIFFERENT Claude agent sessions independently invoked xce-build.sh against the same shared worktree's src/xce.build directory at the same time (confirmed via `ps auxf`: sibling `bash src/tools/xce-build.sh controller` and a separate `... all` invocation running under different parent `claude --output-format stream-json` PIDs). xce-build.sh has no lock/mutex around its unconditional `rm -rf \"$BUILD_DIR\"` + `meson setup` step (same code path as bug-1464), so a peer session's rm -rf tore the build directory out from under this session's own in-progress ninja/meson compile with no error propagation -- the backgrounded process's output just stops, indistinguishable at first glance from a hang.", "fix": "No code change; workflow fix. Before trusting a backgrounded xce-build.sh log that has gone stale, run `ps auxf | grep -E 'xce-build|meson|ninja|cc1'` to check for a sibling xce-build.sh invocation from a different session targeting the same worktree. If found, do not start a third concurrent build -- bounded-poll (`while kill -0 $PEER_PID; do sleep 5; done`, not a passive wait for a background-task notification) until all peer invocations exit, then run exactly one clean blocking `xce-build.sh all` and treat only that run's own output as ground truth. Real fix would be a flock/lockfile around setup_build()'s rm-rf+meson-setup step in xce-build.sh itself so concurrent sessions serialize instead of clobbering.", @@ -17569,22 +6135,25 @@ "race-condition", "concurrent-sessions", "multi-agent", - "workflow", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "workflow" ], "related_bugs": [ "bug-1464" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1486.md" }, { "id": "bug-1487", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-07-31T04:50:00Z", - "error_message": "30-round chaotic HA soak (run-ha-soak.sh --chaotic 30): committed_job_loss=1 at round 25/30 (VERDICT result=FAIL). expect-job reported job 0-6-1785473261-216067 not found on newly-elected primary ctrl-2 (term=27) immediately after the round-25 kill of ctrl-1 (term=26). committed_divergence stayed 0 for the entire run; the job (and all others) was confirmed present on every subsequent verify (rounds 26-30) and the final post-run sweep of all 93 tracked jobs.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "30-round chaotic HA soak (run-ha-soak.sh --chaotic 30): committed_job_loss=1 at round 25/30…\n\n30-round chaotic HA soak (run-ha-soak.sh --chaotic 30): committed_job_loss=1 at round 25/30 (VERDICT result=FAIL). expect-job reported job 0-6-1785473261-216067 not found on newly-elected primary ctrl-2 (term=27) immediately after the round-25 kill of ctrl-1 (term=26). committed_divergence stayed 0 for the entire run; the job (and all others) was confirmed present on every subsequent verify (rounds 26-30) and the final post-run sweep of all 93 tracked jobs.", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", "root_cause": "NOT a controller bug and NOT genuinely-uncommitted-at-kill-time -- test-harness false-positive, root-caused with log/timestamp evidence from the preserved run (/home/claude/tmp/xce-ha-chaos30-run1/logs/ctrl-{1,2,3}.log). ctrl-1 (primary, term=26) majority-committed all 3 round-25 jobs (lsn 101,102,103; self + ctrl-2 ack) between 04:47:41.839 and 04:47:42.052 -- confirmed via commit_advance_locked log lines -- moments before being killed at 04:47:42+. ctrl-2 was elected primary at term=27 at 04:47:52.470 with ha_compute_safe_committed_lsn reporting safe_committed_lsn=99 \"(from 2 granted voters)\" -- the conservative Raft floor, since ctrl-3 (the only other reachable voter) was itself mid-restart/catch-up from round 25s own overlapping-kill design and had not yet caught up past lsn 99. Per the documented Raft Figure-8 safety mechanism (ha_arm_own_term_gate, ha.c:4997-5046: \"the universal commit barrier still requires per-entry majority of a new-term entry\"), a newly-elected primary cannot fold a PRIOR-term entry into its own committed_lsn watermark merely because it holds it durably -- it must first get a fresh majority ack, under its OWN term, for its own-term WAL_NOOP (armed at promotion). ctrl-2s committed_lsn stayed at 100 (excluding lsn 101-103) until ctrl-3s WAL catch-up stream let it ack the term-27 noop at lsn=104, at 04:47:58.702 -- 6.232s after promotion (commit_advance_locked: \"committed_lsn 100 -> 104\"). expect-job (joblist query) reads c->jobmgr, which the apply pump only populates for entries that have reached this committed_lsn watermark -- so the job was genuinely invisible, by design, for that 6.232s window. run-ha-soak.shs verify_classify() (chaotic mode) runs its expect-job check immediately once wait_for_new_primary detects the new primary is SERVING at a higher term -- unlike verify_no_loss (sequential mode; already carries a 3-attempt/2s-settle retry from bug-1422 for a related but distinct client-retry-storm race) and unlike wait_for_full_convergence (which chaotic mode intentionally skips between kills, by design, to land kills inside overlap windows) -- so its single, instantaneous query landed inside the 6.232s barrier window and counted a false LOSS. Comparison: round 24s election (ctrl-1, term=26) closed the identical barrier window in only 167ms because the other voting peer was not mid-catch-up at that moment -- the window duration is directly proportional to how far behind the second required peer is, which chaotic modes overlapping-restart design can occasionally push to several seconds.", "fix": "Added a bounded retry (5 attempts, 2s settle between) around the expect-job/verdict-classify check in verify_classify() (.claude/skills/xce-test-ha/run-ha-soak.sh) -- mirrors verify_no_loss/bug-1422s \"retry the whole check\" precedent, only retrying on a LOSS classification (BUG854/OK still resolve on the first attempt, matching prior semantics). ~8s of total settle margin comfortably covers the measured 6.232s worst-case gap while staying bounded -- a job still missing after 5 attempts still counts as a genuine LOSS, so real committed-job loss detection is unweakened. Verified with a full, fresh 30-round chaotic re-soak (see run2 VERDICT).", @@ -17595,24 +6164,27 @@ "raft-commit-barrier", "apply-decoupling", "false-positive", - "test-harness", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-harness" ], "related_bugs": [ "bug-1422", "bug-1431", "bug-1432" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1487.md" }, { "id": "bug-1489", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-03T04:00:00Z", - "error_message": "gcc: 'unknown type name xce_sched_core_t; did you mean xce_sched_ctx_t?' at core.h:92 (xce_sched_core_init prototype) when compiling scheduler.h consumers (handler.c, main.c, etc).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "gcc: 'unknown type name xce_sched_core_t; did you mean xce_sched_ctx_t?' at core.h:92…\n\ngcc: 'unknown type name xce_sched_core_t; did you mean xce_sched_ctx_t?' at core.h:92 (xce_sched_core_init prototype) when compiling scheduler.h consumers (handler.c, main.c, etc).", "file": "src/scheduler/include/core.h", "root_cause": "core.h defined `struct xce_sched_core_s { ... };` (the full body) but relied entirely on plugin.h's forward declaration (`typedef struct xce_sched_core_s xce_sched_core_t;`) for the typedef name -- core.h itself never declared it. scheduler.h includes core.h BEFORE plugin.h, so in that (the only real) include order, xce_sched_core_t was unresolved by the time core.h's own function prototypes referenced it.", "fix": "Added `typedef struct xce_sched_core_s xce_sched_core_t;` directly in core.h, immediately before the struct body. An identical repeated typedef (same underlying type) is valid C11/gnu17, matching this codebase's existing cluster_t idiom (myconf.h/persist.h/ha.h/cluster_def.h all forward-declare independently) -- so the fix works regardless of which of core.h/plugin.h is included first.", @@ -17621,20 +6193,23 @@ "task-4.1", "core-extraction", "typedef-forward-decl", - "include-order", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "include-order" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1489.md" }, { "id": "bug-1490", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-03T04:00:00Z", - "error_message": "ld: undefined reference to 'reset_sched_obj' when linking xce_scheduler (referenced from plugins/limit.c's limit_job_election()).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ld: undefined reference to 'reset_sched_obj' when linking xce_scheduler (referenced from…\n\nld: undefined reference to 'reset_sched_obj' when linking xce_scheduler (referenced from plugins/limit.c's limit_job_election()).", "file": "src/scheduler/src/schedobj.c", "root_cause": "While rewriting schedobj.c wholesale to relocate the CRUD accessors (sched_get_job/sched_add_job/etc, scheduler-core spec task 4.1) into core.c, the reset_sched_obj() function (unrelated to the relocation, a small xlist/zarray cleanup helper called by plugins/limit.c) was accidentally omitted from the rewritten file -- a transcription miss, not an intentional removal.", "fix": "Re-added reset_sched_obj() verbatim (unchanged body) to schedobj.c, right after make_sched_obj() where it originally lived.", @@ -17643,22 +6218,25 @@ "task-4.1", "core-extraction", "link-error", - "whole-file-rewrite-risk", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "whole-file-rewrite-risk" ], "related_bugs": [ "bug-1489" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1490.md" }, { "id": "bug-1491", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-08-03T06:33:00Z", - "error_message": "AddressSanitizer: SEGV on unknown address 0x000000000000 in dataq_deq_head (lib/dataq.c:15), called from dataq_fini <- xce_sched_core_fini (core.c:126), when a test called xce_sched_core_init() then xce_sched_core_fini() back-to-back on a fresh xce_sched_core_t.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: SEGV on unknown address 0x000000000000 in dataq_deq_head (lib/dataq.c:15)…\n\nAddressSanitizer: SEGV on unknown address 0x000000000000 in dataq_deq_head (lib/dataq.c:15), called from dataq_fini <- xce_sched_core_fini (core.c:126), when a test called xce_sched_core_init() then xce_sched_core_fini() back-to-back on a fresh xce_sched_core_t.", "file": "src/scheduler/testsuite/test_core_jbuf_capacity_pin.c", "root_cause": "xce_sched_core_init() memset()s the whole xce_sched_core_t, leaving core->pend_updates[0]/[1] NULL -- those two dataq_t* are only allocated+dataq_init()d by scheduler.c sched_init() AFTER calling xce_sched_core_init() (see core.c file header comment: pend_updates storage lives in xce_sched_core_t but its dataq_init/killer wiring stays driver-owned in scheduler.c). xce_sched_core_fini() unconditionally calls dataq_fini(core->pend_updates[0]) and [1] with no NULL guard, so calling fini() right after init() without going through sched_init()'s extra setup step dereferences NULL inside dataq_deq_head.", "fix": "Not a production bug (xce_sched_core_fini()'s real caller, scheduler.c sched_fini(), always runs after sched_init() has populated pend_updates) -- fixed the test instead: test_jbuf_capacity_is_runtime_one() (task 4.2) calls only xce_sched_core_init() and reads core.jbuf.ncap, deliberately never calling xce_sched_core_fini() on a fixture built by init() alone. Matches this testsuite directory's established \"fixtures aren't torn down\" convention (test_picker_core_loop_boundary.c's header comment).", @@ -17669,23 +6247,26 @@ "segv", "null-deref", "test-authoring-gotcha", - "pend_updates", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pend_updates" ], "related_bugs": [ "bug-1489", "bug-1490" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1491.md" }, { "id": "bug-1492", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-04T01:04:00Z", - "error_message": "plugin_end_session() checks plugin->begin_session_fn != NULL but calls plugin->end_session_fn(plugin) -- a guard/call field mismatch", + "last_seen": "2026-09-19T09:30:00Z", + "occurrences": 1, + "error_message": "plugin_end_session() checks plugin->begin_session_fn != NULL but calls…\n\nplugin_end_session() checks plugin->begin_session_fn != NULL but calls plugin->end_session_fn(plugin) -- a guard/call field mismatch", "file": "src/scheduler/src/plugin.c", "root_cause": "Pre-existing bug predating the scheduler-core spec (git blame: commit 638173dec, 2026-02-26). plugin_end_session()'s per-plugin loop guards on the wrong struct field before invoking the hook, so a plugin registering only end_session_fn (not begin_session_fn) never has it called; a plugin registering only begin_session_fn (not end_session_fn) would crash via a NULL function-pointer call if this branch were ever reached for it. Discovered incidentally during scheduler-core task 5.2's independent review while verifying the plugin_begin_session/plugin_end_session per-session bookkeeping fan-out is preserved in the new normal-mode driver loop -- not introduced by task 5.2, confirmed untouched by its diff.", "fix": "FIXED (2026-09-19, /xce-audit-invariants scheduler dimension-1 audit, following scheduler-sim-live-allocation). Changed plugin_end_session()'s guard from plugin->begin_session_fn != NULL to plugin->end_session_fn != NULL (src/scheduler/src/plugin.c) -- exactly the one-line fix this entry originally proposed. Confirmed via TDD: added test_end_session_invokes_end_session_fn_when_only_it_is_set() (test_plugin_hook_dispatch.c) registering a plugin with ONLY end_session_fn set, proving it was silently never invoked before the fix (RED, 84/85) and is now correctly invoked (GREEN, 85/85). Duplicate of bug-1682 (same defect, independently re-discovered and re-logged during bug-1681's fix, 2026-08-24) -- this entry (bug-1492, filed 2026-08-04) is the original. Note: this entry's own tags array had been incorrectly stamped \"fixed\"/\"closed-by-scheduler-sim-live-allocation\" ahead of the actual fix landing (a bulk-tagging error found by this same audit) -- now genuinely accurate.", @@ -17695,22 +6276,25 @@ "plugin-c", "guard-call-mismatch", "pre-existing", - "legacy-fcfs-picker", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "legacy-fcfs-picker" ], "related_bugs": [ "bug-1682" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:30:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1492.md" }, { "id": "bug-1493", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-08-04T12:35:00Z", - "error_message": "gcc: 'expected declaration specifiers before sched_snapshot_data_t' cascading into ~20 errors across typemap.h/controller.capnp.h/controller.capnp.c when building lib/controller.capnp.c (scheduler-core task 7.3 SnapshotData C struct).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "gcc: 'expected declaration specifiers before sched_snapshot_data_t' cascading into ~20 errors…\n\ngcc: 'expected declaration specifiers before sched_snapshot_data_t' cascading into ~20 errors across typemap.h/controller.capnp.h/controller.capnp.c when building lib/controller.capnp.c (scheduler-core task 7.3 SnapshotData C struct).", "file": "src/include/typemap.h", "root_cause": "A doc comment on sched_snapshot_data_t read '...raw uint8_t*/size_t pattern...' -- the literal substring '*/' inside the comment text (no space between the uint8_t* pointer-star and the following /size_t) prematurely closed the C block comment. Everything after that point (the rest of the comment prose plus the following struct's opening brace) was parsed as real C tokens, corrupting the sched_snapshot_data_t typedef and cascading into every generated function signature that references it.", "fix": "Reworded the comment to avoid the accidental */ sequence ('uint8_t-pointer + size_t pattern' instead of 'uint8_t*/size_t pattern'). General gotcha: never write 'TypeA*/TypeB' (pointer-star immediately followed by slash) inside a /* */ C comment -- always put a space or word between the * and the /.", @@ -17720,20 +6304,23 @@ "capnp-schema", "typemap.h", "comment-syntax", - "build-break", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "build-break" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1493.md" }, { "id": "bug-1494", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-04T00:00:00Z", - "error_message": "clang-format -i produced bizarre whole-file +1-space indentation drift (starting right after the top #include block, persisting to EOF) in a newly-written test_snapshot_save_load.c, with kill_sched_job() alone expanded to multiple lines while identical sibling one-liners stayed collapsed.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "clang-format -i produced bizarre whole-file +1-space indentation drift (starting right after the…\n\nclang-format -i produced bizarre whole-file +1-space indentation drift (starting right after the top #include block, persisting to EOF) in a newly-written test_snapshot_save_load.c, with kill_sched_job() alone expanded to multiple lines while identical sibling one-liners stayed collapsed.", "file": "src/scheduler/testsuite/test_snapshot_save_load.c", "root_cause": "Same class of bug as bug-1493 (task 7.3): the file's own top-of-file header comment contained the literal substring '(kill_sched_*/plugin_unregister/...)' -- the pointer-star-like '*' immediately followed by '/' prematurely closed the /* */ block comment. Everything after that point was parsed as real C code by clang-format's lexer, which is what produced the indentation drift (a strong diagnostic signal, not just a formatting quirk) -- gcc itself would also have failed to compile this.", "fix": "Reworded 'kill_sched_*/plugin_unregister/sched_worker_release_host_ctx/sched_resolve_job_picker)' to 'kill_sched_* family, plugin_unregister, sched_worker_release_host_ctx, sched_resolve_job_picker)', removing the accidental */ sequence. General lesson (reinforcing bug-1493): a sudden, file-wide clang-format indentation shift starting at a specific line is a strong signal to grep that file for a stray '*/' inside a comment before assuming it's a clang-format config issue -- verified via a small python scanner that walks the file tracking in-comment state and flags any '*/' encountered while not already inside a comment.", @@ -17742,22 +6329,25 @@ "task-8.1", "comment-syntax", "clang-format", - "testsuite", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "testsuite" ], "related_bugs": [ "bug-1493" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1494.md" }, { "id": "bug-1495", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-04T00:00:00Z", - "error_message": "sched_get_user()/sched_get_project() (core.c) always returned NULL for any user/project name not already present in core->users/core->projects, even though both are documented and relied upon (core_events.c JOBS_PUSH, schedobj.c make_sched_job, and task 8.1's own snapshot_load.c) as get-or-create. Surfaced as job->user/job->project being silently NULL for every job referencing a brand-new user/project, discovered while writing test_snapshot_save_load.c's cross-core equivalence test (RED phase: TEST_ASSERT_NOT_NULL(sched_get_project(&core_a, \"proj1\")) failed immediately after a successful JOBS_PUSH).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "sched_get_user()/sched_get_project() (core.c) always returned NULL for any user/project name not…\n\nsched_get_user()/sched_get_project() (core.c) always returned NULL for any user/project name not already present in core->users/core->projects, even though both are documented and relied upon (core_events.c JOBS_PUSH, schedobj.c make_sched_job, and task 8.1's own snapshot_load.c) as get-or-create. Surfaced as job->user/job->project being silently NULL for every job referencing a brand-new user/project, discovered while writing test_snapshot_save_load.c's cross-core equivalence test (RED phase: TEST_ASSERT_NOT_NULL(sched_get_project(&core_a, \"proj1\")) failed immediately after a successful JOBS_PUSH).", "file": "src/scheduler/src/core.c", "root_cause": "Both functions call key_find() then do `if (rc < 0) { return NULL; }` before falling through to the make_sched_user()/make_sched_project() create path. key_find()'s actual not-found signal is a NEGATIVE errno (-ENOENT, confirmed via direct debug instrumentation: rc=-2 for an absent key, rc=1 for a present one) -- NOT rc==0 as the surrounding code assumed. Since -ENOENT < 0, the guard incorrectly treated the EXPECTED not-found outcome as a fatal error and returned NULL before ever reaching the create fallback, making get-or-create unreachable for any genuinely new name. This is a pre-existing bug from earlier tasks (4.1/4.3), not introduced by task 8.1, but it directly breaks task 8.1's own snapshot_load.c users/projects restoration onto a freshly-initialized core (which always starts empty, so every name is 'new').", "fix": "Changed the guard in both functions from `if (rc < 0)` to `if ((rc < 0) && (rc != -ENOENT))`, so only a genuine (non-ENOENT) error short-circuits before the create fallback. Two-line change per function, core.c lines ~341 and ~416. Verified: full scheduler suite 23/23 (up from 22/22 baseline, no regressions), plus the new test's own field-level assertions (users/projects created correctly, priority preserved) now pass.", @@ -17770,20 +6360,23 @@ "key_find", "sched_get_user", "sched_get_project", - "regression-fix", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "regression-fix" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1495.md" }, { "id": "bug-1496", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-04T00:00:00Z", - "error_message": "xce_sched_core_load_snapshot() never restored core->prng_seed from the snapshot -- a freshly loaded core silently kept xce_sched_core_init()'s default (0) instead of the value xce_sched_core_save_snapshot() had captured. Caught by test_snapshot_save_load.c's field-level assertion: TEST_ASSERT_EQUAL_UINT64(0xDEADBEEFULL, core_b.prng_seed) failed with Expected 3735928559 Was 0 after a successful save+load round trip.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_sched_core_load_snapshot() never restored core->prng_seed from the snapshot -- a freshly…\n\nxce_sched_core_load_snapshot() never restored core->prng_seed from the snapshot -- a freshly loaded core silently kept xce_sched_core_init()'s default (0) instead of the value xce_sched_core_save_snapshot() had captured. Caught by test_snapshot_save_load.c's field-level assertion: TEST_ASSERT_EQUAL_UINT64(0xDEADBEEFULL, core_b.prng_seed) failed with Expected 3735928559 Was 0 after a successful save+load round trip.", "file": "src/scheduler/src/snapshot_load.c", "root_cause": "save_snapshot() (snapshot.c) correctly writes `out->prng_seed = core->prng_seed;`, but the corresponding `core->prng_seed = in->prng_seed;` assignment was simply missing from load_snapshot()'s field-copy sequence -- an incomplete round trip for a field the function's own header comment (snapshot.c, 'PRNG seed resolution') explicitly documents as in scope (Req 5 AC 21's 'PRNG state' checklist row).", "fix": "Added `core->prng_seed = in->prng_seed;` to xce_sched_core_load_snapshot() (snapshot_load.c), placed after the jobs-restoration loop and before the internal_state restoration block. Verified via the same field-level assertion, now passing.", @@ -17792,20 +6385,23 @@ "task-8.1", "snapshot_load.c", "prng_seed", - "round-trip-gap", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "round-trip-gap" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1496.md" }, { "id": "bug-1497", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-08-04T00:00:00Z", - "error_message": "ASan SEGV inside dup_val() -> makeString() -> hash_string() while building test_snapshot_save_load.c's cross-core equivalence test fixtures.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ASan SEGV inside dup_val() -> makeString() -> hash_string() while building…\n\nASan SEGV inside dup_val() -> makeString() -> hash_string() while building test_snapshot_save_load.c's cross-core equivalence test fixtures.", "file": "src/scheduler/testsuite/test_snapshot_save_load.c", "root_cause": "Test-side only, not production code: three test-owned xce_val_t resval fixtures were left with `.which` at its memset-zero default. XCE_VAL_STRING happens to be enum value 0 (xce/types.h), so the zeroed `.which` was silently interpreted as \"this is a string\" -- dup_val()/makeString() then reinterpreted the fixture's raw ivalue32 int as a char* pointer and hash_string() dereferenced it, crashing.", "fix": "Explicitly set `.which = XCE_VAL_NUMERIC` on all three test-owned xce_val_t fixtures in test_snapshot_save_load.c before use -- never rely on struct-zero-init to select a union discriminant when a non-zero-valued enum member (XCE_VAL_NUMERIC) is the intended default.", @@ -17816,20 +6412,23 @@ "xce_val_t", "zero-init-union-discriminant", "test-only", - "asan-segv", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "asan-segv" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1497.md" }, { "id": "bug-1498", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "security", "timestamp": "2026-08-05T00:00:00Z", - "error_message": "test_toctou_hardlink_planted_after_validation_rejected FAILED: Expected Non-NULL -- fgets() on the outside 'victim' file returned NULL (file was 0 bytes) after the TOCTOU regression test ran, even though sched_record_start()/the reproduced open-hardening sequence both correctly REJECTED the hard-link swap and returned an error.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_toctou_hardlink_planted_after_validation_rejected FAILED: Expected Non-NULL -- fgets() on the…\n\ntest_toctou_hardlink_planted_after_validation_rejected FAILED: Expected Non-NULL -- fgets() on the outside 'victim' file returned NULL (file was 0 bytes) after the TOCTOU regression test ran, even though sched_record_start()/the reproduced open-hardening sequence both correctly REJECTED the hard-link swap and returned an error.", "file": "src/scheduler/src/recording.c", "root_cause": "sched_record_start()'s TOCTOU-hardened open used O_WRONLY|O_CREAT|O_TRUNC|O_NOFOLLOW. O_TRUNC truncates the target atomically as PART OF the open(2) syscall itself, before any userspace code -- including the fstat()+st_nlink==1 safety check right after open() -- ever runs. Against a hard-linked victim (same inode as the outside file, no symlink involved so O_NOFOLLOW never triggers), open() succeeds and O_TRUNC immediately zeroes the shared inode's content; the subsequent st_nlink>1 check then correctly rejects and closes the fd, but the outside victim file was already corrupted by that point. The check ran too late to matter -- classic 'check-after-side-effect' TOCTOU-adjacent bug, caught by the test's own outside-file-content assertion (added specifically to prove containment, not just that the return code was negative).", "fix": "Removed O_TRUNC from the initial open() flags (now O_WRONLY|O_CREAT|O_NOFOLLOW). After fstat()+st_nlink==1 passes, explicitly call ftruncate(fd, 0) to get the same truncate-on-legitimate-overwrite behavior, but only once safety is verified. Same fix applied to the test's own toctou_hardened_open() helper (a verbatim reproduction of the production sequence, used because a literal multi-threaded race against sched_record_start()'s few-instruction internal window would be flaky).", @@ -17841,20 +6440,23 @@ "hardlink", "O_TRUNC", "check-after-side-effect", - "sched_record_start", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "sched_record_start" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1498.md" }, { "id": "bug-1499", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-05T00:00:00Z", - "error_message": "sched_driver_session_ordering_test TIMEOUT (30s, killed by SIGTERM) after adding task 9.2's recording-write hooks to xce_sched_driver_run_normal_mode_round() -- picker thread, allocator thread, and the main round all blocked forever.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "sched_driver_session_ordering_test TIMEOUT (30s, killed by SIGTERM) after adding task 9.2's…\n\nsched_driver_session_ordering_test TIMEOUT (30s, killed by SIGTERM) after adding task 9.2's recording-write hooks to xce_sched_driver_run_normal_mode_round() -- picker thread, allocator thread, and the main round all blocked forever.", "file": "src/scheduler/src/driver.c", "root_cause": "While inserting the new 'if (sched->recording) { rec_ctx = sched_record_session_begin(...); }' block between plugin_begin_session()/sched_account_jobs() and the decision-drain loop, the pre-existing `xce_core_signal_go(&(sched->core));` call was accidentally dropped from the Edit's old_string/new_string pair -- it was the last line of the block being replaced but was not included in the replacement text. Without xce_core_signal_go(), the picker thread's cbuf_get(&core->wbuf) rendezvous never unblocks, which cascades: the stand-in allocator thread never receives a jobid from cbuf_get(&core->jbuf), so it never pushes the end-of-session NULL decision, so the main thread's own cbuf_get(&core->decisions) drain loop blocks forever -- a 3-way deadlock across all three threads.", "fix": "Restored `xce_core_signal_go(&(sched->core));` immediately after the new recording session-begin block (matching requirements.md's own annotated pseudocode order: build ReplaySession{beginRound} THEN sched_go()). Caught by re-running the full scheduler test suite after the recording hooks were added -- test_driver_session_ordering.c is the ONLY test in the suite that drives a real picker/allocator thread pair through this exact function, so it was the only test capable of catching a signal_go() regression.", @@ -17865,20 +6467,23 @@ "deadlock", "xce_core_signal_go", "accidental-deletion-in-edit", - "xce_sched_driver_run_normal_mode_round", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "xce_sched_driver_run_normal_mode_round" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1499.md" }, { "id": "bug-1500", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-06T00:00:00Z", - "error_message": "test_run_processes_multiple_sessions_in_order_and_no_longer_returns_enosys FAIL: Expected 5000 Was 743938186628040 (a raw-looking CLOCK_MONOTONIC nanosecond value, not the fixture's own beginRound).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_run_processes_multiple_sessions_in_order_and_no_longer_returns_enosys FAIL: Expected 5000 Was…\n\ntest_run_processes_multiple_sessions_in_order_and_no_longer_returns_enosys FAIL: Expected 5000 Was 743938186628040 (a raw-looking CLOCK_MONOTONIC nanosecond value, not the fixture's own beginRound).", "file": "src/scheduler/testsuite/test_sim_event_loop.c", "root_cause": "The test's fixture-building helper called the real sched_record_start() THEN sched_record_stop() to produce a valid header+snapshot prefix (matching test_sim_startup.c's own write_valid_fixture() precedent), then re-opened the same path in append mode ('a') to hand-write additional ReplayEvent/ReplaySession messages after it. But sched_record_stop() (recording.c) unconditionally writes its OWN ReplayEvent{endSession} marker immediately after the snapshot as part of every completed recording (Req 5 AC 20) -- so the fixture already contained a premature end-of-stream marker BEFORE any of the test's own appended events/sessions. sched_sim_driver_run()'s event loop (sim_loop.c, task 11.2) correctly treats ReplayEvent{endSession} as the whole-stream termination signal and stopped reading right there, leaving sim->core's clock at its xce_sched_core_init() default (virtual_clock_base=0, speed_num=1, speed_den=1 -- AC 3's normal-mode row), so xce_core_now() returned an unfrozen, elapsed-wall-clock-driven value instead of the intended frozen 5000.", "fix": "Changed the fixture helper to call ONLY sched_record_start() (never sched_record_stop()), and return the STILL-OPEN FILE* (sched.record_file) so the test appends its own hand-built events/sessions directly onto the same open stream, ending with its own append_end_of_stream_marker() call, then fclose()s it directly -- bypassing sched_record_stop()'s real event/session-recording API entirely for this hand-built-fixture use case.", @@ -17889,20 +6494,23 @@ "test-fixture", "sched_record_stop", "ReplayEvent_body_endSession", - "premature-termination-marker", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "premature-termination-marker" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1500.md" }, { "id": "bug-1501", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-07T00:00:00Z", - "error_message": "gcc: 'expected =, ,, ;, asm or __attribute__ before is' / 'unknown type name is' / 'missing terminating character' cascade starting at sim_verify.c:59, plus 'implicit declaration of function sim_verify_str_eq' further down.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "gcc: 'expected =, ,, ;, asm or __attribute__ before is' / 'unknown type name is' / 'missing…\n\ngcc: 'expected =, ,, ;, asm or __attribute__ before is' / 'unknown type name is' / 'missing terminating character' cascade starting at sim_verify.c:59, plus 'implicit declaration of function sim_verify_str_eq' further down.", "file": "src/scheduler/src/sim_verify.c", "root_cause": "File header doc comment's Comparison-strategy paragraph wrote the literal text 'xce_job_alloc_t*/xce_pendreason_t*' to mean 'an xce_job_alloc_t pointer or an xce_pendreason_t pointer' -- the '*/' substring is C's block-comment CLOSE token, so the /* ... */ doc comment terminated early, right in the middle of prose. Everything from that point until the next real */ (the following function's own closing delimiter) was parsed as C code, producing a cascade of nonsense syntax errors and, once the compiler resynced, an implicit-declaration warning/error for a function whose real prototype/definition had been swallowed into the truncated comment. Exact recurrence of the bug-1493 pattern already logged for typemap.h (same 'accidental */ in a C block comment' root cause).", "fix": "Reworded to 'xce_job_alloc_t* or xce_pendreason_t*' (space before 'or' instead of a bare slash-star-slash-adjacent sequence), eliminating the literal '*/' substring. Ran a repo grep for the same '[A-Za-z_]*/[A-Za-z_]' pattern across both new/modified files (sim_verify.c, sim_loop.c) before rebuilding to rule out a second occurrence.", @@ -17913,22 +6521,25 @@ "block-comment", "accidental-comment-close", "bug-1493-recurrence", - "build-failure", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "build-failure" ], "related_bugs": [ "bug-1493" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1501.md" }, { "id": "bug-1502", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-07T00:00:00Z", - "error_message": "Task 11.5 review round 1 REJECTED: previous implementer's self-reported RED-phase evidence (git stash of 4 'behavioral production files' -> 3 tests failing under meson test) did not reproduce -- reviewer got a hard ninja build failure instead (test_sim_event_loop.c:467 calling sim_process_replay_event_message with 5 args against a stashed-back 4-arg sim_loop.c declaration).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Task 11.5 review round 1 REJECTED: previous implementer's self-reported RED-phase evidence (git…\n\nTask 11.5 review round 1 REJECTED: previous implementer's self-reported RED-phase evidence (git stash of 4 'behavioral production files' -> 3 tests failing under meson test) did not reproduce -- reviewer got a hard ninja build failure instead (test_sim_event_loop.c:467 calling sim_process_replay_event_message with 5 args against a stashed-back 4-arg sim_loop.c declaration).", "file": "src/scheduler/src/sim_loop.c", "root_cause": "git stash scoped to only 4 production files silently desyncs test files (which already call the NEW post-task-11.5 signatures/behavior) against the stashed-back OLD production code -- a stash-based RED probe is not a valid methodology whenever tests and production code changed together in the same commit-to-be. The previous implementer's reported failure counts were therefore not reproducible as described.", "fix": "Replaced the stash approach with a direct, targeted, in-place temporary edit of ONLY the behavioral logic inside sim_loop.c (never touching test files, sim_record.c, sim.c/sim.h, recording_session.c/recording.h, or either meson.build): hardcoded event_src to sim->replay_file (defeats AC16 what-if branching) and wrapped the 3 --sim-output write gates (header/snapshot, event pass-through, session-write via record_ctx) in 'if (0 && ...)' no-ops (defeats AC19). Rebuilt+ran `meson test -C xce.build --suite scheduler`: real RED = 30/32, sched_sim_alt_events_test and sched_sim_record_output_test FAIL with concrete assertion messages. Reverted all 4 edits using the exact original text; confirmed via `git diff src/scheduler/src/sim_loop.c` that only the original (pre-probe) task-11.5 diff remained, no probe residue (grep for 'TEMP RED-PHASE PROBE' and 'if (0 &&' both zero matches). Rebuilt+reran: 32/32 green again.", @@ -17939,20 +6550,23 @@ "red-phase-evidence", "review-rejection", "git-stash-pitfall", - "methodology", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "methodology" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1502.md" }, { "id": "bug-1503", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-08-07T03:11:00Z", - "error_message": "AddressSanitizer: attempting free on address which was not malloc()-ed, inside free_QueueConf() <- free_QueueConf_ptr() <- release_translated_payload() (driver.c) <- xce_sched_driver_translate_pending_events(). Crashing pointer was 56 bytes inside a 61-byte region allocated by makeString()->makeInstance().", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: attempting free on address which was not malloc()-ed, inside free_QueueConf() <…\n\nAddressSanitizer: attempting free on address which was not malloc()-ed, inside free_QueueConf() <- free_QueueConf_ptr() <- release_translated_payload() (driver.c) <- xce_sched_driver_translate_pending_events(). Crashing pointer was 56 bytes inside a 61-byte region allocated by makeString()->makeInstance().", "file": "src/scheduler/testsuite/test_sim_equivalence.c", "root_cause": "makeString() (lib/xstring.c) is backed by the vendored rtti library makeInstance() allocator, which does NOT return a plain malloc()-compatible pointer at offset 0 of its allocation (it returns a pointer into a larger rtti-managed block). Elsewhere in the codebase makeString()-allocated strings are safe because they are only ever released by the SAME rtti-aware teardown path that created them (e.g. kill_sched_* killers, or never freed at all in white-box test fixtures). This new test constructed xce_queue_t/xce_job_t fixtures with q->queue/submit->queue/user/project_name/resreq set via makeString(), then drove them through the REAL production cleanup path release_translated_payload() (driver.c), whose free_QueueConf_ptr()/free_Job_ptr() (controller.capnp.c, capnp-generated) call plain free() on those fields directly -- exactly the mismatch: an rtti-backed allocation freed via plain free().", "fix": "Replaced every makeString() call used to populate xce_queue_t.queue / xce_job_t.submit->{queue,user,project_name,resreq} in test_sim_equivalence.c with strdup(), which returns a plain malloc()-compatible buffer safe for free_QueueConf_ptr()/free_Job_ptr() to free directly. Confirmed clean (no ASan bad-free) after the fix, full scheduler suite (33/33) green.", @@ -17963,20 +6577,23 @@ "asan", "bad-free", "driver.c", - "task-11.6", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-11.6" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1503.md" }, { "id": "bug-1504", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-08-07T08:00:00Z", - "error_message": "sched_sim_startup_test SIGSEGV in write_SnapshotData (lib/controller.capnp.c) -- deterministic crash inside libCapnC_Runtime's read_ptr/capn_setp, discovered during scheduler-core task 12.1 review remediation (full clean rebuild needed for a libedit dependency swap).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "sched_sim_startup_test SIGSEGV in write_SnapshotData (lib/controller.capnp.c) -- deterministic…\n\nsched_sim_startup_test SIGSEGV in write_SnapshotData (lib/controller.capnp.c) -- deterministic crash inside libCapnC_Runtime's read_ptr/capn_setp, discovered during scheduler-core task 12.1 review remediation (full clean rebuild needed for a libedit dependency swap).", "file": "src/schema/controller.capnp", "root_cause": "SnapshotData.internalState was declared `:Data` in the capnp schema. This codebase's capnpc-c backend silently no-ops Data-field encoding inside generated encode_XXX() composite helpers (documented, deliberate limitation -- see controller.capnp's own prior comment citing ReplayHeader.configHash). But encode_SnapshotData_ptr()'s local `struct SnapshotData d;` is never zero-initialized before the (partially no-op) encode_SnapshotData() populates it, so the skipped Data field's capn_ptr is left as UNINITIALIZED STACK GARBAGE. write_SnapshotData() then unconditionally capn_setp()'s that garbage onto the wire. This is genuine UB: whether it crashes depends entirely on what garbage bytes land in that stack slot at each call site -- ReplayHeader.configHash has the identical gap but never crashed (lucky garbage); SnapshotData.internalState's stack layout in this one test's call chain was unlucky. A prior 'rm -rf xce.build' full rebuild (needed to pick up a new nix dependency) perturbed the whole binary's layout enough to flip this from dormant to a deterministic, reproducible SIGSEGV.", "fix": "Changed SnapshotData.internalState from `Data` to `List(UInt8)` in controller.capnp, with a new companion internalStateLen (@11, UInt32) wire field for the maplistcount -- following the exact working precedent of FetchFileReply.content/bytesRead, which this same capnpc-c backend DOES correctly auto-generate real (non-no-op) encode/decode for (capn_new_list8/capn_set8 loop, not a placeholder comment). Updated typemap.h's sched_snapshot_data_t.internal_state_len from size_t to uint32_t to match the wire type, and updated test_e2e_capnp.c's test_snapshot_data_round_trip (which previously worked around the Data no-op with a manual low-level SnapshotData_get/set_internalState accessor) to rely on the now-working ordinary codec bridge instead. No hand-written low-level accessor code needed any more for this field.", @@ -17988,23 +6605,26 @@ "sigsegv", "uninitialized-memory", "controller.capnp", - "snapshot", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "snapshot" ], "related_bugs": [ "bug-1493", "bug-1505" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1504.md" }, { "id": "bug-1505", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-08-07T08:10:00Z", - "error_message": "controller_unit_test SIGSEGV in write_WalCatchupEntry (lib/controller.capnp.c), test_wal_catchup_append_term_fields_round_trip -- discovered incidentally while chasing bug-1504; NOT fixed, logged for its own follow-up.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller_unit_test SIGSEGV in write_WalCatchupEntry (lib/controller.capnp.c)…\n\ncontroller_unit_test SIGSEGV in write_WalCatchupEntry (lib/controller.capnp.c), test_wal_catchup_append_term_fields_round_trip -- discovered incidentally while chasing bug-1504; NOT fixed, logged for its own follow-up.", "file": "src/schema/controller.capnp", "root_cause": "Same root cause as bug-1504 (Data-field codegen no-op leaves an uninitialized stack capn_ptr in the generated encode_XXX_ptr()'s local struct), a DIFFERENT field: WalCatchupEntry.payload (@2: Data), in the controller's HA/WAL-catchup subsystem (bug-847's own territory -- see controller.capnp's comment there), entirely unrelated to scheduler-core. Confirms this is a systemic, project-wide bug class affecting every `:Data` field in controller.capnp whose bytes are populated via the generic encode_XXX()/write_XXX() composite pipeline rather than fully hand-written low-level accessor code -- not a one-off. Known/suspected other Data fields with the same latent gap: ReplayHeader.configHash (dormant so far, no real consumer), CtrlHelloRequest.key, and possibly others not yet audited.", "fix": "FIXED 2026-08-07. Investigated production exposure first: xce_wal_batch_encode_raw()/wal_entry_to_wire() (request.c) never call the generic encode_WalCatchupEntry() at all -- they memset() their own struct and call write_WalCatchupEntry() directly with every field hand-set, deliberately leaving payload unset (bug-847's own comment). So this bug was 100% test-only exposure, zero production risk, confirmed by grepping every caller of encode_WalCatchupEntry_ptr/decode_WalCatchupEntry (only test_e2e_capnp.c uses the generic path). Fix: same as bug-1504 -- changed WalCatchupEntry.payload (@2) and batchPayloads (@10) from Data to List(UInt8) in controller.capnp, with batchPayloadsLen (@11, new) as batchPayloads' maplistcount companion (payload reuses the existing wirePayloadLen @9). Verified batchPayloads' own top-level-single-blob shape means this does not reintroduce bug-847's per-element cross-segment far-pointer loss (that issue is inherent to PER-ELEMENT data regardless of Data vs List(UInt8) spelling -- payload, the per-element field, was already legacy/unpopulated in production before this fix and stays that way; batchPayloads is single top-level, never subject to that issue either way). Updated typemap.h's wal_catchup_entry_t field types (xce_data_t* -> uint8_t*, new batch_payloads_wire_len). Fixed request.c's extract_data_bytes() to accept a raw capn_ptr instead of capn_data (it already only ever used .p internally) so all 4 call sites (WalCatchupEntry.payload/batchPayloads, SnapshotTransfer.payload, per-sub-entry we.payload) work regardless of which Data-shaped wrapper type the accessor returns. Strengthened test_wal_catchup_append_term_fields_round_trip (test_e2e_capnp.c) with real payload/batchPayloads content assertions (previously never exercised at all, since the test's src.payload/batch_payloads were left NULL). Verified: controller_unit_test no longer crashes (3/3 controller suite), scheduler suite unaffected (37/37, unrelated schema area).", @@ -18018,23 +6638,26 @@ "uninitialized-memory", "controller.capnp", "deferred", - "systemic", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "systemic" ], "related_bugs": [ "bug-1504", "bug-847" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1505.md" }, { "id": "bug-1506", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-07T09:05:00Z", - "error_message": "sched_repl_breakpoints_inspection_test: repl_dispatch(sim, \"break JOB_STATE_CHANGE jobid=12345\") stored NULL for the breakpoint's token/filter key/value -- TEST_ASSERT_EQUAL_STRING(\"jobid\", ...) failed with 'Expected jobid Was NULL'.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "sched_repl_breakpoints_inspection_test: repl_dispatch(sim, \"break JOB_STATE_CHANGE jobid=12345\")…\n\nsched_repl_breakpoints_inspection_test: repl_dispatch(sim, \"break JOB_STATE_CHANGE jobid=12345\") stored NULL for the breakpoint's token/filter key/value -- TEST_ASSERT_EQUAL_STRING(\"jobid\", ...) failed with 'Expected jobid Was NULL'.", "file": "src/scheduler/testsuite/test_repl_breakpoints_inspection.c", "root_cause": "makeString() (src/lib/xstring.c) is an interned-xstring allocator backed by a process-global xstr_table that must be initialized via xstring_table_init() before first use -- xce_sched_core_init() (core.c) is the only call site in the scheduler tree and does this transitively. New task 12.2 unit tests built a bare xce_sched_sim_driver_t via memset(0) and called repl_state_ensure()/repl_dispatch() directly for pure command-dispatch tests, without ever calling xce_sched_core_init() first (the real driver flow always does, via sched_sim_driver_start() before xce_repl_start() runs, so this precondition is always satisfied in production -- only the test fixtures skipped it). With xstr_table uninitialized (NULL), makeString()'s internal key_find()/key_add() against a NULL kvmgr container fail gracefully and return NULL rather than crash, so the bug surfaced as silently-NULL breakpoint fields, not an immediate crash.", "fix": "Added xce_sched_core_init(&(sim.core)) as the first statement (right after memset) in every Group A/B test in test_repl_breakpoints_inspection.c that calls repl_dispatch()'s break/delete commands or the repl_break.c matcher functions -- matching the real driver's own precondition ordering.", @@ -18044,22 +6667,25 @@ "repl", "xstring", "test-fixture", - "makeString", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "makeString" ], "related_bugs": [ "bug-1507" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1506.md" }, { "id": "bug-1507", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-08-07T09:10:00Z", - "error_message": "sched_repl_breakpoints_inspection_test: 'free(): invalid pointer' SIGABRT inside repl_breakpoint_free(), reached from repl_cmd_break()'s malformed-filter rejection path (\"break JOB_PUSH nofiltermark\"). gdb backtrace: malloc_printerr <- repl_breakpoint_free (repl_break.c:375) <- repl_cmd_break <- repl_dispatch <- test_break_malformed_filter_rejected.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "sched_repl_breakpoints_inspection_test: 'free(): invalid pointer' SIGABRT inside…\n\nsched_repl_breakpoints_inspection_test: 'free(): invalid pointer' SIGABRT inside repl_breakpoint_free(), reached from repl_cmd_break()'s malformed-filter rejection path (\"break JOB_PUSH nofiltermark\"). gdb backtrace: malloc_printerr <- repl_breakpoint_free (repl_break.c:375) <- repl_cmd_break <- repl_dispatch <- test_break_malformed_filter_rejected.", "file": "src/scheduler/src/repl_break.c", "root_cause": "repl_breakpoint_free() called FREEUP() (a bare free()) on bp->token and each filter's key/value, all of which are makeString()-produced xstrings -- RTTI reference-counted objects (src/lib/xstring.c: xstring_type has custom .reference/.dereference interface functions; the returned pointer is the object's DATA region inside an RTTI-managed allocation, not a plain malloc() chunk). Releasing one requires going through the RTTI dereference path (xstring_dereference(), which only fires the real free when the refcount hits zero via the owning kvtab's key_del), never a bare free(). This codebase's own kill_sched_job()/kill_sched_queue() (schedobj.c) establish the identical precedent for every other makeString()'d field in the scheduler tree (xce_sched_job_t.submit fields, xce_sched_queue_t.name, etc.): none of them are ever explicitly released either -- they live for the process's lifetime, matching this whole test suite's 'detect_leaks=0 by design' convention. My new repl_break.c code was the first place in this file family to call free() directly on a makeString() result.", "fix": "Removed the FREEUP() calls on bp->token/filters[i].key/filters[i].value from repl_breakpoint_free() (repl_break.c) -- it now only frees the plain calloc'd bp->filters array and bp itself, matching kill_sched_job()/kill_sched_queue()'s established never-release-xstrings precedent. Updated repl_internal.h's xce_repl_filter_t doc comment to state this explicitly so a future maintainer doesn't reintroduce the same free() on an RTTI-managed pointer.", @@ -18070,25 +6696,25 @@ "xstring", "heap-corruption", "free", - "rtti", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "rtti" ], "related_bugs": [ "bug-1506" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1507.md" }, { "id": "bug-1508", - "gitea_synced_status": "fixed", - "gitea_issue": 287, "status": "fixed", "status_updated": "2026-09-26", "status_note": "7723a4f4: worker fetchfile accepts only /job__task.out for the request's jobid (exact match), O_NOFOLLOW|O_NONBLOCK, regular single-link file. Also closed the REST route (user out_file=/etc/shadow).", + "component": "worker", + "severity": "unknown", "timestamp": "2026-08-08T00:00:00Z", - "error_message": "No user report / no test failure -- discovered incidentally during doc-cleanup fact-check audit of src/docs/job_peek.md against current code.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "No user report / no test failure -- discovered incidentally during doc-cleanup fact-check audit of…\n\nNo user report / no test failure -- discovered incidentally during doc-cleanup fact-check audit of src/docs/job_peek.md against current code.", "file": "src/worker/src/xcecd_worker.c (handle_fetchfile, jobs-peek fetchfile feature)", "root_cause": "job_peek.md's design called for worker-side security validation on the fetchfile RPC (jobid ownership check, out/err-file path match against the job's actual output paths, path-traversal rejection) before Plan B (CGO/FetchFile) shipped. The validation was never implemented -- handle_fetchfile() opens whatever path it is given with no ownership or path-traversal check.", "fix": "NOT FIXED -- out of scope for the doc-cleanup task that found it. Flagged to user as a follow-up; needs its own security-reviewer pass on handle_fetchfile() (job ownership check + path-traversal rejection, matching the validation job_peek.md originally specified).", @@ -18098,121 +6724,51 @@ "fetchfile", "worker", "jobs-peek", - "needs-fix", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "needs-fix" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "7723a4f4" + ], + "record": ".wolf/bugs/bug-1508.md", + "gitea_issue": 287, + "gitea_synced_status": "fixed" }, { "id": "bug-1509", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-10T07:34:55.093767+00:00", - "error_message": "extralib flake dependency pulls external repo for trivial linked-list code used only by autoscaler scaler.h", - "file": "src/flake.nix,src/nixSupport/xce.nix,src/autoscaler/scaler/include/scaler.h", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "extralib flake dependency pulls external repo for trivial linked-list code used only by autoscaler…\n\nextralib flake dependency pulls external repo for trivial linked-list code used only by autoscaler scaler.h", + "file": "src/flake.nix, src/nixSupport/xce.nix, src/autoscaler/scaler/include/scaler.h", "root_cause": "basiclib/list.h (xcelist_t + list_insert_entry_back) is the only consumer; can be vendored locally in ~30 lines.", "fix": "Port xcelist_t/xcelist_entry_t/list_insert_entry_back to local basic_list.h in autoscaler, remove extralib from flake.nix inputs and xce.nix nativeBuildInputs. ~40 min effort.", "tags": [ "tech-debt", "build", "nix", - "low-risk", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "low-risk" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1510", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-10T13:07:36.020Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/design.md", - "root_cause": "Had `hardware_monitor`", - "fix": "Changed to `worker_process_update()`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1511", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-10T13:09:14.910Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/component-failover-adoption/design.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21924 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1512", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-10T13:10:14.229Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/component-failover-adoption/requirements.md", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 21\u21927 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1513", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-10T13:47:21.938Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u21926 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1509.md" }, { "id": "bug-1514", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-10T18:46:00.000Z", - "error_message": "kiro-validate-design Critical Issue 1: design.md claims sched_job_picker/sched_allocator are 'unaffected' by a blocked xcecli_jobdispatch call, violating Req 3 AC1", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "kiro-validate-design Critical Issue 1: design.md claims sched_job_picker/sched_allocator are…\n\nkiro-validate-design Critical Issue 1: design.md claims sched_job_picker/sched_allocator are 'unaffected' by a blocked xcecli_jobdispatch call, violating Req 3 AC1", "file": ".kiro/specs/component-failover-adoption/design.md, tasks.md, research.md", "root_cause": "design.md/tasks.md/research.md all wrongly claimed xcecli_jobdispatch executes on the sched_allocator thread. Verified against source: it actually runs on scheduler.c's main driver-loop thread (main.c while(1) -> xce_sched_driver_run_normal_mode_round -> driver.c on_decision -> sched_publish_dispatch, scheduler.c:1257), reached only after draining the decisions cbuf (capacity 5). sched_allocator only produces into decisions; it never calls xcecli_jobdispatch. jbuf (picker->allocator handoff) is a fixed capacity-1 scheduler-core invariant (core.c:161, never resizable). So a blocked dispatch call backpressures through decisions (5 slots) then jbuf (1 slot) -- picker/allocator get a bounded ~6-job grace window, then both stall. Not full isolation as originally claimed.", "fix": "Corrected thread attribution and rejoin-call placement in design.md (SchedulerControllerClient section, sequence diagram note, traceability table) and tasks.md (task 3.1/3.2) to point at sched_publish_dispatch/main-loop thread instead of sched_allocator. Documented the real bounded-buffering model (jbuf cap 1 + decisions cap 5 = ~6-job grace window). Added a Req 3 AC1 scope-narrowing note as a Validation Hook -- flagged for human sign-off on requirements.md, not edited directly (requirements phase already approved). Corrected the same wrong claim at its source in research.md (lines 22, 30, 84).", @@ -18221,20 +6777,23 @@ "thread-model", "kiro-validate-design", "scheduler", - "critical-issue", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "critical-issue" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1514.md" }, { "id": "bug-1515", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-10T20:20:00.000Z", - "error_message": "/goal-driven deep re-review of component-failover-adoption requirements.md+design.md found 5 additional distinct defects after bug-1514's fix, several introduced by bug-1514's own Req 7 fix", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "/goal-driven deep re-review of component-failover-adoption requirements.md+design.md found 5…\n\n/goal-driven deep re-review of component-failover-adoption requirements.md+design.md found 5 additional distinct defects after bug-1514's fix, several introduced by bug-1514's own Req 7 fix", "file": ".kiro/specs/component-failover-adoption/requirements.md, design.md, tasks.md, research.md", "root_cause": "(1) Req 3 AC2's '~6-job grace window' was stated as 'at least 6' (a floor) when 6 is actually the best-case ceiling; the real guaranteed floor is 1-2 depending on decisions-cbuf occupancy at the moment of stall -- direction of the bound was backwards. (2) Citation scheduler.c:1253 for sched_publish_pend_reason was off by 2 (actual line 1251). (3) Cross-cutting: the shared 'check and clear needs_rejoin, then call xcecli_join' pattern (worker/scheduler/apiserver) cleared the flag unconditionally rather than only on xcecli_join's own success -- since xcecli_join is itself XCECLI_FAILOVER_CALL-wrapped and can hit its own retry-budget exhaustion, this could leave a component transport-connected but permanently un-joined with nothing left to retry (no requirement/design text previously covered this). (4) The Req 7 scheduler decision-publish retry loop only checked core.stop in the while-condition (after each attempt), so multiple decisions queued behind a stuck one would each burn a full ~100s budget cycle discovering shutdown was requested, compounding shutdown latency to minutes instead of the claimed ~1-2 min bound. (5) While fixing (3) for apiserver's heartbeat-idle-path, the first draft gated the retry on ApiserverBackendHealth.IsDegraded() -- but that state has its own independent TTL-based auto-clear for circuit-breaker purposes; if the TTL expired before an actual rejoin succeeded, the retry would silently stop, reintroducing the exact starvation bug (3) was fixing. Two unrelated state machines (rejoin-tracking vs request-path circuit-breaker) had been conflated onto one flag.", "fix": "(1) Reworded Req 3 AC2 and all design.md/research.md/tasks.md echoes to state the guarantee qualitatively (bounded, occupancy-dependent, resumes with zero loss per Req 7) instead of a wrong fixed number. (2) Fixed the line citation. (3) Added Requirement 2 AC5 (rejoin retry-on-failure, applies to all three components) and updated the shared 'Rejoin callback constraint' section plus all three component sections in design.md and their tasks.md task bullets to clear the flag only on xcecli_join success. (4) Changed the Req 7 retry loop from do/while to while (check core.stop before each attempt, not only after) so only the actively in-flight decision can cost a full budget cycle; queued decisions behind it skip immediately once stop is set. (5) Introduced a dedicated needs_rejoin flag for apiserver's heartbeat path (set on transition, cleared only on join success) fully decoupled from ApiserverBackendHealth's TTL-governed degraded state; updated the sequence diagram, prose, and tasks.md task 4.4 accordingly, plus an explicit note in ApiserverBackendHealth's section warning against this conflation. Also added Req 7 AC6 documenting that the fix does not survive a scheduler process crash/restart (in-memory only, no persistence).", @@ -18247,83 +6806,25 @@ "state-conflation", "scheduler", "apiserver", - "worker", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "worker" ], "related_bugs": [ "bug-1514" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1516", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T11:35:00.157Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/testsuite/meson.build", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u219212 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "build", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1517", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T11:35:50.895Z", - "error_message": "Wrong reference: true should be false", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "Used \"true\" instead of \"false\"", - "fix": "Changed true \u2192 false", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1518", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T12:22:54.348Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7\u219211 lines (2 removed) | Also: ERROR(\"%s: xcecli_join() failed across %d candidat; __func__, sched->cluster_cfg.candidates_count, rc)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1515.md" }, { "id": "bug-1519", "status": "fixed", "status_updated": "2026-09-24", "status_note": "b6a2f5ec (bug-1756)", + "component": "worker", + "severity": "unknown", "timestamp": "2026-08-11T12:49:17.815Z", - "error_message": "==ERROR: AddressSanitizer: Joining already joined thread, aborting.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "==ERROR: AddressSanitizer: Joining already joined thread, aborting\n\n==ERROR: AddressSanitizer: Joining already joined thread, aborting.", "file": "src/worker/src/main.c (xcecd daemon startup)", "root_cause": "UNRESOLVED / not root-caused. xcecd's root check (main.c:288 geteuid()!=0) was bypassed for sandbox testing via `unshare -r` (user-namespace fake root, euid=0 but not real root). Startup then immediately hit an ASAN double-pthread_join abort during/after the HA-cluster-config init log line (main.c:435), before any further logging. Never reproduced under real root -- unknown whether this is a genuine double-join bug in xcecd's daemonize/thread-spawn path (e.g. duplicate cleanup on an early-init error branch) or an artifact of unshare's user-namespace semantics (e.g. a syscall xcecd's threading depends on behaving differently under CLONE_NEWUSER, such as thread creation ordering or signal delivery). Blocked further component-failover-adoption Task 3.3 Item-3 E2E validation from getting a live worker registered, which in turn meant no real job ever produced a scheduler decision (picker/allocator never had a matching candidate to dispatch to or explicitly PEND against), so the scheduler's decision-publish/rejoin code path (task 3.2) could not be exercised via a fully live job in this sandbox.", "fix": "Not fixed -- out of scope for Task 3.3 (which validates scheduler HA reconnect, not worker daemon startup). Worked around by validating scheduler join/failover/reconnect behavior via direct socket-state inspection (ss/proc fd) instead of a full live job pipeline; the decision-publish retry+rejoin logic itself already has white-box unit coverage from task 3.2 (test_scheduler_decision_publish_retry.c) and task 3.3 item 2 (test_scheduler_backpressure.c). If a future session needs a live worker in this sandbox, retry under real root (not unshare -r) first to isolate whether this is userns-specific. UPDATE (2026-08-20, build-time-versioning task 3.3 review): independently reproduced under REAL sudo root (not unshare -r) -- same signature, same timing (fires during later thread-join startup work, strictly AFTER the HA-cluster-config INFO log line, both with and without --peers). This refines/answers the prior 'never reproduced under real root' open question: it is NOT a userns/unshare-specific artifact -- the abort reproduces identically under genuine root. Still not root-caused/fixed; still does not block this session's version-display changes (main.c's new --version/log-line code runs entirely before the point where this abort fires).", @@ -18335,260 +6836,25 @@ "unshare", "sandbox-environment", "unresolved", - "component-failover-adoption", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "component-failover-adoption" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1520", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T13:05:01.694Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 12\u219210 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "b6a2f5ec" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1521", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T13:14:08.399Z", - "error_message": "Wrong reference: test_sched_init_no_peers_default_also_fails_cleanly should be test_all_candidates_unreachable_log_names_every_candidate_host_port", - "file": "src/scheduler/testsuite/test_scheduler_connect.c", - "root_cause": "Used \"test_sched_init_no_peers_default_also_fails_cleanly\" instead of \"test_all_candidates_unreachable_log_names_every_candidate_host_port\"", - "fix": "Changed test_sched_init_no_peers_default_also_fails_cleanly \u2192 test_all_candidates_unreachable_log_names_every_candidate_host_port", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1522", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T13:14:51.964Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/testsuite/test_scheduler_connect.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 11\u219221 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1523", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T14:34:18.839Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/testsuite/test_scheduler_decision_redelivery.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u21927 lines (2 removed) | Also: * Fixtures call xce_sched_core_init()/xce_sched_co; * the full sched_init()) -- the same minimal-fixtu", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1524", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T14:37:43.571Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 26\u219227 lines (5 removed) | Also: /* TEMPORARY RED-PHASE PROBE #2 (task 3.4 evidence; * immediately after use) -- core.stop checked only", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1525", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T23:39:27.643Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "15 lines replaced/restructured", - "fix": "Rewrote 27\u219211 lines (15 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1526", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-11T23:49:56.443Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1527", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T02:41:37.052Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/clib.go", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 21\u219226 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1528", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T02:41:56.690Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/config.go", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 5\u21928 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1529", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T03:09:17.720Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/candidates.go", - "root_cause": "13 lines replaced/restructured", - "fix": "Rewrote 40\u219274 lines (13 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1530", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T04:33:27.866Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had `scheduler.c`", - "fix": "Changed to `xce_parse_peers()`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1531", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T07:05:08.345Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/clib.go", - "root_cause": "40 lines replaced/restructured", - "fix": "Rewrote 83\u2192229 lines (40 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1519.md" }, { "id": "bug-1532", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "crash", "timestamp": "2026-08-12T00:20:00.000Z", - "error_message": "apiserver clib.go tryConnect(): use-after-free on ctx->conf->masters[i] (host address string) on every post-initial-connect failover rotation attempt", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "apiserver clib.go tryConnect(): use-after-free on ctx->conf->masters[i] (host address string) on…\n\napiserver clib.go tryConnect(): use-after-free on ctx->conf->masters[i] (host address string) on every post-initial-connect failover rotation attempt", "file": "src/frontend/apiserver/backend/clib.go", "root_cause": "tryConnect() allocated cAddr/cFrom/cKey via C.CString(), built a Go-function-local xce_client_config_t pointing at them, called xce_backend_init() once, then deferred C.free() on all three strings immediately after that single call returned. xcecli_ctx_init() (src/lib/client.c) stores ctx->conf = cfg as a RAW pointer (never copies the struct), and client_failover.c dereferences ctx->conf->masters[cur_master] on EVERY subsequent failover rotation attempt (client_failover.c:120,386), not just the initial connect -- so any reconnect/rotation after the first tryConnect() call returned read freed memory. A second, independent hazard: cConfig itself was a Go-local var; ctx->conf retained a raw pointer to it invisible to Go GC once tryConnect() returned. (Note: from/key were NOT actually part of this UAF -- xce_ctx_init() strdup()s them immediately at ctx-init time, so freeing cFrom/cKey was safe in isolation; only the masters[] host string and the client_config_t struct address itself were genuinely unsafe.)", "fix": "component-failover-adoption task 4.2: replaced tryConnect() and the dual Go-level connect loops entirely with buildClusterConfig()+RegisterToBackend(), backed by two new package-level (process-lifetime, GC-rooted) vars gClusterCfg (C.xce_cluster_config_t) and gClientCfg (C.xce_client_config_t). Host strings now live in gClusterCfg.candidates[i].hostname (fixed char[256] arrays, no CString/free at all -- xce_cluster_config_to_client_config() aliases pointers directly into this storage). from/key point at gClusterCfg.identity[0]/key[0] (also fixed arrays) rather than separate CStrings. No C.CString/C.free is used anywhere in the connect path except the transient peers-string argument to xce_parse_peers(), which is safely freed right after that one call since xce_parse_peers() copies into gClusterCfg internally.", @@ -18598,200 +6864,23 @@ "apiserver", "failover", "component-failover-adoption", - "task-4.2", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-4.2" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1533", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T07:46:01.677Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had `scheduler.c`", - "fix": "Changed to 's two connect mechanisms into one \u2014 the deleted `", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1534", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T08:45:46.229Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/reconnect_test.go", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 9\u219214 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1535", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T09:15:53.632Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had `scheduler.c`", - "fix": "Changed to `go_backend_rejoin_handler`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1536", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T09:25:02.926Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/heartbeat.go", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 17\u219262 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1537", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T09:25:33.041Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/clib.go", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u219214 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1538", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T09:26:08.038Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/heartbeat_test.go", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 48\u2192214 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1539", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T09:58:02.265Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had `scheduler.c`", - "fix": "Changed to `runHeartbeat()`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1540", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T10:16:08.309Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/clib.go", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 23\u219295 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1541", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T10:17:17.099Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/reconnect.go", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 16\u219226 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1532.md" }, { "id": "bug-1542", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-12T10:18:42.192Z", - "error_message": "health_rejoin_test.go referenced C.E_XCE_NOT_READY directly (via -int(C.E_XCE_NOT_READY)), which requires import \"C\" in a _test.go file -- illegal per golang/go#13938, would have failed to compile", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "health_rejoin_test.go referenced C.E_XCE_NOT_READY directly (via -int(C.E_XCE_NOT_READY)), which…\n\nhealth_rejoin_test.go referenced C.E_XCE_NOT_READY directly (via -int(C.E_XCE_NOT_READY)), which requires import \"C\" in a _test.go file -- illegal per golang/go#13938, would have failed to compile", "file": "src/frontend/apiserver/backend/health_rejoin_test.go", "root_cause": "Wrote a test helper that needed the -E_XCE_NOT_READY errno value, and reached for C.E_XCE_NOT_READY directly without recalling that _test.go files can never import \"C\" themselves (established project constraint, see clib_test.go/reconnect_test.go doc comments) -- caught before ever running go test, by re-reading the file before the first test run.", "fix": "Exposed -E_XCE_NOT_READY as a plain Go int via a new package-level var in clib.go (a cgo file): notReadyRC = -int(C.E_XCE_NOT_READY). health_rejoin_test.go references notReadyRC instead of C.E_XCE_NOT_READY -- same plain-Go-signature test-seam pattern as performRejoinFn (reconnect.go, task 4.3).", @@ -18799,120 +6888,23 @@ "go", "cgo", "test-file-restriction", - "import-C-in-test", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "import-C-in-test" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1543", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T10:20:03.350Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/reconnect_test.go", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 17\u219230 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1544", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T10:51:41.466Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had `sched_init()`", - "fix": "Changed to `callBackend()`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1545", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T12:21:30.458Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had `sched_init()`", - "fix": "Changed to \"zero new code is a valid outcome when genuinely t", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1546", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T12:42:35.605Z", - "error_message": "Significant refactor of ", - "file": "../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-scheduler-framework/93f96538-d154-48c8-b92b-c8f9342a87cf/scratchpad/phase2_failover.sh", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 11\u219213 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1547", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T12:48:36.145Z", - "error_message": "Incorrect value in code", - "file": "../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-scheduler-framework/93f96538-d154-48c8-b92b-c8f9342a87cf/scratchpad/phase2_failover.sh", - "root_cause": "Had \"$TOP/logs/apiserver.stdout.log\"", - "fix": "Changed to \"$APISERVER_LOG\"", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1542.md" }, { "id": "bug-1548", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-12T12:59:31Z", - "error_message": "apiserver never re-registers (application-level xcecli_join with HostRole_apiserver) with the new primary after a live controller failover, even though its transport/write-call connectivity clearly recovers (job/host-control POSTs keep succeeding); observed for 10+ minutes and 20+ default-30s heartbeat ticks with zero occurrences of reconnect.go's own 'checkAndPerformRejoin: rejoin after transparent reconnect succeeded/failed' log lines in apiserver_out.log, across 3 separate write calls (2 job submits + 1 host control) issued after the primary was kill -9'd. TASK 5.2 UPDATE (2026-08-12, second independent live E2E reproduction, cleaner isolation): ran component-failover-adoption task 5.2's own scenario (apiserver-only, no worker/scheduler, 3-node controller cluster at localhost:19991-19993, apiserver -d /home/claude/tmp/xce-e2e-task52/apiserver) specifically to isolate the heartbeat-only idle-period path task 5.1's write-traffic-muddied run could not cleanly separate. Confirmed real election: ctrl-1 (primary, term=2) kill -9'd at 13:33:31Z; ctrl-2 promoted primary at term=3 within 9s (13:33:40Z, raft_cb_become_leader). Zero write requests issued for the full outage window: apiserver_out.log is BYTE-IDENTICAL (23 lines) from before the kill through 268s / ~9 heartbeat ticks after it (well past the task's 2-minute/3-4-tick minimum) -- zero occurrences of 'checkAndPerformRejoin'/'rejoin'/'needsRejoin'/'xcecli_join' anywhere. Diagnostic control issued only AFTER the observation window (13:39:52Z, ~6.5 min post-kill): POST /xce/v1/jobs/create succeeded with a real controller-assigned job ID (73930919093731328) and server-generated success message -- proving the transport had already reached the NEW primary (ctrl-2) -- yet this write call ALSO produced zero rejoin log lines, full apiserver_out.log grep for the same 4 terms still empty after it. This is strictly stronger evidence than task 5.1's: it directly demonstrates the write path's own XCECLI_FAILOVER_CALL/xcecli_invoke_rejoin also did not fire even when explicitly exercised, consistent with the transport having already been silently repaired by an earlier heartbeat tick's XCECLI_FF_CALL before the write call ever needed to retry -- exactly bug-1548's leading hypothesis, not merely 'consistent with' it as task 5.1 could only say. Corroborating bug-1549 downstream symptom also reproduced: GET /xce/v1/jobs immediately after the successful submit returned {\"count\":0,\"rows\":[]} -- the just-submitted job never reached apiserver's local store, consistent with apiserver never having actually re-issued xcecli_join(HostRole_apiserver) against ctrl-2. ctrl-2.log has no apiserver-join lines either direction (join.c logs only rejections, per task 5.1's prior finding -- reconfirmed, not independently informative here). Task 5.2's own observable-completion criterion ('the scenario passes without requiring any client-triggered write request during the outage window' -- i.e. apiserver rejoins and resumes push notifications via the heartbeat-triggered path alone) was NOT met: no rejoin evidence appeared at any point in the heartbeat-only window. Evidence preserved at /home/claude/tmp/xce-e2e-task52/ (apiserver/logs/apiserver_out.log, logs/ctrl-{1,2,3}.log, logon_response.json, diagnostic_job_submit_response.json, diagnostic_joblist_response.json, KILL_TIMESTAMP_UTC.txt, DIAGNOSTIC_CONTROL_START_UTC.txt, DIAGNOSTIC_CONTROL_SUBMIT_UTC.txt).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "apiserver never re-registers (application-level xcecli_join with HostRole_apiserver) with the new…\n\napiserver never re-registers (application-level xcecli_join with HostRole_apiserver) with the new primary after a live controller failover, even though its transport/write-call connectivity clearly recovers (job/host-control POSTs keep succeeding); observed for 10+ minutes and 20+ default-30s heartbeat ticks with zero occurrences of reconnect.go's own 'checkAndPerformRejoin: rejoin after transparent reconnect succeeded/failed' log lines in apiserver_out.log, across 3 separate write calls (2 job submits + 1 host control) issued after the primary was kill -9'd. TASK 5.2 UPDATE (2026-08-12, second independent live E2E reproduction, cleaner isolation): ran component-failover-adoption task 5.2's own scenario (apiserver-only, no worker/scheduler, 3-node controller cluster at localhost:19991-19993, apiserver -d /home/claude/tmp/xce-e2e-task52/apiserver) specifically to isolate the heartbeat-only idle-period path task 5.1's write-traffic-muddied run could not cleanly separate. Confirmed real election: ctrl-1 (primary, term=2) kill -9'd at 13:33:31Z; ctrl-2 promoted primary at term=3 within 9s (13:33:40Z, raft_cb_become_leader). Zero write requests issued for the full outage window: apiserver_out.log is BYTE-IDENTICAL (23 lines) from before the kill through 268s / ~9 heartbeat ticks after it (well past the task's 2-minute/3-4-tick minimum) -- zero occurrences of 'checkAndPerformRejoin'/'rejoin'/'needsRejoin'/'xcecli_join' anywhere. Diagnostic control issued only AFTER the observation window (13:39:52Z, ~6.5 min post-kill): POST /xce/v1/jobs/create succeeded with a real controller-assigned job ID (73930919093731328) and server-generated success message -- proving the transport had already reached the NEW primary (ctrl-2) -- yet this write call ALSO produced zero rejoin log lines, full apiserver_out.log grep for the same 4 terms still empty after it. This is strictly stronger evidence than task 5.1's: it directly demonstrates the write path's own XCECLI_FAILOVER_CALL/xcecli_invoke_rejoin also did not fire even when explicitly exercised, consistent with the transport having already been silently repaired by an earlier heartbeat tick's XCECLI_FF_CALL before the write call ever needed to retry -- exactly bug-1548's leading hypothesis, not merely 'consistent with' it as task 5.1 could only say. Corroborating bug-1549 downstream symptom also reproduced: GET /xce/v1/jobs immediately after the successful submit returned {\"count\":0,\"rows\":[]} -- the just-submitted job never reached apiserver's local store, consistent with apiserver never having actually re-issued xcecli_join(HostRole_apiserver) against ctrl-2. ctrl-2.log has no apiserver-join lines either direction (join.c logs only rejections, per task 5.1's prior finding -- reconfirmed, not independently informative here). Task 5.2's own observable-completion criterion ('the scenario passes without requiring any client-triggered write request during the outage window' -- i.e. apiserver rejoins and resumes push notifications via the heartbeat-triggered path alone) was NOT met: no rejoin evidence appeared at any point in the heartbeat-only window. Evidence preserved at /home/claude/tmp/xce-e2e-task52/ (apiserver/logs/apiserver_out.log, logs/ctrl-{1,2,3}.log, logon_response.json, diagnostic_job_submit_response.json, diagnostic_joblist_response.json, KILL_TIMESTAMP_UTC.txt, DIAGNOSTIC_CONTROL_START_UTC.txt, DIAGNOSTIC_CONTROL_SUBMIT_UTC.txt).", "file": "src/frontend/apiserver/backend/reconnect.go, heartbeat.go, clib.go (component-failover-adoption task 4.3/4.4's checkAndPerformRejoin/needsRejoin mechanism)", "root_cause": "NOT root-caused (out of task 5.1's E2E-validation boundary to fix). Leading hypothesis from live log analysis: XCECLI_FF_CALL (the fire-and-forget macro backing xcecli_heartbeat) can silently self-heal the TRANSPORT (rotate to a live candidate) entirely within a single heartbeat tick without that tick ever returning a failure to Go -- so heartbeat.go's prevFailed-edge detection (onFailureToSuccess) never observes a failure->success transition and never sets needsRejoin. Separately, none of the 3 post-kill write calls (which DO route through XCECLI_FAILOVER_CALL, which per design.md should invoke the rejoin callback mid-call on a genuine failover) show any evidence of having hit a failed connection either -- consistent with the connection having already been silently fixed by a prior heartbeat tick before any write call needed to retry, so XCECLI_FAILOVER_CALL's own rejoin-invoking path is also never exercised. Net effect: needsRejoin is never set by any path, so checkAndPerformRejoin() stays a permanent silent no-op and apiserver's push-registration (join) with the new primary is never renewed. Could not independently confirm via controller-side logs either way -- discovered during this same investigation that the controller's join.c only logs REJECTED/failed joins (WARN/ERROR), never a successful one at any verbosity, so controller logs are not usable evidence for this question in either direction. REVIEW UPDATE (task 5.1 review, 2026-08-12): confirmed via source read that XCECLI_FF_CALL (client_internal.h:290-302) never calls xcecli_invoke_rejoin under any circumstance -- this half of the hypothesis is not speculative, it's structurally true by construction. Elevated confidence this is a REPRODUCIBLE gap, not scenario-specific timing luck: since real elections in this codebase's own logs complete in single-digit seconds while the default heartbeat interval is 30s, a single heartbeat tick landing after the election has already completed will both discover the dead connection AND repair it within that same XCECLI_FF_CALL invocation -- the failure never surfaces to Go's heartbeat.go loop as a separate tick, so the cross-tick prevFailed-edge detection (onFailureToSuccess) structurally cannot observe it. This is likely to recur on MOST real single-election failovers, not just this run's specific timing -- materially risks Req 2 AC4 (component-failover-adoption requirements.md) in production. Recommend escalating past ordinary buglog cadence to a dedicated follow-up spec/task, given it affects the already-approved task 4.3/4.4 mechanism.", "fix": "FIXED (component-failover-adoption tasks.md Phase 6, task 6.1). Root cause confirmed as hypothesized: XCECLI_FF_CALL (client_internal.h:290-302) can rotate+reconnect the transport entirely inside a single call, holding ac->failover.lock, and still return success -- so a failure->success transition on the caller's own return code is not a reliable 'a reconnect happened' signal whenever an election (single-digit seconds) completes faster than the 30s default heartbeat interval, which is the common case. Fix: added a monotonic `_Atomic uint64_t reconnect_gen` field to `endpoint_t` (src/include/netio.h:137), incremented unconditionally on the success path of `xce_ctx_reconnect()` (src/lib/xcectx.c:402) -- confirmed via repo-wide grep as the sole reconnect call-site reachable from `xce_failover_rotate()`, covering both the write-path retry loop and the heartbeat fire-and-forget path. apiserver's heartbeat.go `runHeartbeat()` now compares this generation on every tick (via the existing `xcecli_get_my_endpoint()` accessor, atomic-loaded through clib.go) instead of waiting for a failure->success transition, replacing the old detector entirely. Regression test `TestRunHeartbeat_IdlePeriodRecovery_GenChangeAloneSetsNeedsRejoinEvenWhenAllTicksSucceed` (heartbeat_rejoin_test.go) proves the fix's exact shape. Closed via live E2E re-run of task 5.2's scenario: rejoin fired 22s after primary kill (12s post-election) with zero write traffic for the preceding 5.5 minutes.", @@ -18926,21 +6918,23 @@ "heartbeat", "live-e2e-finding", "task-5.2", - "second-independent-repro", - "resolved", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "second-independent-repro" ], "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1548.md" }, { "id": "bug-1549", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-12T12:59:31Z", - "error_message": "Jobs submitted via apiserver REST API (POST /xce/v1/jobs/create) never appear in apiserver's own job store (GET /xce/v1/jobs / GET /xce/v1/jobs/:id returns empty), even though the job is confirmed genuinely committed on the controller (verified directly via xce_test's `joblist` showing both jobs in PEND state). Reproduced for 2 jobs (one submitted at the moment of a primary kill, one submitted >1 minute after a clean new-primary failover) -- so this is NOT specific to the failover window, appears to be a general gap. By contrast, worker/host-list push (apiserver_push_workers/apiserver_push_workerlist, controller/apiserver.c) DOES work: GET /xce/v1/hosts correctly showed the live-joined worker with real data.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Jobs submitted via apiserver REST API (POST /xce/v1/jobs/create) never appear in apiserver's own…\n\nJobs submitted via apiserver REST API (POST /xce/v1/jobs/create) never appear in apiserver's own job store (GET /xce/v1/jobs / GET /xce/v1/jobs/:id returns empty), even though the job is confirmed genuinely committed on the controller (verified directly via xce_test's `joblist` showing both jobs in PEND state). Reproduced for 2 jobs (one submitted at the moment of a primary kill, one submitted >1 minute after a clean new-primary failover) -- so this is NOT specific to the failover window, appears to be a general gap. By contrast, worker/host-list push (apiserver_push_workers/apiserver_push_workerlist, controller/apiserver.c) DOES work: GET /xce/v1/hosts correctly showed the live-joined worker with real data.", "file": "src/controller/apiserver.c (apiserver_job_new_fun / apiserver_push_joblist hook), src/frontend/apiserver/backend/clib.go (go_push_handler / handleDataset)", "root_cause": "NOT root-caused (out of task 5.1's boundary). apiserver_job_new_fun is wired as a HOOK_DETAIL_JOB_NEW hook meant to fire apiserver_push_joblist()/apiserver_push_jobarray() after a job commits (apply_committed_entry()); controller logs show zero evidence either way since apiserver.c has no INFO-level success logging on this path (only a WARN on push failure, `push job list to api server %s has problem, rc %d`, which also never appeared). Could be: the hook isn't actually registered/invoked in this build, or apiserver's join isn't registered in c->apiservers[] the push fan-out iterates, or something queue/resreq-specific. Discovered opportunistically while probing bug-1548; not independently isolated from bug-1548 (both could share a root cause -- c->apiservers[] fan-out list membership -- or be unrelated). REVIEW UPDATE (task 5.1 review, 2026-08-12): found the likely direct causal chain, narrowing this from 'could share a root cause or be unrelated' to 'very likely a downstream symptom of bug-1548'. src/controller/apiserver.c's on_apiserver_join() unconditionally calls xce_apiserver_push_jobs(c, ep) (bulk job resync) alongside xce_apiserver_push_workers(c, ep), but ONLY fires when a fresh join is actually processed by the controller. c->apiservers[] (the fan-out list push hooks iterate) is per-process, in-memory-only state -- no WAL/config replication (confirmed zero references outside apiserver.c) -- so a NEW primary starts with an empty registration table until each apiserver instance re-joins against it. If bug-1548 means apiserver's rejoin was never re-issued against the new primary, ctrl-2 never ran on_apiserver_join for it, so neither the bulk resync nor any incremental job push could ever reach apiserver -- cleanly explaining both job1 and job2 being empty. Recommend a fix attempt for bug-1548 be verified against this job-push symptom directly (if fixing the rejoin gap makes GET /xce/v1/jobs start populating correctly, that's confirmation bug-1549 was never an independent issue). TASK 5.2 RE-REPRODUCTION (2026-08-12T13:39:52Z): reproduced a third time in an isolated, apiserver-only (no worker/scheduler) idle-period E2E run -- a job submitted well after a clean failover (jobid 73930919093731328, controller-confirmed via the accepted response) still never appeared in GET /xce/v1/jobs. This occurred in the SAME run where bug-1548's rejoin gap was also independently reconfirmed with a clean negative control, strengthening (not just repeating) the downstream-symptom-of-bug-1548 hypothesis: if apiserver never re-joined the new primary at the application level, c->apiservers[] on ctrl-2 never gained an entry for it, so no push (bulk or incremental) could ever reach it -- independent of whether the underlying job-push wiring itself is otherwise correct.", "fix": "FIXED as a downstream symptom of bug-1548 -- confirmed, not just hypothesized. Once apiserver's heartbeat-triggered rejoin (bug-1548's fix) correctly re-issues xcecli_join(HostRole_apiserver) against the new primary, the controller's on_apiserver_join() bulk job resync (xce_apiserver_push_jobs) fires as designed and populates apiserver's local job store. Live E2E re-validation (component-failover-adoption task 6.2): a diagnostic job submitted post-recovery correctly appeared via GET /xce/v1/jobs, with no client-triggered write request required during the outage window.", @@ -18951,164 +6945,25 @@ "job-submit", "component-failover-adoption", "task-5.1", - "live-e2e-finding", - "resolved", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "live-e2e-finding" ], "related_bugs": [ "bug-1548" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1550", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-12T13:22:40.852Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had `sched_init()`", - "fix": "Changed to `xcecd_virtual`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1551", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T02:38:03.922Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had `sched_init()`", - "fix": "Changed to 's bug-1548 finding (heartbeat-only, zero write tra", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1552", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T03:02:32.434Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had `sched_init()`", - "fix": "Changed to `client_failover.c`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-872", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T07:20:20.800Z", - "error_message": "Significant refactor of ", - "file": "src/xceadmin/src/xcecommon/mapping.py", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 8\u219214 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "py", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-873", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T07:20:42.831Z", - "error_message": "Significant refactor of ", - "file": "src/xceadmin/tests/test_mapping.py", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u21925 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "py", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-874", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T07:25:51.662Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_worker_join.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1633", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T07:27:23.041Z", - "error_message": "Significant refactor of ", - "file": "src/controller/worker.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 49\u219267 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1549.md" }, { "id": "bug-1553", "status": "open", "status_updated": "2026-09-24", "status_note": "intermittent (~1 in 3) controller_unit_test SIGSEGV; not isolated, re-seen 5bf123cc", + "component": "controller", + "severity": "crash", "timestamp": "2026-08-13T07:35:00Z", - "error_message": "controller_unit_test (combined 1095-test Unity binary) segfaults intermittently (~1 in 3 full runs observed) somewhere between test_drive_async_config.c and the next testsuite file (test_configcompare.c/test_slowlane.c area) -- no assertion failure, no ASAN report captured before the crash, process just dies (exit 139/SIGSEGV)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller_unit_test (combined 1095-test Unity binary) segfaults intermittently (~1 in 3 full runs…\n\ncontroller_unit_test (combined 1095-test Unity binary) segfaults intermittently (~1 in 3 full runs observed) somewhere between test_drive_async_config.c and the next testsuite file (test_configcompare.c/test_slowlane.c area) -- no assertion failure, no ASAN report captured before the crash, process just dies (exit 139/SIGSEGV)", "file": "src/controller/testsuite/ (combined binary, crash site not yet isolated -- occurs after test_drive_async_config.c's liveness_reserved_* tests)", "root_cause": "Not yet investigated. Reproduced 1/3 runs both BEFORE and AFTER an unrelated worker.c change (update_resval PERROR->rc logging fix) confirming this is pre-existing and unrelated to that fix -- likely order-dependent shared-process state leakage across Unity tests (global fixtures, xstring table, or ASAN heap state) rather than a bug in any specific test file. Two of three runs completed cleanly with 0 failures each time.", "fix": "Not fixed -- out of scope for the update_resval/mxj-auto-detect bugfix session that surfaced it. Needs isolation: bisect which specific test pair triggers it (likely by running testsuite files in smaller batches or under gdb with a core dump), then root-cause the actual memory-safety issue.", @@ -19117,26 +6972,27 @@ "segfault", "controller_unit_test", "order-dependent", - "unity", - "not-fixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "unity" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1553.md" }, { "id": "bug-1554", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "worker", + "severity": "unknown", "timestamp": "2026-08-13T07:36:00Z", - "error_message": "workerlist / resval show maxswap=1024 and maxtmp=2048 for every host regardless of actual hardware, unless overridden by a per-host config resource entry", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "workerlist / resval show maxswap=1024 and maxtmp=2048 for every host regardless of actual…\n\nworkerlist / resval show maxswap=1024 and maxtmp=2048 for every host regardless of actual hardware, unless overridden by a per-host config resource entry", "file": "src/worker/src/xcecd.c", "root_cause": "init_host_attrs() hardcodes host_attrs.maxswap = 1024 and host_attrs.maxtmp = 2048 as placeholder constants (lines ~349-350) -- unlike maxcpus/maxmem, which come from real hardware_monitor_* detection, these two fields have no real detection implemented yet. Confirmed live: a deploy's maxtmp showed exactly 2048 (the hardcoded stub) while maxswap showed 10000 (a per-host config override in hosts.yaml resource list, taking precedence via apply_join_state's mark_host_res loop -- proves the stub IS reachable/live whenever no override exists).", "fix": "FIXED (2026-09-06, worker hardware detection). Real detection replaces both placeholder stubs: maxswap now comes from sysinfo()'s totalswap field (hardware_monitor_get_total_swap(), the same sysinfo() snapshot hardware_monitor_update() already takes for maxmem -- no extra syscall), and maxtmp now comes from a fresh statvfs(\"/tmp\") total-capacity measurement (hardware_monitor_get_disk_info()), mirroring node_information.c's node_info_get_tmp() own statvfs(\"/tmp\") convention but reporting total blocks*frsize instead of free space. init_host_attrs() (xcecd.c) wires both in, with NO synthetic fallback on detection failure (left at 0, matching maxcpus/maxmem's own existing no-fallback behavior) -- a system with swap genuinely disabled reports 0 correctly instead of a fabricated non-zero placeholder. New tests: test_hardware_monitor.c (7 tests, cross-checks each function's output against an independent sysinfo()/statvfs() call made directly in the test) and a new integration test in test_xcecd_startup_order.c pinning that init_host_attrs() picks up whatever these two functions report. Full worker suite: 9/9 pass.", "tags": [ - "fixed", "worker", "stub", "maxtmp", @@ -19144,54 +7000,43 @@ "maxswap" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1634", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T08:06:37.286Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 30\u219241 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1554.md" }, { "id": "bug-1555", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-13T07:40:00Z", - "error_message": "controller log spammed with '[ERROR] update_resval: Success' / 'update_resval: Resource temporarily unavailable' -- misleading, the real failure reason is never logged", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller log spammed with '[ERROR] update_resval: Success' / 'update_resval: Resource…\n\ncontroller log spammed with '[ERROR] update_resval: Success' / 'update_resval: Resource temporarily unavailable' -- misleading, the real failure reason is never logged", "file": "src/controller/worker.c", "root_cause": "update_resvals()/update_resval_list() called PERROR(\"update_resval\") on rc<0, but PERROR (xce_perror -> xce_perror_) formats the process-global errno, not rc. update_resval()'s failure paths (key_find/zarray_put) return raw negative codes directly and never touch errno, so the logged message reflected whatever errno happened to be left over from an unrelated earlier syscall in the same event-loop iteration -- 0 (\"Success\") if nothing had touched it, or a stray EAGAIN if something had. The real reason (e.g. -ENOENT from an unregistered resource name) was silently discarded. Also fixed a second instance of the same errno-vs-rc bug in update_resval()'s r==NULL branch (was `return -errno;`, now `return -ENOENT;`, matching the identical branch in the sibling function mark_host_res()).", "fix": "Replaced PERROR with a local log_update_resval_failure() helper that decodes the actual rc via xce_strerror_r(-rc, ...) and includes the resource name in the message. Regression test: test_update_resvals_logs_actual_rc_not_stale_errno (test_worker_join.c) sets errno=EAGAIN before the call and asserts the captured log contains neither \"Success\" nor \"Resource temporarily unavailable\", contains the resource name, and contains the real ENOENT string -- verified genuine RED (failed against the reverted old code with the exact reported symptom) before GREEN.", "tags": [ "controller", "logging", - "errno", - "fixed" + "errno" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1555.md" }, { "id": "bug-1556", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-08-13T07:41:00Z", - "error_message": "Every host using the documented hosts.yaml default (no max_slot, meaning 'auto-detect from CPU count') gets 0 scheduling slots -- permanently blocks the scheduler from placing any job on that host", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Every host using the documented hosts.yaml default (no max_slot, meaning 'auto-detect from CPU…\n\nEvery host using the documented hosts.yaml default (no max_slot, meaning 'auto-detect from CPU count') gets 0 scheduling slots -- permanently blocks the scheduler from placing any job on that host", "file": "src/xceadmin/src/xcecommon/mapping.py", "root_cause": "hosts.schema.json documents max_slot: -1 as the 'auto-detect' sentinel, but HostConfig.mxj is UInt32 on the wire and cannot represent -1 (pycapnp raises capnp.KjException for a negative assignment). mapping.py's _build_host_config() worked around this by leaving mxj unset when max_slot was absent/negative, which capnp defaults to 0. worker.c's apply_join_state() then does `nslots = hconfig->mxj; if (nslots < 0) nslots = maxcpus;` -- but 0 is never < 0, so the auto-detect fallback never fires, and the host is explicitly configured with 0 slots. Confirmed live via a real deploy: resval -n slots showed value: 0.000000 for the default host. An existing test (test_mapping.py) even asserted this broken behavior as correct (`assert h.mxj == 0`).", "fix": "mapping.py now sets mxj = HOSTCONFIG_MXJ_AUTO_DETECT (0xFFFFFFFF) instead of leaving it unset. The controller's `int mxj` field receives this via capn_read32()'s implicit uint32_t->int conversion, which reinterprets it back to -1 on this project's two's-complement target platforms -- no capnp schema change or C-side change needed, apply_join_state()'s existing `nslots<0` fallback already handles a genuine -1 correctly (now proven by a new C regression test, test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus, since no existing test exercised that branch with a negative mxj before). Updated the stale test assertion, added a new test for the absent-max_slot production case, and regenerated the xceadmin golden fixture (tests/fixtures/full/expected.bin) whose wire bytes changed as a result.", @@ -19200,299 +7045,23 @@ "config", "scheduling", "wire-encoding", - "fixed", "high-severity" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1635", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T08:16:40.941Z", - "error_message": "Significant refactor of ", - "file": "src/schema/controller.capnp", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 14\u21923 lines (11 removed) | Also: # Display-unit hint copied from the resource's own; # at reply-build time -- empty/unset means \"raw nu", - "tags": [ - "auto-detected", - "refactor", - "capnp", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-882", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T08:18:53.065Z", - "error_message": "Significant refactor of ", - "file": "src/controller/opget.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 20\u21929 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1636", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T08:19:26.708Z", - "error_message": "Incorrect value in code", - "file": "src/configs/schema/resources.schema.json", - "root_cause": "Had \"unit\"", - "fix": "Changed to \"resourcemap\"", - "tags": [ - "auto-detected", - "wrong-value", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1637", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T08:20:21.520Z", - "error_message": "Significant refactor of ", - "file": "src/xceadmin/src/xcecommon/mapping.py", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21923 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "py", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1638", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T08:22:12.974Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 8\u219237 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1639", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T09:22:19.958Z", - "error_message": "Significant refactor of ", - "file": "src/worker/src/xcecd.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21927 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1640", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T09:22:43.708Z", - "error_message": "Incorrect value in code", - "file": "src/configs/resources.yaml", - "root_cause": "Had \"available memory in MB (built-in dynamic load ind", - "fix": "Changed to \"available memory in bytes (built-in dynamic load ", - "tags": [ - "auto-detected", - "wrong-value", - "yaml", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1641", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T09:23:07.016Z", - "error_message": "Significant refactor of ", - "file": "src/configs/resources.yaml", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7\u21927 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "yaml", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1642", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T09:23:47.981Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 12\u219216 lines (6 removed) | Also: /* MaxMem is stored/wired as raw bytes (xce_resour; * for the maxmem resource definition) -- humanize ", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1643", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T09:25:25.166Z", - "error_message": "Significant refactor of ", - "file": "src/configs/schema/hosts.schema.json", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 10\u219217 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1644", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T09:25:49.927Z", - "error_message": "Significant refactor of ", - "file": "src/configs/schema/queues.schema.json", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 10\u219217 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-892", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T09:26:55.689Z", - "error_message": "Significant refactor of ", - "file": "src/xceadmin/src/xcecommon/mapping.py", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 16\u219258 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "py", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1645", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T10:45:55.441Z", - "error_message": "Significant refactor of ", - "file": "src/tools/deploy-ha.sh", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 21\u219234 lines (2 removed) | Also: # NOT retroactively update an already-built snapsh; # priority order (as this used to be) silently pic", - "tags": [ - "auto-detected", - "refactor", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1646", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T10:52:01.122Z", - "error_message": "Incorrect value in code", - "file": "src/tools/deploy-ha.sh", - "root_cause": "Had \"Warning: no xceadmin found in $BINARY'", - "fix": "Changed to \"Warning: no xceadmin found alongside $BINARY (che", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1556.md" }, { "id": "bug-1557", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-13T12:00:00Z", - "error_message": "xce_test's `worker` -> `join` command defaults to maxmem=64 -- discovered while writing/verifying detailed `help join` text", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_test's `worker` -> `join` command defaults to maxmem=64 -- discovered while writing/verifying…\n\nxce_test's `worker` -> `join` command defaults to maxmem=64 -- discovered while writing/verifying detailed `help join` text", "file": "src/controller/testsuite/test.c (cmd_join_worker, hostattrs_t attrs initializer)", "root_cause": "attrs.maxmem = 64 was a reasonable default back when maxmem was MB-scale (64 MB); this session's earlier fix converted maxmem's canonical unit to bytes (node_information.c/xcecd.c/resources.yaml) but never touched this test-tool default, which is now a nonsensical 64 BYTES of RAM. Purely a manual-test-tool default (join is used to simulate a worker joining without a real xcecd), not a production code path -- callers who don't pass -m get an obviously-wrong but harmless value.", "fix": "FIXED (2026-09-09, xce-test-cli-ux task 2.6). cmd_join_worker default changed to 64 * 1024 * 1024 (64MB in bytes). Extracted into join_worker_parse_attrs() (join_worker_args.c) with a permanent regression test asserting the corrected default. Verified live: workerlist reports MaxMem: 64.00 MB after a no-'-m' join (was would-be 64.00 B). USAGE_JOIN_WORKER doc update tracked separately under xce-test-cli-ux Requirement 12.2 (task 3.5).", @@ -19501,61 +7070,23 @@ "xce_test", "stale-default", "units", - "not-fixed", - "low-priority", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "low-priority" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-896", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T13:08:50.126Z", - "error_message": "Significant refactor of ", - "file": "src/xceadmin/src/xcecommon/mapping.py", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u21925 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "py", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-897", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-13T13:11:27.503Z", - "error_message": "Wrong reference: write_output should be dump_cluster_config", - "file": "src/xceadmin/src/xceadmin/cli.py", - "root_cause": "Used \"write_output\" instead of \"dump_cluster_config\"", - "fix": "Changed write_output \u2192 dump_cluster_config", - "tags": [ - "auto-detected", - "wrong-reference", - "py", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1557.md" }, { "id": "bug-1558", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-15T11:20:00Z", - "error_message": "make test-backend fails standalone: backend/reconnect.go:6:10: fatal error: client.h: No such file or directory -- FAIL apiserver/backend [build failed], make: *** [Makefile:33: test-backend] Error 1. Hit during /kiro-validate-impl component-failover-adoption full-suite validation.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "make test-backend fails standalone: backend/reconnect.go:6:10: fatal error: client.h: No such file…\n\nmake test-backend fails standalone: backend/reconnect.go:6:10: fatal error: client.h: No such file or directory -- FAIL apiserver/backend [build failed], make: *** [Makefile:33: test-backend] Error 1. Hit during /kiro-validate-impl component-failover-adoption full-suite validation.", "file": "src/frontend/Makefile (test-backend target), src/frontend/apiserver/meson.build", "root_cause": "CGO_CFLAGS/CGO_LDFLAGS for libxce and its transitive deps are constructed only inside apiserver/meson.build (lines 11-49) and set via meson's go_env environment() for its custom_target. They are never exported to a general shell, so the Makefile's test-backend target runs bare `CGO_ENABLED=1 go test ./backend/...` with no include or link paths. Compounding this, xce-build.sh's RUN_TESTS path only invokes `make test` (CGO_ENABLED=0), so the CGO backend suite -- which holds this spec's load-bearing rejoin/heartbeat/candidate-parity tests -- is structurally never exercised by any documented command. Previously noted narratively in component-failover-adoption tasks.md Implementation Notes (task 4.1, task 5.4) but never logged as a bug.", "fix": "FIXED (component-failover-adoption tasks.md Phase 6, non-blocking build-tooling fix). src/frontend/Makefile's test-backend target now self-derives CGO_CFLAGS/CGO_LDFLAGS via `pkg-config --cflags/--libs librtti c-capnproto openssl tinylog` plus -I$(CURDIR)/../include -I$(BUILD_DIR)/lib and -L$(BUILD_DIR)/lib -lxce -lm -lrt -fsanitize=address, mirroring apiserver/meson.build's own construction. BUILD_DIR anchored on $(CURDIR) (the Makefile's own directory), not a path relative to the `cd apiserver` recipe target, avoiding the off-by-one the workaround note flagged. Verified 2026-08-15: `make test-backend` run standalone from a nix develop shell with zero manual env setup -- `ok apiserver/backend 14.652s`, `ok apiserver/backend/auth 0.010s`.", @@ -19567,24 +7098,26 @@ "test-backend", "meson", "makefile", - "component-failover-adoption", - "resolved", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "component-failover-adoption" ], "related_bugs": [ "bug-1440", "bug-1441" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1558.md" }, { "id": "bug-1559", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-15T13:10:00Z", - "error_message": "xce-build.sh scheduler's RUN_TESTS block never builds or runs the 4 new component-failover-adoption task-3.x scheduler test binaries (sched_scheduler_connect_test, sched_scheduler_backpressure_test, sched_scheduler_decision_publish_retry_test, sched_scheduler_decision_redelivery_test) -- it only builds/runs 'xce_scheduler sched_test'. They only got compiled in this session as a side effect of a separate worker build and had to be run directly, bypassing the canonical script.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-build.sh scheduler's RUN_TESTS block never builds or runs the 4 new…\n\nxce-build.sh scheduler's RUN_TESTS block never builds or runs the 4 new component-failover-adoption task-3.x scheduler test binaries (sched_scheduler_connect_test, sched_scheduler_backpressure_test, sched_scheduler_decision_publish_retry_test, sched_scheduler_decision_redelivery_test) -- it only builds/runs 'xce_scheduler sched_test'. They only got compiled in this session as a side effect of a separate worker build and had to be run directly, bypassing the canonical script.", "file": "src/tools/xce-build.sh, src/scheduler/testsuite/meson.build", "root_cause": "xce-build.sh's scheduler case's RUN_TESTS list was never updated when task 3.x added 4 new standalone test executables under src/scheduler/testsuite/meson.build -- same class of gap as bug-1558 (documented commands cover only a subset of the real suite), just for the scheduler component instead of apiserver's CGO path.", "fix": "FIXED (component-failover-adoption tasks.md Phase 6, non-blocking build-tooling fix). src/tools/xce-build.sh's scheduler case's RUN_TESTS block now runs `meson test -C \"$BUILD_DIR\" --suite scheduler --print-errorlogs` instead of invoking sched_test directly by path -- meson test rebuilds by default, so no need to separately enumerate the 4 previously-skipped binaries. Verified 2026-08-15: `xce-build.sh scheduler asan` now runs all 41 scheduler suite tests (was: 1 binary only). NOTE: running the full suite for the first time surfaced a previously-invisible failure, test_recordstart_symlink_escape_replies_error (test_admin_request.c) -- logged separately as bug-1560, unrelated to this fix or to component-failover-adoption.", @@ -19594,183 +7127,25 @@ "scheduler", "xce-build.sh", "component-failover-adoption", - "coverage-gap", - "resolved", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "coverage-gap" ], "related_bugs": [ "bug-1558" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1647", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-15T13:02:22.188Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/apiserver/backend/clib.go", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 14\u219230 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "go", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-901", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-15T13:05:02.816Z", - "error_message": "Significant refactor of ", - "file": "src/frontend/Makefile", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u21926 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "unknown", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-902", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-15T13:06:10.183Z", - "error_message": "Wrong reference: 039 should be 1548", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Used \"039\" instead of \"1548\"", - "fix": "Changed 039 \u2192 1548", - "tags": [ - "auto-detected", - "wrong-reference", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1648", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-15T13:07:00.267Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Had 's own scope to fix (task 5.x is cross-component v", - "fix": "Changed to `/kiro-validate-impl`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1649", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-15T13:07:21.491Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/component-failover-adoption/spec.json", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 20\u219230 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1650", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-15T14:20:32.090Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-906", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-15T14:20:51.010Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/component-failover-adoption/spec.json", - "root_cause": "Had \"2026-08-15T13:10:00Z\"", - "fix": "Changed to \"2026-08-15T14:20:00Z\"", - "tags": [ - "auto-detected", - "wrong-value", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1651", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-15T14:21:07.376Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/component-failover-adoption/spec.json", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 11\u219217 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1559.md" }, { "id": "bug-1560", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "security", "timestamp": "2026-08-15T22:54:35Z", - "error_message": "test_recordstart_symlink_escape_replies_error (src/scheduler/testsuite/test_admin_request.c:850) FAILED: Expected 'invalid record file name' Was 'permission denied accessing record file'. Surfaced only now because bug-1559's fix made xce-build.sh's scheduler case run the full `meson test --suite scheduler` for the first time -- this test was never reached by the old sched_test-only invocation.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 5, + "error_message": "test_recordstart_symlink_escape_replies_error (src/scheduler/testsuite/test_admin_request.c:850)…\n\ntest_recordstart_symlink_escape_replies_error (src/scheduler/testsuite/test_admin_request.c:850) FAILED: Expected 'invalid record file name' Was 'permission denied accessing record file'. Surfaced only now because bug-1559's fix made xce-build.sh's scheduler case run the full `meson test --suite scheduler` for the first time -- this test was never reached by the old sched_test-only invocation.", "file": "src/scheduler/testsuite/test_admin_request.c, src/scheduler/src/recording.c", "root_cause": "Not yet investigated. Test plants a symlink inside record_dir whose target resolves outside record_dir, then issues a recordStart admin request expecting rejection with error string 'invalid record file name' (the canonical-path containment check). Instead the admin path is returning a 'permission denied accessing record file' error, implying some permission-check code path is being hit and short-circuiting before (or instead of) the containment check in this sandbox environment (uid 1000, non-root) -- ordering between the two checks in recording.c not yet confirmed by direct read. Unrelated to component-failover-adoption / bug-1548/1549/1558/1559 -- discovered purely as a side effect of bug-1559's coverage-gap fix making this test reachable.\n\n[UPDATE 2026-08-30] ROOT-CAUSED, confirmed by direct code reading and cross-checked against a sibling unit test. sched_validate_record_filename() (recording.c)'s canonical-path containment check (path_is_contained() branch, hit when a pre-existing symlink inside record_dir resolves outside it) deliberately returns -EACCES, not -EINVAL -- -EINVAL is reserved for Step 1's malformed-filename check (bare '/' or '..' in the filename itself, rejected before any filesystem access). This is NOT environment-dependent (not a uid-1000-vs-root permission-check ordering issue as originally suspected) -- test_recording_path_validation.c's own direct unit test for the identical scenario (test_symlink_inside_record_dir_escaping_outside_is_rejected) already asserts TEST_ASSERT_EQUAL_INT(-EACCES, rc) and has always passed. The bug was purely in test_admin_request.c's test_recordstart_symlink_escape_replies_error, which asserted the wrong expected error string (\"invalid record file name\", the -EINVAL string) against production code that has always, correctly, returned -EACCES (\"permission denied accessing record file\") for this exact scenario.", "fix": "FIXED (2026-08-30). Corrected test_recordstart_symlink_escape_replies_error's expected error string in src/scheduler/testsuite/test_admin_request.c from \"invalid record file name\" to \"permission denied accessing record file\", matching the already-correct, already-independently-verified production behavior. No production code changed -- this was purely a wrong test assertion, not a real product bug (the symlink-escape IS correctly rejected; only the test's expectation of WHICH rejection string was wrong). Verified: full `meson test --suite scheduler` run clean, Ok:55 Fail:0, all 55 test binaries pass including sched_admin_request_test (16/16 cases in test_admin_request.c PASS).", @@ -19782,123 +7157,26 @@ "coverage-gap-discovery", "xce-build.sh", "set-e-propagation", - "fixed", "test-only-bug", "not-environment-dependent" ], "related_bugs": [ "bug-1559" ], - "occurrences": 5, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1652", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T03:42:14.435Z", - "error_message": "Missing error handling in unknown", - "file": "reports/aug08-aug15-2026.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-910", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T03:57:07.719Z", - "error_message": "Incorrect value in code", - "file": "reports/aug08-aug15-2026.md", - "root_cause": "Had `build.py`", - "fix": "Changed to `3afe3d5`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1653", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T04:29:20.063Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/config-upgrade-snapshot/spec.json", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 21\u219229 lines (3 removed) | Also: \"updated_at\": \"2026-07-29T09:20:00Z\",; \"phase\": \"tasks-generated\",", - "tags": [ - "auto-detected", - "refactor", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1654", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T04:51:52.141Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 11\u219216 lines (11 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1655", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T04:54:26.304Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: j == NULL", - "fix": "Added guard clause: if (j == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1560.md" }, { "id": "bug-1561", "status": "fixed", "status_updated": "2026-09-24", "status_note": "a02d4777", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-16T00:00:00.000Z", - "error_message": "controller_unit_test: test_conf_trim_keeps_only_n_versions and test_conf_get_saved_tag_no_config (test_config_repl.c) FAIL during an unrelated ASAN controller build/test run (bug-451 fix session)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 14, + "error_message": "controller_unit_test: test_conf_trim_keeps_only_n_versions and test_conf_get_saved_tag_no_config…\n\ncontroller_unit_test: test_conf_trim_keeps_only_n_versions and test_conf_get_saved_tag_no_config (test_config_repl.c) FAIL during an unrelated ASAN controller build/test run (bug-451 fix session)", "file": "src/controller/testsuite/test_config_repl.c", "root_cause": "Not investigated -- discovered incidentally while verifying an unrelated fix (bug-451, persist_job_hook_fun's RUSAGE arm). Neither test_config_repl.c nor config_repl.c were touched this session (confirmed via git diff, zero changes). Coincides with a burst of 'persist: flusher fdatasync FAILED (errno=5 Input/output error)' ERROR lines in the same test run, matching this project's well-documented overlayfs sandbox fdatasync artifact class (same family as bug-924, bug-1470, bug-1553) rather than a genuine regression.", "fix": "NOT FIXED -- out of scope for the bug-451 session that surfaced it. Needs confirmation this is the same overlayfs artifact (re-run on a real fs) before treating it as a real regression in test_config_repl.c's config-version-trim/tag-save logic. Re-observed 2026-08-17 during resource-value-verification task 1.1 (resval_typeresolve extraction) full controller ASAN build/test run: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was ), same accompanying persist_flush/flusher fdatasync errno=5 burst. Zero diff in test_config_repl.c/config_repl.c this session either (task 1.1 boundary was testsuite/resval_typeresolve.{c,h}, testsuite/test_resval_typeresolve.c, meson.build only) -- further corroborates the overlayfs sandbox artifact theory over a genuine regression, still unconfirmed on a real fs. Re-observed again 2026-08-17 during resource-value-verification task 2.1 (resupdate fallback + -t flag wiring, GREEN phase): identical 2 tests, identical Expected/Was values, identical fdatasync errno=5 burst, in the same full `/xce-build controller` run that also confirmed resval_typeresolve_unit_test's own Unity summary at 4 Tests 0 Failures (task 2.1's GREEN evidence). Task 2.1's diff was scoped to testsuite/resval_typeresolve.{c,h} and testsuite/test.c only (confirmed via git diff --stat) -- test_config_repl.c/config_repl.c untouched, per this bug's own do-not-chase guidance in tasks.md's Implementation Notes.\n\nRe-observed a 5th time 2026-08-18 during a full `nix develop ./src --command bash src/tools/xce-build.sh all asan build-asan --incremental` verification run on the scheduler-framework branch (post bug-1584/1585 scheduler fixes). Identical signature: same 2 tests, same Expected/Was values (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. This run recompiled ZERO C source files (ninja reported \"no work to do\" for all C targets -- only the xceadmin custom target/wheel regenerated), so the failure cannot be attributed to any change in this session -- strongest evidence yet that this is purely an environment (overlayfs fdatasync) artifact, not a code regression. Scheduler suite's own single failure in the same run (test_recordstart_symlink_escape_replies_error, bug-1560) is separate and expected. Re-observed a 6th time 2026-08-19 during a controller ASAN build/test run verifying the StartTime/FinishTime persist.c fix (raft-apply-decoupling job-fact timestamp stamping, unrelated files persist.c/test_apply_committed_entry.c) -- identical 2 tests, identical Expected/Was values, identical fdatasync errno=5 burst. Session diff confirmed scoped to persist.c/test_apply_committed_entry.c only -- further corroborates environment-artifact theory.\n\nRe-observed a 7th time 2026-08-21 verifying a pure output-formatting edit to src/controller/testsuite/test.c (raw %lu jobid printf sites replaced with fmt_jobid()+%s, 10 call sites, zero logic change) via `xce-build.sh controller asan --incremental` on the scheduler-framework branch. Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to test.c only (git diff --stat) -- test_config_repl.c/config_repl.c untouched. controller/xce_test (the binary test.c itself compiles into) built and linked cleanly with zero warnings/errors tied to the fmt_jobid change; only controller_unit_test (a separate binary, testsuite_test_config_repl.c.o) carries these 2 pre-existing failures. Further corroborates environment-artifact theory over regression.\n\nRe-observed an 8th time 2026-08-23 during scheduler-dispatch-commit task 3.3's full controller ASAN build/test verification run (job.c dispatch-failure retry restructuring + new test_dispatch_fail_retry.c). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to job.c and testsuite/ files only (git diff --stat) -- test_config_repl.c/config_repl.c untouched. Further corroborates environment-artifact theory over regression.\n\nRe-observed a 9th time 2026-08-23 during scheduler-dispatch-commit task 4.1's full controller ASAN build/test verification run (job.c handle_jobdispatch() double-notify removal + new test_jobdispatch_no_double_notify.c). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to job.c (handle_jobdispatch() loop body only) and new/edited testsuite files -- test_config_repl.c/config_repl.c untouched. Further corroborates environment-artifact theory over regression.\n\nRe-observed a 10th time 2026-08-23 during the scheduler-dispatch-commit feature's final GO/NO-GO release-gate verification run (`xce-build.sh controller asan build-asan --incremental`, tree fully committed, zero uncommitted diff in src/controller/job.c, handler.c, or testsuite/). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. A dedicated scoped rerun of controller_unit_test with -t filters covering only the scheduler-dispatch-commit feature's own groups (test_dispatch_commit, test_dispatch_fail_terminal, test_dispatch_fail_retry, test_jobdispatch_no_double_notify, test_commit_barrier_reply, test_wal_job_dispatch_fact, test_job_dispatch, test_job_dispatch_fail_count) came back 62 Tests 0 Failures 0 Ignored, exit 0 -- confirms these 2 pre-existing failures are fully isolated to test_config_repl.c and do not touch this feature's own test surface.\n\n[UPDATE 2026-08-26, xce-test-rc-file task 1.1] Re-observed again, identical signature (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was , same fdatasync errno=5 burst) in both the meson-driven controller test run and a standalone direct controller_unit_test invocation, in a session whose only diff was adding a new, unrelated rcfile.c/rcfile.h/test_rcfile.c module (xce-test-rc-file spec task 1.1) plus meson.build/controller_test.c wiring -- config_repl.c/test_config_repl.c untouched by this session's own diff (though config_repl.c already carried unrelated pre-existing uncommitted changes from prior work at session start). Further corroborates: not caused by this task.\n\nRe-observed 2026-08-29 during a debug-profile (non-ASAN, -O0 -g3) controller build/test run -- same 2 tests, same failure shape, but this time WITHOUT ASAN instrumentation and without the accompanying fdatasync errno=5 burst previously blamed on overlayfs -- weakens the overlayfs-artifact theory as the sole explanation. New candidate root cause found by reading the actual test/production code: cr_setup() (test_config_repl.c:62-71) keys its temp dir on getpid()+time(NULL) (second resolution), and conf_trim_old_versions() (config_repl.c:1027-1079) sorts by st.st_mtime (also second resolution) -- both can collide within one wall-clock second despite the test's own writes being only ~15ms apart. NOT YET fixed or confirmed as the sole cause (both theories -- overlayfs fdatasync artifact and second-resolution timestamp collision -- remain plausible and are not mutually exclusive); still open.\n\n[UPDATE 2026-08-30, scheduler-framework session] FIXED. Two distinct, independently-confirmed root causes, both found by direct reading of the current source (not the overlayfs-fdatasync theory explored above -- that theory is now superseded for this bug, though it likely still explains other unrelated flaky-test sightings in this project's history):\n\n1. conf_trim_old_versions() (src/controller/config_repl.c, ~line 1013-1079) sorted candidate config version files by st.st_mtime, a time_t with SECOND resolution. test_conf_trim_keeps_only_n_versions writes 3 version files usleep(15000) (15ms) apart -- well under one second, so all 3 routinely land in the same st_mtime second, making the qsort() comparator see ties. qsort() is not stable, so a tied ordering can push the current-pointer file (protected from deletion by conf_trim_old_versions(), with the deletion simply skipped for it -- no compensating extra delete) past index keep_n, leaving keep_n+1 files on disk instead of keep_n. This exactly matches the observed 'Expected 2 Was 3' failure. FIX: changed ver_entry_t.mtime from time_t to struct timespec, populated from st.st_mtim (POSIX.1-2008 nanosecond-resolution mtime, standard on Linux/glibc, no new includes needed -- sys/stat.h and time.h were already included), and updated ver_entry_cmp_desc() to compare tv_sec first and tv_nsec as the tiebreak, preserving the original descending-order (newest-first) comparison direction exactly. This is a genuine, if minor, production correctness fix to conf_trim_old_versions() (config version retention, the sole conf_save_config() call site) -- config pushes landing in the same wall-clock second could previously retain a non-deterministic count of versions; they now sort by true write order at filesystem-provided resolution.\n\n2. cr_setup() (src/controller/testsuite/test_config_repl.c, ~line 62-72) built its per-test tmpdir path from only getpid() (constant for the whole binary run) and time(NULL) (second resolution), so any two cr_setup() calls within the same wall-clock second collided on an identical path. cr_teardown()'s synchronous rm -rf makes within-run sequential reuse safe by itself, but the same collision-prone naming means a stale directory left behind by any earlier interrupted run (ASAN abort, timeout, SIGKILL -- none of which reach cr_teardown()) could be silently inherited by a later run reusing the same PID (common on Linux) in the same second. test_conf_get_saved_tag_no_config expects a genuinely empty fresh directory (conf_get_saved_tag() returns 0 cleanly when conf/current is absent -- that function's own logic was confirmed correct, not the bug); the only way it observes a nonzero garbage tag is a directory that was not actually empty. FIX: added a static monotonically-increasing seq counter to cr_setup()'s snprintf, guaranteeing every cr_setup() call within one test binary invocation gets a unique tmpdir regardless of PID/wall-clock collisions. Verified g_tmpdir's 256-byte buffer comfortably fits the added _%d suffix.\n\nRED reproduction: confirmed on the FIRST attempt, before any fix -- `nix develop ./src --command bash src/tools/xce-build.sh controller asan` failed with exactly the historical signature: test_conf_trim_keeps_only_n_versions Expected 2 Was 3 (test_config_repl.c:258) and test_conf_get_saved_tag_no_config Expected 0 Was 11954236611186853435 (test_config_repl.c:273). No further repro attempts were needed.\n\nGREEN + repeat-run evidence: after applying both fixes, `xce-build.sh controller asan --incremental` passed clean (1220 Tests 0 Failures). Re-ran the controller suite an additional 18 times across two invocation methods (11 via `meson test -C src/xce.build --suite controller \"controller test\"`, labeled runs A-N; 4 via direct controller_unit_test binary invocation with meson's own ASAN/MSAN/UBSAN/MALLOC_PERTURB_ env vars replicated) to build confidence beyond a single green run, since this was a flake, not a deterministic failure. 16 of these fully completed with '1220 Tests 0 Failures 0 Ignored', explicitly re-confirming both target tests PASS every time they were checked. Two runs were inconclusive due to this session's own tooling (a bash-tool timeout killing the process, once after confirming both target tests had already PASSED at the same log position as every clean run, before an unrelated later test simply ran slower than usual that one time) -- not test failures. One run ('run D') exited status 1 after only 14.04s (vs. the consistent ~38.8-39.1s for every other run), whose detailed log was lost to meson's testlog.txt overwrite before it could be inspected (a mistake in this session's own harness, corrected for all subsequent runs by capturing full output per-run to unique files). This anomaly could not be reproduced again despite 12 further attempts after switching to per-run log capture, and no evidence tied it to config_repl.c/test_config_repl.c specifically -- a genuine single Unity assertion failure would not shorten total runtime this much, since Unity continues running the remaining ~1219 tests after one failure (confirmed by the original RED run itself, which had 2 failures and still ran the full ~39s), so a 14s exit is more consistent with an early crash/abort in some other, unidentified part of this large (1220-test) ASAN suite than with a config-version-trim or tag-read regression from this fix. Given zero reproductions in 12 subsequent attempts, explicit target-test PASS confirmation in every run that could be checked (16/16), and this project's own prior 13-occurrence documented history of unrelated environment-driven flakiness affecting this exact large test binary in this sandbox, this single anomaly is recorded here for transparency but is not attributed to this fix. No other regressions were observed in any of the 16 fully-completed runs.\n\nFiles touched: src/controller/config_repl.c (production fix), src/controller/testsuite/test_config_repl.c (test-harness fix).", @@ -19909,205 +7187,29 @@ "overlayfs", "fdatasync", "sandbox-environment", - "unconfirmed", - "not-fixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "unconfirmed" ], "related_bugs": [ "bug-924", "bug-1470", "bug-1553" ], - "occurrences": 14, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-912", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T05:04:02.838Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 37\u219286 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fix_commits": [ + "a02d4777" ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1656", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T05:22:23.386Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_e2e_checkpoint.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1657", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T05:25:35.899Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u219217 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-915", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T06:54:02.070Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ha->standby_endpoints", - "fix": "Added guard clause: if (ha->standby_endpoints)", - "tags": [ - "auto-detected", - "guard-clause", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-916", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T07:03:34.559Z", - "error_message": "Significant refactor of ", - "file": "src/controller/worker.c", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 12\u219223 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-917", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T09:54:06.127Z", - "error_message": "Significant refactor of ", - "file": "src/controller/worker.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 15\u21928 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1658", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T11:12:53.822Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/admin-config-tool/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-919", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T12:12:34.395Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/spec.json", - "root_cause": "Had \"All tasks 1.x-11.x done (verified 2026-06-09 gap ", - "fix": "Changed to \"CLOSED 2026-08-16. All 3 known deviations from th", - "tags": [ - "auto-detected", - "wrong-value", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1659", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-16T12:12:57.617Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/resval-id-decoupling/spec.json", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 21\u219222 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "json", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "record": ".wolf/bugs/bug-1561.md" }, { "id": "bug-1562", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-08-17T03:40:00.000Z", - "error_message": "3-node HA bootstrap never elects a primary: ctrl-1 (only node with local config) goes completely silent ~22s after boot -- no crash, no log output, cluster.log shows no further activity even after ctrl-2/ctrl-3 join minutes later. Downstream symptom: worker gets 'SSL_do_handshake: Broken pipe' in a loop and ctrl-2/ctrl-3 log repeated 'incorrect worker idx -1' ERRORs from handle_endpoint_error (harmless noise -- join.c's handle_join_ stamps priv->role=HostRole_worker before the ha_no_primary_known guard can reject the join, so u.idx is still -1 when the rejected connection is torn down).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "3-node HA bootstrap never elects a primary: ctrl-1 (only node with local config) goes completely…\n\n3-node HA bootstrap never elects a primary: ctrl-1 (only node with local config) goes completely silent ~22s after boot -- no crash, no log output, cluster.log shows no further activity even after ctrl-2/ctrl-3 join minutes later. Downstream symptom: worker gets 'SSL_do_handshake: Broken pipe' in a loop and ctrl-2/ctrl-3 log repeated 'incorrect worker idx -1' ERRORs from handle_endpoint_error (harmless noise -- join.c's handle_join_ stamps priv->role=HostRole_worker before the ha_no_primary_known guard can reject the join, so u.idx is still -1 when the rejected connection is torn down).", "file": "src/controller/ha.c (ha_on_primary_ctrl_fd_rdhup, 5.10b debounce branch)", "root_cause": "Live-reproduced locally (build.dbg xce_controller, fresh 3-node deploy via deploy-ha.sh, ctrl-1 started ~45s before ctrl-2/ctrl-3 to match the user's actual timing) -- byte-for-byte matches the user's ctrl-1.log. ha_on_primary_ctrl_fd_rdhup's 5.10b debounce branch (ha.c ~8573-8589, added 2026-07-01 commit 0d70d5e4, bug-1310 task 5.10b) returns WITHOUT calling ha_arm_heartbeat_timeout_vtask() when a debounced RDHUP lands within the 2x-election-timeout cooldown window, unlike the function's other two early-return branches: the 5.6 election-in-progress guard (checks role==CTRL_ELECTING || pre_vote_in_progress, i.e. actually verifies a live timer is still running elsewhere) and the 5.2 spurious-suppression branch (explicitly re-arms via ha_arm_heartbeat_timeout_vtask before returning). The 5.10b branch only checks elapsed time since last_failover_ms -- it never checks whether an election is ACTUALLY still in flight. On a lone cold-booting node (n_peers up but none reachable yet), the first PreVote round loses within milliseconds (all connect attempts fail fast -> counted as no-votes), which re-arms the liveness watchdog once (10s). When that watchdog fires again ~10s later, last_failover_ms is still inside the 2x-election-timeout cooldown (observed 9805ms < 10000ms), so it lands in the debounce branch even though no election is in flight anymore -- and returns without re-arming anything. That is the LAST timer ever scheduled in the process: /proc//task/*/wchan confirmed every vtask worker thread idling in futex_do_wait (empty work queue, not deadlocked) and the io_worker thread parked in do_epoll_wait with zero further epoll events (lsof showed only the listen socket + epoll fd, no in-flight connections) for the remainder of the run. Node is permanently stuck CTRL_STANDBY with zero further election attempts -- other config-less nodes correctly wait forever for a primary that will never appear, so the whole cluster is permanently leaderless. Timing-dependent: reproducible only when the solo bootstrap window is long enough for a second liveness-timeout to land inside the cooldown after the first PreVote round already concluded (confirmed NOT reproducible with a tight ~0.5s inter-node start stagger, where a real peer answers before the trap window closes) -- explains why this presented as 'used to work' without any actual regression: bug has been latent since 2026-07-01, only the user's own start timing changed.", "fix": "FIXED (TDD RED->GREEN). Added test_rdhup_debounce_still_rearms_liveness_watchdog (test_ha_rdhup_integration.c, next to the sibling test_rdhup_within_cooldown_is_debounced) -- asserts ha->last_activity moves off a sentinel after a debounced RDHUP, the same delaymgr-independent observable test_ha_arm_no_crash_no_thread (test_ha_liveness.c) already uses to prove ha_arm_heartbeat_timeout_vtask ran without a live delaymgr thread. RED confirmed pre-fix (assertion failed with the bug-1562 message). Fix: ha_on_primary_ctrl_fd_rdhup's 5.10b debounce branch (ha.c ~8578-8598) now calls ha_arm_heartbeat_timeout_vtask(ha, c) before returning, mirroring the function's own 5.2 spurious-suppression branch pattern -- safe with c==NULL (unit-test mode) since the arm function stamps ha->last_activity unconditionally before its internal c-null/delaymgr-running check gates only the actual timer scheduling. GREEN: controller_unit_test 1102 tests, 2 Failures (both pre-existing, unrelated overlayfs fdatasync EIO sandbox flakiness in test_config_repl.c -- test_conf_trim_keeps_only_n_versions and test_conf_get_saved_tag_no_config -- confirmed via 'persist: flusher fdatasync FAILED (errno=5 Input/output error)' immediately preceding both in the test log; identical to the cerebrum.md-documented sandbox quirk, present before this change too). LIVE END-TO-END VALIDATION (per this project's live-soak-validation-discipline convention -- unit-green alone is not proof): rebuilt xce_controller (src/xce.build), redeployed the identical 3-node scenario that hung before (ctrl-1 solo 45s, then ctrl-2, then ctrl-3 -- byte-for-byte the same timing that previously froze the process forever). Post-fix ctrl-1.log shows the debounce branch now immediately followed by 'HA: standby liveness poll armed' (previously the log went dead right after the debounce line), the watchdog continuing to retry every ~10s through several more PreVote-loses/debounce cycles while alone, and once ctrl-2/ctrl-3 came online: 'HA FSM (raft): -> CTRL_PRIMARY (term=2, first_boot=0)' -- ctrl-1 successfully elected primary, and both controller/ctrl-2/conf and controller/ctrl-3/conf received the pushed cluster.bin + current config. Full cluster convergence confirmed on the exact scenario that previously froze permanently.", @@ -20118,82 +7220,25 @@ "dropped-timer", "bootstrap-hang", "raft-ha-core-safety", - "5.10b", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "5.10b" ], "related_bugs": [ "bug-1310" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1660", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T03:45:49.314Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1661", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T04:59:39.812Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1662", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T05:01:33.795Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1562.md" }, { "id": "bug-1563", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-17T04:59:00.000Z", - "error_message": "40-round chaotic /xce-test-ha soak: ctrl-2's own log shows 309,671 'persist: flusher fdatasync FAILED (errno=22 Invalid argument) -- durable_lsn held at 163 (tip=164 NOT made durable)' ERROR lines, 706 of them landing within a single 705ms window (04:52:54.000-04:52:54.705) on the freshly-promoted primary -- a sustained sync failure the flusher never recovered from or backed off on for the rest of the run. The soak's own verdict still reports PASS (committed_divergence=0, committed_job_loss=0) because the other two standbys' acks alone satisfy majority and the bug-924 fdatasync-error exclusion swallows the count -- but this specific pattern (a stuck LSN, EINVAL not the documented EIO, hundreds of retries per second) is a distinct, real defect from the known overlayfs sandbox quirk, not just noise.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "40-round chaotic /xce-test-ha soak: ctrl-2's own log shows 309,671 'persist: flusher fdatasync…\n\n40-round chaotic /xce-test-ha soak: ctrl-2's own log shows 309,671 'persist: flusher fdatasync FAILED (errno=22 Invalid argument) -- durable_lsn held at 163 (tip=164 NOT made durable)' ERROR lines, 706 of them landing within a single 705ms window (04:52:54.000-04:52:54.705) on the freshly-promoted primary -- a sustained sync failure the flusher never recovered from or backed off on for the rest of the run. The soak's own verdict still reports PASS (committed_divergence=0, committed_job_loss=0) because the other two standbys' acks alone satisfy majority and the bug-924 fdatasync-error exclusion swallows the count -- but this specific pattern (a stuck LSN, EINVAL not the documented EIO, hundreds of retries per second) is a distinct, real defect from the known overlayfs sandbox quirk, not just noise.", "file": "src/controller/persist.c (fsync_worker)", "root_cause": "fsync_worker's loop (persist.c) only calls pthread_cond_timedwait (its ONLY throttle/backoff mechanism) when flusher_work_pending_locked(p) is false. But that predicate is purely 'durable_lsn < tip' -- it cannot distinguish 'genuine fresh work just landed' from 'still stuck from the previous failed attempt', because flusher_flush_once()'s fail-loud gate (correct, unchanged) deliberately never advances durable_lsn on a failed fdatasync. Once fdatasync starts failing with no new appends arriving (a standby/primary transition mid-chaotic-kill with a broken local fd, or any sustained real disk error in production), durable_lsn < tip stays true forever, so 'pending' is permanently true and the loop skips the wait on every single iteration -- flusher_flush_once (and thus fdatasync) gets called as fast as the CPU/lock overhead allows, pinning one core and flooding the log. Confirmed by direct code read of flusher_work_pending_locked (dur < tip, no failure-awareness) and fsync_worker's loop structure (wait only in the !pending branch), and reproduced deterministically in a unit test (test_force_fsync_fail seam) before any live soak was re-run.", "fix": "FIXED (TDD RED->GREEN). Added test_flusher_flush_attempt_count (persist.h, XCE_TESTING-only atomic counter, incremented once per flusher_flush_once() call) and test_flusher_backs_off_on_sustained_fsync_failure (test_term_safety_adapter.c): forces every fdatasync to fail via the existing test_force_fsync_fail seam on a PRIMARY persist_t with wal_sync_ms=20, appends one entry (primary's append path defers all sync to the async flusher, unlike a standby's inline persist-before-ACK sync -- see test_on_persist_entry_failure_propagates_no_ack), waits 300ms, and asserts attempt count stays under 60 (~15 expected at a 20ms cadence). RED confirmed pre-fix (assertion failed -- unbounded spin). Fix: fsync_worker's loop (persist.c) now also waits (same pthread_cond_timedwait/wal_sync_ms cadence the idle path already uses, no new tunable) when the just-completed flush WAS pending but made no progress (after == before) -- i.e. specifically the failure case, never the legitimate 'nothing to sync' idle case (pending was false there, left untouched) nor a successful flush (after > before, still loops immediately to catch a fresh append with zero latency, unchanged fast path). GREEN: test_flusher_backs_off_on_sustained_fsync_failure passes with attempts=15 (was unbounded/thousands pre-fix in the same 300ms window). Full controller_unit_test: 1103 tests, 2 failures -- both pre-existing, unrelated overlayfs fdatasync EIO sandbox flakiness in test_config_repl.c (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), confirmed unchanged from before this fix. LIVE VALIDATION: re-ran the 40-round chaotic /xce-test-ha soak that originally surfaced this, with the rebuilt binary -- excluded_bug924_fdatasync dropped from 304,689 to 0 in the re-run (VERDICT mode=chaotic rounds=40 jobs=123 committed_divergence=0 committed_job_loss=0 result=PASS). Also re-ran the 40-round sequential soak for completeness (PASS: 40 rounds, 123 jobs, zero job loss, all 3 nodes led) -- unaffected by this change as expected, confirming no regression. Both full 40-round soaks (sequential and chaotic) now pass cleanly end to end.", @@ -20205,182 +7250,25 @@ "backoff", "durability", "chaotic-soak", - "xce-test-ha", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "xce-test-ha" ], "related_bugs": [ "bug-661" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1663", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T06:41:48.625Z", - "error_message": "Significant refactor of ", - "file": "src/tools/deploy-ha.sh", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 40\u219293 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-927", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T06:44:51.034Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-deploy-ha/SKILL.md", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 65\u2192132 lines (7 removed) | Also: # 1. Stop running controllers; bash \"$TOP/scripts/cleanup.sh\"", - "tags": [ - "auto-detected", - "refactor", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1664", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T06:51:13.785Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-build/SKILL.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u219226 lines (2 removed) | Also: does not need to run from inside this repo or the ; **not** bundle its dependencies (`click`, `pycapnp | Also: ## Installing the xceadmin wheel elsewhere; `/xce-build xceadmin` always leaves a fresh, teste", - "tags": [ - "auto-detected", - "refactor", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1665", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T06:52:21.795Z", - "error_message": "Significant refactor of ", - "file": "src/xceadmin/pyproject.toml", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 6\u21929 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "toml", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1666", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T07:00:05.291Z", - "error_message": "Significant refactor of ", - "file": "src/tools/deploy-ha.sh", - "root_cause": "13 lines replaced/restructured", - "fix": "Rewrote 22\u219244 lines (13 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1667", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T07:00:21.077Z", - "error_message": "Incorrect value in code", - "file": "src/tools/deploy-ha.sh", - "root_cause": "Had \"xceadmin installed at $TOP/xceadmin-venv -- use i", - "fix": "Changed to \"xceadmin installed at $TOP/xceadmin-site -- use i", - "tags": [ - "auto-detected", - "wrong-value", - "sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-932", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T07:15:26.246Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1668", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T07:15:58.937Z", - "error_message": "Wrong reference: fire_count should be g_durable_advance_fire_count", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Used \"fire_count\" instead of \"g_durable_advance_fire_count\"", - "fix": "Changed fire_count \u2192 g_durable_advance_fire_count", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1563.md" }, { "id": "bug-1564", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-17T07:15:00.000Z", - "error_message": "controller_unit_test: test_durable_majority_commit_end_to_end FAIL: Expected 0 Was -110 (commit_submit(WAIT) must return 0 after durable-majority) -- deterministic 3/3 direct runs of the binary, not a flake. Surfaced while live-testing an unrelated /xce-deploy-ha enhancement, right after rebuilding controller on top of bug-1563's fix.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller_unit_test: test_durable_majority_commit_end_to_end FAIL: Expected 0 Was -110…\n\ncontroller_unit_test: test_durable_majority_commit_end_to_end FAIL: Expected 0 Was -110 (commit_submit(WAIT) must return 0 after durable-majority) -- deterministic 3/3 direct runs of the binary, not a flake. Surfaced while live-testing an unrelated /xce-deploy-ha enhancement, right after rebuilding controller on top of bug-1563's fix.", "file": "src/controller/persist.c (persist_flush)", "root_cause": "persist_flush() (the synchronous durability point tests and clean-shutdown use in place of waiting for the background flusher) advances durable_lsn directly under p->lock but never calls p->on_durable_advance -- only fsync_worker's own successful flusher_flush_once() call fires that callback (persist.c, 'after > before' gate). Any WAIT-blocked commit caller (commit_engine's COMMIT_MODE_WAIT) that relies on on_durable_advance to be re-evaluated is therefore never woken by a persist_flush()-driven advance -- only by fsync_worker's own next successful cycle. This gap was LATENT and never triggered before bug-1563: pre-1563, fsync_worker's un-throttled retry loop (the livelock bug-1563 fixed) spun continuously while work was pending, so it happened to notice any state change (e.g. a test clearing test_force_fsync_fail) and independently re-attempt+succeed+fire the callback within microseconds, completely masking that persist_flush's own advance was silently un-observed. bug-1563's fix (correct: fsync_worker now backs off up to wal_sync_ms between failed attempts instead of spinning) removed that accidental fast-repoll cover, so a persist_flush()-only advance during a slow-cadence window (make_durability_persist uses wal_sync_ms=5000 deliberately, 'durability comes from the explicit flush, not the timer') now genuinely times out the 2000ms-budget WAIT with nothing left to wake it for the remaining ~4.7s fsync_worker was still asleep.", "fix": "FIXED (TDD RED->GREEN). Added test_persist_flush_fires_durable_advance_callback (test_commit_durability.c, next to test_durable_majority_commit_end_to_end): registers a counting callback via persist_set_durable_cb, appends an entry, asserts persist_flush() fires it exactly once on a genuine advance and not again on a no-op flush. RED confirmed against the pre-fix persist_flush (0 fires). Fix: persist_flush() now fires p->on_durable_advance(p->durable_cb_ctx) after releasing p->lock, gated on advanced==true (only when durable_lsn actually moved) -- identical gate and identical lock-ordering discipline (strictly outside p->lock, so the callback's own ce->lock acquisition has no p->lock held underneath it) to fsync_worker's own existing callback-firing code. GREEN: both test_persist_flush_fires_durable_advance_callback and the originally-broken test_durable_majority_commit_end_to_end pass, confirmed deterministic across 3 direct repeat runs of controller_unit_test. Full suite: 1104 tests, 2 failures (both pre-existing, unrelated overlayfs fdatasync sandbox flakiness in test_config_repl.c, unchanged from before this fix).", @@ -20391,22 +7279,25 @@ "callback", "wait-timeout", "regression", - "bug-1563-followup", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "bug-1563-followup" ], "related_bugs": [ "bug-1563" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1564.md" }, { "id": "bug-1565", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-17T07:16:00.000Z", - "error_message": "src/tools/xce-build.sh's setup_build() does an unconditional `rm -rf \"$BUILD_DIR\"` + fresh `meson setup` for EVERY component case (controller, worker, xceadmin, etc), not just when actually needed -- running `/xce-build xceadmin` (or any single-component target) after `/xce-build controller` in the SAME build dir silently deletes the already-built xce_controller/xce_test/etc binaries, leaving only that one component's fresh output. Discovered live while enhancing /xce-deploy-ha: a working xce_controller from an earlier turn vanished after an unrelated `/xce-build xceadmin` run in the same xce.build dir, breaking a live-deploy test with 'No such file or directory' for the controller binary.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "src/tools/xce-build.sh's setup_build() does an unconditional `rm -rf \"$BUILD_DIR\"` + fresh `meson…\n\nsrc/tools/xce-build.sh's setup_build() does an unconditional `rm -rf \"$BUILD_DIR\"` + fresh `meson setup` for EVERY component case (controller, worker, xceadmin, etc), not just when actually needed -- running `/xce-build xceadmin` (or any single-component target) after `/xce-build controller` in the SAME build dir silently deletes the already-built xce_controller/xce_test/etc binaries, leaving only that one component's fresh output. Discovered live while enhancing /xce-deploy-ha: a working xce_controller from an earlier turn vanished after an unrelated `/xce-build xceadmin` run in the same xce.build dir, breaking a live-deploy test with 'No such file or directory' for the controller binary.", "file": "src/tools/xce-build.sh (setup_build)", "root_cause": "setup_build() (xce-build.sh, called at the top of every component case branch) always does `rm -rf \"$BUILD_DIR\"; meson setup \"$BUILD_DIR\" ...` before compiling -- a full wipe-and-reconfigure regardless of which single component was requested. Since `meson compile -C \"$BUILD_DIR\" ` only rebuilds that one target afterward, every OTHER previously-built target in the same dir (e.g. xce_controller, when the requested component was xceadmin) is gone until something explicitly rebuilds it too. Not investigated further / not fixed -- out of scope for the /xce-deploy-ha enhancement session that found it; flagged here for whoever owns xce-build.sh next. A likely fix direction: only rm -rf+reconfigure when MESON_OPTS actually differ from the existing build dir's recorded options (or on first setup / explicit request), otherwise just `meson compile -C \"$BUILD_DIR\" ` against the existing configured dir.", "fix": "FIXED (opt-in, not a default-behavior change). Added --incremental to xce-build.sh (parsed out of the argument list before the existing component/profile/build_dir positional parsing, so it can appear anywhere and never disturbs that parsing). When passed AND the target build dir already has a build.ninja (i.e. it's already configured), setup_build() runs `meson setup --reconfigure \"$BUILD_DIR\" \"${MESON_OPTS[@]}\"` in place instead of `rm -rf` + fresh `meson setup` -- preserving any other already-built component's binaries in that dir. First-ever build of a dir (no build.ninja yet) still does a normal fresh setup regardless of the flag, since there's nothing to reconfigure. Default (flag omitted) is completely unchanged -- still always wipes. Deliberately NOT made the default: the known missing-dependency-edge races in the capnp-generation custom_target (see recurring buglog entries on 'controller.capnp.h: No such file or directory') are papered over by always wiping (worst case: a loud, retry-fixable clean-build failure); without the wipe the same gap could instead cause SILENT staleness (a schema edit not triggering regeneration, build reports success against stale generated code) -- an --incremental caller is explicitly opting into that tradeoff. Live-validated: built controller (xce_controller present), then `xceadmin --incremental` (wheel built, xce_controller's mtime unchanged -- survived); confirmed the flag works regardless of argument position (`--incremental xceadmin` and `xceadmin --incremental` both work); confirmed the DEFAULT (no flag) still wipes as before (xce_controller correctly gone after a plain `xceadmin` rebuild). Also updated the /xce-build skill and build-deployment.md steering doc to document the new flag.", @@ -20415,140 +7306,23 @@ "build-tooling", "footgun", "meson", - "discovery", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "discovery" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-936", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T08:02:07.121Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/persistence.md", - "root_cause": "Had `on_durable_advance(ctx)`", - "fix": "Changed to `fdatasync`", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1669", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T08:03:27.006Z", - "error_message": "Significant refactor of ", - "file": ".kiro/steering/build-deployment.md", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 32\u219248 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-938", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T08:18:40.345Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-build/SKILL.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 11\u219212 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-939", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T08:19:13.328Z", - "error_message": "Wrong operator: && should be <<", - "file": ".claude/skills/xce-build/SKILL.md", - "root_cause": "Used \"&&\" instead of \"<<\"", - "fix": "Changed operator && \u2192 <<", - "tags": [ - "auto-detected", - "operator-fix", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-940", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T08:19:30.812Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/build-deployment.md", - "root_cause": "Had 're gone until something explicitly rebuilds them ", - "fix": "Changed to 're gone until something explicitly rebuilds them ", - "tags": [ - "auto-detected", - "wrong-value", - "md", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1670", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T08:24:49.130Z", - "error_message": "Incorrect value in code", - "file": "src/xceadmin/tests/test_cli.py", - "root_cause": "Had \", event_count=0, started_at=None)\n\n monkeypatc", - "fix": "Changed to \", event_count=0, started_at=0)\n\n monkeypatch.s", - "tags": [ - "auto-detected", - "wrong-value", - "py", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1565.md" }, { "id": "bug-1566", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-08-17T08:30:00.000Z", - "error_message": "User-reported: `./scripts/xceadmin scheduler record status` (from a /xce-deploy-ha deployment) failed with 'Error: cannot connect to localhost:9991: [Errno 111] Connection refused' -- the deployment's actual --peers were localhost:9901/9902/9903, not 9991.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported: `./scripts/xceadmin scheduler record status` (from a /xce-deploy-ha deployment)…\n\nUser-reported: `./scripts/xceadmin scheduler record status` (from a /xce-deploy-ha deployment) failed with 'Error: cannot connect to localhost:9991: [Errno 111] Connection refused' -- the deployment's actual --peers were localhost:9901/9902/9903, not 9991.", "file": "src/xceadmin/src/xceadmin/cli.py, src/tools/deploy-ha.sh", "root_cause": "xceadmin's --controller option (config push, and all 3 scheduler record start/stop/status subcommands) defaulted unconditionally to a hardcoded module constant _DEFAULT_CONTROLLER = 'localhost:9991' with no way to override it except typing --controller on every single invocation. /xce-deploy-ha's generated scripts/xceadmin wrapper (added earlier this session) was a bare passthrough (`exec python3 -m xceadmin.cli \"$@\"`) that never told xceadmin what the actual deployed cluster's real ports were -- so any xceadmin command that didn't explicitly pass --controller silently tried the generic, deployment-agnostic default instead of this specific deployment's real peers, which have nothing to do with 9991 unless the deploy happened to use that exact port by coincidence.", "fix": "FIXED (TDD RED->GREEN). Added envvar=\"XCEADMIN_CONTROLLER\" to all 4 --controller click.option() decorators in cli.py (naming matches the existing XCEADMIN_SCHEMA_PATH convention in xcecommon/schema.py) -- click's built-in option-resolution precedence is CLI flag > envvar > default, so an explicit --controller still always wins. Added test_controller_option_falls_back_to_xceadmin_controller_env_var (RED pre-fix: asserted the env var value reaches the command, failed with the hardcoded default instead) and test_controller_flag_overrides_xceadmin_controller_env_var (regression lock for the override guarantee) to test_cli.py. deploy-ha.sh's generated scripts/xceadmin wrapper now exports XCEADMIN_CONTROLLER=\"${PEER_ARRAY[0]}\" (this deployment's own first --peers candidate) before exec'ing xceadmin. GREEN: full xceadmin test suite (261 tests via direct pytest, 258 passed + 3 skipped via the meson-driven build) all pass, no regressions. LIVE VALIDATION: rebuilt via `/xce-build xceadmin --incremental` (bug-1565's new flag, keeping the already-built controller/worker/scheduler binaries), redeployed with --peers localhost:9901,9902,9903 (matching the user's report exactly), started the cluster + scheduler, ran the user's EXACT command (`scripts/xceadmin scheduler record status`) in a completely stripped environment (env -i, no nix, no network) with NO --controller flag: correctly reached localhost:9901 (confirmed by the error changing from a raw connection-refused on the wrong port to a real application-level 'scheduler not connected' before the scheduler was started, then a clean 'recording: no' / exit 0 once it was) instead of failing against the wrong hardcoded port.", @@ -20558,20 +7332,23 @@ "deploy-ha", "config", "default-port", - "envvar", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "envvar" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1566.md" }, { "id": "bug-1567", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-17T09:00:00Z", - "error_message": "xce_scheduler -i -s : WARN 'failed to resolve job picker plugin (rc=-2)' on load; all pre-existing jobs stay PEND forever, nothing ever dispatches in simulation/interactive mode", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "xce_scheduler -i -s : WARN 'failed to resolve job picker plugin (rc=-2)' on load; all…\n\nxce_scheduler -i -s : WARN 'failed to resolve job picker plugin (rc=-2)' on load; all pre-existing jobs stay PEND forever, nothing ever dispatches in simulation/interactive mode", "file": "src/scheduler/src/sim.c, src/scheduler/src/sim_loop.c", "root_cause": "Two compounding, both explicitly documented 'Known scope boundary' gaps in the scheduler-core sim driver (task 11.1/11.2, .kiro/specs/scheduler-core): (1) sched_sim_driver_start() (sim.c) loads the initial snapshot via xce_sched_core_load_snapshot() BEFORE any picker plugin is registered into core->plugins -- only job_pick_plugin's NAME is threaded through from the ReplayHeader (sim.h:206-216's own doc comment). sched_resolve_job_picker() therefore returns -ENOENT (=-2), producing the WARN, and the picker-rebuild walk (on_queue_update/on_job_push) inside xce_sched_core_load_snapshot() is silently skipped for every job/queue in the initial snapshot. The real picker only gets registered moments later, in sim_register_default_picker() (sim_loop.c:381, called from sched_sim_driver_run() right after start() returns) -- but nothing re-pushes the already-loaded jobs/queues into it, so any job present in the replay file's INITIAL snapshot is permanently invisible to the fcfs picker's next_job(). (2) Independently, sim_picker_thread_fn() (sim_loop.c:448) has its own documented scope boundary: real resource-matching allocation (sched_make_allocation()) needs a live xce_scheduler_t* that the sim driver deliberately doesn't have, and no core-only equivalent exists yet (deferred to the separate, not-yet-designed .kiro/specs/scheduler-allocation spec) -- so EVERY candidate the picker selects is unconditionally turned into a PEND decision; dispatch is not implemented in simulation mode at all today, regardless of resource availability.", "fix": "FIXED (TDD, RED->GREEN). New file src/scheduler/src/sim_picker.c: sim_bootstrap_picker() (replaces sim_loop.c's sim_register_default_picker(), moved out because sim_loop.c was already at its documented 800-line 'zero line-budget headroom' cap) registers the real builtin_fcfs picker AND then replays the picker-rebuild walk (on_queue_update per queue, on_job_push per pending job) that xce_sched_core_load_snapshot() had to skip at initial-snapshot-load time. Wired into scheduler_srcs, sched_sim_isolation_lib, and all 9 white-box test TUs that #include sim_loop.c directly (test_sim_isolation.c mechanically re-verified). New regression test: test_sim_event_loop.c's test_job_present_only_in_initial_snapshot_still_reaches_the_picker -- builds a fixture with a job baked into the INITIAL snapshot (not a ReplayEvent), asserts it gets a live PEND decision. RED confirmed by temporarily no-op'ing the rebuild walk (test failed as expected), GREEN with the real fix restored. Full scheduler suite green (41/41, only pre-existing unrelated bug-1560 skipped/excluded from that count). Note: finding (2) from the original diagnosis (sim mode never dispatches, only PENDs -- scheduler-allocation spec gap) is NOT fixed here, by design -- out of scope, a whole separate not-yet-designed spec.", @@ -20583,19 +7360,23 @@ "job-picker", "scheduler-core", "known-scope-boundary", - "fixed", "tdd" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1567.md" }, { "id": "bug-1568", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-17T09:00:00Z", - "error_message": "xce_scheduler -i -s : 'info worker ' shows total=0.00 available=0.00 for every resource, even though the live cluster (xce_test resval/workerlist) reports real non-zero values for the same worker -- CONFIRMED ROOT CAUSE (superseding this entry's earlier 'architecture gap' theory, which was wrong -- corrected after the user pointed to worker.c's real mark_host_res()/update_val_res() write path)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "xce_scheduler -i -s : 'info worker ' shows total=0.00 available=0.00 for every…\n\nxce_scheduler -i -s : 'info worker ' shows total=0.00 available=0.00 for every resource, even though the live cluster (xce_test resval/workerlist) reports real non-zero values for the same worker -- CONFIRMED ROOT CAUSE (superseding this entry's earlier 'architecture gap' theory, which was wrong -- corrected after the user pointed to worker.c's real mark_host_res()/update_val_res() write path)", "file": "src/scheduler/src/repl_info.c, src/scheduler/src/snapshot_wire.c, src/scheduler/src/snapshot_load.c, src/controller/sched.c", "root_cause": "CONFIRMED via live value-level TRACE instrumentation (not just counts this time): xce_core_push_events()'s XCE_CORE_EVENT_WORKERS_PUSH handler (src/scheduler/src/core_events.c) calls sched_add_worker() UNCONDITIONALLY on every push -- and sched_add_worker() (src/scheduler/src/core.c:270) is a plain key_add() (ADD-ONLY semantics: fails -EEXIST if the worker name is already registered, never updates in place). A worker connecting triggers MULTIPLE WORKERS_PUSH events in quick succession (confirmed live: sched.c's collect_resval()/xcesched_push_workers() fire more than once during join sequencing -- e.g. an early push before src/controller/worker.c's apply_join_state()->add_hostattrs_resvals()/set_host_slots() has finished writing real hostattrs-derived values into w->host.resvals, carrying stale/zero data, followed by a later push carrying the real values). The FIRST push's sched_add_worker() call succeeds and registers the worker (confirmed: 'sched_add_worker(\"pc13v\") ok (idx=0)' at 10:02:34.848, immediately preceded by a TRACE showing it received maxmem=0.000000). The SECOND push's freshly-built xce_sched_worker_t -- this time carrying the REAL values (confirmed: 'resource maxmem received fvalue=33182089216.000000', matching xce_test's own real 30.9GB reading, at 10:02:36.849 -- silently fails sched_add_worker() with rc=-17 (-EEXIST) one line later in the SAME log, and its return value was unchecked until this investigation's own instrumentation added the WARN (core_events.c, added this session) that finally surfaced it. The correctly-valued worker object is discarded/leaked; core->workers['pc13v'] permanently keeps the FIRST, zero-valued object. This is not limited to the initial-join race -- because there is no 'update an existing worker in place' event type (unlike jobs, which have XCE_CORE_EVENT_JOB_STATE_CHANGE/JOB_UPDATE for in-place mutation of an already-added job), EVERY subsequent WORKERS_PUSH for an already-registered worker (sched_worker_change's ongoing per-event pushes throughout normal cluster operation, not just at join) hits the identical silent EEXIST-discard. The scheduler's view of a worker's resources freezes forever at whichever push happened to land first and win the race, and never updates again for the lifetime of the process. UPDATE: after fixing sched_add_worker's EEXIST discard, live re-testing showed 'info worker' STILL displayed 0.00 for every resource despite value-level TRACE evidence confirming the real number (e.g. slots=22, maxmem=33182089216) survived the FULL round trip intact -- controller push -> scheduler apply -> snapshot save (20/20 captured) -> snapshot load/decode (20/20 restored, correct fvalue) -- so the remaining gap had to be in the DISPLAY layer itself, unconnected to the push/save/load pipeline. Found: repl_info_worker_one()/repl_info_resource_one() (src/scheduler/src/repl_info.c) checked `avail_v->which == XCE_VAL_NUMERIC` to decide whether an xce_val_t held a real number. XCE_VAL_NUMERIC (xce/types.h, value 1) is a RESOURCE-DEFINITION value-type constant (xce_resource_t.valtype's own enum) -- a completely different enum from xce_val_t.which's real domain, Value_u_which (controller.capnp.h, Value_u_fvalue=5). Every producer in this codebase sets xce_val_t.which to Value_u_fvalue, never XCE_VAL_NUMERIC, so the comparison was ALWAYS false for a genuinely-numeric resval -- 'info worker'/'info resource' printed total=0.00/available=0.00 unconditionally, regardless of whether the underlying data was correct (which it was, once bug-1567's picker fix and the sched_add_worker EEXIST fix were both in place). This display bug is what made the user's originally-reported symptom persist even after both upstream fixes.", "fix": "FIXED (TDD, RED->GREEN), TWO independent root causes, both confirmed and fixed: (1) src/scheduler/src/core_events.c: XCE_CORE_EVENT_WORKERS_PUSH now goes through a new core_upsert_worker() (mirrors the existing core_upsert_queue() add-or-update precedent) -- looks up the worker by name first (sched_get_worker()), updates status+resvals in place if found (via a new shared core_apply_worker_resvals() helper, safe without a wipe step since zarray_put() already frees any existing value at a given index), else builds+adds a new one via the original core_build_worker()/sched_add_worker() path. New regression test in test_core_events.c: two WORKERS_PUSH events for the SAME worker name with DIFFERENT resource values must both apply -- resvals end up at the SECOND push's values, not stuck at the first, no duplicate/orphaned worker object. (2) src/scheduler/src/repl_info.c: all 4 `->which == XCE_VAL_NUMERIC` checks (2 in repl_info_worker_one(), 2 in repl_info_resource_one()) corrected to `->which == Value_u_fvalue` (added #include \"controller.capnp.h\"). New regression test in test_repl_breakpoints_inspection.c: `info worker` on a worker with a real numeric resval (42.5) must display it, not fall back to 0.00. Both RED confirmed before the fix (unit tests), then GREEN after. Live-validated end-to-end against a real deployed cluster (controller + xcecd_virtual worker + scheduler), reproducing the user's EXACT original command sequence (deploy, record, `xce_scheduler -i -s `, `info worker pc13v`) -- now shows the real values matching the user's own report exactly: slots=22.00, maxmem=33182089216.00 (30.9GB), mem=17151664128.00 (16GB), etc. Full scheduler suite green (41/41 excluding pre-existing bug-1560); controller suite green (1102/1104, the same pre-existing bug-1561 pair, confirmed unrelated).", @@ -20609,121 +7390,26 @@ "confirmed-bug", "sched_add_worker", "key_add-eexist", - "fixed", "tdd" ], "related_bugs": [ "bug-1567" ], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1671", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T09:14:31.784Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/sim_loop.c", - "root_cause": "47 lines replaced/restructured", - "fix": "Rewrote 52\u219212 lines (47 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-946", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T09:14:35.731Z", - "error_message": "Wrong reference: sim_register_default_picker should be sim_bootstrap_picker", - "file": "src/scheduler/src/sim_loop.c", - "root_cause": "Used \"sim_register_default_picker\" instead of \"sim_bootstrap_picker\"", - "fix": "Changed sim_register_default_picker \u2192 sim_bootstrap_picker", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-947", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T10:14:50.237Z", - "error_message": "Wrong reference: test_job_rusage_applies_state_when_hook_is_null should be test_workers_push_twice_updates_existing_worker_in_place", - "file": "src/scheduler/testsuite/test_core_events.c", - "root_cause": "Used \"test_job_rusage_applies_state_when_hook_is_null\" instead of \"test_workers_push_twice_updates_existing_worker_in_place\"", - "fix": "Changed test_job_rusage_applies_state_when_hook_is_null \u2192 test_workers_push_twice_updates_existing_worker_in_place", - "tags": [ - "auto-detected", - "wrong-reference", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-1672", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T10:16:21.240Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/core_events.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 74\u2192151 lines (3 removed) | Also: xce_sched_worker_t* worker; = core_build_worker(core, ev->u.workers_push.worke | Also: * WORKERS_PUSH core_build_worker() + sched_; * No picker hook (Req 1 AC 2:", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" - }, - { - "id": "bug-949", - "status": "fixed", - "status_updated": "2026-09-24", - "timestamp": "2026-08-17T10:31:46.356Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/repl_info.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21929 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c", - "fixed", - "closed-by-scheduler-sim-live-allocation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1568.md" }, { "id": "bug-1569", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-17T10:50:00Z", - "error_message": "TWO related, still-open problems on a live normal-mode (non-sim) deployment, found while verifying bug-1567/bug-1568's fixes actually let a real job dispatch: (A) 'info worker'/xceadmin-recorded snapshot still shows 0.00 for 'mem'/'r1m'/'r15m'/'r15s'/'ut'/'pg'/'ls'/'it'/'swp'/'io'/'tmp' (the 11 periodic resource_sync 'load index' metrics) even though xce_test's live 'resval -n mem -m pc13v' / 'resval -n r15s -m pc13v' show correct non-zero values on the SAME worker at the SAME time -- hostattrs-derived resources (slots/maxcpus/maxmem/maxswap/maxtmp/cpuf/ndisks, all fixed correctly by bug-1568) display fine; only the periodic/dynamic ones are still wrong. (B) A submitted job (single or array, queue 'xce', resreq 'type == any', worker 'pc13v' with 22 real slots available) never leaves PEND -- xce_sched_driver_run_normal_mode_round logs 'decision = (nil)' on EVERY round, forever (60+ consecutive rounds observed), meaning the scheduler never produces even a PEND decision, let alone DISPATCH, for any job.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "TWO related, still-open problems on a live normal-mode (non-sim) deployment, found while verifying…\n\nTWO related, still-open problems on a live normal-mode (non-sim) deployment, found while verifying bug-1567/bug-1568's fixes actually let a real job dispatch: (A) 'info worker'/xceadmin-recorded snapshot still shows 0.00 for 'mem'/'r1m'/'r15m'/'r15s'/'ut'/'pg'/'ls'/'it'/'swp'/'io'/'tmp' (the 11 periodic resource_sync 'load index' metrics) even though xce_test's live 'resval -n mem -m pc13v' / 'resval -n r15s -m pc13v' show correct non-zero values on the SAME worker at the SAME time -- hostattrs-derived resources (slots/maxcpus/maxmem/maxswap/maxtmp/cpuf/ndisks, all fixed correctly by bug-1568) display fine; only the periodic/dynamic ones are still wrong. (B) A submitted job (single or array, queue 'xce', resreq 'type == any', worker 'pc13v' with 22 real slots available) never leaves PEND -- xce_sched_driver_run_normal_mode_round logs 'decision = (nil)' on EVERY round, forever (60+ consecutive rounds observed), meaning the scheduler never produces even a PEND decision, let alone DISPATCH, for any job.", + "file": "", "root_cause": "(A) FULLY ROOT-CAUSED AND FIXED. The periodic resval update path is DIFFERENT from the join-time hostattrs push bug-1568 fixed: src/controller/opupd.c's do_update_resvals() applies a worker's periodic report locally using the CORRECT connection-resolved worker `w`, but then re-fires HOOK_DETAIL_RESOURCE_UPDATE with the wire `resvals` list's OWN unmodified `.host` field, forwarded unchanged by src/controller/sched.c's sched_resource_hook_fun() -> notify() -> scheduler's handler.c/driver.c/core_events.c XCE_CORE_EVENT_RESVALS_DELTA case, which does `sched_get_worker(core, rv->host)`. The true source of the wrong `.host` was WORKER-side, not xcecd_virtual-specific: src/worker/src/resource_sync.c's collect_load_indices() tagged every periodic report with gethostname() (the raw OS/container hostname -- 'xce' in this sandbox) instead of xcecd_ctx.virtual_name (the worker's actual join identity from -n/--name, e.g. 'pc13v'), which is what every OTHER worker-keyed lookup (mark_host_res()/sched_get_worker()) uses. FIXED (this session): collect_load_indices() now tags .host with xcecd_ctx.virtual_name, falling back to gethostname() only when virtual_name is unset. IMPORTANT CORRECTION to this entry's earlier note: after applying this fix, 'info worker'/'resval' via the INTERACTIVE REPL (`xce_scheduler -i -s `) appeared to still show 0.00 for mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp -- this was a TEST-METHODOLOGY ARTIFACT, not a residual bug: the REPL starts PAUSED at the initial snapshot and requires an explicit `step`/`continue` command to advance the simulated clock past the recorded resvalsDelta ReplayEvent before querying state; querying immediately after load (as this investigation's own test scripts did, e.g. `echo -e \"info worker pc13v\\nquit\"` with no step/continue first) only ever shows the pre-periodic-report baseline. Live-confirmed end-to-end via -vvv scheduler.log (schedUpdateResvals notify arrives, zero 'unknown worker'/'unknown resource' WARNs, xceadmin scheduler dump shows the resvalsDelta event correctly captured with all 11 metrics @pc13v) AND via `step N` in the REPL (mem=16786444288.00, r1m=0.64, r15m=0.99, r15s=0.64, ut=2.08, pg=463.20, it=3341.19, tmp=65732259840.00 -- all correct; ls/swp/io legitimately 0 on this idle sandbox). (B) FULLY ROOT-CAUSED AND FIXED -- THREE independent causes stacked on the same symptom, each one masking the next (each was only reachable once the prior one was fixed), all live-confirmed via gdb backtraces on real SIGSEGVs: B1 (legacy xce_sched_fcfs picker active) was RULED OUT via the main.c diagnostic INFO log -- confirmed 'active job picker plugin: builtin_fcfs'. B2 (resource-starvation) was RULED OUT -- dynamic resvals confirmed correctly flowing into live core state (thread A's fix), yet decisions stayed nil. The TRUE causes: (B-i) scheduler.c's sched_init_queues() -- the scheduler's ONE-TIME synchronous initial-config-push handler (handler.c's scheduler_req_handler, Request_body_configpushmsg) -- only called the pre-scheduler-core-rewrite plugin_queue_update() fan-out, which reaches a plugin's OLD job_elect_fn-era `.hooks.queue_hook` only; builtin_fcfs has no `.hooks`, only `picker_iface`, so on_queue_update() was NEVER called for any of the cluster's initial queues. builtin_fcfs's own internal per-queue candidate registry (fcfs_plugin_data_t.queues) stayed permanently empty from process start, so next_job() found zero candidates every session forever -- exactly 'decision=(nil) forever', regardless of resources/readiness. Every OTHER queue-registration path (core_events.c's core_upsert_queue(), used by QUEUE_UPDATE and a later CONFIG_PUSH config upgrade) already fired on_queue_update() correctly -- this was the one remaining call site still skipping it. (B-ii) Once B-i was fixed and a job first reached the picker, core_events.c's core_build_job() was found to never construct job->schedobj -- a DELIBERATE, DOCUMENTED omission from the scheduler-core rewrite ('no picker hook reads job->schedobj... belongs to the allocation path (scheduler-allocation spec), out of this task's scope') whose assumption was wrong: scheduler.c's sched_allocator() thread (plugin_job_election() + sched_make_allocation()) dereferences job->schedobj directly with no NULL check, and no 'scheduler-allocation spec' construction step exists anywhere in the live codebase -- every job built via JOBS_PUSH has had schedobj==NULL since the rewrite, reproduced live as a SIGSEGV in plugins/limit.c's limit_job_election() (obj->configver dereference on NULL obj), gdb-confirmed. (B-iii) Once B-i and B-ii were both fixed, a job reached genuine allocation (sched_make_allocation returned 1, a real DISPATCH) for the first time ever, and crashed a THIRD time inside resource.c's put_resvals() (gdb: NULL deref on xvals[0]) during post-dispatch resource-charge accounting -- an off-by-one: `if (idxs[i] <= 0) continue;` incorrectly treated resource index 0 ('slots', almost always the first resource resmgr registers) as an invalid sentinel, so the 'create a zero-value entry if missing' step was skipped for it, leaving xvals[0] NULL for the unconditional `xvals[i]->u.fvalue = vals[i]` write two lines later. get_resvals() (its read-side sibling in the same file) has no such guard, confirming this was a bug, not intentional design.", "fix": "(A) FIXED -- src/worker/src/resource_sync.c's collect_load_indices() now tags .host with xcecd_ctx.virtual_name (fallback: gethostname() if unset). TDD: RED test confirmed the bug (test_collect_load_indices_tags_host_with_virtual_name_not_gethostname expected 'pc13v', got 'xce'), GREEN after fix; new src/worker/testsuite/test_resource_sync.c (2 tests) added to src/worker/testsuite/meson.build, full worker suite green (3/3). Live-validated on a real 3-process deployment (controller+scheduler+xcecd_virtual): 'unknown worker' WARN eliminated, resvalsDelta event correctly recorded/replayed with real values. (B) FIXED, all 3 stacked causes: (B-i) src/scheduler/src/scheduler.c's sched_init_queues() now resolves the active picker once and calls `picker->picker_iface->on_queue_update(picker, q)` for every queue from the initial config, in addition to the pre-existing plugin_queue_update() legacy fan-out (kept for job_elect_fn-only plugin compat) -- best-effort, logs+continues on a picker-resolve failure the same way core_upsert_queue() does. (B-ii) src/scheduler/src/core_events.c: new core-scoped core_build_schedobj()/core_build_primary_schedobj()/core_build_compound_schedobj()/core_build_alter_schedobj() (a port of schedobj.c's make_sched_obj()+3 helpers, parameterized WITHOUT xce_scheduler_t* since the original's `sched` param was verified dead/unused in its own body -- preserves this file's zero-controller-I/O boundary), wired into core_build_job() right after job->resreq is parsed. (B-iii) src/scheduler/src/resource.c's put_resvals(): `idxs[i] <= 0` -> `idxs[i] < 0` (idx 0 is a legitimate resource index, not a sentinel), plus a defensive NULL guard added to the second loop for the (now-unreachable except on OOM) calloc-failure case get_resvals() already guards against. All 3 confirmed via TWO independent live 1-node deployments (controller+scheduler+xcecd_virtual, real jobsub via xce_test): first with temporary WARN-level diagnostics in builtin_fcfs.c pinpointing exactly where each fix took effect (removed before commit, confirmed via `grep DIAG` returning empty), then a clean run with a 3-element job array -- ALL elements reached state DONE (PEND -> DISPATCH -> RUN -> DONE), scheduler process stayed alive throughout (no crash), decision log showed a real non-nil decision pointer (previously always (nil)). Root-caused the two post-B-i crashes via gdb `bt full` on the live SIGSEGVs, not guesswork. Full scheduler suite green (40/41, the 1 failure is bug-1479-unrelated pre-existing symlink-escape filename-validation test flakiness, confirmed unaffected -- same failure present both before and after this fix's stash/rebuild).", "tags": [ @@ -20740,23 +7426,26 @@ "off-by-one", "sigsegv", "fully-fixed", - "test-methodology-artifact", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-methodology-artifact" ], "related_bugs": [ "bug-1567", "bug-1568" ], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1569.md" }, { "id": "bug-1570", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-17T23:45:00Z", - "error_message": "resource-value-verification task 2.1 round-1 REJECTED: `resupdate -n mem -v 5 -v 8` silently overwrote to 8 (last-write-wins) instead of rejecting the duplicate -v and keeping the first value 5", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "resource-value-verification task 2.1 round-1 REJECTED: `resupdate -n mem -v 5 -v 8` silently…\n\nresource-value-verification task 2.1 round-1 REJECTED: `resupdate -n mem -v 5 -v 8` silently overwrote to 8 (last-write-wins) instead of rejecting the duplicate -v and keeping the first value 5", "file": "src/controller/testsuite/test.c", "root_cause": "cmd_resupdate()'s deferred-finalization refactor (needed so a trailing -t can still apply to its pending -n/-v pair) rewrote the 'v' case to `if (resname != NULL) { resval_str = optarg; }` -- it dropped the pre-existing guard that rejected a second -v for the same pending -n. Baseline (pre-task-2.1, git show 98b8c34b^) fired set_resval() immediately on -v and reset resname=NULL, so a second -v naturally hit the 'no res name found' else-branch; the deferred model keeps resname non-NULL across multiple -v's, so the missing resval_str==NULL check let the second -v silently clobber the first.", "fix": "Changed the 'v' case guard to `if (resname != NULL && resval_str == NULL)` -- only accept a value into the pending slot if a name is pending AND no value has already been accepted for it; a duplicate -v now prints the same `ignore res value \\`%s' as no res name found` warning and the first value is kept. Verified via full case-by-case trace against the git-show baseline (duplicate -v, two full -n/-v pairs, -n flushing a valueless prior -n, stray -v with no -n, trailing bare -n, and the new -t override case) plus a live single-node controller smoke test: `resupdate -n mem -v 5 -v 8` on a real joined worker prints the ignore-warning for `8` and `resval -n mem -m localhost` confirms the stored value is 5.000000, not 8.", @@ -20768,22 +7457,25 @@ "last-write-wins", "resource-value-verification", "task-2.1", - "round-2-remediation", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "round-2-remediation" ], "related_bugs": [ "bug-1569" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1570.md" }, { "id": "bug-1571", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-17T00:00:00.000Z", - "error_message": "apiserver logs \"Backend not configured -- running in stub mode\" and never calls RegisterToBackend, even though the generated app.yaml backend.peers list is fully populated with a valid HA candidate list.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "apiserver logs \"Backend not configured -- running in stub mode\" and never calls RegisterToBackend…\n\napiserver logs \"Backend not configured -- running in stub mode\" and never calls RegisterToBackend, even though the generated app.yaml backend.peers list is fully populated with a valid HA candidate list.", "file": "src/tools/deploy-ha.sh", "root_cause": "resource-value-verification task 3.1 (xce-deploy-ha apiserver step): the new deploy-ha.sh Step-3 apiserver-config generation block initially wrote backend.address: \"\" and backend.port: 0 in the generated app.yaml, reasoning that backend.peers (non-empty) takes priority over address/port inside RegisterToBackend per backend/config.go's own doc comment. That reasoning only covers WHICH candidate RegisterToBackend picks once called -- it does not cover WHETHER it is called at all. apiserver/cmd/main.go:58 gates the entire RegisterToBackend call behind `BackendYamlConfig.Address != \"\"`, checked before Peers is ever consulted. A blank address left the apiserver permanently in stub mode regardless of a fully-populated peers list. Caught live during this task's own scenario-B verification run (grep of apiserver.log showed the stub-mode line instead of the expected \"Backend connection established\").", "fix": "deploy-ha.sh's apiserver app.yaml generation now also populates backend.address/backend.port from the first --peers candidate (host/port split via bash parameter expansion) alongside the full backend.peers list, so main.go's gate passes and RegisterToBackend still uses the full peers candidate list for actual connection/failover. Re-verified live: apiserver.log now shows \"Attempting backend connection\" / \"Backend connection established\" / \"Backend connection registered!\".", @@ -20793,21 +7485,23 @@ "xce-deploy-ha", "backend-config", "go", - "resource-value-verification", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "resource-value-verification" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1571.md" }, { "id": "bug-1572", "status": "fixed", "status_updated": "2026-09-24", "status_note": "c9acd502 (bug-1585)", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-08-18T01:30:00Z", - "error_message": "xce_scheduler silently exits (no ASAN report captured -- backgrounded via setsid nohup ... >/dev/null 2>&1) a few seconds after a worker's SECOND periodic resource_sync report for the same resource(s); first report processes fine, second triggers the crash with zero \"resval applied\" TRACE lines logged for that batch and \"handle_scheduler_leave\" observed on the controller ~1s later.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_scheduler silently exits (no ASAN report captured -- backgrounded via setsid nohup ...…\n\nxce_scheduler silently exits (no ASAN report captured -- backgrounded via setsid nohup ... >/dev/null 2>&1) a few seconds after a worker's SECOND periodic resource_sync report for the same resource(s); first report processes fine, second triggers the crash with zero \"resval applied\" TRACE lines logged for that batch and \"handle_scheduler_leave\" observed on the controller ~1s later.", "file": "src/scheduler/src/driver.c, src/scheduler/src/core_events.c", "root_cause": "Discovered live during resource-value-verification task 4.1 round-2 remediation smoke test (unrelated to the fix being validated -- reproduces identically with round-1's code, since round-1 only added a TRACE line AFTER the same zarray_put call). xce_core_push_events()'s XCE_CORE_EVENT_RESVALS_DELTA case (core_events.c ~line 786) does `zarray_put(&(w->resvals), res->idx, rv->val)` -- storing the decoded delta's xce_val_t* pointer DIRECTLY into scheduler-owned state, WITHOUT a MOVE_PTR/copy (unlike core_apply_config_resvals()'s sibling handling of the same ambiguity, which explicitly does `MOVE_PTR(val, nv->val); zarray_put(&(core->sharedInst), r->idx, val);`). Meanwhile driver.c's release_translated_payload() (called at driver.c:586 with applied=(push_rc>=0) after every successfully-pushed batch) has a case XCE_SCHED_OP_UPDATE_RESVALS: `free_ResValList_ptr(&(req->u.resvalupdate.resvals))` with NO `if (!applied)` guard -- unlike the sibling XCE_SCHED_OP_UPDATE_CONFIG case, which explicitly skips the free when applied (\"core->config now owns it permanently -- do not free\"). Net effect: every successfully-applied RESVALS_DELTA batch immediately frees the exact xce_val_t objects that were just aliased into w->resvals[idx], leaving a dangling pointer there. The dangling pointer is inert until the SAME (worker, resource) index is updated again -- at that point zarray_put's killer (kill_sched_resval -> free_Value) is invoked on the already-freed pointer, a double-free that aborts the ASAN-instrumented process. This is why it manifests specifically on a worker's second periodic report, not the first.", "fix": "NOT FIXED -- out of scope for resource-value-verification task 4.1 round-2 remediation, whose boundary was explicitly restricted to src/scheduler/src/core_events.c's Value_u_which else-branch bug only. Needs its own task: either (a) MOVE_PTR the value out of rv before zarray_put in core_events.c's RESVALS_DELTA case (mirroring core_apply_config_resvals()'s existing pattern) so release_translated_payload()'s free no longer touches live scheduler state, or (b) add an `if (!applied)` guard to driver.c's XCE_SCHED_OP_UPDATE_RESVALS case (mirroring XCE_SCHED_OP_UPDATE_CONFIG) so a successfully-applied batch is never freed at all -- (a) is likely preferable since RESVALS_FULL's sibling case has the identical unguarded-free shape and should be audited too. Worked around during round-2 validation by running the smoke test's resupdate quickly after deploy, before any real worker's second periodic report could trigger the crash.", @@ -20822,20 +7516,25 @@ "resource-value-verification", "task-4.1", "pre-existing", - "unresolved", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "unresolved" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "c9acd502" + ], + "record": ".wolf/bugs/bug-1572.md" }, { "id": "bug-1573", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-18T01:20:00Z", - "error_message": "resource-value-verification task 4.1 round-1 REJECTED by independent review: the RESVALS_DELTA TRACE logging in xce_core_push_events() assumed any non-string Value_u_which tag was Value_u_fvalue and read rv->val->u.fvalue unconditionally in the else branch -- Value_u_bvalue (and ivalue32/uvalue32/ivalue64/uvalue64) are real, reachable tags at this exact path (worker.c's mark_host_res() sets Value_u_bvalue for any XCE_VAL_BOOLEAN resource, routed through the same update_val_res()/resupdate RPC) that would read the wrong C union member.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "resource-value-verification task 4.1 round-1 REJECTED by independent review: the RESVALS_DELTA…\n\nresource-value-verification task 4.1 round-1 REJECTED by independent review: the RESVALS_DELTA TRACE logging in xce_core_push_events() assumed any non-string Value_u_which tag was Value_u_fvalue and read rv->val->u.fvalue unconditionally in the else branch -- Value_u_bvalue (and ivalue32/uvalue32/ivalue64/uvalue64) are real, reachable tags at this exact path (worker.c's mark_host_res() sets Value_u_bvalue for any XCE_VAL_BOOLEAN resource, routed through the same update_val_res()/resupdate RPC) that would read the wrong C union member.", "file": "src/scheduler/src/core_events.c", "root_cause": "Value_u_which (src/lib/controller.capnp.h:835-843) has 7 tags: ivalue32/uvalue32/ivalue64/uvalue64/svalue/fvalue/bvalue. Round-1's TRACE logic in the XCE_CORE_EVENT_RESVALS_DELTA case only distinguished svalue (explicit check) vs everything else (assumed fvalue, read .u.fvalue unconditionally in the else branch) -- a false dichotomy. Two directly-relevant precedents in this codebase already handle this correctly: update_val_res() (worker.c:107-117) explicitly checks `== Value_u_fvalue` with a generic 'non-numeric' fallback for anything else, and the sibling function core_apply_worker_resvals() 360 lines above in the SAME FILE (core_events.c:439-443) explicitly checks `== Value_u_fvalue` with no else at all (silent skip). Round-1 did not follow either precedent.", "fix": "Rewrote the RESVALS_DELTA TRACE block (core_events.c, XCE_CORE_EVENT_RESVALS_DELTA case, right after zarray_put) to explicit if/else-if: Value_u_svalue -> TRACE with value=%s; Value_u_fvalue -> TRACE with value=%f (reading .u.fvalue only once .which is confirmed); anything else (ivalue32/uvalue32/ivalue64/uvalue64/bvalue) -> silently skipped, mirroring core_apply_worker_resvals()'s established convention for the same ambiguity in this file (reviewer-approved Option B -- this feature's resource set is entirely fvalue/svalue today per resval_typeresolve.c). Verified: full scheduler build green (xce-build.sh scheduler), all 41 scheduler tests pass except the pre-existing unrelated bug-1560 (sched_admin_request_test's symlink-escape case); live smoke test via xce-deploy-ha + xce_test worker-context join/resupdate -n r15m -v 12345.5 confirmed unchanged output `resval applied host=task41r2-worker resource=r15m value=12345.500000` in scheduler.log, format-identical to round-1's (expected -- the fix only changes behavior for non-fvalue/non-svalue tags, which this smoke test doesn't exercise).", @@ -20847,23 +7546,25 @@ "resupdate", "resource-value-verification", "task-4.1", - "round-2-remediation", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "round-2-remediation" ], "related_bugs": [ "bug-1572" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1573.md" }, { "id": "bug-1574", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test resupdate string/type crash: set_resval() lacks svalue case, vals[] uninitialized (src/harness/test.c)", + "component": "controller", + "severity": "crash", "timestamp": "2026-08-17T18:55:00Z", - "error_message": "xce_test segfaults (SIGSEGV, exit 139) immediately on 'resupdate -n model -v ' (and by the same code path, -n type) run in the worker context against a live controller -- no output printed before the crash", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_test segfaults (SIGSEGV, exit 139) immediately on 'resupdate -n model -v ' (and by the…\n\nxce_test segfaults (SIGSEGV, exit 139) immediately on 'resupdate -n model -v ' (and by the same code path, -n type) run in the worker context against a live controller -- no output printed before the crash", "file": "src/controller/testsuite/test.c", "root_cause": "Pre-existing, predates the resource-value-verification feature entirely (confirmed via git show on the pre-task-2.1 baseline commit 98b8c34b^: the original set_resval()'s switch(valtype) already had no case for Value_u_svalue, same shape as today). set_resval() only assigns vals[idx].name inside a case block of switch(valtype); Value_u_svalue (type/model) has no case, so for these two names vals[idx].name is never assigned and keeps whatever uninitialized garbage was already on the local stack array `xce_resval_t vals[MAX_N_RES]` (never zero-initialized) in cmd_resupdate(). That garbage pointer is then passed unconditionally into xcecli_update_resvals()'s wire encoder inside &valptrs[0], which (most likely) dereferences it as a C string, causing the segfault. Task 2.1's own comment at test.c (near set_resval, 'the switch falls through with vals[idx].name/.val->which left unset ... intentionally not fixed by this task, see CONCERNS') already flagged this exact gap and deliberately preserved it byte-for-byte per Requirement 1.4, rather than introducing it. Discovered live during resource-value-verification task 4.2's validation (apiserver trace-log addition, live-testing the string-value code path with `resupdate -n model -v CustomModelX`) -- unrelated to task 4.2's own change (src/frontend/apiserver/backend/handler.go), which only touches Go/apiserver code.", "fix": "NOT FIXED -- out of boundary for task 4.2 (scope: src/frontend/apiserver/backend/handler.go only). Belongs to test.c/set_resval()/resval_typeresolve.c territory (tasks 2.1/2.2 of this same spec, or a dedicated follow-up). Worked around during task 4.2 live validation by using a numeric resource (r15m) for the primary trace-log confirmation and avoiding type/model live resupdate calls thereafter. Likely fix: zero-initialize `xce_resval_t vals[MAX_N_RES]` in cmd_resupdate() (e.g. `= {0}`) so an unassigned .name is NULL instead of garbage, PLUS give set_resval() a real Value_u_svalue case that sets vals[idx].name and vals[idx].val->u.svalue (currently the string value passed to resupdate for type/model is silently never stored in the union either -- a second, related but distinct defect covered by the same missing switch case).", @@ -20876,23 +7577,26 @@ "Value_u_svalue", "test.c", "pre-existing", - "resource-value-verification", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "resource-value-verification" ], "related_bugs": [ "bug-1570", "bug-1573" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1574.md" }, { "id": "bug-1575", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-08-18T03:15:00Z", - "error_message": "run-resource-verify.sh's trap-based cleanup() (EXIT/INT/TERM) non-deterministically left the scheduler and/or apiserver process running after the script was interrupted mid-deployment via an external SIGTERM -- confirmed processes still alive (e.g. `xce_scheduler --peers ... --logfile .../scheduler.log`, `/tmp/apiserver -d .../apiserver`) seconds after the orchestrator's own `wait` returned with exit code 143. The SAME cleanup() code, reached via a normal fail()/exit-1 path (no external signal involved) inside the same single foreground invocation, worked perfectly every time (verified zero leftover processes).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "run-resource-verify.sh's trap-based cleanup() (EXIT/INT/TERM) non-deterministically left the…\n\nrun-resource-verify.sh's trap-based cleanup() (EXIT/INT/TERM) non-deterministically left the scheduler and/or apiserver process running after the script was interrupted mid-deployment via an external SIGTERM -- confirmed processes still alive (e.g. `xce_scheduler --peers ... --logfile .../scheduler.log`, `/tmp/apiserver -d .../apiserver`) seconds after the orchestrator's own `wait` returned with exit code 143. The SAME cleanup() code, reached via a normal fail()/exit-1 path (no external signal involved) inside the same single foreground invocation, worked perfectly every time (verified zero leftover processes).", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh", "root_cause": "cleanup() issued four SEPARATE sequential `pkill -9 -f -- \"\"` invocations (one per component: controller/worker/scheduler/apiserver), each forking+execing its own `pkill` subprocess. When cleanup() ran as a normal function call/return within the script's own top-to-bottom control flow (e.g. from fail()), all four completed reliably. When cleanup() instead ran from on_exit as the EXIT trap fired by an asynchronously-delivered SIGTERM (via a separate `trap 'exit 143' TERM` translating the signal into an explicit exit), the sequence of pkill forks was intermittently and non-deterministically cut short partway through (confirmed via added debug instrumentation: one run stopped after the 1st pkill, another after the 3rd, another after the 2nd -- no fixed failure point, consistent with a timing race rather than a deterministic pattern-matching bug). Root cause not fully isolated at the syscall level (ruled out: pattern-matching correctness -- each individual pkill pattern was independently verified correct via direct pgrep; ruled out: cgroup pids/resource exhaustion -- process count stayed low, ~20-25 total throughout). Most likely a bash/signal-trap interaction where starting new child processes (fork/exec for each pkill) from within a trap handler invoked asynchronously (as opposed to synchronous top-level exit) has some window of unreliability in this environment.", "fix": "Rewrote cleanup() to (1) combine all four per-component patterns into ONE `pkill -9 -f` call using extended-regex alternation (`\"pat1|pat2|pat3|pat4\"`) instead of four separate invocations -- fewer fork/exec round trips through the trap-handler code path -- and (2) wrap it in a retry-until-verified loop (`pgrep -f` check, up to 10 attempts at 0.2s intervals) so a slow/racy single kill attempt can no longer silently leave a process behind. Verified reliable across 5+ repeated live runs deliberately interrupted via external SIGTERM at different points mid-deployment (right after controller-only confirmation, and after controller+scheduler confirmation) -- zero leftover processes every time after the fix, versus non-deterministic leftovers before it.", @@ -20905,22 +7609,25 @@ "race-condition", "run-resource-verify.sh", "resource-value-verification", - "bash", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "bash" ], "related_bugs": [ "bug-1436" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1575.md" }, { "id": "bug-1576", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-08-18T03:15:00Z", - "error_message": "Multiple interactive diagnostic Bash calls (`pgrep -af \"\"`, `pkill -9 -f -- \"\"`, `ps -eo pid,args | grep \"\"`) unexpectedly matched -- and in one case SIGKILLed -- the CURRENTLY-RUNNING Bash tool call's own wrapper shell process, not just the intended target process(es). One instance: `pkill -9 -f -- \"rrv_dummy_marker_TEST1|...\"` killed the invoking Bash call itself mid-script, truncating all output after the pkill line with no further commands executing (looked like a mysterious 'exit code 1, script just stops' failure).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Multiple interactive diagnostic Bash calls (`pgrep -af \"\"`, `pkill -9 -f -- \"\"`…\n\nMultiple interactive diagnostic Bash calls (`pgrep -af \"\"`, `pkill -9 -f -- \"\"`, `ps -eo pid,args | grep \"\"`) unexpectedly matched -- and in one case SIGKILLed -- the CURRENTLY-RUNNING Bash tool call's own wrapper shell process, not just the intended target process(es). One instance: `pkill -9 -f -- \"rrv_dummy_marker_TEST1|...\"` killed the invoking Bash call itself mid-script, truncating all output after the pkill line with no further commands executing (looked like a mysterious 'exit code 1, script just stops' failure).", "file": "N/A (diagnostic/interactive Bash tool usage, not a repo source file)", "root_cause": "This harness wraps every Bash tool invocation as roughly `bash -c \"... && eval '' < /dev/null && ...\"`. The wrapper's own process argv (visible via /proc//cmdline and matched by `pgrep -f`/`pkill -f`/`ps ... | grep`) therefore contains the FULL LITERAL TEXT of the submitted script -- including any pattern string typed directly into that same script (e.g. inside the pgrep/pkill command's own quoted argument, or in an unrelated echo/comment). Any `pgrep -f`/`pkill -f` search run from within a Bash tool call will self-match (and, for pkill, potentially self-kill) that call's own wrapper process whenever the search pattern also appears as literal text anywhere else in the SAME submitted command block -- which is nearly unavoidable, since typing the pattern to search for IS typing it into the submitted script.", "fix": "Not a code fix (methodology only). When diagnosing process state from an interactive Bash tool call: (1) prefer reading /proc//cmdline directly via a `for p in /proc/[0-9]*; do ...; done` scan and filtering in a way that doesn't require typing the exact search substring inline where it could also match the wrapper's own transcript (e.g. write the pattern to a variable/file first, or accept that the wrapper itself will show up as one extra line and visually distinguish it by its distinctive `bash -c \"source .../shell-snapshots/... && eval '...'\"` shape); (2) never assume a `pgrep -f`/`pkill -f` result of exactly 1 match confirms a target is unique -- it may be exactly 1 SELF-match with zero real matches; (3) for any pkill run interactively for diagnostic purposes (not from inside a dedicated script/process whose own argv doesn't contain the pattern), sanity-check the matched PID's `ps -o cmd` BEFORE trusting the pkill's effect, since a self-kill silently truncates the rest of that same Bash call with no distinguishing error message beyond a generic nonzero/truncated-output exit.", @@ -20930,22 +7637,25 @@ "self-match", "diagnostic-methodology", "bash-tool", - "sandbox-artifact", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "sandbox-artifact" ], "related_bugs": [ "bug-1575" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1576.md" }, { "id": "bug-1577", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "crash", "timestamp": "2026-08-17T00:00:00Z", - "error_message": "No explicit error -- discovered via live testing task 6.1 (resource-value-verification): xce-deploy-ha's deploy-ha.sh Step 3 opt-in-by-presence worker branch (predates this feature) generates start-worker.sh whenever a worker binary (xcecd_virtual) happens to exist alongside the controller build, and generated start-all.sh always launches it. run-resource-verify.sh's own deployment (task 5.1+6.1) unconditionally hit this since a full `/xce-build all` leaves xcecd_virtual built. Result: a REAL worker daemon ('worker-1') ran for the whole duration of every xce-test-resources run, silently violating Requirement 2.1's design intent ('no real worker daemon process required') and periodically re-reporting its own resources (~every 15s) -- a live risk of triggering the still-open bug-1572 (scheduler double-free/crash on a resource's 2nd periodic report) if a run happened to last that long.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "No explicit error -- discovered via live testing task 6.1 (resource-value-verification)…\n\nNo explicit error -- discovered via live testing task 6.1 (resource-value-verification): xce-deploy-ha's deploy-ha.sh Step 3 opt-in-by-presence worker branch (predates this feature) generates start-worker.sh whenever a worker binary (xcecd_virtual) happens to exist alongside the controller build, and generated start-all.sh always launches it. run-resource-verify.sh's own deployment (task 5.1+6.1) unconditionally hit this since a full `/xce-build all` leaves xcecd_virtual built. Result: a REAL worker daemon ('worker-1') ran for the whole duration of every xce-test-resources run, silently violating Requirement 2.1's design intent ('no real worker daemon process required') and periodically re-reporting its own resources (~every 15s) -- a live risk of triggering the still-open bug-1572 (scheduler double-free/crash on a resource's 2nd periodic report) if a run happened to last that long.", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh", "root_cause": "deploy-ha.sh's worker-detection/start-all.sh generation is out of this task's edit boundary (xce-deploy-ha files are done/approved for this feature) and has no flag to skip the worker step -- it only conditionally starts one when a binary is present, with no opt-out.", "fix": "start_all() now exports XCE_WORKER_BIN=/bin/true before invoking the generated start-all.sh. The generated start-worker.sh already reads ${XCE_WORKER_BIN:-} at RUN time from its own inherited environment, so pointing it at a real no-op binary makes the worker step exit immediately with no persistent process, without touching deploy-ha.sh itself. Verified via a /proc/*/exe-based watcher (not pgrep -f text matching, which self-matches -- see bug-1576) across a full default-resource-set run: zero xcecd_virtual/xcecd/xcecd_worker processes at any point post-fix, versus one running the entire time pre-fix.", @@ -20954,22 +7664,25 @@ "xce-deploy-ha", "worker-daemon", "bug-1572", - "task-6.1", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-6.1" ], "related_bugs": [ "bug-1572" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1577.md" }, { "id": "bug-1578", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-08-18T04:45:00Z", - "error_message": "Controller (xce_controller) reproducibly exits via SIGSEGV (si_addr=NULL, si_code=1/SEGV_MAPERR) within ~1s of the last resupdate follow-up verification reply completing, when the synthetic worker then disconnects ('quit'). ROOT-CAUSED via live repro (2/2) with an LD_PRELOAD signal-handler shim (gdb reliably masked the crash for both ASAN and debug/-O0 builds -- ptrace overhead perturbs timing enough to avoid the window; ~20s under gdb = no crash, every time). Resolved backtrace (addr2line against the debug-symbol'd ASAN binary): encode_NameValue <- encode_NameValue_list <- encode_Worker <- capnp_encode_worker (persist.c) <- persist_worker_hook_fun <- xce_run_hooks <- mark_worker_unavail <- handle_worker_leave <- handle_endpoint_error (cluster.c) <- io_worker_dispatch_batch <- io_worker (netio.c).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "Controller (xce_controller) reproducibly exits via SIGSEGV (si_addr=NULL, si_code=1/SEGV_MAPERR)…\n\nController (xce_controller) reproducibly exits via SIGSEGV (si_addr=NULL, si_code=1/SEGV_MAPERR) within ~1s of the last resupdate follow-up verification reply completing, when the synthetic worker then disconnects ('quit'). ROOT-CAUSED via live repro (2/2) with an LD_PRELOAD signal-handler shim (gdb reliably masked the crash for both ASAN and debug/-O0 builds -- ptrace overhead perturbs timing enough to avoid the window; ~20s under gdb = no crash, every time). Resolved backtrace (addr2line against the debug-symbol'd ASAN binary): encode_NameValue <- encode_NameValue_list <- encode_Worker <- capnp_encode_worker (persist.c) <- persist_worker_hook_fun <- xce_run_hooks <- mark_worker_unavail <- handle_worker_leave <- handle_endpoint_error (cluster.c) <- io_worker_dispatch_batch <- io_worker (netio.c).", "file": "src/controller/sched.c (sched_worker_change(), ~line 548-553) -- crash SITE is persist.c/controller.capnp.c (capnp_encode_worker/encode_Worker/encode_NameValue_list/encode_NameValue) but the DEFECT is in sched.c", "root_cause": "ROOT-CAUSED, PRE-EXISTING (predates resource-value-verification by ~5 months, NOT introduced by tasks 2.1/2.2/6.1). sched_worker_change() (src/controller/sched.c) aliases w->host.resvals_to_sched/nresvals_to_sched directly to a local xlist_t resvalist's backing array (resvalist.data), pushes it to the scheduler connection, then calls xlist_fini(&resvalist) -- which frees that backing array -- WITHOUT first resetting w->host.resvals_to_sched to NULL / nresvals_to_sched to 0. The correct reset-before-free code exists in the function but is wrapped in a dead `#if 0 /*FIXME*/ ... #endif` block, introduced already-disabled by commit acfbec875 (2026-03-03, 'save works'). This leaves w->host.resvals_to_sched a DANGLING pointer and nresvals_to_sched a stale nonzero count on the worker_t struct for the rest of that worker's lifetime. commit e5fde7b5 (2026-05-20, 'fix(persist): zero nresvals_to_sched after free_name_val_list') attempted to fix exactly this class of bug but edited code INSIDE the already-dead #if 0 block, so the fix silently never took effect. When the worker later disconnects, handle_worker_leave() -> mark_worker_unavail() fires the WORKER_DEAD hook -> persist_worker_hook_fun() -> capnp_encode_worker() -> encode_Worker() -> encode_NameValue_list(), which iterates the stale count over the freed array and dereferences a garbage/NULL entry in encode_NameValue() -> SIGSEGV. Only manifests once a scheduler has connected at least once: on_scheduler_join() (sched.c) registers sched_worker_hook_fun into c->hooksys (gated by a `static int hook_installed` one-shot flag) ONLY on the first scheduler join -- a controller with no scheduler ever connecting never reaches the buggy sched_worker_change(), so w->host.resvals_to_sched stays NULL/0 forever and persist_worker_hook_fun's later read is always safe. This exactly matches run-resource-verify.sh's controller+scheduler+apiserver deployment shape and explains the original isolation matrix precisely. The sibling function apiserver_worker_change() (src/controller/apiserver.c ~line 479-481) does the reset correctly (NOT wrapped in #if 0), which is why apiserver-only interaction never crashed.", "fix": "FIX IDENTIFIED, NOT YET APPLIED (out of task 6.1's boundary -- lives entirely in sched.c, a file no task in this feature touches). In src/controller/sched.c's sched_worker_change(), replace the dead block:\n #if 0\n /*FIXME*/\n free_name_val_list(&(w->host.resvals_to_sched), &(w->host.nresvals_to_sched));\n #endif\n xlist_fini(&resvalist);\nwith the exact working pattern already used by apiserver_worker_change():\n w->host.nresvals_to_sched = 0;\n w->host.resvals_to_sched = NULL;\n xlist_fini(&resvalist);\nIMPORTANT: do NOT simply uncomment the existing free_name_val_list() call as-is -- that would introduce a DOUBLE FREE instead. free_name_val_list() (src/lib/utils.c:412) already deep-frees every entry AND the outer array (FREEUP(resvals) where resvals == w->host.resvals_to_sched == the identical pointer value as resvalist.data), then sets w->host.resvals_to_sched itself to NULL -- but resvalist.data (the xlist_t's own separate field, still holding that same now-freed address) is untouched by that call, so the very next line's xlist_fini(&resvalist) -> FREEUP(resvalist.data) would free the identical address a second time. This is almost certainly why the original author disabled it with #if 0 rather than deleting it. Secondary/lower-severity, NOT part of this fix: both sched_worker_change() and apiserver_worker_change() leak the individual xce_name_val_t entries collected into resvalist -- xlist_fini() only frees the outer array (FREEUP(l->data)), never each entry's .name/.val/nv allocations -- worth a separate follow-up. Also worth a wider audit: other readers/freers of w->resvals_to_sched (opget.c:128 do_reply_workerlist(), apiserver.c, dataex.c) may have the same latent dangling-pointer exposure, only the WORKER_DEAD->persist path was confirmed live in this repro. FIX APPLIED 2026-08-18: src/controller/sched.c sched_worker_change() -- replaced the dead `#if 0 /*FIXME*/ free_name_val_list(...) #endif` block with `w->host.nresvals_to_sched = 0; w->host.resvals_to_sched = NULL;` immediately before `xlist_fini(&resvalist);` -- matching apiserver_worker_change()'s proven pattern exactly (src/controller/apiserver.c:480-482). Verified: (1) xce-build.sh controller -- clean build, 1102/1104 controller tests pass (the 2 failures are pre-existing bug-1561 overlayfs/fdatasync sandbox artifacts, unrelated); (2) live repro via run-resource-verify.sh --keep-running -- deployed controller+scheduler+apiserver, joined synthetic worker 'resverify-worker', ran resupdate for 9 resource values (all accepted), worker disconnected ('quit') triggering handle_worker_leave -> sched_worker_change (pushing 17 resvals to scheduler, twice, situation=HOOK_AFTER_EVENT) -- the exact previously-fatal sequence. Controller stayed alive and logging normally for ~2 minutes past the disconnect (previously died within 1-10s); no SEGV, no ASAN report, no core file. No new unit test added -- sched_worker_change() is a static function with no existing test coverage and no testable-static-function harness pattern in this codebase (confirmed via codegraph), and the crash path requires a live scheduler connection + hooksys + persist/capnp encoding chain across sched.c/cluster.c/persist.c -- disproportionate scaffolding for a 2-line fix; live-repro evidence used instead, per this codebase's established xce-test-ha-class convention. UNIT TEST ADDED 2026-08-18 (follow-up task, closes the gap this entry originally flagged as missing): src/controller/testsuite/test_sched_worker_change.c (new standalone binary, sched_worker_change_unit_test, wired into src/controller/meson.build mirroring sched_push_unit_test's real-sched.c+dataex.c+egress.c linking strategy). Test 1 fires XCE_CONTROLLER_HOOK_WORKER/HOOK_DETAIL_WORKER_DEAD via xce_run_hooks() (the same production entry point handle_worker_leave()->mark_worker_unavail() uses) on a worker_t with a populated w->host.resvals, then asserts w->host.resvals_to_sched==NULL && nresvals_to_sched==0. Test 2 goes further: after the same hook fire, it calls the REAL production encoder encode_Worker() (controller.capnp.c) directly on the worker -- the exact function bug-1578's crash backtrace bottoms out in via capnp_encode_worker/persist_worker_hook_fun -- so a regression is caught as an ASAN heap-use-after-free, not just a failed assertion. RED/GREEN verified live: temporarily re-wrapping the fix in the original dead `#if 0 ... #endif` block reproduced a genuine ASAN heap-use-after-free (encode_NameValue_list, controller.capnp.c:3408, freed by xlist_fini<-sched_worker_change<-sched_worker_dead<-sched_worker_hook_fun) -- confirming the earlier claim in this same entry ('no testable-static-function harness pattern exists in this codebase for it') was wrong; xce_run_hooks() through the real installed hooksys is exactly that harness, already proven by sched_push_unit_test for the JOB hook and now for the WORKER hook too. Restored the approved fix afterward (verified via git diff byte-identical to the pre-revert state) and confirmed GREEN again.", @@ -20997,7 +7710,6 @@ "task-6.1", "task-6.2-blocker", "use-after-free", - "fixed", "unit-test-added" ], "related_bugs": [ @@ -21005,16 +7717,20 @@ "bug-1574", "bug-1577" ], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1578.md" }, { "id": "bug-1579", "status": "wontfix", "status_updated": "2026-09-24", "status_note": "not a bug: intended host-attrs reset on disconnect; script fixed b673b7c9", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-08-18T09:00:00Z", - "error_message": "run-resource-verify.sh's new controller-side check (task 6.2, query_controller_resvals()) reported a live MISMATCH for the default resource set's 'cpuf' resource: 'controller check: cpuf -> MISMATCH: injected=9.5 observed=0.000000', even though inject_resources()'s own resupdate call reported 'cpuf=9.5 -> accepted' moments earlier in the same run.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "run-resource-verify.sh's new controller-side check (task 6.2, query_controller_resvals()) reported…\n\nrun-resource-verify.sh's new controller-side check (task 6.2, query_controller_resvals()) reported a live MISMATCH for the default resource set's 'cpuf' resource: 'controller check: cpuf -> MISMATCH: injected=9.5 observed=0.000000', even though inject_resources()'s own resupdate call reported 'cpuf=9.5 -> accepted' moments earlier in the same run.", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh (not a bug in this file's new logic -- see root_cause); src/controller/worker.c (add_hostattrs_resvals(), apply_worker_dead_mutation()) is the production code responsible for the behavior being (correctly) observed", "root_cause": "NOT A BUG -- documented, intentional controller production behavior colliding with this test script's own multi-process session structure. inject_resources() (task 6.1) runs join+resupdate+quit in ONE xce_test process; the 'quit' disconnects the synthetic worker's TCP connection to the controller at the end of that same script. query_controller_resvals() (task 6.2) then runs its resval query in a SEPARATE, LATER xce_test process/session -- by which point the synthetic worker is already gone from the controller's perspective. 'cpuf' is one of worker.c's N_HOST_ATTRS resources (add_hostattrs_resvals(), worker.c:260-287: type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks), which mark_worker_unavail() -> apply_worker_dead_mutation() (worker.c:704-716) deliberately RESETS (reset_hostattrs_unknown() then re-applies add_hostattrs_resvals()) whenever a worker is marked dead/disconnected -- correct, intentional production behavior for a REAL worker daemon that has genuinely gone away (you don't want a dead worker still advertising stale cpu capacity for scheduling). Every OTHER default resource (r15m, pg, ls, it, swp, io, tmp, mem -- none of which are in worker.c's N_HOST_ATTRS list) is a generic, non-host-attrs resval that persists correctly across worker disconnect and reported MATCH. Confirmed via live isolation: re-running with the exact same 8 non-cpuf resources reported 8/8 MATCH; re-running the mixed accept/reject scenario (mem=5, not-a-real-resource=1) correctly reported mem MATCH and skipped the rejected resource entirely (no query attempted). design.md's own sequence diagram does not show a worker disconnect between the resupdate leg and the resval-query leg, implying the intended flow keeps the synthetic worker connected throughout verification -- task 6.1's single-shot 'join;resupdate;quit' session structure deviates from that, but only resources that collide with worker.c's host-attrs reset list are affected.", "fix": "DEFAULT-SET INSTANCE FIXED (task 6.2 remediation round 2): DEFAULT_RESOURCE_ARGS in run-resource-verify.sh now uses \"r1m=9.5\" instead of \"cpuf=9.5\" as the second already-supported regression-coverage name (mem=8.5 kept as-is -- mem is NOT host-attrs, confirmed safe). r1m is confirmed via resval_typeresolve.c's resinfo[] table to be a genuinely pre-existing resource name, and via src/worker/src/resource_sync.c to be a periodic/dynamic load-index resource (not in worker.c's N_HOST_ATTRS list), so it preserves the original 'already-supported name for regression coverage' intent without the disconnect-reset problem. Live-validated: default (no-args) set now reports a clean N/N MATCH with no false cpuf mismatch. Original two options re-evaluated: option (a) -- drop the host-attrs name from the default set -- is what was taken here. Option (b) -- keep the synthetic worker's xce_test session connected across all per-layer checks instead of quitting after inject_resources() -- was investigated live and found NOT to be a simple call-reordering fix: xce_test's REPL has no command path from the `worker` context back to the `client` context without first calling leave_ctx() (via `..`/cmd_up), which performs the exact same disconnect that `quit` does. Making option (b) work would require either a C-level xce_test change (e.g. a context-preserving command) or a materially riskier bash coprocess restructuring of this script -- both explicitly out of scope for this remediation. The underlying production behavior (add_hostattrs_resvals() / mark_worker_unavail() resetting host-attrs resources on worker disconnect) remains correct and untouched, and this note is preserved for any FUTURE task or user-supplied custom resource=value argument that names a host-attrs resource (type, model, cpuf, maxcpus, maxmem, maxswap, maxtmp, ndisks) -- those still hit this same class of false-MISMATCH if queried after the synthetic worker disconnects.\n\nREAL FIX APPLIED (follow-up session, 2026-08-18): the underlying disconnect-timing gap this workaround left open for any host-attrs name is now actually closed, not just avoided for the default set. run-resource-verify.sh's inject_resources() now inserts a `!bash \"$SELF_SCRIPT\" --internal-controller-check ` line (xce_test's `!` escape, preprocess_and_run(), test.c ~line 950-998 -- a single top-level-REPL-loop call site that works identically in ANY context including `worker`, with no leave_ctx()/disconnect) BETWEEN `resupdate` and `quit`, inside the SAME xce_test worker session. This invokes a new internal_controller_check() function (hidden dispatch at the bottom of the script, before `main \"$@\"`) that runs a nested client-context xce_test session and queries resval for every requested name WHILE the synthetic worker's connection is still open -- so a host-attrs resource is never reset to baseline before it is checked. Raw observed values are written to logs/controller-check-raw.tsv; query_controller_resvals() (still called at the same point in main(), after inject_resources() returns) now reads that file instead of running its own live query -- a pure sequencing change, the MATCH/MISMATCH comparison logic and TSV formats are byte-for-byte unchanged. DEFAULT_RESOURCE_ARGS restored `cpuf=9.5` (dropped the r1m workaround) as live proof: 'controller check: cpuf -> MATCH' now, not the old false MISMATCH. Live-validated (5 separate runs): default 9-resource set incl. cpuf clean 9/9 MATCH at both controller and scheduler layers (~5-7s total runtime); mixed accept/reject with a host-attrs custom name (cpuf=3.25 alongside a rejected unknown resource) correctly MATCHes cpuf and skips the rejected name at both layers; --keep-running and mid-injection SIGTERM (landing WHILE the nested `!`-escape check was actively running) both verified clean, zero orphaned xce_test processes (full /proc/[0-9]*/cmdline scan, not just pgrep). Option (b) from the original analysis below (keep the session connected) turned out to be achievable after all via this `!`-escape mechanism, without any C-level xce_test change -- the earlier conclusion that it needed one was specific to trying to move BETWEEN contexts (worker -> client) without disconnecting, which is genuinely not possible; running an entirely separate NESTED xce_test client session via a shell escape while remaining IN the worker context sidesteps that limitation entirely.", @@ -21032,23 +7748,25 @@ "mark_worker_unavail", "not-a-bug", "default-resource-set", - "task-7.1-followup", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-7.1-followup" ], "related_bugs": [ "bug-1578" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1579.md" }, { "id": "bug-1580", "status": "open", "status_updated": "2026-09-24", "status_note": "intermittent controller_unit_test \"2 Failures\" with all cases PASS; not investigated", + "component": "controller", + "severity": "crash", "timestamp": "2026-08-18T10:39:52Z", - "error_message": "meson test 'xce:controller / controller test' (controller_unit_test) reports '1104 Tests 2 Failures 0 Ignored' / exit status 2, but grep of the full raw testlog for ':FAIL:' or a per-test [FAIL] marker finds ZERO matches -- every individual Unity test case visibly printed PASS. Reproduced deterministically 2/2 times (once via meson test, once via a direct standalone re-run of the binary with ASAN_OPTIONS=detect_leaks=0), unlike bug-1553's documented ~1-in-3 intermittent SIGSEGV (exit 139) signature -- this is exit 2 with no crash indication, a distinct manifestation.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "meson test 'xce:controller / controller test' (controller_unit_test) reports '1104 Tests 2…\n\nmeson test 'xce:controller / controller test' (controller_unit_test) reports '1104 Tests 2 Failures 0 Ignored' / exit status 2, but grep of the full raw testlog for ':FAIL:' or a per-test [FAIL] marker finds ZERO matches -- every individual Unity test case visibly printed PASS. Reproduced deterministically 2/2 times (once via meson test, once via a direct standalone re-run of the binary with ASAN_OPTIONS=detect_leaks=0), unlike bug-1553's documented ~1-in-3 intermittent SIGSEGV (exit 139) signature -- this is exit 2 with no crash indication, a distinct manifestation.", "file": "src/controller/testsuite/ (combined controller_unit_test binary; discovered running xce-build.sh all for the resource-value-verification bug-1579 follow-up fix -- zero C files touched this session, confirmed via git status)", "root_cause": "Not investigated. Most likely the same class of issue cerebrum.md's Unity-harness note already flags for this codebase (multiple UnitySetTestFile()-sharing test-group files in one binary can desync the aggregate PASS/FAIL counter from the printed per-test lines) rather than a real regression -- no code change in this session could have caused it (only .claude/skills/xce-test-resources/run-resource-verify.sh, a bash script, was edited). Possibly related to / the same underlying fragility as bug-1553 (order-dependent shared Unity/ASAN state across test files in the same binary) but with a different observed symptom (miscounted summary vs. an outright segfault).", "fix": "Not fixed -- out of scope for the resource-value-verification bug-1579 follow-up (bash-script-only boundary). Verified NOT a regression from this session's change (no C files modified) and does not block or relate to the run-resource-verify.sh live validation, which was carried out and passed independently. Needs the same isolation bug-1553 needs: bisect which specific linked test-group file(s) desync the counter.", @@ -21057,23 +7775,25 @@ "controller_unit_test", "unity", "aggregate-count-mismatch", - "not-fixed", - "pre-existing", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pre-existing" ], "related_bugs": [ "bug-1553" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1580.md" }, { "id": "bug-1581", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-08-18T12:10:00Z", - "error_message": "aggregate_verdict() synthetic self-test scenario (c) [one layer UNREACHABLE] reported unreachable=2 instead of 1, and the apiserver layer of an accepted+reachable record showed api_ok=2.5/api_obs=(empty) after IFS=$'\\t' read -- fields silently shifted by one position.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "aggregate_verdict() synthetic self-test scenario (c) [one layer UNREACHABLE] reported…\n\naggregate_verdict() synthetic self-test scenario (c) [one layer UNREACHABLE] reported unreachable=2 instead of 1, and the apiserver layer of an accepted+reachable record showed api_ok=2.5/api_obs=(empty) after IFS=$'\\t' read -- fields silently shifted by one position.", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh", "root_cause": "aggregate_verdict()/build_verdict_record() (task 7.1, new this session) originally joined/split its 9-field verdict record with a literal tab character (IFS=$'\t' read), matching every other TSV parse already in this file. But this is the FIRST record format in the file where a field can be genuinely empty (an UNREACHABLE or reachable-but-empty-value *_observed field) -- and bash treats tab (like space/newline) as \"IFS whitespace\" for field splitting even when IFS is set to ONLY a tab character, so two adjacent tab delimiters around an empty field silently COLLAPSE into one, shifting every field after it. Every prior TSV in this file (injection-results.tsv, *-check-results.tsv) never carries an empty field, which is why this bash gotcha never surfaced before.", "fix": "Switched the record-format delimiter from a literal tab to ASCII Unit Separator (0x1F, $'\u001f', VERDICT_FIELD_SEP) in build_verdict_record()/aggregate_verdict() only -- not a whitespace-class character, so bash never collapses adjacent delimiters or strips empty fields. Caught immediately by this task's own required synthetic self-test invocations (per task 7.1's TDD expectation to sanity-check aggregate_verdict() with synthetic records before relying on it), before any live-run validation -- all 4 required synthetic scenarios (match/mismatch/unreachable/mixed) plus edge cases now pass.", @@ -21083,19 +7803,23 @@ "tsv-parsing", "resource-value-verification", "task-7.1", - "aggregate_verdict", - "fixed" + "aggregate_verdict" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1581.md" }, { "id": "bug-1582", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-08-18T13:00:00Z", - "error_message": "run-resource-verify.sh task 7.1 round-1 review REJECTED: independent reviewer killed the live scheduler process mid-run (after deployment confirmed up, before its trace-log poll completed) and got VERDICT line `unreachable=0 mismatches=1` plus FAILURE line `status=MISMATCH ... observed=` -- should have been `unreachable=1 mismatches=0` and `status=UNREACHABLE ... observed=`.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "run-resource-verify.sh task 7.1 round-1 review REJECTED: independent reviewer killed the live…\n\nrun-resource-verify.sh task 7.1 round-1 review REJECTED: independent reviewer killed the live scheduler process mid-run (after deployment confirmed up, before its trace-log poll completed) and got VERDICT line `unreachable=0 mismatches=1` plus FAILURE line `status=MISMATCH ... observed=` -- should have been `unreachable=1 mismatches=0` and `status=UNREACHABLE ... observed=`.", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh", "root_cause": "main()'s record-building loop derived sched_ok/api_ok (feeding aggregate_verdict()) by testing whether the SCHEDULER_OBSERVED[$vname]/APISERVER_OBSERVED[$vname] associative-array KEY EXISTED (`[ -n \"${arr[k]+x}\" ]`). But query_scheduler_resvals()/query_apiserver_resvals() (tasks 6.3/6.4, already-approved) unconditionally SET these array entries for every accepted+checked resource -- including on a genuine UNREACHABLE/timeout, where the value is just set to the empty string \"\". Since the key always existed, sched_ok/api_ok were always 1 regardless of reachability, so aggregate_verdict() never saw the real UNREACHABLE status -- it fell through to comparing the injected value against an empty string, which reads as MISMATCH instead of UNREACHABLE. Contradicted Requirement 6.3, design.md's Error Handling (\"Layer unreachable ... labeled UNREACHABLE distinctly from MISMATCH ... not a silent skip\"), and this task's own SKILL.md documentation of the same behavior. Note: the analogous ctrl_ok derivation (from CONTROLLER_OBSERVED array-key existence) is NOT affected -- query_controller_resvals()/internal_controller_check() has no separate UNREACHABLE concept at all (a query failure there aborts the whole run via fail()), so array-key existence is an accurate proxy for that layer specifically.", "fix": "Round-2 remediation: derive sched_ok/api_ok from SCHEDULER_CHECK_RESULTS[$vname]/APISERVER_CHECK_RESULTS[$vname] instead (already correctly populated by tasks 6.3/6.4 with exactly \"MATCH\", \"MISMATCH: injected=... observed=...\", or \"UNREACHABLE: ...\") -- a leading \"UNREACHABLE:\" means ok=0, anything else (including a defensive __UNCHECKED__ fallback for a missing key, which should never happen given both query_*_resvals() functions iterate the same accepted-name list this loop reads) means ok=1 with the real observed value. Also extracted main()'s inline record-building loop into a standalone build_verdict_records_from_tsv() function (populates global VERDICT_RECORDS) so a synthetic test can exercise main()'s ACTUAL data-marshaling code (not just aggregate_verdict() called directly with hand-built records, which is what round-1's task 8.2-style synthetic tests did and which structurally could never have caught this bug). Live-reproduced the reviewer's exact scenario (kill scheduler right after deployment confirms up, before inject_resources()/query_scheduler_resvals() ever runs) and confirmed the VERDICT line now correctly shows `unreachable=9 mismatches=0` (all 9 default-set resources) with `status=UNREACHABLE ... observed=` FAILURE lines, apiserver/controller layers unaffected (still MATCH). New synthetic regression test (scenario E, using build_verdict_records_from_tsv() directly) added and confirmed to fail against a reverted copy of the old buggy logic (unreachable=0 mismatches=2) and pass against the fix (unreachable=1 mismatches=1 across a mixed good/schedbad/apibad/rejected batch) -- proving the test is meaningful, not tautological.", @@ -21108,23 +7832,25 @@ "round-1-rejected", "round-2-fixed", "sched_ok", - "api_ok", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "api_ok" ], "related_bugs": [ "bug-1581" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1582.md" }, { "id": "bug-1583", "status": "fixed", "status_updated": "2026-09-27", "status_note": "70360ccf", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-18T13:35:00.000Z", - "error_message": "Live task 8.1 run (resource-value-verification, comprehensive e2e validation): injecting resupdate -n totally-fake-resource-xyz -v 1 (a deliberately-unregistered resource name) alongside 9 genuinely-registered names -- the controller correctly rejects it (resval_typeresolve/update_resval's key_find(c->resmgr,...) returns -ENOENT, never written to w->host.resvals, confirmed empty via a direct resval query) and the scheduler correctly discards it (core_events.c's RESVALS_DELTA handler logs 'xce_core_push_events RESVALS_DELTA: unknown resource totally-fake-resource-xyz, skipping' at WARN and never applies it) -- but the apiserver's OnResourceLocationUpdated (src/frontend/apiserver/backend/handler.go:92) has NO such validation: it logged 'resource-location stored {\"resource\": \"totally-fake-resource-xyz\", \"host\": \"resverify-worker\", \"value\": 1}' and unconditionally called store.ResourceLocations.Put() for it, live-confirmed in apiserver.log.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Live task 8.1 run (resource-value-verification, comprehensive e2e validation): injecting resupdate…\n\nLive task 8.1 run (resource-value-verification, comprehensive e2e validation): injecting resupdate -n totally-fake-resource-xyz -v 1 (a deliberately-unregistered resource name) alongside 9 genuinely-registered names -- the controller correctly rejects it (resval_typeresolve/update_resval's key_find(c->resmgr,...) returns -ENOENT, never written to w->host.resvals, confirmed empty via a direct resval query) and the scheduler correctly discards it (core_events.c's RESVALS_DELTA handler logs 'xce_core_push_events RESVALS_DELTA: unknown resource totally-fake-resource-xyz, skipping' at WARN and never applies it) -- but the apiserver's OnResourceLocationUpdated (src/frontend/apiserver/backend/handler.go:92) has NO such validation: it logged 'resource-location stored {\"resource\": \"totally-fake-resource-xyz\", \"host\": \"resverify-worker\", \"value\": 1}' and unconditionally called store.ResourceLocations.Put() for it, live-confirmed in apiserver.log.", "file": "src/frontend/apiserver/backend/handler.go", "root_cause": "do_update_resvals() (src/controller/opupd.c:10-51) fires XCE_CONTROLLER_HOOK_RESOURCE/HOOK_DETAIL_RESOURCE_UPDATE with the FULL, RAW, UNFILTERED xce_resval_list_t from the client RPC -- BEFORE update_resval_list() runs its own per-item validity check, and the HOOK_AFTER_EVENT fire (the one both sched_resource_hook_fun (sched.c:609-639) and apiserver_resource_hook_fun (apiserver.c:545-581) act on) still passes that same unfiltered list, since update_resval_list() only applies/logs per-item failures internally -- it never mutates or filters the list object itself. So both the scheduler and the apiserver receive a raw push containing EVERY submitted resource name, valid or not, regardless of whether the controller's own update_resval()->key_find(c->resmgr,...) accepted it. The scheduler's core_events.c independently re-validates each pushed name against its own resource table before applying (WARN + skip on unknown) -- but OnResourceLocationUpdated (handler.go:92-115) does zero such validation; it stores whatever ResourceName/Location/value it's handed, unconditionally. This is a genuine 3-way inconsistency: controller rejects+never-stores, scheduler rejects+discards+logs, apiserver silently accepts+stores forever.", "fix": "NOT FIXED -- discovered incidentally during resource-value-verification task 8.1's comprehensive live e2e validation run (deliberately injecting one unregistered resource name alongside the default 9-resource set to prove the run-level FAIL path). Explicitly out of task 8.1's boundary (validation-only, no code changes) and out of the resource-value-verification feature's boundary generally (design.md 'Out of Boundary' section explicitly excludes 'the controller's resval-forwarding hooks' and scopes handler.go's own change to exactly one logger.Debug() call after the pre-existing store.ResourceLocations.Put() -- the Put() call and its lack of validation both predate this feature). A real fix would add a registered-resource-name check to OnResourceLocationUpdated (or, more centrally, filter the notify() payload in do_update_resvals()/sched_resource_hook_fun/apiserver_resource_hook_fun to only include items update_resval_list() actually accepted) before persisting to store.ResourceLocations -- worth a dedicated follow-up task, does not block resource-value-verification's own completion (the skill's own verdict logic already correctly ignores/skip-checks rejected names via injection-results.tsv's accept/reject accounting, so this apiserver-side phantom-store gap does not affect task 8.1's or 7.1's PASS/FAIL correctness).", @@ -21135,20 +7861,25 @@ "data-integrity", "resource-value-verification", "handler.go", - "OnResourceLocationUpdated", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "OnResourceLocationUpdated" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "70360ccf" + ], + "record": ".wolf/bugs/bug-1583.md" }, { "id": "bug-1584", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T02:47:00Z", - "error_message": "User-reported: `xceadmin scheduler record start/status/stop` always shows 'started at:' near 1970-01-01 (e.g. '1970-01-04T20:11:26') instead of the real start time, even though 'file'/'events'/'recording' fields are correct.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported: `xceadmin scheduler record start/status/stop` always shows 'started at:' near…\n\nUser-reported: `xceadmin scheduler record start/status/stop` always shows 'started at:' near 1970-01-01 (e.g. '1970-01-04T20:11:26') instead of the real start time, even though 'file'/'events'/'recording' fields are correct.", "file": "src/scheduler/src/core_clock.c, src/scheduler/include/core.h, src/scheduler/src/core.c, src/scheduler/src/scheduler.c, src/scheduler/testsuite/test_clock.c", "root_cause": "xce_sched_core_init() (core.c) seeds the scheduler-core's clock anchors to a neutral (virtual_clock_base=0, real_clock_base=0, speed_num=1, speed_den=1) default. The existing doc comments in core.c/core.h (task 4.5, scheduler-core spec) claimed this 'degrades to real wall-clock time' -- that claim is simply wrong: with both anchors at 0, xce_core_now()'s elapsed computation (wall_ns - real_clock_base) collapses to the raw CLOCK_MONOTONIC reading itself, i.e. nanoseconds since an arbitrary reference point (typically system boot), NOT nanoseconds since the Unix epoch. Only the simulation driver (sim.c's sim_set_virtual_clock(), called from sched_sim_driver_start()) ever seeds a genuine epoch anchor -- the normal-mode driver (scheduler.c's sched_init()) never did. recording.c's sched_record_start()/sched_record_stop() stamp record_start_ts and every ReplayEvent/SnapshotEntry ts via xce_core_now(), and controller.capnp's SchedAdminReply.startedAt is explicitly documented as 'nanoseconds since epoch' (which xceadmin's cli.py _format_timestamp_ns() renders via datetime.fromtimestamp(ns/1e9, tz=utc)) -- so a normal-mode scheduler's uptime-since-boot value gets rendered as a date near 1970. Independently confirmed already known and documented as a latent issue for a DIFFERENT symptom (tlog logging integration, design.md line ~793 / scheduler-core tasks.md Implementation Notes item #6) but nobody had connected it to the recording subsystem's user-facing 'started at' field before this report.", "fix": "Added xce_core_set_wallclock_anchor(xce_sched_core_t*) to core_clock.c -- the normal-mode counterpart to sim.c's sim_set_virtual_clock() -- which reads CLOCK_REALTIME (the epoch anchor for virtual_clock_base) and CLOCK_MONOTONIC (the matching real_clock_base xce_core_now()'s own elapsed computation is defined against) back-to-back and stores them as a group (speed_num=speed_den=1). Declared in core.h. Wired into scheduler.c's sched_init(), called once immediately after xce_sched_core_init() succeeds, before anything can observe xce_core_now(). Corrected the misleading 'degrades to real wall-clock time' doc comments in core.c and core.h to state the actual (monotonic-since-boot) behavior of the neutral default and point to the new function. Simulation mode and every core.c-including test fixture that constructs a bare core directly are unaffected (they never call the new function, so they keep the (0,0,1,1) neutral default or sim_set_virtual_clock()'s own anchors exactly as before). TDD: added test_xce_core_set_wallclock_anchor_seeds_real_epoch_time (brackets xce_core_now() immediately after the call between two back-to-back CLOCK_REALTIME reads -- fails loudly pre-fix since the bracketed value would be a small monotonic-since-boot number, not a real ~1.7e18ns epoch value) and test_xce_core_set_wallclock_anchor_null_core_is_a_safe_noop to test_clock.c. Verified via /xce-build scheduler asan --incremental: build clean, test_clock 100% green (17/17 including both new tests). One pre-existing, unrelated failure in the same suite run (test_admin_request's test_recordstart_symlink_escape_replies_error, bug-1560, a sandbox uid-permission-ordering issue) confirmed unaffected -- git status confirms test_admin_request.c was not touched by this fix. Not yet live-validated against a real xceadmin CLI run end-to-end (`xceadmin scheduler record start/status`) -- recommend a live /xce-deploy-ha smoke check before considering this fully closed.", @@ -21162,20 +7893,23 @@ "epoch-time", "monotonic-clock", "core-clock", - "user-reported", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "user-reported" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1584.md" }, { "id": "bug-1585", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T02:47:00Z", - "error_message": "User-reported, live `xce_scheduler -i -s /tmp/x1.replay` REPL session against a real .replay file recorded from a live normal-mode deployment: `info summary`/`info worker all` show '0 jobs (0 pending, 0 running)' immediately after load, even though the scheduler log and xce_test confirm jobs were actually submitted/present during that recording. Startup also logs a WARN: `xce_sched_core_load_snapshot failed to resolve job picker plugin (rc=-2); picker rebuild will be skipped for this load`.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "User-reported, live `xce_scheduler -i -s /tmp/x1.replay` REPL session against a real .replay file…\n\nUser-reported, live `xce_scheduler -i -s /tmp/x1.replay` REPL session against a real .replay file recorded from a live normal-mode deployment: `info summary`/`info worker all` show '0 jobs (0 pending, 0 running)' immediately after load, even though the scheduler log and xce_test confirm jobs were actually submitted/present during that recording. Startup also logs a WARN: `xce_sched_core_load_snapshot failed to resolve job picker plugin (rc=-2); picker rebuild will be skipped for this load`.", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_RESVALS_FULL/RESVALS_DELTA handlers -- root cause), src/scheduler/src/main.c (--record-snapshot-interval CLI wiring), src/scheduler/src/repl.c (banner clarification), src/scheduler/testsuite/test_resval_ownership.c (new regression test), src/scheduler/testsuite/test_core_events.c/test_core_full_cycle.c/test_driver_decision_cb.c/test_driver_session_ordering.c/test_driver_translate.c (dup_val link stub + stale-comment/fixture fix)", "root_cause": "TWO stacked, independently-confirmed causes, both live-reproduced this session against a real controller+scheduler+xcecd_virtual deployment (peers localhost:19991, worker-1):\n\n(A) FULLY ROOT-CAUSED AND FIXED -- a genuine heap-use-after-free crash, NOT the picker-resolve WARN (confirmed a red herring, per this entry's own prior analysis of xce_sched_core_load_snapshot()) and not purely a test-methodology gap. core_events.c's XCE_CORE_EVENT_RESVALS_FULL (line ~750) and XCE_CORE_EVENT_RESVALS_DELTA (line ~786) handlers did `zarray_put(&(zarray), idx, rv->val)` -- a SHALLOW store of the wire-decoded xce_val_t* pointer (confirmed via zarray_put_ex(), src/lib/zarray.c: `array->data[idx] = data[i]`, no copy). The zarray now owns that pointer and hands it to its killer (kill_sched_resval, schedobj.c) the next time that index is replaced. But driver.c's release_translated_payload() unconditionally frees the ENTIRE decoded ResValList/ResourceVal array (including that same rv->val) immediately after xce_core_push_events() returns, on EVERY drain, regardless of adoption. Result: the very next periodic resval report for the same worker+resource -- which happens every resource_sync interval (~1-15s) on literally any live deployment -- evicts the now-dangling first value via zarray_put(), crashing the scheduler with ASAN heap-use-after-free inside free_Value() (lib/controller.capnp.c:2898). Live-reproduced TWICE against a real deployment during this session's own repro attempts (first crash: PID 68752, silently lost because stderr went to /dev/null under start-scheduler.sh's nohup redirection; second crash, WITH stderr captured this time: PID 73919, full ASAN backtrace obtained -- xce_core_push_events -> zarray_put -> kill_sched_resval -> free_Value, use-after-free on a pointer release_translated_payload had freed one drain earlier). This exact hazard had ALREADY been noticed and explicitly documented as a 'PRE-EXISTING production ownership characteristic' in test_driver_translate.c's own test comment (task 5.1) but never fixed, because that test's kill_sched_resval is a no-op stub (its own comment wrongly generalizes 'fixtures are never torn down so these killers are never invoked' -- true for the OTHER kvmgr datakillers which fire only on core teardown, but NOT true for a zarray's killer, which fires on every REPLACEMENT of an occupied slot, not just teardown -- that blind spot is exactly why no existing test caught this). This crash means a live deployment's scheduler process is a near-certain crash within one or two resval-report cycles of any real session, corrupting/truncating whatever recording was in progress -- the practical dominant explanation for why a `.replay` file recorded from a real live session so often ends up missing jobs the operator knows were submitted.\n\n(B) CONFIRMED BY DESIGN, NOT A DEFECT (bug-1569 precedent, re-verified live this session after A's fix) -- once the crash was fixed, a full clean recording session was captured live: submitted 3 jobs via xce_test while `xceadmin scheduler record start` was active, waited for real DISPATCH decisions, `xceadmin scheduler record stop`, then `xceadmin scheduler dump`. The dump showed the SnapshotEntry (initial) genuinely reflecting whatever core state existed at the exact instant `record start` was issued (6 pre-existing jobs in this session's case, since jobs from an earlier test batch were still live) -- ReplayEvents (jobsPush/jobUpdate/resvalsDelta) for state that changed AFTER that instant correctly appear later in the stream, and only become visible in the REPL after `step`/`continue` (querying `info jobs`/`info summary` immediately after `[sim] ready.`, as the user's own transcript shows, only ever shows the starting snapshot's baseline -- exactly bug-1569's own prior finding for resvals, now confirmed to apply identically to job counts). Live-confirmed via `step N`: job count correctly grew from 6 to 9 once the REPL stepped past the recorded jobsPush/dispatch events, with correct per-job states (RUN/PEND) matching xce_test's live joblist at that exact point. The REAL underlying operator-facing gap this session's own analysis (and tasks.md's task-9.3 review note) already flagged: `--record-snapshot-interval` had NO CLI wiring anywhere in main.c, even though the struct field (xce_scheduler_t.record_snapshot_interval) and its consumption logic (recording_events.c's sched_record_write_events(), task 9.3) were already fully implemented and unit-tested -- so an operator had no way to ever get a periodically-refreshed starting snapshot; a `.replay` file with no periodic checkpointing legitimately has exactly ONE SnapshotEntry (written at `record start` time), and the REPL correctly starting from that baseline is expected behavior, not a bug.", "fix": "(A) FIXED -- core_events.c's RESVALS_FULL/RESVALS_DELTA handlers now `dup_val(rv->val)` before `zarray_put()` (both call sites), matching the deep-copy-before-adopt convention this codebase already established for the identical hazard on the snapshot-load side (snapshot_load.c: `zarray_put(&(core->sharedInst), r->idx, dup_val(rv->val))` / `zarray_put(&(w->resvals), r->idx, dup_val(nv->val))`). `dup_val` (defined in snapshot_wire.c) is now also `extern`-declared and consumed by core_events.c. TDD: new src/scheduler/testsuite/test_resval_ownership.c (2 tests, wired into meson.build as 'scheduler resval ownership test') white-box-includes core.c/core_clock.c/core_events.c/driver.c with a REAL (non-stubbed) kill_sched_resval (schedobj.c's implementation verbatim) plus a real dup_val reimplementation -- RED confirmed the exact live ASAN backtrace (heap-use-after-free in free_Value via zarray_put's eviction path) on the pre-fix code; GREEN after the fix, with an added assertion the evicted-and-replaced resval reads back correctly (no data corruption, not just 'didn't crash'). The other 5 pre-existing white-box test files that #include core_events.c (test_core_events.c, test_core_full_cycle.c, test_driver_decision_cb.c, test_driver_session_ordering.c, test_driver_translate.c) each needed a small real dup_val reimplementation added (link-satisfying, same 'reimplement the one small function' idiom as their existing sched_resolve_job_picker copies) since core_events.c now references it; test_driver_translate.c's own `test_drain_translates_and_pushes_all_ten_op_types` additionally needed its full_val/delta_val fixtures to set `->which = XCE_VAL_NUMERIC` explicitly (a bare calloc() leaves `which` at 0 == XCE_VAL_STRING, which dup_val() now actually inspects -- exposed a latent test-fixture gap, not a production bug) and its stale 'PRE-EXISTING production ownership characteristic ... unsafe to dereference' comment was corrected to reflect the fix. Live-reproduced end-to-end post-fix: restarted the scheduler against the same live deployment, let 22+ periodic resval cycles for the same worker+resource pass with zero crash (previously guaranteed to crash on the 2nd cycle), then ran a full record-start/submit-jobs/record-stop/dump/REPL-step cycle cleanly. Full scheduler suite green (41/42, only bug-1560 unrelated). (B) FIXED -- `--record-snapshot-interval N` is now a real CLI flag on main.c's argp table (OPT_RECORD_SNAPSHOT_INTERVAL=262), mirroring `--record-dir`'s exact pattern: parsed with the same strtoull()+validation shape --sim-speed already uses, assigned to `sched.record_snapshot_interval` right alongside `sched.record_dir`. No dedicated unit test was added (main.c defines its own `main()`, so it cannot be white-box #included into a test TU without a symbol collision, and this codebase's own precedent -- --record-dir itself -- has zero dedicated unit test either, relying on code review + the consumption side's existing coverage); verified instead via a genuine live E2E run: restarted the scheduler with `--record-snapshot-interval 3`, submitted 5 jobs across a record start/stop cycle, and confirmed via `xceadmin scheduler dump` that THREE periodic 'SnapshotEntry (checkpoint)' entries now appear mid-stream (previously only ever one, at record-start), each fully populated (14 jobs) -- then loaded the file via `-i -s FILE` and confirmed `[sim] ready.` now shows all 14 jobs immediately, with zero stepping required. Also added a one-line repl_print_banner() clarification (repl.c): '[sim] (jobs above reflect this snapshot only -- step/continue to replay later events)', printed between the loaded-snapshot line and the ready-prompt line -- verified this doesn't break test_repl_execution_control.c's own banner assertion (a strstr() substring/ordering check, unaffected by an inserted line).", @@ -21196,24 +7930,26 @@ "record-snapshot-interval", "fully-fixed", "live-reproduced", - "tdd", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "tdd" ], "related_bugs": [ "bug-1569", "bug-1560" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1585.md" }, { "id": "bug-1586", "status": "wontfix", "status_updated": "2026-09-24", "status_note": "not a bug: resval getters mode-agnostic, live values confirmed non-zero", + "component": "worker", + "severity": "security", "timestamp": "2026-08-19T02:47:00Z", - "error_message": "User-reported, same live REPL session: `info worker all` shows `total=0.00 available=0.00` for mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io (the periodic/dynamic resval metrics) on a worker that should have real non-zero values. User's own hypothesis: this worker was running in 'virtual host mode' (xcecd_virtual, a synthetic/mocked worker used for sandbox testing, not a real xcecd against real host hardware) which may never populate these metrics with real readings; wants to validate against a real (non-virtual) xcecd worker instead, but notes the sandbox/container this session runs in needs a fix to make `sudo` work before that's possible (real xcecd's periodic resource collection -- disk/load/mem readings -- may need elevated privileges not currently available in this container).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "User-reported, same live REPL session: `info worker all` shows `total=0.00 available=0.00` for…\n\nUser-reported, same live REPL session: `info worker all` shows `total=0.00 available=0.00` for mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io (the periodic/dynamic resval metrics) on a worker that should have real non-zero values. User's own hypothesis: this worker was running in 'virtual host mode' (xcecd_virtual, a synthetic/mocked worker used for sandbox testing, not a real xcecd against real host hardware) which may never populate these metrics with real readings; wants to validate against a real (non-virtual) xcecd worker instead, but notes the sandbox/container this session runs in needs a fix to make `sudo` work before that's possible (real xcecd's periodic resource collection -- disk/load/mem readings -- may need elevated privileges not currently available in this container).", "file": "src/worker/src/resource_sync.c (collect_load_indices), src/worker/src/node_information.c (the 10+1 node_info_get_* getters), src/worker/meson.build (xcecd vs xcecd_virtual target definitions)", "root_cause": "CONFIRMED, INVESTIGATION-ONLY (no code bug found -- design is correct, working as intended): `sudo -n true` fails in this sandbox (`sudo: unable to initialize PAM: Critical error - immediate abort`, exit 1) -- confirmed non-interactively this session, real (non-virtual) xcecd cannot be brought up here (its check_user() unconditionally requires geteuid()==0 for the production binary). Fell back to code-level determination, and it is conclusive: worker/meson.build shows `xcecd` and `xcecd_virtual` are compiled from the IDENTICAL source list (`xcecd_srcs`, including resource_sync.c and node_information.c verbatim) -- the only difference is the `-DVIRTUAL_CLUSTER` preprocessor define, which (grepped across every worker .c/.h file) gates ONLY root-privilege/sqlite-db-recovery/dbus-manager/process-execution-related code paths (xcecd.c, xcecd_db.c, controller_connector.c, xcecd_dbus_manager.c, xcecd_worker_utils.c, main.c's check_user()) -- resource_sync.c and node_information.c have ZERO VIRTUAL_CLUSTER conditionals anywhere. Read every one of the 11 node_info_get_*() getters (node_information.c): all are REAL, unconditional system-reads -- read_system_loadavg() parses /proc/loadavg (r1m/r15m/r15s), read_memory_info() parses /proc/meminfo (mem), node_info_get_tmp() calls statvfs('/tmp'), node_info_get_io()'s read_disk_io_rate() parses /proc/diskstats, node_info_get_ls() walks utmp via setutent()/getutent(), node_info_get_it() reads utmp activity or /proc/uptime as a fallback -- none are stubbed, mocked, or synthesized for virtual mode. collect_load_indices() (resource_sync.c) calls these same getters unconditionally regardless of xcecd vs xcecd_virtual. This directly corroborates bug-1569 thread A's own prior, independent live finding on this exact question (already in this buglog, closed): 'step N in the REPL ... mem=16786444288.00, r1m=0.64, r15m=0.99, r15s=0.64, ut=2.08, pg=463.20, it=3341.19, tmp=65732259840.00 -- all correct; ls/swp/io legitimately 0 on this idle sandbox' -- using this SAME xcecd_virtual binary. This session's own bug-1585 live repro work independently reconfirmed it fresh, twice: (1) the scheduler log during the bug-1585 deployment showed genuinely non-zero periodic resvals flowing from worker-1 (xcecd_virtual) -- e.g. 'resval applied host=worker-1 resource=mem value=19967893504.000000', 'resource=r15s value=4.610000', 'resource=tmp value=801655111680.000000', 'resource=it value=5739.550000'; (2) after fixing bug-1585 and loading a periodic-checkpoint snapshot via the REPL, 'info worker worker-1' showed mem=18382995456.00, r1m=4.93, r15m=5.73, r15s=4.93, ut=24.26, pg=1413.87, it=5768.14, tmp=797803061248.00 -- all real, plausible non-zero values; only ls/swp/io read 0.00, exactly matching bug-1569's own 'legitimately 0 on this idle sandbox' explanation (no logged-in users in utmp, no swap configured, negligible disk I/O delta in a short idle window -- not a bug, a correct reading of an idle/minimal sandbox). CONCLUSION: the user's reported all-zero values were almost certainly the SAME REPL-query-before-step/continue methodology gap bug-1569 already root-caused and bug-1585 (this session) re-confirmed and fixed the recording-side half of -- NOT a virtual-vs-real xcecd distinction. xcecd_virtual is not designed to stub/mock these 10 metrics; it collects them for real, identically to production xcecd.", "fix": "NO CODE FIX -- none needed; this is a confirmed non-bug (design is correct, resource_sync.c/node_information.c's collection code is genuinely virtual/real-mode-agnostic, verified by full source read plus two independent fresh live confirmations this session of real non-zero xcecd_virtual readings, corroborating bug-1569's own prior identical conclusion). If a live real-vs-virtual xcecd comparison is still wanted for extra certainty, it requires fixing this sandbox's `sudo`/PAM configuration first (out of scope for a code-level investigation) -- but given resource_sync.c/node_information.c have zero VIRTUAL_CLUSTER branching and every getter is a genuine unconditional /proc or utmp read, a real xcecd would be expected to read the SAME live host counters xcecd_virtual already does (same process, same kernel, same /proc), just without the artificial worker-name/slot-count synthesis xcecd_virtual's OTHER (unrelated) code paths add. No further action recommended unless a NEW live report of persistently-zero values is seen AFTER confirming step/continue was used correctly.", @@ -21228,24 +7964,26 @@ "investigation-only", "confirmed-non-bug", "code-inspection", - "test-methodology-artifact", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-methodology-artifact" ], "related_bugs": [ "bug-1569", "bug-1585" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1586.md" }, { "id": "bug-1587", "status": "fixed", "status_updated": "2026-09-24", "status_note": "c9acd502", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T02:47:00Z", - "error_message": "User feedback (UX, not a functional defect): the REPL's startup line `[sim] loaded: 0 jobs, snapshot @ ts=331886899213871` prints the snapshot timestamp as a raw nanosecond integer -- would be more readable as a human-readable date/time.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User feedback (UX, not a functional defect): the REPL's startup line `[sim] loaded: 0 jobs…\n\nUser feedback (UX, not a functional defect): the REPL's startup line `[sim] loaded: 0 jobs, snapshot @ ts=331886899213871` prints the snapshot timestamp as a raw nanosecond integer -- would be more readable as a human-readable date/time.", "file": "src/scheduler/src/repl.c:183", "root_cause": "repl.c's startup message (`fprintf(stdout, \"[sim] loaded: %d jobs, snapshot @ ts=%llu\\n\", njobs, ...)`) formats the raw uint64 nanosecond value directly with no conversion -- purely a display/formatting gap, not a logic defect. Now that bug-1584 makes a normal-mode-recorded .replay file's timestamps genuine epoch nanoseconds (previously they were CLOCK_MONOTONIC-since-boot, which would have made a 'human-readable' rendering of this same field misleading/wrong before that fix), converting this to a readable date is now meaningful for normal-mode recordings; simulation-mode-authored .replay files may still show a discrete/simulated ts depending on how --sim-start-ts was chosen, but a calendar rendering of an epoch-ns value is well-defined either way.", "fix": "NOT YET FIXED. Straightforward: convert the raw `ts` to a formatted string (matching core_clock.c's existing ns_to_tlog_time()-style gmtime_r() conversion, or a simple strftime over ts/1e9) before printing, likely showing both the human-readable form and the raw ns value for debugging (`snapshot @ ts= ()`). Low risk, single call site (repl.c:183) -- worth checking whether other REPL/dump output (repl_info.c, xceadmin's `scheduler dump`) has the same raw-ns-only presentation and would benefit from the same treatment while this is being fixed.", @@ -21255,24 +7993,28 @@ "ux", "timestamp", "formatting", - "open", - "not-yet-fixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "not-yet-fixed" ], "related_bugs": [ "bug-1584" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "c9acd502" + ], + "record": ".wolf/bugs/bug-1587.md" }, { "id": "bug-1588", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-19T02:47:00Z", - "error_message": "User feedback (UX/observability, not a functional defect), from logs/ctrl-1.log: job IDs printed in controller log lines are large, non-sequential-looking 64-bit numbers (e.g. `4631574334564990976`, `4631574334564990977`, `4631574334633148416`) that are hard to visually scan/compare/diff -- would be better rendered in a more human-readable form. Affected call sites (src/controller/job.c): `dispatch_a_job_()` (job.c:841, 'dispatch_a_job_ jobid '), `on_job_start()` (job.c:1662, 'on_job_start jobid '), `on_job_finish()` (job.c:1766, 'on_job_finish job finish jobid exit code '). Likely other job.c/controller log call sites print the same raw jobid and would benefit from the same treatment (not yet enumerated).", - "file": "src/controller/job.c (dispatch_a_job_ ~841, on_job_start ~1662, on_job_finish ~1766) -- likely more sites, not yet grepped exhaustively; PLUS (open, not yet fixed) src/worker/src/{xcecd_dbus_manager,xcecd,main,xcecd_worker,controller_connector,uv_ipc,xcecd_cgroups,xcecd_worker_utils}.c and src/controller/testsuite/test.c -- see fix field for full enumeration", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "User feedback (UX/observability, not a functional defect), from logs/ctrl-1.log: job IDs printed…\n\nUser feedback (UX/observability, not a functional defect), from logs/ctrl-1.log: job IDs printed in controller log lines are large, non-sequential-looking 64-bit numbers (e.g. `4631574334564990976`, `4631574334564990977`, `4631574334633148416`) that are hard to visually scan/compare/diff -- would be better rendered in a more human-readable form. Affected call sites (src/controller/job.c): `dispatch_a_job_()` (job.c:841, 'dispatch_a_job_ jobid '), `on_job_start()` (job.c:1662, 'on_job_start jobid '), `on_job_finish()` (job.c:1766, 'on_job_finish job finish jobid exit code '). Likely other job.c/controller log call sites print the same raw jobid and would benefit from the same treatment (not yet enumerated).", + "file": "src/controller/job.c (dispatch_a_job_ ~841, on_job_start ~1662, on_job_finish ~1766) -- likely more sites, not yet grepped exhaustively; PLUS (open, not yet fixed) src/worker/src/{xcecd_dbus_manager, xcecd, main, xcecd_worker, controller_connector, uv_ipc, xcecd_cgroups, xcecd_worker_utils}.c and src/controller/testsuite/test.c -- see fix field for full enumeration", "root_cause": "CONFIRMED, FIXED. The codebase already has a well-established human-readable jobid formatter -- fmt_jobid()/fmt_jobid_ex() (src/lib/utils.c:466-507, declared src/include/utils.h:132-133) -- used consistently across scheduler.c, handler.c, plugins/limit.c, plugins/fcfs.c, repl_info.c (scheduler), process_manager.c, uv_proc.c, xcecd_worker_utils.c (worker), and xce_test/waldump.c (controller testsuite tooling). It renders idgen()'s packed 64-bit jobid (src/lib/idgen.c, bit layout in src/include/config.h: 2 tag bits + 6 cluster-id bits + 4 apiserver-id bits + 32 timestamp-seconds bits + 20 sequence bits, packed via MAKE_JOBID/unpacked via the EXTRACT_ID_* macros in src/include/utils.h) as 'cluster-apiserver-timestamp-sequence' (e.g. '3-1-1755590313-976'), the SAME format REGEX_JOBID/parse_jobid() already parse back from user input and xcecli already displays to end users -- there is no separate 'new' human-readable scheme to design, job.c was simply the one place that never adopted the existing one, printing the raw packed integer via a mismatched `%ld` (a uint64_t formatted as a signed long -- a latent type/format mismatch, incidentally fixed as a side effect since the fix replaces the whole format specifier with %s).", "fix": "Fixed. src/controller/job.c: all 8 TRACE() call sites that printed a raw jobid now call fmt_jobid(jobidStr, sizeof(jobidStr), ) into a local `char jobidStr[64]` (matching scheduler.c's established buffer-size convention) immediately before the TRACE() call, and print `%s` instead of `%ld`: dispatch_a_job_() (~841), on_job_start() (~1662), on_job_finish() (~1766), on_job_fail() (~2026), on_job_stop() (~2055), on_job_resume() (~2090), on_job_postdone() (~2123), set_preason() (~2297). fmt_jobid is already transitively visible in job.c via request.h's #include \"utils.h\" -- no new include needed. Purely a log-formatting change, zero logic/behavior change -- verified via /xce-build controller asan --incremental: job.c compiles with zero errors/warnings (compiled twice, for xce_controller and controller_unit_test), full controller suite green (1104/1104 Unity assertions, 5/5 meson test cases), git diff confirms job.c is the only file touched in src/controller/. Not live-validated against a real deployment's log output (no functional/behavioral risk to validate beyond the build+test suite, since this only changes what a string looks like) -- if desired, a live run would show e.g. 'dispatch_a_job_ jobid 3-1-1755590313-976' instead of 'dispatch_a_job_ jobid 4631574334564990976'.\n\n---\n\nRemaining sites enumerated 2026-08-21 via `ag -i job src | grep '%lu'` (job.c's 8 sites from this bug are already fixed; these are NOT yet fixed):\n\nWORKER (src/worker/src/) -- still print the raw uint64 jobid via %lu instead of calling fmt_jobid():\n- xcecd_dbus_manager.c:423\n- xcecd.c:60\n- main.c:167,227,245\n- xcecd_worker.c: ~25 sites between lines 83-1314 (rusage, VNC, exdata, port assignment, signal/pause/resume, CRIU checkpoint status messages)\n- controller_connector.c: ~20 sites between lines 508-1857 (job dispatch, remote task, job-control action, rusage-to-controller messages)\n- uv_ipc.c: ~20 sites between lines 77-1541 (client mapping, signal/checkpoint ack, rusage, job completion/failure)\n- xcecd_cgroups.c:1703 (rusage summary log line; xcecd_cgroups.c:1405's `xcecd-worker-%lu.service` unit name and xcecd_worker.c:1209's `%s/%lu` checkpoint dir / xcecd_exdata.c:408's numeric-string exdata substitution are identifier/path construction, not human-facing log text -- left as-is pending a decision on whether those should change too)\n- xcecd_worker_utils.c: 8 sites between lines 240-504 -- correction to this bug's original note: original text claimed xcecd_worker_utils.c \"already use fmt_jobid() consistently\", but `grep -c fmt_jobid( src/worker/src/xcecd_worker_utils.c` now returns 0; that claim does not hold for the current file state (process_manager.c and uv_proc.c's fmt_jobid() usage, also cited in the original note, is still confirmed correct)\n\nCONTROLLER TESTSUITE (src/controller/testsuite/test.c) -- interactive test-tool printf() output, lower priority than production daemon logs:\n- lines 2599, 2649, 3966, 4043, 4048, 4053, 4155, 4180, 4294, 4353\n\nExplicitly OUT of scope -- these matched the `%lu` grep but are not job IDs, so must NOT be converted: src/controller/job.c:1332 and src/controller/persist.c:6376,6530 all format `lsn` (WAL log sequence number), not a jobid; src/worker/src/main.c:160 formats a `client ID`, not a jobid.\n\nSuggested fix pattern (matches this bug's job.c fix): at each site, add `char jobidStr[64]; fmt_jobid(jobidStr, sizeof(jobidStr), );` immediately before the log call, and replace the `%lu` for that argument with `%s` / jobidStr. utils.h (which declares fmt_jobid) needs to be confirmed reachable in each of these worker files (uv_ipc.c, controller_connector.c, xcecd_worker.c, xcecd_dbus_manager.c, xcecd.c, main.c, xcecd_worker_utils.c) before starting -- not yet verified.\n\nNot yet fixed. Logged as a follow-up scope expansion of this same bug per user request (\"log an issue to fix all job ids which are shown as uint64 to human readable string via fmt_jobid()\"), 2026-08-21.\n\n--- FOLLOW-UP FIXED 2026-08-21 ---\n\nAll remaining sites enumerated above are now fixed too (user request: \"make sure that all job ids which have been shown will use human readable format ... please also include src/controller/testsuite/test.c, as we use it test controller heavily\"). Same fmt_jobid()+%s pattern as job.c, applied via 5 parallel edit passes (one per file/group) each independently instructed with the exact site list and the exclusion list below, then independently reviewed line-by-line against the originals by the orchestrating session (not just trusted on the subagents' self-reports) before building.\n\nFiles fixed (99 call sites total across 9 files):\n- src/worker/src/xcecd_worker.c -- 30 sites (worker_signal_callback needed two buffers, jobidStr + curJobidStr, for its two distinct job-id values on one line)\n- src/worker/src/controller_connector.c -- 21 sites (handle_fetchfile: only the jobid= half of a two-%lu line converted, offset= left alone)\n- src/worker/src/uv_ipc.c -- ~20 sites (add_job_client_mapping: only the job_id half of a two-%lu line converted, client->id left alone)\n- src/worker/src/xcecd_worker_utils.c -- 9 sites\n- src/worker/src/controller_connector.c, xcecd_dbus_manager.c, xcecd.c, main.c, xcecd_cgroups.c -- remaining small sites (main.c's 4 sites all genuine job ids despite one message's misleading \"client ID\" wording -- verified against the function parameter type/usage, not just the string)\n- src/controller/testsuite/test.c -- 10 sites, matching this file's own pre-existing fmt_jobid()-with-'- 1'-sizeof convention\n\nDeliberately left unconverted (confirmed non-jobid or non-log-text, do not touch):\n- src/controller/job.c:1332 and src/controller/persist.c:6376,6530 -- format `lsn` (WAL log sequence number), not a jobid\n- src/worker/src/controller_connector.c:1376 -- the `offset=%lu` half of a fetchfile log line\n- src/worker/src/uv_ipc.c -- the `client->id` half of `add_job_client_mapping`'s log line\n- src/worker/src/xcecd_worker.c:1231 (ckpt_dir path), xcecd_cgroups.c:1405 (`xcecd-worker-%lu.service` unit name), xcecd_exdata.c:408 (`XCE_JOBID` env var set for the job's own process) -- all raw-numeric identifier/path/protocol construction, not human-facing log text; changing these would be a behavioral change (on-disk paths, systemd unit names, a documented env var format), not a log-formatting change\n\nVerification:\n- /xce-build worker asan --incremental: SUCCESS, 430/430 ninja targets, zero warnings/errors related to fmt_jobid/jobidStr in any of the 8 edited worker files, worker meson test suite 4/4 OK (resource_sync, xcecd_startup_order, dbus_workers_update, controller_connector tests).\n- /xce-build controller asan --incremental: SUCCESS, test.c compiled with zero warnings/errors, controller meson test suite 4/5 OK -- the 1 failure (test_config_repl.c, 2 Unity assertions) is the pre-existing, previously-logged bug-1561 overlayfs-sandbox fdatasync flake (7th occurrence, `git diff --stat` confirms config_repl.c/test_config_repl.c untouched by this change) -- not caused by this fix.\n- Every diff was read and checked line-by-line against the pre-edit originals by the orchestrating session (argument-to-specifier order, buffer scoping/reuse correctness, exclusion list respected) before either build ran -- not just accepted on the editing subagents' self-reports.\n\nNot fixed / explicitly out of scope: no attempt was made to reconcile the on-disk checkpoint-dir/systemd-unit-name/XCE_JOBID raw-numeric conventions with fmt_jobid() -- flagged above as a separate judgment call if ever wanted, not part of this fix.", "tags": [ @@ -21282,22 +8024,25 @@ "jobid", "ux", "job.c", - "open", "not-yet-investigated", - "fixed", "worker", "partial-fix" ], "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1588.md" }, { "id": "bug-1589", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "worker", + "severity": "unknown", "timestamp": "2026-08-19T03:20:00Z", - "error_message": "User-reported, live 3-node deployment log analysis: a 5-element job array (base_jobid=0-7-1787111880-0, array_size=5) was fully expanded and all 5 elements were dispatched by the controller (job.c's dispatch_a_job_ fires for [0]-[4], confirmed in logs/ctrl-1.log) and elected by the scheduler's limit plugin (logs/scheduler.log's limit_job_election fires for [0]-[4]), but only element [0] ever produced an on_job_start log line (job.c:1673) in either log file -- elements [1]-[4] were dispatched but never reported as started. User's own hypothesis: may be related to the worker running in 'virtual host mode' (xcecd_virtual) rather than a real xcecd worker.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "User-reported, live 3-node deployment log analysis: a 5-element job array…\n\nUser-reported, live 3-node deployment log analysis: a 5-element job array (base_jobid=0-7-1787111880-0, array_size=5) was fully expanded and all 5 elements were dispatched by the controller (job.c's dispatch_a_job_ fires for [0]-[4], confirmed in logs/ctrl-1.log) and elected by the scheduler's limit plugin (logs/scheduler.log's limit_job_election fires for [0]-[4]), but only element [0] ever produced an on_job_start log line (job.c:1673) in either log file -- elements [1]-[4] were dispatched but never reported as started. User's own hypothesis: may be related to the worker running in 'virtual host mode' (xcecd_virtual) rather than a real xcecd worker.", "file": "src/worker/src/xcecd_dbus_manager.c (dbus_workers_update() -- THE fix), src/worker/src/main.c (confirms master_job_status_callback/ipc_master path is UNREACHABLE for xcecd_virtual -- and for real xcecd too), src/worker/testsuite/test_dbus_workers_update.c (new regression test), src/controller/job.c (on_job_start/dispatch_a_job_ -- read/traced, confirmed innocent, not touched)", "root_cause": "CONFIRMED via source trace + live reproduction (both hypotheses from the prior static-reading pass were REFUTED). (1) Code path: worker/meson.build shows xcecd_virtual == xcecd_srcs + main.c, built with -DVIRTUAL_CLUSTER -DDBUS_MANAGER -- and the REAL xcecd target uses the SAME xcecd_srcs + main.c with -DDBUS_MANAGER only. Because DBUS_MANAGER is defined for BOTH binaries, main.c's run loop (`#ifndef DBUS_MANAGER worker_process_update(...) #else dbus_workers_update(...) #endif`, main.c:603-616) ALWAYS takes the dbus_workers_update() branch for both -- process_worker_manager.c's worker_process_update() (the 'looks less obviously broken' alternative flagged in the original hypothesis) is dead code, never called, in either build. Separately, ipc_master_init() is `#ifndef VIRTUAL_CLUSTER`-gated (main.c:571-579) so for xcecd_virtual no IPC-master unix-socket listener is ever created; master_job_status_callback() (main.c:163) and the single global `static xcecd_job_t job` (main.c:32) are therefore UNREACHABLE for xcecd_virtual (no forked job-runner ever connects to report MSG_JOB_STATUS over that socket) -- the original hypothesis about a missing `job.job_id = job_status->job_id` assignment there was investigating dead code for this deployment shape; it may still be a latent issue for a REAL xcecd with genuinely concurrent forked job-runners, but is NOT bug-1589's cause and was not touched. (2) Real root cause: dbus_workers_update() (xcecd_dbus_manager.c:178, called from main.c's run loop every ~10ms) iterates xcecd_ctx.worker_list -- one xcecd_worker_t node per concurrently-dispatched job -- with `xcecd_worker_t* worker = *worker_list; while (worker) { ... worker = worker->next; }`. Several branches inside the loop body used a bare `continue` that re-tests the SAME `worker` pointer instead of advancing (the VIRTUAL_CLUSTER JOB_STARTING->JOB_RUNNING branch, xcecd_dbus_manager.c:201 pre-fix; the exit_code==0 and exit_code<0 VIRTUAL_CLUSTER completion-check branches, pre-fix lines 241/268; the real-mode 'job done but running not received' branch, pre-fix line 291) -- and worse, the WORKER_DONE/WORKER_EXITED early check (xcecd_dbus_manager.c:205-209 pre-fix) did `worker = worker->next; return true;`, which ABORTED THE ENTIRE FUNCTION the moment ANY worker in the list was already finished, discarding the scan of every worker queued behind it (also a latent bug: `return true` == `return 1` == XCECD_ERROR on a function whose only real caller only special-cases XCECD_DB_ERROR, so this wasn't even surfaced as an error). Net effect on a job array: once dbus_workers_update() started making progress on the HEAD of xcecd_ctx.worker_list (element [0], first added via add_worker()), every OTHER concurrently-dispatched element ([1]-[4]) was starved of update_worker_status() -- the function that fires xcecd_reply_startjob() i.e. the on_job_start report -- for as long as element [0] stayed un-reaped from the list (which for a long-running job is indefinite, matching the user's 'never, in either log file' observation for a job array of `sleep N`-style long jobs; for short/instant jobs the same bug still starves [1]-[4] but only for the ~10ms tick until cleanup_worker_list() reaps the finished head and the next element becomes head in turn). This is a WORKER-side bug only -- the controller's on_job_start()/record_job_start_fact() (job.c:1665-1663) and its job_state_is_backward() guard (job.c:43) were traced and confirmed to correctly key everything by jobid with no cross-job interference; they simply never received a report for [1]-[4] because the worker never sent one.", "fix": "FIXED. xcecd_dbus_manager.c's dbus_workers_update(): capture `xcecd_worker_t* next = worker->next;` once at the top of every loop iteration (right after `while (worker) {`), and changed every early-exit path in the loop body to `worker = next; continue;` instead of a bare `continue` (re-tests the same node forever) or the old `worker->worker_status == WORKER_DONE/WORKER_EXITED` branch's `return true` (aborted the whole scan on the first finished worker). This fixes traversal for BOTH VIRTUAL_CLUSTER and real (non-VIRTUAL_CLUSTER) branches of the function -- the real-mode `kill(pid,0)!=0` / 'job done but running not received' branch had the identical bare-continue bug and would starve a real xcecd's worker_list the same way under concurrent dispatch. No change to the state-machine logic itself (update_worker_status(), check_worker_sleep_time_elapsed(), the systemd dbus_is_worker_active()/get_systemd_job_exit_code() calls) -- purely a traversal/control-flow fix, matching the 'minimal, correct, no symptom-papering' guidance; considered and rejected converting worker_list to a per-jobid array (process_manager.c's processes[] precedent) since the linked list itself was never the problem, only its traversal. TDD: new src/worker/testsuite/test_dbus_workers_update.c (2 tests, white-box #includes xcecd_dbus_manager.c with -DVIRTUAL_CLUSTER -DDBUS_MANAGER matching xcecd_virtual's real build flags, stubs update_worker_status()/is_systemd_job_active()/get_systemd_job_exit_code()/stop_systemd_job()/make_job_sd_unit_name()/start_systemd_job()/get_systemd_job_pid()/check_worker_sleep_time_elapsed() to isolate the loop's own control flow) wired into src/worker/testsuite/meson.build. RED-verified live: `git stash push -- src/worker/src/xcecd_dbus_manager.c`, rebuild, both tests FAIL ('Expected 0 Was 1' -- the started-call-seen assertions correctly detect elements [1]-[4] never receiving their start transition, and a DONE head worker correctly blocks the scan); `git stash pop`, rebuild, both tests PASS (GREEN). Live end-to-end proof (not just the unit test): deployed a fresh 3-controller + scheduler + xcecd_virtual cluster (/xce-deploy-ha pattern, dedicated dir /tmp/xce-deploy-bug1589), submitted `jobsub -u rshen -q normal -c \"sleep 300\" -a 5` (5-element array, rshen used specifically to avoid limits.yaml's 'limit1' 3-slot cap on all-users-except-rshen, matching the user's original unconstrained-concurrency scenario) -- ctrl-1.log showed `dispatch_a_job_` AND `on_job_start` for ALL FIVE elements [0]-[4] (previously only [0] would ever show on_job_start), scheduler.log showed `limit_job_election` for all 5 (as it already did pre-fix), a follow-up `joblist` showed all 5 elements in state RUN, and xcecd-worker-1.log showed the `Job ID running - sending status to controller` line (the exact xcecd_reply_startjob trigger) for 5 DISTINCT job ids within milliseconds of each other. Cluster torn down after capturing evidence (self-started, self-stopped, per boundaries). Note: this repro session also observed pre-existing, unrelated 'Failed to connect to system bus' ERROR spam from dbus_is_worker_active()/get_systemd_job_exit_code() polling every ~10ms per still-running virtual job (expected -- no real dbus session in this sandbox, and VIRTUAL_CLUSTER never creates a real systemd unit to poll in the first place) -- cosmetic log noise, not a correctness bug, pre-existing before this fix and unrelated to bug-1589's actual defect; flagged here for visibility, NOT fixed (out of this bug's scope). Also flagged, NOT fixed (out of scope, no buglog entry created per explicit task boundary): dbus_worker_clean() (xcecd_dbus_manager.c) has an apparently-inverted `#ifndef VIRTUAL_CLUSTER return true;` guard -- as written, the REAL (non-virtual) xcecd never actually runs its own systemd-unit cleanup body, while VIRTUAL_CLUSTER mode does (but finds nothing to clean, since no real unit ever existed) -- looks backwards from the evident intent, worth a separate look. Full suite after fix: worker 4/4 GREEN (resource_sync, xcecd_startup_order, dbus_workers_update, controller_connector). Controller 1101/1104 (3 unrelated FAILs in test_config_repl.c -- config version trimming/tag lookup/marker supersession, a file this fix never touches and that was confirmed clean/unmodified by this session; controller/ was not touched at all by this fix). Scheduler: bug-1560's known symlink-escape failure (expected, unrelated) plus 3 NEW failures in test_sim_verify.c/test_repl_breakpoints_inspection.c -- traced via `git status` to src/scheduler/src/driver.c and src/scheduler/src/recording_session.c being concurrently modified, uncommitted, by ANOTHER session's in-flight work in this shared worktree (unrelated 'bug-1591' job.c/driver.c changes observed mid-flight during this session's builds) -- outside this bug's boundary (scheduler/core files are explicitly off-limits for this task) and not caused by this worker-only fix; not investigated further.", @@ -21309,7 +8054,6 @@ "concurrency", "job-id", "controller", - "fixed", "root-caused", "dbus-workers-update", "linked-list-traversal", @@ -21318,15 +8062,20 @@ "related_bugs": [ "bug-1586" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1589.md" }, { "id": "bug-1590", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T03:40:00Z", - "error_message": "User-reported: `(sim) info worker all` in the scheduler's interactive REPL shows `resource type: total=0.00 available=0.00` and `resource model: total=0.00 available=0.00` -- but `type`/`model` are STRING-valued resources (e.g. OS type, hardware model name), not numeric, so 0.00 is meaningless. User also notes boolean-typed resources exist and asked to verify resource-value type-correct formatting broadly, across both REPL output and log lines, not just this one command.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "User-reported: `(sim) info worker all` in the scheduler's interactive REPL shows `resource type…\n\nUser-reported: `(sim) info worker all` in the scheduler's interactive REPL shows `resource type: total=0.00 available=0.00` and `resource model: total=0.00 available=0.00` -- but `type`/`model` are STRING-valued resources (e.g. OS type, hardware model name), not numeric, so 0.00 is meaningless. User also notes boolean-typed resources exist and asked to verify resource-value type-correct formatting broadly, across both REPL output and log lines, not just this one command.", "file": "src/scheduler/src/repl_info.c (repl_info_worker_one ~287-335, repl_info_resource_one ~410-496); src/scheduler/src/core_events.c (~838-861, TRACE 'resval applied'); src/controller/worker.c (~107-119); src/controller/dataex.c (~86-90)", "root_cause": "CONFIRMED via source read across the whole stack (C scheduler/controller/worker + Go apiserver/xcecli), several distinct severities under one theme -- 'resource values are not type-dispatched when formatted for display/log, several call sites assume every resource is numeric (Value_u_fvalue)':\n\n(1) MOST SEVERE, matches the exact user report -- `repl_info_worker_one()` (repl_info.c:329-334): `avail = (avail_v->which == Value_u_fvalue) ? avail_v->u.fvalue : 0.0;` then unconditionally `fprintf(..., \"total=%.2f available=%.2f\", ...)` -- ANY resource whose wire value is Value_u_svalue (string, e.g. `type`/`model`) or Value_u_bvalue (boolean) silently renders as `0.00`, indistinguishable from a genuinely-zero numeric resource. Comment at line 324-328 references bug-1568 (a DIFFERENT, already-fixed bug about comparing against the wrong enum) but bug-1568's fix never added string/bool handling -- it only made the numeric case correct.\n(2) Same defect, `repl_info_resource_one()` (repl_info.c:436-444, `info resource `): the total/available aggregation loop only accumulates when `->which == Value_u_fvalue` (correctly guarded, no UB) -- but silently EXCLUDES string/bool resources from the sum entirely, so `info resource type`/`info resource model` would also show `total=0.00 available=0.00` despite `valtype=string` being correctly labeled two lines below it (line 489-495) -- an internally inconsistent-looking display (correct type label, wrong/misleading value).\n(3) Related, narrower gap in scheduler logging -- `core_events.c`'s bug-1573 fix (~838-861, the 'resval applied host=... resource=... value=...' TRACE line) DOES correctly branch Value_u_svalue -> '%s' vs Value_u_fvalue -> '%f', but its own comment self-documents that Value_u_bvalue (boolean) is 'silently skipped here rather than guessed at' -- confirmed still true as of this reading: no case handles Value_u_bvalue, so a boolean resource update produces NO trace line at all (silent omission, not a wrong value, but incomplete visibility for exactly the resource kind the user flagged).\n(4) Minor/lower severity, controller-side -- `worker.c` (~107-119) has an `else` branch for non-fvalue that logs `which=%d (non-numeric)` (informative about the presence of a non-numeric update, but never decodes/prints the actual string or boolean value); `dataex.c` (~86-90) has NO else branch at all -- silently skips the TRACE entirely for any non-fvalue resource, weakest visibility of the four sites but least user-facing (debug-level TRACE only).\n\nPOSITIVE finding, scoped this from being a systemic whole-stack issue: the Go apiserver's `cNameValToWorkerResval()` (`src/frontend/apiserver/backend/convert.go:325-370`) is a CORRECT, COMPLETE reference implementation -- it switches on all 7 `Value_u_which` wire tags (ivalue32/ivalue64/uvalue32/uvalue64/fvalue all -> NUMERIC/`NumericValue`, svalue -> STRING/`StringValue`, bvalue -> BOOLEAN/`BooleanValue`, matching `models.ResourceDistributionInfo`'s already-correctly-typed fields, `src/frontend/apiserver/models/resources.go:118-126`), plus a `default:` WARN log for any unrecognized tag. `src/frontend/xcecli/models/resource.go`'s `ResourceValue.UnmarshalJSON()` (client-side) also correctly handles string/bool/numeric JSON values. So the production REST API (apiserver) and `xcecli` CLI resource-value display path are NOT affected -- this bug is confined to the scheduler's own C-side simulation-debugging REPL (`repl_info.c`, the user-facing symptom) and a handful of C-side TRACE/debug log call sites in the scheduler/controller (lower severity, debug-visibility-only gaps).", "fix": "Fixed. src/scheduler/src/repl_info.c: repl_info_worker_one() (`info worker |all`) now switches on the resval's real Value_u_which wire tag -- Value_u_svalue prints `resource : value=`, Value_u_bvalue prints `resource : value=true|false`, Value_u_fvalue keeps the existing `total=%.2f available=%.2f` numeric display unchanged. repl_info_resource_one() (`info resource `) got the analogous fix one layer up: the per-worker aggregation loop now tracks the first-seen string/boolean value (in addition to the existing numeric total/available sum), and the final print branches on the resource DEFINITION's authoritative `r->valtype` (XCE_VAL_STRING/XCE_VAL_BOOLEAN/XCE_VAL_NUMERIC, xce/types.h) to decide which of `value=` (with a 'reported by N worker(s)' count) vs. the numeric total/available pair to print -- total/available is a consumable-quantity concept that is only meaningful for a numeric resource, so a string/boolean resource no longer gets a fabricated total=0.00/available=0.00 pair at all. TDD: added test_info_worker_reports_string_and_boolean_resource_values and test_info_resource_reports_string_value to test_repl_breakpoints_inspection.c, both constructing a real WORKERS_PUSH event with a string- and boolean-valued resource through the actual xce_core_push_events() path (same fixture idiom the existing bug-1568 numeric-resource test already established), asserting the new `value=` lines appear and no `total=` line is ever emitted for a non-numeric resource. Verified via a full non-incremental /xce-build scheduler asan run (fresh wipe+reconfigure, not reusing any cached object): zero compiler errors, both new tests confirmed PASS in testlog.txt, full suite Ok:41/Fail:1 out of 42 -- the sole failure is the pre-existing unrelated bug-1560 (test_admin_request's symlink-escape test). core_events.c's separately-noted boolean-TRACE gap (item 3 in this entry's root_cause) and worker.c/dataex.c's non-numeric TRACE decode gaps (item 4) were NOT touched -- left as documented, lower-priority follow-ups, not part of this fix's scope.", @@ -21337,24 +8086,27 @@ "type-dispatch", "display-bug", "logging", - "open", - "not-yet-fixed", - "fixed" + "not-yet-fixed" ], "related_bugs": [ "bug-1568", "bug-1573" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1590.md" }, { "id": "bug-1591", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T03:45:00Z", - "error_message": "User-reported: `(sim) info queue all` in the scheduler's interactive REPL prints raw packed jobids instead of the human-readable form, e.g. `[sim] queued job: jobid=4645085138445664257 priority=0` instead of the 'cluster-apiserver-timestamp-sequence' form (e.g. '3-1-1755590313-976') already used elsewhere in this codebase (bug-1588 fixed the same class of issue in src/controller/job.c's TRACE lines).", - "file": "src/scheduler/src/repl_info.c (queued job: ~230, assigned job: ~356), src/scheduler/src/repl_break.c (~665), src/scheduler/src/sim_loop.c (~379,383,387), src/scheduler/src/sim_verify.c (~293,325,335,346,373,380,389), src/scheduler/src/driver.c (~652), src/scheduler/src/core_events.c (~872,907,979), src/scheduler/src/recording_session.c (~173) -- full raw-jobid-print inventory for the scheduler tree, not yet fixed", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "User-reported: `(sim) info queue all` in the scheduler's interactive REPL prints raw packed jobids…\n\nUser-reported: `(sim) info queue all` in the scheduler's interactive REPL prints raw packed jobids instead of the human-readable form, e.g. `[sim] queued job: jobid=4645085138445664257 priority=0` instead of the 'cluster-apiserver-timestamp-sequence' form (e.g. '3-1-1755590313-976') already used elsewhere in this codebase (bug-1588 fixed the same class of issue in src/controller/job.c's TRACE lines).", + "file": "src/scheduler/src/repl_info.c (queued job: ~230, assigned job: ~356), src/scheduler/src/repl_break.c (~665), src/scheduler/src/sim_loop.c (~379, 383, 387), src/scheduler/src/sim_verify.c (~293, 325, 335, 346, 373, 380, 389), src/scheduler/src/driver.c (~652), src/scheduler/src/core_events.c (~872, 907, 979), src/scheduler/src/recording_session.c (~173) -- full raw-jobid-print inventory for the scheduler tree, not yet fixed", "root_cause": "Same class as bug-1588 (already fixed for src/controller/job.c): fmt_jobid()/fmt_jobid_ex() (src/lib/utils.c, declared src/include/utils.h:132-133) is the established human-readable jobid formatter already used elsewhere in this exact scheduler tree -- scheduler.c, handler.c, plugins/limit.c, plugins/fcfs.c, and even repl_info.c ITSELF at other call sites (e.g. line 119/143/184, per bug-1588's original grep) already call it correctly. The two sites the user's `info queue all` output actually exercises -- repl_info.c's 'queued job: jobid=%llu' (~230, the queue-dump command, `info queue |all`) and its sibling 'assigned job: jobid=%llu' (~356, inside repl_info_worker_one, `info worker |all`) -- were simply never converted, the same oversight bug-1588 already found and fixed in job.c. A broader grep of src/scheduler/src/ for the raw `jobid=%llu`/`jobid: %llu` pattern (see `file` field above) found ~15 more sites across repl_break.c (breakpoint-hit message), sim_loop.c/sim_verify.c (batch --sim DISPATCH/PEND/PREEMPT decision output and --sim-verify divergence reports), driver.c/core_events.c/recording_session.c (WARN/ERROR log lines for unknown-jobid and preemption-not-yet-implemented conditions) -- all likely candidates for the same fix, not yet individually confirmed to be in-scope (some may be deliberately terse WARN/ERROR paths where the raw id is acceptable; needs a per-site judgment call when this is picked up, not a blanket mechanical replace).", "fix": "Fixed. All 15 raw jobid=%llu sites converted to fmt_jobid() across src/scheduler/src/repl_info.c (2), repl_break.c (1), sim_loop.c (3), sim_verify.c (7), driver.c (1), recording_session.c (1), core_events.c (3) -- same mechanical pattern as bug-1588's job.c fix. This intentionally changed observable REPL/--sim/--sim-verify output text, which broke 5 test binaries' assertions that hardcoded the OLD raw-numeric jobid format (e.g. `strstr(captured, \"jobid=4242\")`); updated all affected assertions in test_sim_alt_events.c, test_repl_execution_control.c, test_sim_event_loop.c, test_sim_verify.c, and test_repl_breakpoints_inspection.c to compute their expected marker strings dynamically via fmt_jobid() too, matching the existing precedent already used in src/controller/testsuite/test.c. Verified via two /xce-build scheduler asan --incremental runs: first run caught a genuine test regression (5 unexpected failures, all traced to the hardcoded-format issue above, not a production bug); after the test fixes, final run is Ok:41/Fail:1 out of 42 -- all 10 affected tests confirmed individually passing, the only remaining failure is the pre-existing unrelated bug-1560 (test_admin_request's symlink-escape test). One build-tooling wrinkle encountered along the way (not a code bug): an --incremental ninja run once silently skipped recompiling one changed test object, giving a stale Ok:40/Fail:2 result until the object was force-rebuilt -- worth a non-incremental rebuild if an --incremental result ever looks surprising.", "tags": [ @@ -21364,22 +8116,25 @@ "human-readable", "logging", "ux", - "open", - "not-yet-fixed", - "fixed" + "not-yet-fixed" ], "related_bugs": [ "bug-1588" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1591.md" }, { "id": "bug-1592", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "worker", + "severity": "unknown", "timestamp": "2026-08-19T07:38:00Z", - "error_message": "grep/tail: Permission denied on $TOP/logs/xcecd.log after starting the real (production) xcecd worker via sudo", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "grep/tail: Permission denied on $TOP/logs/xcecd.log after starting the real (production) xcecd…\n\ngrep/tail: Permission denied on $TOP/logs/xcecd.log after starting the real (production) xcecd worker via sudo", "file": "src/worker/src/main.c", "root_cause": "xcecd's main log file (config.log_file, opened by the logging library at daemon startup while running as root) is created with default umask perms (verified live: root:root, mode 640) and never chmod'd permissive -- unlike xcecd_worker.log (explicit open()+chmod(0666) at main.c:536-543) and the domain socket (chmod(config.domain_sock_file, 0666) at main.c:578). Any verification command (grep/tail/cat) run as the invoking non-root user against xcecd.log fails with Permission denied.", "fix": "Documented as a known behavior in .claude/skills/xce-deploy-ha/SKILL.md: all log reads against the real worker's xcecd.log must be prefixed with sudo (sudo grep .../sudo tail ...), same as the sudo kill already required to stop the root-owned process. Not a code fix -- xcecd.log intentionally stays root-owned; only the skill's documented verification commands changed.", @@ -21390,20 +8145,23 @@ "sudo", "permissions", "logging", - "real-worker", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "real-worker" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1592.md" }, { "id": "bug-1593", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T08:00:00Z", - "error_message": "User-reported (live REPL testing): `info resource all` and `info resource ` aggregate every worker's per-host resource values (total=sum(available), available=sum) for EVERY resource, including host-based ones like cpus/mem -- a fabricated, meaningless number for a per-host consumable already correctly shown by `info worker `. User asked that `info resource` instead show resource metadata and be restricted to globally-shared resources only.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (live REPL testing): `info resource all` and `info resource ` aggregate every…\n\nUser-reported (live REPL testing): `info resource all` and `info resource ` aggregate every worker's per-host resource values (total=sum(available), available=sum) for EVERY resource, including host-based ones like cpus/mem -- a fabricated, meaningless number for a per-host consumable already correctly shown by `info worker `. User asked that `info resource` instead show resource metadata and be restricted to globally-shared resources only.", "file": "src/scheduler/src/repl_info.c (repl_info_resource_one, repl_info_resource)", "root_cause": "xce_resource_t (xce/types.h) carries a `flags` bitmask (XCE_RESF_BUILTIN/DYNAMIC/GLOBAL/SHARED/EXTERNAL/RELEASE/DEFAULT_HOST/CONSUMABLE/LIMIT) that distinguishes a genuinely cluster-wide resource (GLOBAL/SHARED, stored once in core->sharedInst, indexed by r->idx -- the same pool RESVALS_FULL/RESVALS_DELTA event handling and snapshot save/load already treat as the defining store) from a host-based/per-worker-distributed resource (stored per-worker in w->resvals/w->resInUse, already correctly surfaced by repl_info_worker_one()). repl_info_resource_one() never consulted r->flags or core->sharedInst at all -- it iterated sim->core.workers unconditionally for every resource regardless of kind, summing w->resvals[r->idx] across the whole cluster. For a host-based resource (the common case -- cpus, mem, etc.) this total/available is not a meaningful quantity (no single cluster-wide capacity exists for a per-host resource); for a genuinely shared resource it also happened to work by accident (since a shared resource's value is typically replicated identically into every worker's resvals array too), but even then never reported the resource's own metadata (desc, valtype, flags, interval) the user asked for.", "fix": "Fixed as part of task 12.5 (scheduler-core spec, Requirement 5 AC 22 amendment). repl_info_resource_one() rewritten to print resource DEFINITION metadata (name, desc, valtype, human-readable flags list, interval) plus, only for a resource flagged XCE_RESF_GLOBAL or XCE_RESF_SHARED, its current value read directly from core->sharedInst[r->idx] (never aggregated across workers). A host-based resource's metadata is still printed but with an explicit note directing the operator to `info worker ` for its actual per-host values -- no fabricated total/available. repl_info_resource() (`info resource all`) now skips any resource whose flags do not include GLOBAL or SHARED, per the user's explicit request to keep this command scoped to cluster-wide resources. New tests in test_repl_breakpoints_inspection.c construct a real GLOBAL/SHARED resource via xce_core_push_events() and a plain host-based one, asserting the metadata line and shared value appear for the former, the host-based one is excluded from `info resource all`, and both appear correctly for `info resource ` with the host-based one omitting a fabricated total/available pair.", @@ -21414,21 +8172,25 @@ "shared-resource", "host-based", "metadata", - "display-bug", - "fixed" + "display-bug" ], "related_bugs": [ "bug-1590" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1593.md" }, { "id": "bug-1594", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T08:00:00Z", - "error_message": "User-reported: in the scheduler's --interactive REPL, `continue` with no breakpoints set runs the simulation to completion and the process then exits -- the operator never gets a chance to inspect the final simulated state (info job/queue/worker/resource/summary). Related: the simulator should never terminate except via an explicit `quit`.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported: in the scheduler's --interactive REPL, `continue` with no breakpoints set runs the…\n\nUser-reported: in the scheduler's --interactive REPL, `continue` with no breakpoints set runs the simulation to completion and the process then exits -- the operator never gets a chance to inspect the final simulated state (info job/queue/worker/resource/summary). Related: the simulator should never terminate except via an explicit `quit`.", "file": "src/scheduler/src/sim_loop.c (sched_sim_driver_run), src/scheduler/src/repl.c (repl_dispatch, new xce_repl_on_replay_complete), src/scheduler/include/repl.h, src/scheduler/src/repl_internal.h", "root_cause": "requirements.md's own AC 22 pseudocode for the REPL integration never re-enters the REPL after the replay loop's `for each event in replay_file` reaches EOF -- it prints two informational lines (\"replay complete\"/\"output written to\") and falls straight through to sched_sim_driver_run()'s return, which unwinds back to main() and the process exits normally. This matches the literal pseudocode but is a genuine UX gap the requirement text never called out: `continue` (mode=XCE_REPL_MODE_CONTINUE, no breakpoint match) is the ONLY path that can reach EOF without another repl_command_loop() re-entry already pausing execution first (step's budget-exhaustion pause and every breakpoint hit both already re-enter the REPL mid-stream; only a completed `continue` run had no re-entry point at all).", "fix": "Fixed as part of task 12.5. Added xce_repl_on_replay_complete() (repl.c) + its sim->interactive-gated inline wrapper xce_repl_check_replay_complete() (repl.h), called from sim_loop.c immediately after the replay for(;;) loop breaks (EOF or AC20 end-of-stream marker), before the --sim-output file is closed. It prints the existing completion banner, sets a new sim->repl_state->replay_complete flag, and calls repl_command_loop(sim) so the process blocks at the `(sim) ` prompt instead of returning. repl_dispatch()'s `step`/`continue` branches now check replay_complete first: if set, they print \"[sim] replay already complete -- no more events to process. Use 'quit' to exit.\" and return false (re-prompt) instead of returning true (which would have handed control back to a driver loop that no longer exists). The only remaining exit paths are repl_cmd_quit() (explicit `quit`) and readline() returning NULL on stdin EOF (already treated as an implicit quit, pre-existing documented behavior, unchanged). Batch (non-interactive) `--sim` mode is unaffected -- xce_repl_check_replay_complete() is a no-op when sim->interactive is false, mirroring every other REPL hook's existing gating. New tests in test_repl_execution_control.c: scripted stdin `continue\\ninfo summary\\nquit\\n` against a fixture with zero breakpoints asserts the process does NOT exit after 'replay complete' and that `info summary` after it reports the final job/session state correctly; a second test asserts `step`/`continue` issued after replay_complete print the re-prompt message and the REPL stays alive.", @@ -21438,21 +8200,25 @@ "continue", "quit", "exit-behavior", - "ux", - "fixed" + "ux" ], "related_bugs": [ "bug-1596" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1594.md" }, { "id": "bug-1595", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T08:00:00Z", - "error_message": "User-reported: in each `step`, it would be better to show the decisions/events that occur in the current session in the sequence they actually occur -- currently only decisions are printed, input events (JOB_PUSH, JOB_STATE_CHANGE, WORKERS_PUSH, etc.) are completely silent unless a breakpoint happens to match one.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported: in each `step`, it would be better to show the decisions/events that occur in the…\n\nUser-reported: in each `step`, it would be better to show the decisions/events that occur in the current session in the sequence they actually occur -- currently only decisions are printed, input events (JOB_PUSH, JOB_STATE_CHANGE, WORKERS_PUSH, etc.) are completely silent unless a breakpoint happens to match one.", "file": "src/scheduler/src/repl_break.c (xce_repl_on_event)", "root_cause": "sim_loop.c already calls xce_repl_check_event()/xce_repl_on_event() for every decoded ReplayEvent, in stream order, strictly before it joins the batch xce_core_push_events() later applies -- but xce_repl_on_event() (repl_break.c) only ever prints anything when a breakpoint matches; there is no unconditional per-event echo. sim_print_decision() (sim_loop.c), by contrast, IS called unconditionally for every drained decision. Net effect: an operator using `step`/`continue` without breakpoints sees decisions appear with no visible cause -- the events that produced them (a job arriving, a job's state changing, a worker reporting in) are invisible.", "fix": "Fixed as part of task 12.5. xce_repl_on_event() (repl_break.c) now unconditionally prints a one-line human-readable summary of every event before running the (unchanged) breakpoint-match check -- e.g. \"[sim] event: JOB_PUSH njobs=2 jobids=[, ]\", \"[sim] event: JOB_STATE_CHANGE jobid= state=RUN\", \"[sim] event: WORKERS_PUSH nworkers=1 names=[worker-1]\", one line per xce_core_event_type_t kind. Since xce_repl_on_event() is only ever invoked when sim->interactive (via repl.h's existing gate), this adds zero output to batch `--sim` mode. repl_job_state_name() (repl_info.c) was made non-static and declared in repl_internal.h so both repl_info.c's `info job` and repl_break.c's new event summary render the same state name for JOB_STATE_CHANGE. Printed order is now event(s) (as decoded) -> BEGIN_ROUND -> decision(s) (as drained), matching sim_loop.c's actual processing order. New tests in test_repl_breakpoints_inspection.c step through a fixture with a JOBS_PUSH and a JOB_STATE_CHANGE event and assert both event summary lines appear, in order, before the resulting decision line.", @@ -21462,19 +8228,23 @@ "step", "continue", "event-visibility", - "ux", - "fixed" + "ux" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1595.md" }, { "id": "bug-1596", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T08:00:00Z", - "error_message": "User-reported: `info job \"0-15-1787129769-0[0]\"` (jobid copied from `info jobs`' own output, wrapped in double quotes out of habit) returns \"[sim] info job: no such job 0 (0-0-0-0)\" even though the job is genuinely present -- the identical jobid without quotes works.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported: `info job \"0-15-1787129769-0[0]\"` (jobid copied from `info jobs`' own output…\n\nUser-reported: `info job \"0-15-1787129769-0[0]\"` (jobid copied from `info jobs`' own output, wrapped in double quotes out of habit) returns \"[sim] info job: no such job 0 (0-0-0-0)\" even though the job is genuinely present -- the identical jobid without quotes works.", "file": "src/scheduler/src/repl.c (repl_split_tokens)", "root_cause": "repl_info_job() already correctly calls parse_jobid() (lib/utils.c) with a strtoull() fallback -- confirmed NOT the bug. parse_jobid()'s two regexes (REGEX_JOBARRAY_ID/REGEX_JOBID, src/include/config.h) both require a whole-string anchor match (pmatch[0].rm_so==0 && pmatch[0].rm_eo==len). repl_split_tokens() (repl.c) is a plain whitespace tokenizer with no quote-awareness, so a quoted argument's token literally includes the surrounding `\"` characters. Those quote characters break the anchor match (the regex still matches the interior substring, but not starting at offset 0 / ending at len), parse_jobid() returns -EINVAL, and the strtoull() fallback then parses a string beginning with `\"` -- not a valid numeric prefix -- returning 0. MAKE_JOBID(0,0,0,0,0) formats back as \"0-0-0-0\", exactly matching the reported output.", "fix": "Fixed as part of task 12.5. repl_split_tokens() (repl.c) now strips one matching pair of leading/trailing `\"` or `'` characters from each extracted token (length >= 2, first == last == a quote char) before returning it -- applies uniformly to every REPL command argument (info job/queue/worker/resource names, break filters), not special-cased to `info job`. New test in test_repl_breakpoints_inspection.c asserts `info job \"\"` (double-quoted) and `info job ''` (single-quoted) both resolve the same job as the unquoted form.", @@ -21484,22 +8254,25 @@ "jobid", "quoting", "parsing", - "ux", - "fixed" + "ux" ], "related_bugs": [ "bug-1591" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1596.md" }, { "id": "bug-1597", "status": "fixed", "status_updated": "2026-09-24", "status_note": "obsolete: src/frontend/Makefile removed (6fabdba9)", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-19T00:00:00Z", - "error_message": "backend/reconnect.go:6:10: fatal error: client.h: No such file or directory (compilation terminated) -> make: *** [Makefile:33: build-server] Error 1", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "backend/reconnect.go:6:10: fatal error: client.h: No such file or directory (compilation…\n\nbackend/reconnect.go:6:10: fatal error: client.h: No such file or directory (compilation terminated) -> make: *** [Makefile:33: build-server] Error 1", "file": "src/frontend/Makefile (build-server target)", "root_cause": "build-server's recipe is a bare `cd apiserver && go build -o /tmp/apiserver ./cmd/main.go` with no CGO_CFLAGS/CGO_LDFLAGS, so cgo has no -I path to src/include (client.h) or BUILD_DIR/lib (controller.capnp.h) and no -L/-l for libxce + its pkg-config deps. Unlike build-server, both test-backend (same Makefile, bug-1558) and apiserver/meson.build derive CGO_CFLAGS/CGO_LDFLAGS via pkg-config (librtti, c-capnproto, openssl, tinylog) plus -I$(BUILD_DIR)/lib / -L$(BUILD_DIR)/lib -lxce -- build-server was apparently never given the same treatment, so it only works if CGO_CFLAGS/CGO_LDFLAGS happen to already be exported in the shell (e.g. by meson's custom_target env), not under a bare `nix develop ../ --command make build-server`. Reproduced identically with and without an unrelated comment-only change staged elsewhere in the same package (apiserver/backend/handler.go), confirming this is pre-existing and unrelated to that change.", "fix": "Not fixed (out of scope for the verification task that found it -- reported only). Workaround that does compile cleanly (exit 0): run the build-server go build with the same self-derived CGO env as test-backend, e.g. inside `nix develop ../ --command bash -c '...'`: BUILD_DIR=$(pwd)/../xce.build; CGO_CFLAGS=\"-fsanitize=address -g $(pkg-config --cflags librtti c-capnproto openssl tinylog) -I$(pwd)/../include -I$BUILD_DIR/lib\"; CGO_LDFLAGS=\"-fsanitize=address -g $(pkg-config --libs librtti c-capnproto openssl tinylog) -lm -lrt -L$BUILD_DIR/lib -lxce\"; then CGO_ENABLED=1 CGO_CFLAGS=\"$CGO_CFLAGS\" CGO_LDFLAGS=\"$CGO_LDFLAGS\" go build -o /tmp/apiserver ./cmd/main.go. Requires BUILD_DIR/lib/libxce.a to already exist (prior xce-build.sh lib run). Proper fix would mirror bug-1558's FRONTEND_CGO_CFLAGS/FRONTEND_CGO_LDFLAGS into the build-server recipe in src/frontend/Makefile.", @@ -21510,22 +8283,27 @@ "makefile", "build-server", "nix", - "pre-existing", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pre-existing" ], "related_bugs": [ "bug-1558" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "6fabdba9" + ], + "record": ".wolf/bugs/bug-1597.md" }, { "id": "bug-1598", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-19T00:00:00Z", - "error_message": "AddressSanitizer: stack-use-after-return on address inside `sim` (xce_sched_sim_driver_t) at cbuf_get, thread T2 (sim_picker_thread_fn), reading test_run_deactivates_tlog_virtual_clock_logging_on_completion's stack frame after it returned", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: stack-use-after-return on address inside `sim` (xce_sched_sim_driver_t) at…\n\nAddressSanitizer: stack-use-after-return on address inside `sim` (xce_sched_sim_driver_t) at cbuf_get, thread T2 (sim_picker_thread_fn), reading test_run_deactivates_tlog_virtual_clock_logging_on_completion's stack frame after it returned", "file": "src/scheduler/testsuite/test_sim_event_loop.c", "root_cause": "sched_sim_driver_run() (src/scheduler/src/sim_loop.c) spawns sim_picker_thread_fn on a real pthread that is, by this codebase's own documented design (that function's own header comment), never joined or cancelled -- no sim-driver teardown/fini function exists yet. It loops forever on cbuf_get(&sim->core.wbuf, ...), reading `sim` for the lifetime of the process. Every white-box test in test_sim_event_loop.c that calls sched_sim_driver_run()/sched_sim_driver_start() with a stack-local `xce_sched_sim_driver_t sim;` was therefore ALWAYS a latent stack-use-after-return once its test function returned -- it simply hadn't been caught yet, because no prior test ordering/stack-frame-reuse pattern had triggered ASAN's shadow-memory detector. Adding one new test (test_run_deactivates_tlog_virtual_clock_logging_on_completion, part of the scheduler-core design-review follow-up wiring tlog virtual-clock activation into sim.c/sim_loop.c) between two existing tests shifted the stack layout enough to make the pre-existing latent bug fire deterministically (reproduced on 2 independent runs).", "fix": "Changed the fixture from stack-local to `static xce_sched_sim_driver_t sim;` storage in three places: test_run_processes_multiple_sessions_in_order_and_no_longer_returns_enosys(), test_run_deactivates_tlog_virtual_clock_logging_on_completion() (the test that first exposed it), and the run_sim_and_capture_stdout() helper (used once, by test_job_present_only_in_initial_snapshot_still_reaches_the_picker()). Safe specifically because: (a) sched_sim_driver_start() unconditionally memset()s the whole struct as its first action, so no stale-data leak across runs; (b) each affected function is called exactly once per test-binary process (verified via grep, no re-entrancy), so no second call's memset can ever stomp a still-live zombie thread's cbuf mutex/condvar state from a first call -- static would be UNSAFE on a helper called more than once while an earlier zombie thread might still be alive. Does NOT fix the underlying thread-lifecycle gap (still no join/cancel) -- that is a separate, larger, already-documented-as-deferred architectural item (a real sim-driver teardown/fini function). Verified via 2 independent direct runs of the rebuilt sched_sim_event_loop_test binary post-fix: 4 Tests, 0 Failures, clean exit code 0 both times, no ASAN abort.", @@ -21536,21 +8314,23 @@ "thread-lifecycle", "test-only", "sim-driver", - "design-review-followup", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "design-review-followup" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1598.md" }, { "id": "bug-1599", "status": "fixed", "status_updated": "2026-09-24", "status_note": "sim_loop.c comment reworded", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-20T02:50:00Z", - "error_message": "sched_no_bare_printf_test / test_scheduler_src_has_no_bare_printf_calls FAIL: '.../scheduler/src/sim_loop.c calls bare printf() -- xce_scheduler link-wraps printf ...' -- even though sim_print_decision() in sim_loop.c already correctly uses fprintf(stdout, ...) for all 3 PEND/DISPATCH/PREEMPT lines (verified via grep: lines 398/402/406 all fprintf(stdout, ...), zero bare printf( calls in actual code).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "sched_no_bare_printf_test / test_scheduler_src_has_no_bare_printf_calls FAIL…\n\nsched_no_bare_printf_test / test_scheduler_src_has_no_bare_printf_calls FAIL: '.../scheduler/src/sim_loop.c calls bare printf() -- xce_scheduler link-wraps printf ...' -- even though sim_print_decision() in sim_loop.c already correctly uses fprintf(stdout, ...) for all 3 PEND/DISPATCH/PREEMPT lines (verified via grep: lines 398/402/406 all fprintf(stdout, ...), zero bare printf( calls in actual code).", "file": "src/scheduler/testsuite/test_no_bare_printf.c (new regression test) flags src/scheduler/src/sim_loop.c", "root_cause": "The new regression test's scan_dir_for_bare_printf()/contains_bare_printf_call() (test_no_bare_printf.c) is a naive whole-file text scan with NO comment/string-literal awareness (documented as such in its own header comment, matching test_core_isolation.c's precedent). The doc comment directly above sim_print_decision() in sim_loop.c (lines 375-387, added as part of this same fix, explaining WHY fprintf(stdout,...) is used instead of printf()) contains the literal prose substrings ' plain printf() (2026-08-20 fix)' (line 375) and ' plain printf() call' (line 377) -- both have a space immediately before 'printf(', which the scanner's rule (char before 'printf(' must not be letter/digit/underscore) matches as a bare-printf violation. The scanner cannot distinguish prose describing printf() from an actual call to it.", "fix": "NOT YET FIXED -- reported to caller for triage per explicit instruction not to fix. Likely fix: reword the sim_loop.c doc comment (lines 375/377) to avoid the literal pattern 'printf(' immediately -- e.g. write it as \"printf\" and \"(\" separately, or reference the wrapped __wrap_printf() symbol name instead of the bare word, or otherwise break up the substring so the naive scanner does not match it. Do NOT weaken the scanner itself to add comment-awareness without considering that test_core_isolation.c deliberately uses the same simple/strict genre.", @@ -21561,21 +8341,23 @@ "sim_loop.c", "test_no_bare_printf", "text-scan", - "regression-test", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "regression-test" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1599.md" }, { "id": "bug-1600", "status": "fixed", "status_updated": "2026-09-24", "status_note": "ddee884c", + "component": "lib", + "severity": "crash", "timestamp": "2026-08-19T21:05:00Z", - "error_message": "xce_scheduler crashed with free(): invalid pointer / ASAN bad-free during controller failover retry (3x config-push retry loop); 6 crash-loop occurrences on 2026-08-17, 2 more on 2026-08-19 (core dumps confirmed via gdb bt full and coredumpctl, crash site consistently makeGroup() at src/lib/group.c:234)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 8, + "error_message": "xce_scheduler crashed with free(): invalid pointer / ASAN bad-free during controller failover…\n\nxce_scheduler crashed with free(): invalid pointer / ASAN bad-free during controller failover retry (3x config-push retry loop); 6 crash-loop occurrences on 2026-08-17, 2 more on 2026-08-19 (core dumps confirmed via gdb bt full and coredumpctl, crash site consistently makeGroup() at src/lib/group.c:234)", "file": "src/lib/group.c", "root_cause": "makeGroup() existing-group update path (the branch taken when a config-push re-applies a group name that already exists, e.g. every retried/duplicate config push) freed xce_g_member_t.u.name via FREEUP() (raw free()). u.name is allocated by makeString() (make_g_member(), group.c:77), an rtti-managed pointer returned by makeInstance()/objAlloc() that is NOT the malloc()-returned base address (it is an offset into a combined header+data block) and is refcounted, not malloc-owned. Calling free() on it is undefined behavior -> heap corruption, surfacing later as glibc/ASAN bad-free aborts on unrelated allocations. group_killer() (same file) already released the identical field correctly via dereference(). The buggy FREEUP() path is hit on every config re-push for a group with existing members (e.g. controller failover retrying a config push), which explains the crash correlating with the 3-attempt xcecli_retry_next() failover loop even though no thread concurrency is required to trigger it.", "fix": "Changed both FREEUP(m->u.name) call sites inside makeGroup()'s existing-group branch (group.c, the g->members loop and the g->ex_members loop) to dereference(m->u.name), matching group_killer()'s established correct pattern for releasing rtti makeString() pointers. Added regression test test_reconfigure_same_group_members (src/lib/testsuite/test_group.c) that calls makeGroup() twice with the same group name/members, reproducing the exact ASAN bad-free at group.c:234 pre-fix (confirmed RED), and passing clean post-fix (confirmed GREEN, full lib asan suite Ok:1 Fail:0, no regressions). NOTE: a separate, related latent issue was also found but NOT fixed here (out of scope, no concurrency needed to explain the observed crash) - the entire existing-group update branch in makeGroup() only takes RTTI_LOCK_READ (a genuine shared pthread_rwlock_rdlock) while mutating shared group state, and sched_init_config()/plugin_config_update() (the config-push handler chain) hold no serializing lock at all, so two vtask worker threads (scheduler runs a 4-vcpu pool) could in principle race on the same group concurrently -- most plausibly triggered by xce_ctx_reconnect() reusing the same endpoint struct and endpoint_init() resetting ep->busy=0 before an already-dispatched handler finishes. This should be tracked as a follow-up.", @@ -21591,22 +8373,27 @@ "group.c", "config-push", "failover", - "production", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "production" ], "related_bugs": [ "bug-1503" ], - "occurrences": 8, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "ddee884c" + ], + "record": ".wolf/bugs/bug-1600.md" }, { "id": "bug-1601", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-20T07:26:00Z", - "error_message": "User request: info resource/info worker did not distinguish consumable from non-consumable resources -- repl_info_worker_one() showed a fabricated total=/available= pair for EVERY numeric per-worker resource regardless of XCE_RESF_CONSUMABLE, and repl_info_resource_one() never reported a used/in-use amount for a CONSUMABLE GLOBAL/SHARED resource at all; the resource definition line was also missing ordtype.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User request: info resource/info worker did not distinguish consumable from non-consumable…\n\nUser request: info resource/info worker did not distinguish consumable from non-consumable resources -- repl_info_worker_one() showed a fabricated total=/available= pair for EVERY numeric per-worker resource regardless of XCE_RESF_CONSUMABLE, and repl_info_resource_one() never reported a used/in-use amount for a CONSUMABLE GLOBAL/SHARED resource at all; the resource definition line was also missing ordtype.", "file": "src/scheduler/src/repl_info.c", "root_cause": "repl_info_worker_one() unconditionally computed/printed total=avail+inUse available=avail for any Value_u_fvalue resval, never checking XCE_RESF_CONSUMABLE -- a non-consumable numeric attribute (nothing ever allocates against it) got a misleading total/available split instead of a single current value. repl_info_resource_one() (the GLOBAL/SHARED case) only ever read core->sharedInst (the available pool) and never core->sharedInUse (the parallel in-use pool that already exists and is populated by the real resource-accounting code, resource.c), so a CONSUMABLE cluster-wide resource never showed how much was in use. The definition line also omitted xce_resource_t.ordtype (INCR/DECR/NA).", "fix": "repl_info_resource_one(): added ordtype= to the definition line; for a GLOBAL/SHARED resource additionally flagged XCE_RESF_CONSUMABLE, the numeric value line now also reports used= (read from core->sharedInUse at the same idx) alongside value= (avail+used); non-consumable stays a single value= line. repl_info_worker_one(): the per-resource numeric line now looks up the matching xce_resource_t (by idx) from the already-built res_arr and gates the value/used split on XCE_RESF_CONSUMABLE -- consumable prints value=avail+used used=inUse (renamed from the old total=/available= labels for consistency with the resource-level display), non-consumable prints a single value=avail line. Added test_info_resource_reports_consumable_value_and_used and test_info_worker_reports_consumable_value_and_used (new, seed core->sharedInUse / worker->resInUse directly via dup_val()+zarray_put, mirroring the allocation pattern the real accounting code and core_events.c already use); updated 3 pre-existing tests whose asserted strings changed shape (test_info_worker_reports_resource_values, test_info_resource_reports_string_value, test_info_resource_named_lookup_reports_host_based_note). Full scheduler suite: 42/43 Ok, only the pre-existing unrelated bug-1560 failure remains.", @@ -21617,23 +8404,26 @@ "info-worker", "consumable", "display", - "feature", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "feature" ], "related_bugs": [ "bug-1590", "bug-1593" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1601.md" }, { "id": "bug-1602", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-20T08:30:00Z", - "error_message": "User request: scheduler simulation mode should show event time and session when events occur, and loading a replay file should show its total session count up front.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User request: scheduler simulation mode should show event time and session when events occur, and…\n\nUser request: scheduler simulation mode should show event time and session when events occur, and loading a replay file should show its total session count up front.", "file": "src/scheduler/src/repl_break.c, src/scheduler/src/sim.c", "root_cause": "Feature gap, not a defect: repl_print_event_summary() (interactive REPL event echo) printed only the event type/payload, no clock time or session number; sched_sim_driver_start() never reported how many sessions a loaded .replay file contains before stepping through it.", "fix": "repl_print_event_summary() now prefixes each echoed event with \"[sim] t= () session=\". Non-obvious correctness point found via sim_loop.c inspection (not assumption): events are read+echoed individually AS THEY STREAM from the file, and the ReplaySession/begin_round message for that batch -- which is what increments rs->session_count via xce_repl_on_begin_round() -- comes AFTER those events in file order (file layout is event,event,...,ReplaySession). So every echoed event always belongs to the UPCOMING session; the function reports session_count+1, not the raw count. A first attempt using the raw count failed its own new test (session=0 instead of session=1) and was corrected after tracing the real sim_loop.c call order (xce_repl_check_event at line ~593 vs xce_repl_check_begin_round at ~631) rather than trusting the repl.h doc comment's summary.\n\nAdded sim_prescan_session_count() (sim.c): scans sim->replay_file from the current position using msg_root_has_shape() shape-only classification (ReplayEvent/ReplaySession/SnapshotEntry, matching sim_loop.c main loop's own three-way dispatch) to count ReplaySession messages, then rewinds via fseek back to the captured ftell() position -- mirrors sim_locate_and_load_snapshot()'s own established rewind pattern. Confirmed safe via research: capn_init_fp() in this codebase is always unpacked (packed=0), reading via a single plain fread() per message with no internal read-ahead buffer, so ftell/fseek stay exactly in sync with message boundaries. Wired into sched_sim_driver_start() right after sim_set_virtual_clock(): prints \"[sim] replay file contains N sessions\" once at load, non-fatal on scan failure. User explicitly chose this pre-scan approach over a recording-time trailer message (would have required a schema change and only worked for newly-recorded files) via AskUserQuestion.\n\nAdded test_replay_load_reports_total_session_count (2-session fixture) and extended test_step_echoes_events_before_session_decisions with t=/session=1 assertions in test_repl_breakpoints_inspection.c. Full scheduler suite: 42/43 Ok, only pre-existing unrelated bug-1560 remains (now 3 occurrences).", @@ -21644,22 +8434,25 @@ "replay", "session", "feature", - "observability", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "observability" ], "related_bugs": [ "bug-1595" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1602.md" }, { "id": "bug-1603", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-20T09:20:00Z", - "error_message": "build-time-versioning task 2.2 REJECTED by reviewer: `nix build .#xcecore_debug` (no flags, the exact documented/task-Observable command) failed at evaluation with `error: attribute 'currentTime' missing`; also the raw epoch-seconds string didn't satisfy Requirement 1.2's absolute-UTC-form requirement.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "build-time-versioning task 2.2 REJECTED by reviewer: `nix build .#xcecore_debug` (no flags, the…\n\nbuild-time-versioning task 2.2 REJECTED by reviewer: `nix build .#xcecore_debug` (no flags, the exact documented/task-Observable command) failed at evaluation with `error: attribute 'currentTime' missing`; also the raw epoch-seconds string didn't satisfy Requirement 1.2's absolute-UTC-form requirement.", "file": "src/flake.nix, src/nixSupport/default.nix, src/nixSupport/xce.nix, src/nixSupport/xcecli.nix", "root_cause": "First attempt computed myBuildTime = builtins.toString builtins.currentTime at flake-evaluation time (src/flake.nix), threaded through nixSupport/default.nix into xce.nix/xcecli.nix's mesonFlags. builtins.currentTime is stripped entirely under a flake's default pure evaluation (not just fixed to a dummy value -- the attribute itself is removed from `builtins`), so any nix build/eval that forces it fails unless --impure is passed. This also produced a raw epoch-seconds string, not an ISO-8601 UTC string like xce-build.sh's (task 2.1) `date -u +%Y-%m-%dT%H:%M:%SZ` output.", "fix": "Moved the build-time computation OUT of Nix expression evaluation and INTO each derivation's build phase. xce.nix and xcecli.nix each gained a `preConfigure` hook that runs `date -u +%Y-%m-%dT%H:%M:%SZ` (stdenv's bundled coreutils, always available in the build sandbox, unaffected by pure-eval, no --impure/git/network needed) and appends `-Dbuild_time=$buildTime` to `mesonFlagsArray` (verified against the real nixpkgs meson setup-hook.sh: mesonConfigurePhase runs `runHook preConfigure` BEFORE it reads mesonFlags/mesonFlagsArray via `concatTo`, so the appended flag is picked up at configure time). The hook `exit 1`s if `date` produces empty output, mirroring xce-build.sh's own non-empty-check discipline. Since build_time is no longer an eval-time concept, `myBuildTime` was removed entirely from flake.nix/nixSupport/default.nix/the xce.nix+xcecli.nix function signatures and their eval-time asserts -- only myCommitId (a genuine eval-time/flake-level value from self.shortRev/self.dirtyShortRev) still threads through. Live-verified: `nix eval --json .#packages.x86_64-linux.xcecore_debug.mesonFlags` (no --impure) succeeds; `nix build .#xcecore_debug` (no flags) proceeds through evaluation+configure+570/614 ninja targets (all major C binaries linked) before failing only on an unrelated pre-existing sandbox network restriction (autoscaler/bcctool's Go module fetch from goproxy.cn, nothing to do with this fix); build log shows `-Dbuild_time=2026-08-20T09:13:29Z` reaching meson configure and meson's own summary printing `build_time : 2026-08-20T09:13:29Z` (real, non-empty, ISO-8601 UTC). Same confirmed for xcecli.nix (`-Dbuild_time=2026-08-20T09:15:46Z`). `commit_id=bca6397-dirty` confirmed unaffected/still working in both.", @@ -21671,21 +8464,25 @@ "build-time-versioning", "meson", "preConfigure", - "mesonFlagsArray", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "mesonFlagsArray" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "bca6397" + ], + "record": ".wolf/bugs/bug-1603.md" }, { "id": "bug-1604", "status": "open", "status_updated": "2026-09-24", "status_note": "concurrent xce-build.sh runs race on rm -rf of shared BUILD_DIR (no lock)", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-20T10:24:39Z", - "error_message": "rm: cannot remove 'xce.build/controller/controller_unit_test.p': Directory not empty (xce-build.sh setup_build()'s rm -rf \\$BUILD_DIR step aborted the script under set -euo pipefail before meson setup ran)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "rm: cannot remove 'xce.build/controller/controller_unit_test.p': Directory not empty (xce-build.sh…\n\nrm: cannot remove 'xce.build/controller/controller_unit_test.p': Directory not empty (xce-build.sh setup_build()'s rm -rf \\$BUILD_DIR step aborted the script under set -euo pipefail before meson setup ran)", "file": "src/tools/xce-build.sh", "root_cause": "CORRECTED after further evidence (see fix note): initial overlayfs-flakiness theory was WRONG. The real cause is xce-build.sh defaulting BUILD_DIR to the same \"xce.build\" for every component and every invocation, with setup_build() doing an unconditional rm -rf \"$BUILD_DIR\" + fresh meson setup on every non-incremental run. When two xce-build.sh invocations run concurrently in the SAME repo working directory against DIFFERENT components (observed live: this session's \"xce-build.sh scheduler asan\" raced against a second, independently-running \"xce-build.sh controller asan\" process, PID 655158, started ~1 min later in the same cwd), they contend for the identical xce.build/ path -- one process's rm -rf can hit a directory the other process is actively writing into (\"Directory not empty\"), and later, a second clean-build pass can silently delete and replace a just-verified binary out from under a caller who believes their build is done. Confirmed live: xce_scheduler at src/xce.build/scheduler/xce_scheduler, verified present and fresh via ls -la and 'xce_scheduler --version' immediately after this session's scheduler build/test run, was gone minutes later -- the concurrent controller build's rm -rf had removed and was in the process of regenerating xce.build/ for its own component.", "fix": "No code fix applied (out of scope for the build-verification task this session was doing) -- flagging as a real, reproducible hazard rather than env flakiness. Practical mitigation for future sessions/agents: never assume xce.build/ (or any non-incremental xce-build.sh output) is stable once another concurrent xce-build.sh process might be running in the same working directory/worktree -- check 'ps aux | grep xce-build.sh' (or pass a distinct BUILD_DIR / use --incremental) before relying on a binary's presence or mtime surviving past the moment it was checked. Corrects this entry's own original overlayfs-race theory, which was a plausible but incorrect guess made before the concurrent process was discovered.", @@ -21698,24 +8495,25 @@ "sandbox-flake", "scheduler", "shared-build-dir", - "xce-build.sh", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "xce-build.sh" ], "related_bugs": [ "bug-1561" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1604.md" }, { "id": "bug-1605", "status": "cannot-reproduce", "status_updated": "2026-09-24", "status_note": "transient build failure, passed on immediate retry; likely bug-1604 collision", - "repro_attempts": "1 retry, clean", + "component": "lib", + "severity": "unknown", "timestamp": "2026-08-20T10:27:00.000Z", - "error_message": "controller ASAN build: two failures in the same ninja run -- (1) controller/config_repl.c:1921:28: error: 'RequestReply_data_probeconfigtagreplymsg' undeclared (first use in this function); did you mean 'RequestReply_data_upgradeconfigreplymsg' -- and (2) lib/controller.capnp.c:25013:1: fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory, compilation terminated; ninja: build stopped: subcommand failed.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller ASAN build: two failures in the same ninja run -- (1) controller/config_repl.c:1921:28…\n\ncontroller ASAN build: two failures in the same ninja run -- (1) controller/config_repl.c:1921:28: error: 'RequestReply_data_probeconfigtagreplymsg' undeclared (first use in this function); did you mean 'RequestReply_data_upgradeconfigreplymsg' -- and (2) lib/controller.capnp.c:25013:1: fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory, compilation terminated; ninja: build stopped: subcommand failed.", "file": "src/lib/controller.capnp.c (generated), src/controller/config_repl.c (consumer)", "root_cause": "Matches the capnp custom_target's documented missing-dependency-edge race (see comment block at top of src/tools/xce-build.sh, 'retry once before debugging'): config_repl.c was compiled against a stale/partially-regenerated controller.capnp.h that lacked the 'probeconfigtagreplymsg' enumerant even though src/schema/controller.capnp:596 defines it (probeconfigtagreplymsg @20: ProbeConfigTagReply), and lib/controller.capnp.c's own compile then hit an I/O race writing its .d dependency file, most likely because the generator custom_target re-ran/rewrote lib/ mid-build without ninja having a correct dependency edge ordering the two. Not caused by the concurrent main.c/testsuite/test.c argp_program_version diff (those files are unrelated to config_repl.c/capnp). CORRECTION (same session, minutes later): discovered via `ps aux` a THIRD live instance of a concurrent `nix develop ./src --command bash src/tools/xce-build.sh controller asan` process actively doing rm -rf xce.build + fresh meson compile in this SAME shared worktree at 10:38, i.e. immediately after this bug's own two build attempts (10:23-10:33) -- and .wolf/memory.md/cerebrum.md already documented an earlier occurrence THIS SAME SESSION of exactly this hazard (bug-1604: two concurrent xce-build.sh runs sharing default BUILD_DIR=xce.build race on rm -rf, one silently wiping/regenerating the other's in-progress build, corrupting or vanishing already-verified binaries). The 'known capnp codegen dependency-edge race' explanation above is therefore NOT confirmed as the true root cause -- a concurrent cross-session xce-build.sh collision (bug-1604) is at least as likely, and possibly the actual cause, of both the stale-header undeclared-enum error and the .d-file open I/O failure (a concurrent rm -rf mid-compile would produce exactly both symptoms). Do not treat this entry as proof the capnp custom_target race is real without ruling out concurrent-session collision first -- check `ps aux | grep xce-build.sh` before attributing a build failure to codegen internals in this worktree.", "fix": "Confirmed transient: re-ran the exact same command (nix develop ./src --command bash src/tools/xce-build.sh controller asan) a second time with zero source changes in between and the full clean build compiled successfully (202/202, zero errors). Not root-caused/fixed in the underlying capnp custom_target dependency graph -- xce-build.sh's own comments already document this as a known, unfixed gap and recommend exactly this retry-once workaround.", @@ -21729,24 +8527,28 @@ "ninja", "pre-existing", "race", - "transient", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "transient" ], "related_bugs": [ "bug-1439", "bug-1493", "bug-1604" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1605.md", + "repro_attempts": "1 retry, clean" }, { "id": "bug-1606", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-20T16:00:00Z", - "error_message": "grep: invalid option -- 'P' (BusyBox usage dump) when a standalone .sh script (run via `bash script.sh`, not typed as an inline Bash-tool command) calls `grep -oP '...\\K...'`", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "grep: invalid option -- 'P' (BusyBox usage dump) when a standalone .sh script (run via `bash…\n\ngrep: invalid option -- 'P' (BusyBox usage dump) when a standalone .sh script (run via `bash script.sh`, not typed as an inline Bash-tool command) calls `grep -oP '...\\K...'`", "file": "src/tools/test-version-startup-consistency.sh", "root_cause": "This coding-agent harness's interactive shell shadows plain `grep` with a `.bashrc`-sourced shell FUNCTION that shells out to a PCRE-capable (ugrep-backed) binary -- but that function is interactive-shell-only and is NOT inherited by a script invoked as `bash script.sh` (a separate, non-interactive process), which falls back to the real `grep` on PATH: BusyBox in this environment, with no -P/PCRE support at all. `grep -oP '(?<=x)y'` typed directly as a Bash-tool command therefore worked fine, masking the incompatibility until the same call was placed inside a committed script file and run standalone.", "fix": "Replaced the grep -oP '\"gitCommit\":\\s*\"\\K[^\"]*' style extraction with POSIX BRE `sed -n 's/.*\"gitCommit\":[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p'`, which BusyBox sed supports. General rule: any shell script meant to be run standalone (not just typed inline) must assume POSIX-only grep/sed (BRE, -E, -F, -o) -- no -P/PCRE, no lookbehind/\\K.", @@ -21757,20 +8559,23 @@ "sandbox", "portability", "sed", - "build-time-versioning", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "build-time-versioning" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1606.md" }, { "id": "bug-1607", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-20T16:15:00Z", - "error_message": "`git status` showed src/frontend/apiserver/go.sum modified (+65 lines) after a direct `go build ./cmd/main.go` invocation that never touched go.mod and produced a successful, working binary -- no warning or error printed by the build itself", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "`git status` showed src/frontend/apiserver/go.sum modified (+65 lines) after a direct `go build…\n\n`git status` showed src/frontend/apiserver/go.sum modified (+65 lines) after a direct `go build ./cmd/main.go` invocation that never touched go.mod and produced a successful, working binary -- no warning or error printed by the build itself", "file": "src/frontend/apiserver/go.sum", "root_cause": "A plain `go build` (default `-mod` setting, `go env GOFLAGS` confirmed empty) blocks go.mod EDITS under Go 1.16+'s readonly-by-default policy, but does NOT by itself block go.sum EXTENSION for modules already implied by go.mod's requirements -- resolving the full module graph for cgo-enabled apiserver added ~65 unrelated indirect-dependency /go.mod-hash-only lines to the committed go.sum as a silent side effect.", "fix": "Pass `-mod=readonly` explicitly on every direct `go build` invocation against this repo (src/tools/test-version-startup-consistency.sh's apiserver build step). Confirmed live: the identical build still succeeds under explicit -mod=readonly with zero go.sum diff, so nothing was genuinely missing -- the extra entries were pure side effect of the laxer default. Reverted the accidental go.sum change via `git checkout -- src/frontend/apiserver/go.sum` before re-running with the fix.", @@ -21780,20 +8585,23 @@ "go-modules", "build", "side-effect", - "build-time-versioning", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "build-time-versioning" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1607.md" }, { "id": "bug-1608", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "hang", "timestamp": "2026-08-20T15:57:00Z", - "error_message": "`xce-build.sh apiserver ` (meson -> go_build custom_target) hung indefinitely (8-10+ minutes, zero progress) building xcecli's Go dependencies; no error, no timeout, just stuck", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "`xce-build.sh apiserver ` (meson -> go_build custom_target) hung indefinitely…\n\n`xce-build.sh apiserver ` (meson -> go_build custom_target) hung indefinitely (8-10+ minutes, zero progress) building xcecli's Go dependencies; no error, no timeout, just stuck", "file": "src/tools/go_build", "root_cause": "go_build hardcodes `GOPROXY=https://goproxy.cn` plus a FRESH, per-invocation GOCACHE/GOPATH under meson's own $TMPDIR sandbox, bypassing the shell's already-populated default $GOPATH/pkg/mod module cache entirely. Confirmed via `ss -tnp` that the resulting `go` process had an ESTABLISHED (not refused/reset) TCP connection to 8.45.176.179:443 (=goproxy.cn, independently reachable fine via plain curl/`/dev/tcp` from the same shell) sitting in futex_do_wait with near-zero CPU time -- a genuinely stuck data path in this sandbox's networking for that specific fresh-cache+custom-GOPROXY combination, not mere slowness (never progressed no matter how long left running across 3 separate attempts). A plain `go list -m all` from an ordinary `nix develop` shell (no wrapper, default cache) resolved the identical apiserver module graph in under a second.", "fix": "Workaround only (go_build/Makefile NOT modified -- this is a sandbox-specific networking quirk, not a real defect in those files, and out of this task's scope to fix): for a standalone real apiserver/xcecli build in this sandbox, invoke `go build` directly against the default (already-populated) module cache with `GOPROXY=off` plus the same CGO_CFLAGS/CGO_LDFLAGS pkg-config derivation apiserver's own meson.build/test-backend already use (see bug-1597's documented fix for the exact derivation), instead of routing through xce-build.sh apiserver / go_build / make build-server. Diagnose a suspected hang via `ss -tnp` + `/proc//wchan` before assuming it just needs more time.", @@ -21805,22 +8613,25 @@ "hang", "sandbox", "meson", - "build-time-versioning", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "build-time-versioning" ], "related_bugs": [ "bug-1597" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1608.md" }, { "id": "bug-1609", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-20T21:10:00Z", - "error_message": "User-reported (live REPL session, xce_scheduler -i -s ): after 'continue' with zero breakpoints set, output is just as verbose as single-stepping -- every decoded event still prints a '[sim] t=... session=...' + '[sim] event: ...' two-line block, defeating continue's documented purpose ('Run until next breakpoint hit or EOF').", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (live REPL session, xce_scheduler -i -s ): after 'continue' with zero…\n\nUser-reported (live REPL session, xce_scheduler -i -s ): after 'continue' with zero breakpoints set, output is just as verbose as single-stepping -- every decoded event still prints a '[sim] t=... session=...' + '[sim] event: ...' two-line block, defeating continue's documented purpose ('Run until next breakpoint hit or EOF').", "file": "src/scheduler/src/repl_break.c", "root_cause": "xce_repl_on_event() (repl_break.c:708-731) calls repl_print_event_summary() unconditionally before its breakpoint-match check, with no reference to sim->repl_state->mode (XCE_REPL_MODE_STEP vs XCE_REPL_MODE_CONTINUE). This was intentional for bug-1595 ('in each step, ... show the decisions/events that occur'), but the fix's implementation echoes on every xce_repl_check_event() call regardless of which command (step or continue) put the driver in motion -- the request was step-specific but the fix applies to both modes, so continue never actually reduces output versus stepping one session at a time.", "fix": "FIXED (2026-09-02, scheduler-sim-repl-verbosity spec, task 3.2 -- design.md's Event Echo Gate component, Requirement 1). Close to the originally proposed shape, refined per design.md's fuller classification: xce_repl_on_event() (repl_break.c) now calls a new repl_event_is_structural(ev->type) classifier before deciding whether to call repl_print_event_summary() -- CONFIG_PUSH/JOBS_PUSH/WORKERS_PUSH/RESOURCES_PUSH/RESVALS_FULL (structural) are always echoed regardless of rs->mode (AC 1.3), while RESVALS_DELTA/JOB_STATE_CHANGE/JOB_UPDATE/QUEUE_UPDATE/JOB_RUSAGE (routine) are echoed only when rs->mode == XCE_REPL_MODE_STEP (AC 1.1/1.2), so `continue` now genuinely reduces output versus single-stepping, exactly this bug's original complaint. Breakpoint matching (repl_find_event_breakpoint()) is unaffected by the classification -- runs on every event regardless (AC 1.4), same as before. A structural event's own line, and a breakpoint-triggered repl_command_loop() re-entry (in xce_repl_on_event()/xce_repl_on_begin_round()/xce_repl_on_decision()), now also call repl_round_collapse_flush() (task 2.2, repl_rollup.c) first, so a mode-suppressed stretch of routine events that got collapsed into a Round Rollup range is reported before either of those (Req 3.5/3.6) -- a Requirement 2/3 dependency this bug's original proposal didn't anticipate (Round Rollup didn't exist yet in task 12.5's original codebase). TDD: task 3.1's RED-phase test_repl_event_echo.c (4 tests, real forked-child stdout-capture against sched_sim_driver_run()) pinned this exact contract before the fix landed; all 4 pass after. Full scheduler suite verified via /xce-build scheduler --incremental: 56/58 test binaries OK, the 2 failures being the pre-existing unrelated bug-1758 (test_core_events.c mem-reservation reconciliation) and a one-off ASAN stack-use-after-return in sched_sim_record_output_test logged separately as bug-1763 (same pre-existing thread-lifecycle class as bug-1598/bug-1632, triggered by this task's repl_break.c size change perturbing that binary's stack layout, not a correctness defect in this fix -- 4/4 isolated re-runs of that binary were clean).", @@ -21831,23 +8642,26 @@ "continue", "ux", "scheduler-sim-diagnostics", - "scheduler-sim-repl-verbosity", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "scheduler-sim-repl-verbosity" ], "related_bugs": [ "bug-1595", "bug-1763" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1609.md" }, { "id": "bug-1610", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-20T21:10:00Z", - "error_message": "User-reported (same live REPL session): under 'continue', an unchanged PEND decision for the same jobid prints an identical '[sim] PEND jobid=...' line every scheduling round it remains pending -- e.g. jobs [22]-[29] (8 jobs) print their 8 byte-identical PEND lines 5+ times in a row across consecutive rounds with zero new information each time.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (same live REPL session): under 'continue', an unchanged PEND decision for the same…\n\nUser-reported (same live REPL session): under 'continue', an unchanged PEND decision for the same jobid prints an identical '[sim] PEND jobid=...' line every scheduling round it remains pending -- e.g. jobs [22]-[29] (8 jobs) print their 8 byte-identical PEND lines 5+ times in a row across consecutive rounds with zero new information each time.", "file": "src/scheduler/src/sim_loop.c", "root_cause": "sim_print_decision() (sim_loop.c:388-411) unconditionally prints one DISPATCH/PEND/PREEMPT line per drained xce_decision_t with no de-duplication and no session/round context attached to the line itself -- called every round a job's allocation is (re-)attempted, so a job that stays PEND for N consecutive rounds produces N identical lines. xce_repl_on_decision() (repl_break.c:776-811), called alongside it, already tallies last_n_pend/last_n_dispatch/last_n_preempt per round for 'info session' -- that rollup exists but is never surfaced automatically, only on explicit 'info session'.", "fix": "FIXED (2026-09-02, scheduler-sim-repl-verbosity spec, task 4.2 -- design.md's `sim_print_decision` Gate component, option (b) of this bug's own original proposal, now that the Round Rollup -- task 2.2, repl_rollup.c -- already provides the per-round summary line to fall back on). sim_loop.c's decision-drain loop (sched_sim_driver_run()) now calls a new file-local sim_print_decision_gate(sim, d) instead of calling sim_print_decision(d) directly; the gate skips the bare per-decision print only when sim->interactive && sim->repl_state->mode == XCE_REPL_MODE_CONTINUE, leaving step mode and non-interactive/headless runs completely unaffected (Req 5.1, 5.4) -- the Round Rollup's own per-round summary becomes the sole source of decision output under continue. Implementation wrinkle discovered and resolved during this task: design.md describes this as an 'inline-wrapper-style gate ... consistent with repl.h's established convention', but repl.h's existing wrappers (xce_repl_check_decision() and siblings) only ever test the PUBLIC sim->interactive field inline and delegate all repl-internal-state logic to an extern function implemented in repl.c/repl_break.c (the only files with repl_internal.h access, where xce_sched_repl_state_t's full definition -- including `mode` -- actually lives; sim.h/repl.h deliberately keep repl_state an opaque forward-declared pointer). That pattern cannot be reused verbatim here because the function being gated, sim_print_decision(), is itself static (private to sim_loop.c) -- repl.h cannot wrap a conditional call to a symbol it cannot see, and moving the check into repl.h would require repl.h to include repl_internal.h, leaking the full repl_state struct into every other repl.h consumer. Resolved by having sim_loop.c include repl_internal.h directly (a private header already scoped to src/scheduler/src/, so this stays an intra-directory dependency, not a new external one) and implementing sim_print_decision_gate() as a file-local static inline wrapper co-located with sim_print_decision() -- same 'wrapper hides the condition, call site stays one bare call expression' style repl.h's wrappers use, just not physically in that shared header since the wrapped function isn't extern. No new extern symbol crosses out of sim_loop.c for this purpose, so test_sim_isolation.c's nm(1) undefined-symbol layer and source-text-scan layer are both unaffected. repl.h itself was left untouched, matching design.md's own File Structure Plan line for it ('no functional change expected'). TDD: task 4.1's RED-phase test_sim_print_decision_gate.c (3 tests, real forked-child/direct-call stdout capture) pinned this exact contract before the fix landed; all 3 pass after (continue-mode suppression, step-mode-unaffected regression guard, headless-batch-unaffected regression guard). sched_repl_round_rollup_test (7/7) and sched_repl_event_echo_test (4/4) reverified green post-fix. Full-suite verification via /xce-build scheduler --incremental surfaced 3 UNEXPECTED additional failures beyond the pre-existing bug-1758, confirmed (via a controlled before/after rebuild swapping only sim_loop.c) to be caused by this fix itself, not a defect in it: test_repl_execution_control.c's test_pre_replay_pause_blocks_before_any_event_is_processed, and test_repl_breakpoints_inspection.c's test_event_breakpoint_pauses_repl_at_matching_event and test_decision_breakpoint_pauses_after_decision_output -- all three pre-date this spec and assert on the bare '[sim] PEND jobid=...' line's presence in a continue-mode capture, i.e. they pin exactly the bug-1610 behavior this fix intentionally removes. Left unmodified per this task's explicit no-test-file-editing boundary; flagged to the parent controller/reviewer as a real, reproducible side effect requiring a follow-up task to update those 3 assertions (e.g. swap their continue scripts for step, or assert on the Round Rollup's summary line instead) rather than a regression in this fix's own logic. **Remediation (2026-09-02, task 4.2 remediation, post-review):** the fix above only covered ONE of sim_loop.c's two direct sim_print_decision() call sites -- sched_sim_driver_run()'s main decision-drain loop. Review found a second, direct, unguarded call inside sim_reconcile_unaccounted_jobs() (added by the sibling scheduler-sim-decision-parity/scheduler-sim-eager-reconciliation specs after this bug's original fix landed, unrelated in authorship but real production code reporting DISPATCH decisions through this exact same channel) -- and that reconciliation path is actually a PRIMARY mechanism by which DISPATCH decisions surface in replay mode at all (most live-drain-loop activity is PEND), so a round where reconciliation fired was still showing both the bare per-decision line AND the Round Rollup's summary counting the same dispatch: this bug's own duplication, reproduced via the second call site design.md's original Boundary Commitments never accounted for (it only ever named \"the existing call site\", singular). Fixed by routing that call site through the identical sim_print_decision_gate(sim, d) too (same function, same condition, no new symbol) -- sim_loop.c:760's bare sim_print_decision(d) became sim_print_decision_gate(sim, d). TDD: two new tests added to test_sim_reconcile_unaccounted.c (test_reconcile_continue_mode_with_repl_state_suppresses_bare_decision_line / test_reconcile_step_mode_with_repl_state_still_prints_bare_decision_line), using the repl_state_ensure()-based direct-call fixture technique test_repl_round_rollup.c's own fixtures establish; the pre-existing test_reconcile_reports_reconstructed_decision_through_existing_channel (which deliberately leaves sim->repl_state NULL) was empirically re-verified to still pass unmodified, confirming the gate's NULL-safety default (\"never load-bearing, degrade to print\") covers it. Full-suite verification via /xce-build scheduler --incremental: sched_sim_reconcile_unaccounted_test now 16/16 (was 14/14), sched_sim_print_decision_gate_test 3/3, sched_repl_round_rollup_test 7/7, sched_repl_event_echo_test 4/4, sched_repl_execution_control_test 6/6, sched_repl_breakpoints_inspection_test 30/30 -- all green, confirming this bug's own previously-flagged test_repl_execution_control.c/test_repl_breakpoints_inspection.c follow-up (3 tests pinning the original bug-1610 behavior) was already resolved by an earlier task before this remediation. The only other full-suite failure remains the pre-existing, unrelated bug-1758 (sched_core_events_test). This closes the duplication gap completely -- both production call sites now gated.", @@ -21857,23 +8671,25 @@ "verbosity", "duplicate-output", "pend", - "scheduler-sim-diagnostics", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "scheduler-sim-diagnostics" ], "related_bugs": [ "bug-1609" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1610.md" }, { "id": "bug-1611", "status": "fixed", "status_updated": "2026-09-24", "status_note": "e2bb98c8, 984a6869", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-21T15:10:00Z", - "error_message": "Discovered analyzing a real 108-session replay (xce_scheduler -i -s scheduler-replay-20260820-070103.replay, continue to EOF): sim_print_decision() (sim_loop.c) never printed a single DISPATCH or PREEMPT line across the entire replay -- only PEND -- even though 22 of the 30 jobs pushed at round 32 clearly transitioned to running (confirmed by ongoing JOB_UPDATE/JOB_RUSAGE ticks for jobids [0]-[21] every few rounds through round 107). An operator watching only decision output has zero confirmation those 22 jobs ever dispatched; they just silently stop appearing in PEND.", + "last_seen": "2026-09-19T14:00:00Z", + "occurrences": 3, + "error_message": "Discovered analyzing a real 108-session replay (xce_scheduler -i -s…\n\nDiscovered analyzing a real 108-session replay (xce_scheduler -i -s scheduler-replay-20260820-070103.replay, continue to EOF): sim_print_decision() (sim_loop.c) never printed a single DISPATCH or PREEMPT line across the entire replay -- only PEND -- even though 22 of the 30 jobs pushed at round 32 clearly transitioned to running (confirmed by ongoing JOB_UPDATE/JOB_RUSAGE ticks for jobids [0]-[21] every few rounds through round 107). An operator watching only decision output has zero confirmation those 22 jobs ever dispatched; they just silently stop appearing in PEND.", "file": "src/scheduler/src/sim_loop.c", "root_cause": "Not yet root-caused with certainty. Plausible, benign explanation: in --sim/-s REPLAY mode, jobs already resolved (dispatched) in the ORIGINAL recorded run may be replayed purely as state deltas (JOB_UPDATE/JOB_RUSAGE) without the live picker/allocator re-deriving a fresh XCE_DECISION_DISPATCH -- only jobs still outstanding (PEND) get freshly re-decided every round, which is why PEND is the only decision op ever observed. Unconfirmed against sim_loop.c's actual replay-vs-live-decide branching; needs a source read of how sim_loop.c distinguishes replayed-decision playback from freshly-computed decisions before concluding this is 'working as designed' vs a real visibility gap.", "fix": "Not yet fixed. If confirmed benign (dispatch legitimately has no fresh decision object in replay mode), the round-summary rollup being designed for bug-1609/1610 should not label its dispatch count misleadingly (a permanent 'X dispatch' column reading 0 for an entire replay looks like a bug even when it isn't) -- consider deriving a running-job delta from info-jobs-equivalent state instead of the decision tally for replay-mode display. If NOT benign (a real decision-visibility gap), this is a separate, more serious finding than the formatting issues in bug-1609/1610.\n\n--- STATUS UPDATE 2026-08-24 (xce-audit-invariants scheduler audit) ---\n\nRoot cause confirmed and the literal mechanism is now built: sim_reconcile_unaccounted_jobs() (sim_loop.c:566-709, landed via scheduler-sim-decision-parity tasks 2.1-3, 2026-08-22) reconstructs and reports a decision for a RUN-state job whose DISPATCH decision was only in the recorded replay stream. BUT its trigger gate (sim_loop.c:592) requires JOB_IS_RUN(job->state) before searching, and the search only scans the CURRENT round's decoded_session (decoded fresh at sim_loop.c:871, freed at line 999 -- no cross-round persistence). So reconciliation only fires when a job's DISPATCH decision and its RUN confirmation land in the exact same replay round. This bug's own original scenario (dispatch at round 32, RUN confirmation through round 107) never matches -- the round-32 DISPATCH decision is gone by the time the job reaches RUN state. The scheduler-sim-decision-parity spec's own task 4.3 implementation notes explicitly acknowledge this residual does NOT fix bug-1611's dominant real-world manifestation. A follow-up spec already exists for this exact residual: .kiro/specs/scheduler-sim-eager-reconciliation/ (created 2026-08-22, phase=requirements-generated, ready_for_implementation=false -- design/tasks not yet generated). Its diagnosis: decouple allocation CAPTURE from the JOB_IS_RUN gate (capture as soon as a DISPATCH decision is seen for a job with job->accounts == NULL, regardless of state), leaving sched_account_jobs()'s existing JOB_IS_RUN + job->accounts != NULL gate to keep deferring actual charging until state transitions to RUN. Fix location: sim_reconcile_unaccounted_jobs() (sim_loop.c:566-709), trigger gate at line 592, plus its ownership-transfer logic (609-693) needs a double-capture guard since capture would no longer imply state==RUN. Do NOT fix inline as a drive-by patch per scheduler-driver-equivalence.md's testing discipline -- promote scheduler-sim-eager-reconciliation to design/tasks instead.\n\n--- STATUS UPDATE 2026-08-27 (scheduler-sim-eager-reconciliation task 1.1) ---\n\nCore production fix landed: sim_reconcile_unaccounted_jobs() trigger gate (sim_loop.c:592) had !JOB_IS_RUN(job->state) removed, leaving job->accounts == NULL as the sole capture-eligibility gate -- exactly the diagnosis from the 2026-08-24 status update. Match/capture/report mechanics (609-693) were confirmed already safe for early (pre-RUN) capture without changes (see scheduler-sim-eager-reconciliation/research.md's uncharge-safety research log -- sched_uncharge_account() has zero production callers, accounting is reset-and-resum not incremental, so no double-capture guard beyond the existing job->accounts != NULL check was needed). Old test test_reconcile_skips_non_run_state_job (which asserted the bug as correct behavior) replaced with test_reconcile_captures_non_run_state_job_when_dispatch_recorded (inverted expectation). RED/GREEN verified: new test failed under the old gate (job->allocs stayed NULL, confirmed via a real build+run before the fix), then passed after the one-line fix; full sched_sim_reconcile_unaccounted_test suite green (10/10, all prior regression tests unchanged). This is task 1.1 of the spec only -- task 1.2 (capture-safety unit coverage), task 2.1 (resInUse deferral integration test), and task 3.x (contention/--sim-verify/full-regression validation) are still pending before the spec itself is implementation-complete. CORRECTED STATUS UPDATE 2026-09-19 (superseding the immediately-preceding note in this same entry, which itself worked from stale premises): scheduler-sim-eager-reconciliation (spec completed 2026-09-11, BEFORE this session's audit ran) already decoupled sim_reconcile_unaccounted_jobs()'s trigger from the JOB_IS_RUN gate -- the function now fires on job->accounts == NULL alone, regardless of PEND/RUN state (sim_loop.c:560-572), so a job's DISPATCH decision is captured in the SAME round it is recorded, never deferred until a later, separately-recorded RUN-state confirmation event. This closes the exact real-world scenario that motivated this bug (dispatch round 32, confirmation round 107) for good -- the previous note's claimed \"residual gap\" (a job already-RUN before the replay window starts) is not a real gap either: such a job's allocation is populated directly from the snapshot at load time (xce_sched_core_load_snapshot(), confirmed field-complete for job->allocs/accounts by this same audit's dimension-4 pass), so job->accounts is already non-NULL immediately post-snapshot-load and such a job is never a reconciliation candidate in the first place. Combined with scheduler-sim-reconcile-terminal-guard (bug-1764, excludes terminal jobs from false rematch) and scheduler-sim-live-allocation (the live picker/allocator now handles most jobs directly, reconciliation is a narrow safety net), this entry is FULLY FIXED. STATUS: FIXED, not PARTIALLY FIXED.", @@ -21884,9 +8700,7 @@ "dispatch", "decision-visibility", "scheduler-sim-diagnostics", - "scheduler-sim-eager-reconciliation", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "scheduler-sim-eager-reconciliation" ], "related_bugs": [ "bug-1764", @@ -21894,15 +8708,23 @@ "bug-1610", "bug-1609" ], - "occurrences": 3, - "last_seen": "2026-09-19T14:00:00Z" + "fix_commits": [ + "e2bb98c8", + "984a6869" + ], + "record": ".wolf/bugs/bug-1611.md" }, { "id": "bug-1612", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-21T00:00:00Z", - "error_message": "Adversarial review REJECTED dirty-content-fingerprint task 4.3 (src/frontend/Makefile DIRTY_FINGERPRINT): when `git diff HEAD` failed but the following `git ls-files | sort | xargs cat` pipe still succeeded, the computation silently proceeded and emitted a fabricated non-empty fingerprint instead of degrading to empty.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Adversarial review REJECTED dirty-content-fingerprint task 4.3 (src/frontend/Makefile…\n\nAdversarial review REJECTED dirty-content-fingerprint task 4.3 (src/frontend/Makefile DIRTY_FINGERPRINT): when `git diff HEAD` failed but the following `git ls-files | sort | xargs cat` pipe still succeeded, the computation silently proceeded and emitted a fabricated non-empty fingerprint instead of degrading to empty.", "file": "src/frontend/Makefile", "root_cause": "The `$(shell ...)` block wrote both steps into the scratch file via `if { git diff HEAD 2>/dev/null; git ls-files -z ... | sort -z | xargs -0 -r cat -- 2>/dev/null; } >\"$fp_tmp\" 2>/dev/null; then`. POSIX shell only propagates the exit status of the LAST command in a `{ cmd1; cmd2; }` sequential group (and `sh` here has no `pipefail`), so a `git diff HEAD` failure was silently discarded whenever the trailing ls-files/sort/xargs pipe still exited 0 -- unlike xce-build.sh's task 2.1 reference implementation, which tracks per-step success explicitly via an `fp_ok` flag.", "fix": "Restructured the DIRTY_FINGERPRINT $(shell ...) block to gate each step individually, mirroring xce-build.sh's fp_ok discipline: `git diff HEAD >\"$fp_tmp\"` is now its own `if ! ...; then fp_ok=0; fi` step, and the ls-files/sort/xargs pipe (appending via `>>`) is a separate `if [ \"$fp_ok\" = 1 ] && ! ...; then fp_ok=0; fi` step; hashing only proceeds if fp_ok stayed 1 through both. Verified via a wrapper `git` on PATH that fails only `git diff` (all other subcommands pass through to the real git): before the fix this produced a fabricated hash (e.g. f2cd51ec) instead of the reference implementation's empty result; after the fix it resolves to empty, matching xce-build.sh under the identical injected failure. Cross-checked the happy path remains byte-identical to xce-build.sh's standalone computation (e.g. 96726026), the sha256sum-missing fallback (Requirement 5.1) still degrades to empty, GIT_COMMIT's own `-dirty` suffix is unaffected in all three scenarios, and a real `nix develop`-driven `make build-cli` + `go version -m` shows the matching fingerprint (f60eb4be) embedded in the -ldflags.", @@ -21913,21 +8735,23 @@ "dirty-content-fingerprint", "task-4.3", "code-review-rejection", - "fp_ok", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fp_ok" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1612.md" }, { "id": "bug-1613", "status": "fixed", "status_updated": "2026-09-24", "status_note": "e2bb98c8 (scheduler-sim-decision-parity)", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-21T07:25:00Z", - "error_message": "User-reported (xce_scheduler -i -s , 'info worker '): every CONSUMABLE resource's used= value reads 0.00 for a worker regardless of how many jobs are actually dispatched and running on it, e.g. 'resource slots: value=22.00 used=0.00' and 'resource mem: value=17286225920.00 used=0.00' with jobs clearly active (confirmed via ongoing JOB_RUSAGE ticks in the same session). Confirmed real, not a display bug.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 4, + "error_message": "User-reported (xce_scheduler -i -s , 'info worker '): every CONSUMABLE resource's…\n\nUser-reported (xce_scheduler -i -s , 'info worker '): every CONSUMABLE resource's used= value reads 0.00 for a worker regardless of how many jobs are actually dispatched and running on it, e.g. 'resource slots: value=22.00 used=0.00' and 'resource mem: value=17286225920.00 used=0.00' with jobs clearly active (confirmed via ongoing JOB_RUSAGE ticks in the same session). Confirmed real, not a display bug.", "file": "src/scheduler/src/sim_loop.c", "root_cause": "sched_account_jobs() (scheduler.c:1255, exported via scheduler.h) is the ONLY mechanism that re-populates per-worker (w->resInUse) and shared (core->sharedInUse) resource-in-use accounting after each session's resval reset zeroes them (core.h's own doc comment on xce_core_begin_session()/xce_core_signal_go(): 'the picker thread must never observe a session whose resource-in-use accounting has been reset to zero ... but not yet re-summed from currently running jobs (sched_account_jobs())'). This obligation was split out (scheduler-core spec task 5.2) into xce_core_begin_session()+xce_core_signal_go() specifically so a driver can call sched_account_jobs() in between; the normal-mode driver does this correctly (driver.c:739-749, xce_sched_driver_run_normal_mode_round(): xce_core_begin_session() -> plugin_begin_session()+sched_account_jobs() -> xce_core_signal_go()). The simulation driver (sim_loop.c:680) was never migrated to this split -- it still calls the older monolithic xce_core_go() (core_session.c:402-409, itself now just xce_core_begin_session()+xce_core_signal_go() called back-to-back with zero interleaving slot), and sim_loop.c/sim.c/sim_verify.c/sim_record.c contain NO call to sched_account_jobs() anywhere (grep-confirmed). Consequently every session's resval reset in simulation/replay mode is never followed by re-accumulation, so w->resInUse/core->sharedInUse stay at their zero-initialized value for the entire run -- even a job freshly dispatched via sched_commit_decision()'s direct sched_charge_account() call (scheduler.c:1438-1443) would have that charge wiped by the very next session's reset with nothing to re-sum it. Broader concern (not yet verified): the live allocator thread's own capacity checks (sched_try_alloc_on_host_ -> sched_check_host_satisfy/sched_meet_host_selection, resource.c) may read w->resInUse to compute free capacity -- if so, the simulation driver could be scheduling against permanently-zero 'in use' data, i.e. believing full capacity is always available regardless of what is actually running, which would be a correctness risk for scheduler-core's documented normal-mode/sim-mode decision-equivalence invariant, not merely a REPL display cosmetic. This needs confirmation before being treated as proven.", "fix": "Not yet fixed -- investigation only, per user request. Proposed direction: call sched_account_jobs(sched) in sim_loop.c's round loop, in the same relative position driver.c uses (after xce_core_begin_session()'s resval reset, before signaling the picker/allocator thread) -- likely requires migrating sim_loop.c from the monolithic xce_core_go() to the split xce_core_begin_session()+xce_core_signal_go() calls, mirroring xce_sched_driver_run_normal_mode_round()'s exact pattern (driver.c:739-749). Should be paired with confirming/refuting the broader allocator-capacity-check concern above before scoping a fix.\n\n--- CONFIRMED 2026-08-21 (scheduler-sim-decision-parity spec research) ---\n\nThe \"broader concern (not yet verified)\" above is CONFIRMED, not theoretical: `sched_check_host_satisfy_()`/`sched_check_shared_satisfy_()` (resource.c) read `w->resInUse`/`core->sharedInUse` directly as inputs to the live allocator's own per-host capacity check (`sched_try_alloc_on_host_`'s viability test). With `resInUse` permanently zero in sim/replay mode, simulation-mode capacity checks can systematically pass where normal-mode execution of the same recorded state would fail -- this is a scheduling-decision-correctness bug under resource contention, not merely a REPL display cosmetic.\n\nAlso determined to likely share one root cause with bug-1611 (see that entry): sim_loop.c decodes the recorded decision stream (`decoded_session->decisions`) every round but only ever uses it for optional `--sim-verify` comparison, never to reconstruct state (job->allocs/job->accounts) or report a decision for a job already resolved before the replay reaches it.\n\nThis is now elevated to an enforced steering invariant: `.kiro/steering/scheduler-driver-equivalence.md` (\"normal-mode and sim-mode must produce identical decisions AND identical resource-accounting state, not just a matching decision sequence\").\n\nA dedicated fix spec has been scoped (deliberately not folded into the closed scheduler-core spec or the narrower scheduler-sim-repl-verbosity spec): `.kiro/specs/scheduler-sim-decision-parity/` (requirements approved; design generated, not yet approved; tasks not yet generated -- next step is /kiro-validate-design or /kiro-spec-tasks). Chosen approach: reconstruct job->allocs/job->accounts for already-resolved RUN-state jobs from the recorded decision stream at the round-boundary accounting point, keyed/deduped on job->accounts == NULL (the same signal sched_account_jobs() already uses), and feed the reconstructed decision through the same on_decision/xce_repl_check_decision reporting path a fresh decision uses -- fixing bug-1613's accounting gap and bug-1611's decision-visibility gap as one change. See that spec's research.md for full architecture-option evaluation and risks (--sim-verify interaction, snapshot-restored jobs with no in-window recorded decision).\n\n--- PARTIAL FIX 2026-08-22 (scheduler-sim-decision-parity task 2.1) ---\n\nsim_loop.c's round boundary now calls the relocated sched_account_jobs(&(sim->core)) (core.h, task 1.1) in exactly the position driver.c uses: xce_core_go() replaced with the split xce_core_begin_session() -> [reconciliation insertion point, not yet built] -> sched_account_jobs() -> xce_core_signal_go() sequence. Proven via a new spy-based regression test (test_sim_session_ordering.c): 0 calls pre-fix (RED, matches this bug's own symptom exactly), 2 calls (one per replayed round) with core->session_active==1 post-fix (GREEN). Interactive and batch sim runs share this one call site (Req 5.1), no change to normal-mode behavior (Req 5.3) -- full scheduler suite 45/46 OK post-fix, only pre-existing unrelated bug-1560 fails.\n\nSTILL NOT FULLY OBSERVABLE END-TO-END: confirmed while implementing this task that job->accounts/job->allocs is ALWAYS NULL for every RUN-state job in sim mode today -- sim_picker_thread_fn() never emits a live DISPATCH (unconditionally PENDs every candidate), and sched_commit_decision() (the only function that ever sets job->accounts) is only ever called from scheduler.c's real sched_allocator() thread, which sim mode never runs. sched_account_jobs()'s round-boundary re-charge therefore has nothing to charge yet in isolation -- 'info worker' will still show used=0.00 until Recorded-Decision Reconciliation (scheduler-sim-decision-parity tasks 2.2-2.4, not yet implemented) populates job->accounts from the recorded decision stream. This task deliberately leaves a documented insertion point for that work (sim_loop.c comment, right before the sched_account_jobs() call) rather than building it now (out of task 2.1's own scope). Do not close this bug until tasks 2.2-2.4 land.\n\n--- CONFIRMED FIXED 2026-08-24 (xce-audit-invariants scheduler audit) ---\n\nscheduler-sim-decision-parity's remaining tasks (2.2-4.3) are now implementation-complete (spec.json phase=implementation-complete, updated_at 2026-08-22T13:10:00Z). sim_reconcile_unaccounted_jobs() (sim_loop.c:566-709) now searches each round's decoded replay decisions for RUN-state jobs with job->accounts == NULL, takes ownership of the matching DISPATCH allocation, builds job->accounts via make_job_account(), and feeds a reconstructed decision through the same reporting path a live decision uses -- called at sim_loop.c:948, immediately before sched_account_jobs() at line 950 (itself already relocated to take xce_sched_core_t* -- scheduler.c:1258 -- with no remaining signature mismatch against xce_sched_sim_driver_t). Live-verified: 'info worker' shows nonzero used= at multiple checkpoints (.wolf/memory.md line 3342). Full accounting-parity gap closed. Do not reopen for the same-round-only limitation -- that residual gap is bug-1611's concern, tracked separately (see bug-1611 and scheduler-sim-eager-reconciliation spec).", @@ -21940,22 +8764,27 @@ "sched_account_jobs", "correctness", "repl", - "confirmed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "confirmed" ], "related_bugs": [ "bug-1611" ], - "occurrences": 4, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "e2bb98c8" + ], + "record": ".wolf/bugs/bug-1613.md" }, { "id": "bug-1614", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-08-21T08:50:00Z", - "error_message": "pytest FAILED xceadmin/repo-mirror/src/xceadmin/tests/test_cli.py::test_cli_version_flag_includes_build_info_fields -- AssertionError: assert '2bf62a64-dirty' in 'xceadmin, version 0.1.0 (2bf62a64-857bd5b5, 2026-08-21T08:49:15Z)\\n' (1 failed, 266 passed, 3 skipped) -- blocks meson's xceadmin-build.stamp custom_target, so no wheel is produced.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "pytest FAILED…\n\npytest FAILED xceadmin/repo-mirror/src/xceadmin/tests/test_cli.py::test_cli_version_flag_includes_build_info_fields -- AssertionError: assert '2bf62a64-dirty' in 'xceadmin, version 0.1.0 (2bf62a64-857bd5b5, 2026-08-21T08:49:15Z)\\n' (1 failed, 266 passed, 3 skipped) -- blocks meson's xceadmin-build.stamp custom_target, so no wheel is produced.", "file": "src/xceadmin/tests/test_cli.py", "root_cause": "dirty-content-fingerprint task 5.2's _compose_version_display() strip-and-append rule (cli.py:59-75, already implemented, matches design.md/tasks.md 5.2 exactly and all its OWN dedicated tests pass: test_compose_version_display_strips_dirty_suffix_and_appends_fingerprint, ..._anchored_suffix_not_blind_substring, test_compose_version_includes_fingerprint_when_build_info_has_one) now legitimately conflicts with an older, pre-existing task-2.2 test (test_cli_version_flag_includes_build_info_fields, added under Requirement 2.2 before this feature existed) that still asserts the RAW _build_info.GIT_COMMIT string (e.g. '2bf62a64-dirty') appears verbatim in `xceadmin --version` output. On a dirty tree with a non-empty DIRTY_FINGERPRINT, the composed output now correctly shows '2bf62a64-857bd5b5' (dirty suffix stripped, fingerprint appended per spec) instead of containing the literal '...-dirty' substring, so the old assertion is now stale and fails. tasks.md 5.2 is still unchecked ([ ]) -- this stale test update was left out of scope / not yet done.", "fix": "RESOLVED CONCURRENTLY, not by this run: the first `xce-build.sh xceadmin asan ` invocation (BUILD_TIME 2026-08-21T08:49:15Z) failed exactly as described above (1 failed, 266 passed, 3 skipped). While investigating, found `src/xceadmin/tests/test_cli.py` in the LIVE working tree already carried an uncommitted rewrite of test_cli_version_flag_includes_build_info_fields (now computes the expected display via the same strip-and-append contract cli.py's _compose_version_display() uses, instead of asserting raw GIT_COMMIT verbatim) -- but its mtime (08:50:55Z) POSTDATES the first build's BUILD_TIME, meaning some other concurrent process/session applied this fix mid-investigation, not this build-verification agent. Re-ran the identical xce-build.sh command a second time against the now-updated tree (BUILD_TIME 2026-08-21T08:55:40Z, dirty_fingerprint recomputed to c0c2a0b7 since tracked-file content changed) and it now passes clean: '267 passed, 3 skipped in 7.65s', wheel built at .../xceadmin/dist/xceadmin-0.1.0-py3-none-any.whl. Installed that wheel alone into a fresh throwaway venv (nix develop wrapper, matching this project's established pattern) and confirmed `xceadmin --version` -> 'xceadmin, version 0.1.0 (2bf62a64-c0c2a0b7, 2026-08-21T08:55:40Z)', exit 0 -- dirty suffix correctly stripped and replaced with the fingerprint, not doubled up. Task 5.2 appears functionally complete now; tasks.md still shows it unchecked ([ ]) as of this writing -- someone should flip it once the concurrent editor's change is reviewed/committed.", @@ -21969,20 +8798,25 @@ "regression", "stale-test", "meson", - "concurrent-edit-observed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "concurrent-edit-observed" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "2bf62a64" + ], + "record": ".wolf/bugs/bug-1614.md" }, { "id": "bug-1615", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-21T09:37:18Z", - "error_message": "src/tools/test-dirty-fingerprint-distinguishability.sh: line 344: iso_src: unbound variable", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "src/tools/test-dirty-fingerprint-distinguishability.sh: line 344: iso_src: unbound variable\n\nsrc/tools/test-dirty-fingerprint-distinguishability.sh: line 344: iso_src: unbound variable", "file": "src/tools/test-dirty-fingerprint-distinguishability.sh", "root_cause": "capture_version()'s `local iso_src=\"$1\" binary=\"$iso_src/$BUILD_DIR_NAME/scheduler/xce_scheduler\"` put two assignments in ONE `local` statement where the second referenced the first. Bash expands every word of a single `local` command before ANY of that command's assignments take effect, so `$iso_src` in the second word's value expanded against the OLD (not-yet-declared) `iso_src`, which under `set -u` is an unbound-variable error rather than silently empty.", "fix": "Split into two separate `local` statements (`local iso_src=\"$1\"` then `local binary=\"$iso_src/...\"` on its own line) so the second statement's expansion happens after the first has already taken effect. Confirmed via `bash -n` plus two full real end-to-end script runs (both exit 0, byte-identical fingerprints reproduced across runs) after the fix.", @@ -21993,20 +8827,23 @@ "unbound-variable", "dirty-content-fingerprint", "task-6.1", - "src/tools", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "src/tools" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1615.md" }, { "id": "bug-1616", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-21T00:00:00Z", - "error_message": "src/tools/test-dirty-fingerprint-nix-production-path.sh: line 191: flag: unbound variable", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "src/tools/test-dirty-fingerprint-nix-production-path.sh: line 191: flag: unbound variable\n\nsrc/tools/test-dirty-fingerprint-nix-production-path.sh: line 191: flag: unbound variable", "file": "src/tools/test-dirty-fingerprint-nix-production-path.sh", "root_cause": "extract_flag_from_json()'s `local json=\"$1\" flag=\"$2\" needle=\"-D${flag}=\"` recurred the exact same-statement `local` cross-reference gotcha already documented as bug-1615 (test-dirty-fingerprint-distinguishability.sh's capture_version()): bash expands every word of a single `local` command before any of that command's own assignments take effect, so `${flag}` in the third word's value expanded against the not-yet-declared `flag`, an unbound-variable error under `set -u`.", "fix": "Split into two separate `local` statements (`local json=\"$1\" flag=\"$2\"` then `local needle=\"-D${flag}=\"` on its own line), with an inline comment citing bug-1615 explicitly so a future reader recognizes the pattern immediately instead of re-discovering it. Confirmed via `bash -n` plus a full real end-to-end script run (exit 0).", @@ -22018,22 +8855,25 @@ "dirty-content-fingerprint", "task-6.3", "src/tools", - "recurring-pattern", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "recurring-pattern" ], "related_bugs": [ "bug-1615" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1616.md" }, { "id": "bug-1617", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-21T00:00:00Z", - "error_message": "mktemp: Invalid argument", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "mktemp: Invalid argument\n\nmktemp: Invalid argument", "file": "src/tools/test-dirty-fingerprint-nix-production-path.sh", "root_cause": "Used a GNU-coreutils-only mktemp template shape (`.../name.XXXXXX.log`, trailing literal suffix after the X run) assuming GNU mktemp's rule that XXXXXX only needs to be present, with a literal suffix like `--suffix` allowed after it in some historical GNU versions. This sandbox's `mktemp` is BusyBox's implementation, which requires the template to literally END with XXXXXX (no trailing suffix at all, and no `--suffix` flag support either) -- confirmed via `mktemp --help` showing BusyBox v1.36.1's usage text.", "fix": "Dropped the `.log` suffix from the template entirely (`.../xce-dirty-fp-nix-build-A.XXXXXX` with nothing after the X run) rather than reaching for `--suffix` (also BusyBox-unsupported). The isolated worktree's own top-level mktemp template (no suffix) already worked correctly and was the right pattern to copy for the two new build-log mktemp calls.", @@ -22044,20 +8884,23 @@ "portability", "dirty-content-fingerprint", "task-6.3", - "src/tools", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "src/tools" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1617.md" }, { "id": "bug-1618", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-21T00:00:00Z", - "error_message": "error: pathspec 'flake.nix' did not match any file(s) known to git", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "error: pathspec 'flake.nix' did not match any file(s) known to git\n\nerror: pathspec 'flake.nix' did not match any file(s) known to git", "file": "src/tools/test-dirty-fingerprint-nix-production-path.sh", "root_cause": "`git -C \"$ISO_DIR\" checkout -- \"$EDIT_FILE_REL\"` used the isolated worktree's ROOT ($ISO_DIR) as the -C directory while EDIT_FILE_REL was just \"flake.nix\" (unqualified), but the edited file actually lives at $ISO_DIR/src/flake.nix -- git resolves pathspecs relative to the -C directory, so \"flake.nix\" alone did not match anything from the worktree root. Unlike the sibling task 6.1/6.2 scripts (which edit README.md directly at the worktree ROOT, where this exact pattern is correct), this script's edit target is one level deeper inside src/, which the copied pattern did not account for.", "fix": "Changed both `git -C \"$ISO_DIR\" checkout -- \"$EDIT_FILE_REL\"` call sites to `git -C \"$ISO_SRC\" checkout -- \"$EDIT_FILE_REL\"` (ISO_SRC = $ISO_DIR/src), so the pathspec correctly resolves relative to the flake's own source directory where flake.nix actually lives.", @@ -22067,20 +8910,23 @@ "worktree", "dirty-content-fingerprint", "task-6.3", - "src/tools", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "src/tools" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1618.md" }, { "id": "bug-1619", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-21T00:00:00Z", - "error_message": "FAIL: this script itself contains the string '--impure' -- the Observable requires no --impure flag anywhere.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "FAIL: this script itself contains the string '--impure' -- the Observable requires no --impure…\n\nFAIL: this script itself contains the string '--impure' -- the Observable requires no --impure flag anywhere.", "file": "src/tools/test-dirty-fingerprint-nix-production-path.sh", "root_cause": "The script's own self-check for 'no --impure flag used anywhere' did a blind whole-file `grep -q -- '--impure' \"$0\"`. The script's own header documentation and status messages deliberately discuss '(no --impure)' in prose extensively (to explain the Observable to a reader) -- a false positive against the script's own comments/log strings, not an actual nix invocation using the flag.", "fix": "Narrowed the check to `grep -A2 -E '^[[:space:]]*nix (eval|build|develop|log)\\b' \"$0\" | grep -q -- '--impure'` -- only lines that begin a real nix command invocation (plus 2 following lines, to cover this script's own multi-line backslash-continued invocations), not prose. Verified the narrower grep correctly isolates exactly the script's two real `nix eval`/`nix build` invocation sites and neither contains --impure.", @@ -22091,20 +8937,23 @@ "false-positive", "dirty-content-fingerprint", "task-6.3", - "src/tools", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "src/tools" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1619.md" }, { "id": "bug-1620", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-21T15:56:00Z", - "error_message": "ERROR: Unknown option: \"dirty_fingerprint\".", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ERROR: Unknown option: \"dirty_fingerprint\"\n\nERROR: Unknown option: \"dirty_fingerprint\".", "file": "src/tools/xce-build.sh (setup_build, --incremental path)", "root_cause": "`xce-build.sh worker asan --incremental` runs `meson setup --reconfigure \"$BUILD_DIR\" -Dcommit_id=... -Dbuild_time=... -Ddirty_fingerprint=...` against a pre-existing xce.build dir that was configured before commit a860141b (feat(dirty-content-fingerprint): add dirty_fingerprint meson option). Meson 1.9.1 validates -D flags on `--reconfigure` against the OLD cached coredata BEFORE re-reading meson.options to pick up newly-added options, so any brand-new option name passed via -D on the same reconfigure invocation is rejected as unknown -- even though the option genuinely exists in the current src/meson.options (confirmed: `intro-buildoptions.json` in the stale build dir only listed commit_id/build_time, not dirty_fingerprint). Pure staleness in the incremental build dir vs. the current option surface, unrelated to any C source edit.", "fix": "Ran `meson setup --reconfigure xce.build` once with NO -D overrides first -- this successfully re-reads meson.options and registers the new dirty_fingerprint option in cached coredata (confirmed via intro-buildoptions.json). A subsequent normal `xce-build.sh worker asan --incremental` (which does pass -Ddirty_fingerprint=...) then succeeded cleanly. General recipe: if `--incremental` reconfigure fails with 'Unknown option: \"X\"' for an option that IS defined in src/meson.options, the build dir predates that option's introduction -- do one plain `meson setup --reconfigure ` with no -D args to register it, then retry the real build. Rebuilding without --incremental (full rm -rf + fresh setup) also fixes it but wipes any other component's binaries sharing that dir (see bug-1565).", @@ -22115,22 +8964,25 @@ "dirty_fingerprint", "xce-build.sh", "worker", - "stale-build-dir", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "stale-build-dir" ], "related_bugs": [ "bug-1565" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1620.md" }, { "id": "bug-1621", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "leak", "timestamp": "2026-08-22T08:12:00Z", - "error_message": "LeakSanitizer (run with ASAN_OPTIONS=detect_leaks=1, overriding this test suite's own detect_leaks=0 convention): 'Direct leak of 8 byte(s) ... #1 make_host_account ../lib/xcealloc.c:119 #2 make_job_account ../lib/xcealloc.c:188' -- an xce_host_account_t's `host` field (strdup'd in make_host_account(), src/lib/xcealloc.c:119) is never freed.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "LeakSanitizer (run with ASAN_OPTIONS=detect_leaks=1, overriding this test suite's own…\n\nLeakSanitizer (run with ASAN_OPTIONS=detect_leaks=1, overriding this test suite's own detect_leaks=0 convention): 'Direct leak of 8 byte(s) ... #1 make_host_account ../lib/xcealloc.c:119 #2 make_job_account ../lib/xcealloc.c:188' -- an xce_host_account_t's `host` field (strdup'd in make_host_account(), src/lib/xcealloc.c:119) is never freed.", "file": "src/lib/misc.c (free_host_account)", "root_cause": "free_host_account() (src/lib/misc.c) frees acct->accounts[] (each via free_res_account()) and then FREEUP(acct) itself, but never FREEUP(acct->host) -- the strdup'd host string make_host_account() (src/lib/xcealloc.c:119, `haccount->host = strdup(halloc->host);`) allocates. Pre-existing, NOT introduced by scheduler-sim-decision-parity task 2.3: confirmed by tracing the leak's allocation stack, which bottoms out in make_job_account()/make_host_account() (xcealloc.c) -- the exact same tree-building code the LIVE dispatch path already uses (scheduler.c's sched_commit_decision() -> make_job_account()), so this leak equally exists on the live, production dispatch path today, just previously undetected because src/scheduler/testsuite/meson.build runs this whole suite with ASAN_OPTIONS=detect_leaks=0 (fixtures are deliberately not torn down in this test family, per test file header conventions -- masks this along with every other intentional fixture leak).", "fix": "FIXED (2026-09-08, follow-up session). Applied exactly the fix this entry already recommended: added FREEUP(acct->host); to free_host_account() (src/lib/misc.c), immediately before FREEUP(acct) -- mirrors free_res_account()'s existing FREEUP(acct->resname) pattern in the same file. Discovered as the remaining residual leak while fixing bug-1723's test-cleanup gaps: after adding the missing free_JobAlloc_ptr()/free_HostAlloc_list() calls those two tests needed, xcelib_test still leaked 12 bytes/2 allocations (exactly 2x \"host0\"/\"host1\", 6 bytes each) at make_host_account (xcealloc.c:119) via make_job_account -- this exact function/line. Since free_host_account() is called from free_job_account(), which real production dispatch code uses (scheduler.c, core_events.c), this fix has genuine production value, not just test-cleanliness. Verified: xcelib_test now 1/1 Ok with zero LeakSanitizer reports; full xce-build.sh all asan --incremental: 94/94 Ok, zero regressions.", @@ -22144,21 +8996,25 @@ "resource-accounting", "scheduler-sim-decision-parity", "pre-existing", - "out-of-scope", - "fixed" + "out-of-scope" ], "related_bugs": [ "bug-1723" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1621.md" }, { "id": "bug-1622", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-22T09:40:00Z", - "error_message": "TEST_ASSERT_EQUAL_INT(0, rc) failed: Expected 0 Was 1, in test_sim_round_boundary_reconciliation.c calling get_resvals(&(w->resInUse), 1, &residx, got)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "TEST_ASSERT_EQUAL_INT(0, rc) failed: Expected 0 Was 1, in test_sim_round_boundary_reconciliation.c…\n\nTEST_ASSERT_EQUAL_INT(0, rc) failed: Expected 0 Was 1, in test_sim_round_boundary_reconciliation.c calling get_resvals(&(w->resInUse), 1, &residx, got)", "file": "src/scheduler/testsuite/test_sim_round_boundary_reconciliation.c", "root_cause": "resource.c's get_resvals(zarray_t*, int nres, int*, float*) returns `nres` on success (not 0) -- confirmed by reading its body (resource.c:17-51, `return nres;` at the end). Wrongly assumed a 0-on-success/negative-on-error convention (like most other functions in this codebase, e.g. put_resvals's own callers use `rc >= 0`), and asserted TEST_ASSERT_EQUAL_INT(0, rc) instead of TEST_ASSERT_EQUAL_INT(1, rc) for a 1-resource query.", "fix": "Changed the assertion to TEST_ASSERT_EQUAL_INT(1, rc) with an inline comment noting get_resvals() returns nres on success. Not a production bug -- purely a test-authoring mistake caught immediately by the RED/GREEN cycle (this was the FIRST failure seen after restoring the task-3 sim_loop.c wiring, so it briefly looked like the fix hadn't worked before the real root cause -- a test bug, not a reconciliation bug -- was found).", @@ -22168,20 +9024,23 @@ "resource.c", "get_resvals", "return-convention", - "scheduler-sim-decision-parity", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "scheduler-sim-decision-parity" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1622.md" }, { "id": "bug-1623", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-22T12:40:00Z", - "error_message": "Test-authoring bug while building scheduler-sim-decision-parity task 4.3 (test_sim_e2e_original_motivation.c): initial version asserted resInUse checkpoints of 17.00/18.00/20.00/12.00 (derived from the RECORDING driver own in-memory resInUse arithmetic) but the actual replay-side info worker output showed only 12.00 (ANCHOR own charge) at every checkpoint -- the first three assertions failed.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Test-authoring bug while building scheduler-sim-decision-parity task 4.3…\n\nTest-authoring bug while building scheduler-sim-decision-parity task 4.3 (test_sim_e2e_original_motivation.c): initial version asserted resInUse checkpoints of 17.00/18.00/20.00/12.00 (derived from the RECORDING driver own in-memory resInUse arithmetic) but the actual replay-side info worker output showed only 12.00 (ANCHOR own charge) at every checkpoint -- the first three assertions failed.", "file": "src/scheduler/testsuite/test_sim_e2e_original_motivation.c", "root_cause": "Recorded-Decision Reconciliation (sim_reconcile_unaccounted_jobs(), sim_loop.c, scheduler-sim-decision-parity tasks 2.2-2.4/3) matches a RUN-state, unaccounted job against a DISPATCH decision recorded in THAT SAME replayed round only -- never an earlier or later round (each round decoded_session is decoded fresh and freed immediately after that round own processing). A job that arrives PEND and is genuinely live-dispatched by a real allocator in round K, then confirmed RUN via a JOB_STATE_CHANGE event in a LATER round K+1, can never be reconciled during replay: round K+1 own decoded decisions do not contain a match (the real DISPATCH is in round K own data), so the job stays RUN+unaccounted forever (Requirement 4 graceful degradation, correctly, not a bug in production). This is structurally impossible to avoid for ANY organically-arriving job, since a job cannot simultaneously be a fresh PEND candidate for a live picker (required to ever get a real DISPATCH decision) AND already-RUN-before-the-picker-runs (required for reconciliation to even consider it) in the same round -- confirmed by direct source read of sim_loop.c/driver.c/builtin_fcfs.c before writing any test code, then independently reconfirmed empirically by running the real test binary. Only ONE job in the whole fixture (ANCHOR_ID) was deliberately engineered so its arrival+RUN-confirmation events AND its own injected DISPATCH decision land in the exact same round (round 1) -- every other churn job (J2-J12) arrives and is genuinely live-dispatched by the test own capacity-aware allocator thread in ITS OWN arrival round, so its dispatch can never be reconciled later, exactly like every other real production job whose confirmation lags its dispatch by more than zero rounds.", "fix": "Changed all four info-worker checkpoint assertions from the RECORDING-side arithmetic (17.00/18.00/20.00/12.00, i.e. what a normal-mode continuation of the same session would show) to the REPLAY-side reality: a constant used=12.00 (ANCHOR_ID own charge only) at all four checkpoints spanning the 24-round replay, since no churn job dispatch is ever reconciled. Added an explicit non-tautology guard (used=0.00 never appears) and an extensive header-comment/round-table-comment explanation of why churn-job charges never surface during replay, so a future reader does not mistake this for an oversight. Also independently reconfirmed via a live discriminating-power check: commenting out sim_loop.c own sim_reconcile_unaccounted_jobs() call site made the ANCHOR_ID DISPATCH-visibility assertion fail for the right reason; restoring it (git diff clean on production files afterward) made the whole suite green again.", @@ -22191,23 +9050,26 @@ "replay", "reconciliation", "test-authoring", - "scheduler-sim-decision-parity", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "scheduler-sim-decision-parity" ], "related_bugs": [ "bug-1611", "bug-1613" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1623.md" }, { "id": "bug-1624", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-22T19:53:00Z", - "error_message": "None yet observed at runtime -- found via code trace while answering an architectural question about job RUN-state boundaries and over/under-commit safety.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "None yet observed at runtime -- found via code trace while answering an architectural question…\n\nNone yet observed at runtime -- found via code trace while answering an architectural question about job RUN-state boundaries and over/under-commit safety.", "file": "src/scheduler/src/scheduler.c, src/scheduler/src/core_session.c, src/scheduler/src/resource.c", "root_cause": "sched_account_jobs() (scheduler.c:1258-1292) re-derives core->sharedInUse/worker->resInUse every round from scratch, but gates on JOB_IS_RUN(job->state) before checking job->accounts. A job that has been DISPATCHED (job->accounts set synchronously at commit-decision time via sched_commit_job_alloc(), scheduler.c:1446) but not yet confirmed RUN by the worker is still job->state==PEND, so it is skipped by this resum. Since xce_core_begin_session() (core_session.c:66-108) unconditionally zeroes sharedInUse/resInUse every round (normal-mode cadence: 1s, main.c:453-460), that job's charge -- correct within the round it was dispatched -- is wiped at the very next round and not restored until the RUN confirmation is processed. The admission check other jobs go through (sched_check_host_satisfy_/sched_check_shared_satisfy_, resource.c:252-370, called from seqjob_allocator.c:130) reads exactly this same resInUse/sharedInUse, with no separate reserved/committed pool (xce_sched_job_t has only state/allocs/accounts, sched/types.h:96-116) -- so once wiped, the resource looks free again and can be granted to a second job, a genuine over-commit/double-booking risk under any dispatch-to-worker-confirmation latency exceeding ~1 round (ordinary for real worker RPC + fork/exec).", "fix": "FIXED (2026-08-23, scheduler-dispatch-commit tasks 2.1 + 3.1). Scheduler side (task 2.1, commit 8607348c): sched_commit_decision()'s DISPATCH branch now sets job->state = JOB_STAT_RUN synchronously, in the same operation that already sets job->accounts/job->allocs -- no separate fix to sched_account_jobs()'s JOB_IS_RUN gate was needed; the job simply reaches RUN from the round it's dispatched, so the existing gate naturally includes it every subsequent round regardless of confirmation latency. Controller side (task 3.1, commit d9aedc4b): dispatch_a_job_() gates the job.allocs commit + JOB_STAT_RUN transition together on xce_startjob_req() succeeding, keeping both sides consistent. Regression-tested end-to-end: test_dispatch_commit_survives_several_rounds_before_confirmation and test_dispatch_commit_multi_round_gap_matches_same_round_confirmation (src/scheduler/testsuite/test_scheduler_dispatch_commit.c, task 5.1) prove resInUse survives a 5-round gap with no confirmation event, and match a same-round-confirmation baseline; RED-phase independently verified via a real revert/rebuild/rerun of the fix by both the task 5.1 implementer and its reviewer. Full HA soak (sequential + chaotic, 10 rounds each) confirms no new job loss or divergence, task 5.4.", @@ -22219,23 +9081,29 @@ "normal-mode", "sched_account_jobs", "JOB_IS_RUN", - "race-condition", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "race-condition" ], "related_bugs": [ "bug-1611", "bug-1613" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "8607348c", + "d9aedc4b" + ], + "record": ".wolf/bugs/bug-1624.md" }, { "id": "bug-1625", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-23T05:20:00Z", - "error_message": "None yet observed at runtime -- found via code trace while designing scheduler-dispatch-commit. Referenced throughout that spec's requirements.md/research.md/design.md by this id; being logged now after noticing it was never actually appended to buglog.json.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "None yet observed at runtime -- found via code trace while designing scheduler-dispatch-commit\n\nNone yet observed at runtime -- found via code trace while designing scheduler-dispatch-commit. Referenced throughout that spec's requirements.md/research.md/design.md by this id; being logged now after noticing it was never actually appended to buglog.json.", "file": "src/scheduler/src/handler.c", "root_cause": "The controller already reports dispatch-RPC failure back to the scheduler via NotifyData_data_jobDispatchFail (notify_dispatch(), src/controller/job.c:1108-1125), but sched_handle_notify()'s switch (handler.c:499-544) has no case for it -- it falls to default: rc = -EINVAL. A dispatch failure is therefore silently dropped by the scheduler: job->accounts/job->allocs (set synchronously at dispatch-decision-commit time by sched_commit_decision()) are never released, leaving stale committed-resource bookkeeping behind indefinitely.", "fix": "FIXED (2026-08-23, scheduler-dispatch-commit task 2.2, commit 281a026d). Added case NotifyData_data_jobDispatchFail to sched_handle_notify()'s switch (handler.c), applying JOB_STAT_PEND via the existing XCE_SCHED_OP_JSTATE_CHANGE enqueue path -- exactly as designed. No new cleanup code needed: core_events.c's existing free-on-non-RUN branch releases job->accounts/job->allocs automatically once the enqueued PEND change is applied. Regression-tested: test_handle_notify_job_dispatch_fail_enqueues_pend (src/scheduler/testsuite/test_handlers.c) proves the notify enqueues the correct op; independently reviewed and approved.", @@ -22245,23 +9113,27 @@ "dispatch-failure", "notify", "unhandled-event", - "resource-leak", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "resource-leak" ], "related_bugs": [ "bug-1624" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "281a026d" + ], + "record": ".wolf/bugs/bug-1625.md" }, { "id": "bug-1626", "status": "fixed", "status_updated": "2026-09-27", "status_note": "0c1bac44", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-23T05:20:00Z", - "error_message": "None yet observed at runtime -- found via code trace during /kiro-validate-design review of scheduler-dispatch-commit.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "None yet observed at runtime -- found via code trace during /kiro-validate-design review of…\n\nNone yet observed at runtime -- found via code trace during /kiro-validate-design review of scheduler-dispatch-commit.", "file": "src/controller/job.c", "root_cause": "apply_job_fail_transition() (job.c:1804-1836) is explicitly documented as safe to call only \"as long as the caller already holds job under RTTI_LOCK_WRITE\" (a genuine rwlock, my_rwlock_wlock(), confirmed via src/lib/jobj.c:708-725's job_lock()). Its existing caller mark_job_fail() (job.c:1844) is itself called from job_ctrl() (job.c:965) at line 1083 -- but job_ctrl()'s only lock on the job object across its entire body (traced lines 965-1090) is lock(j, RTTI_LOCK_READ) at line 999, released via unlock(j) at line 1016, well before the mark_job_fail() call at line 1083. No RTTI_LOCK_WRITE is ever taken in this path. Separately, dispatch_a_job_() (job.c:831-963) mutates j->job.allocs under only RTTI_LOCK_READ on the containing maps (c->jobmgr, c->workermgr) and RTTI_LOCK_READ on the worker w -- never RTTI_LOCK_WRITE on the job j itself. RTTI_LOCK_READ is a genuine reader lock (my_rwlock_rlock()) that does not exclude concurrent readers from each independently mutating the same object, so if two threads process job-mutating requests for the same job concurrently, this is a real data race, not just an inaccurate comment. Contrast: many OTHER call sites in the same file (e.g. job.c:768, 1687, 1795, 2058, 2089, 2127, 2169, 2206, 2293, 2340, 2416) DO correctly take RTTI_LOCK_WRITE before mutating job state, confirming RTTI_LOCK_WRITE is this codebase's established, intentional convention for job-state mutation -- the job_ctrl()/mark_job_fail() and dispatch_a_job_() call sites appear to be inconsistent outliers, not a different intentional design.", "fix": "NOT YET FIXED -- discovered as a pre-existing gap while validating scheduler-dispatch-commit's design (that spec plans to add new mutations, including a WAL-fact-durable one, to dispatch_a_job_(), and reuse mark_job_fail() for its terminal dispatch-failure path -- both inherit this open question). Needs its own investigation: either (a) confirm some other serialization guarantee (e.g. single-threaded-per-job request processing) makes RTTI_LOCK_READ sufficient in practice and the RTTI_LOCK_WRITE comments/convention are overcautious, or (b) these call sites are missing a genuine lock upgrade and should take RTTI_LOCK_WRITE like their sibling call sites do. Flagged back to scheduler-dispatch-commit's design review as Critical Issue 1 rather than fixed inline, since it is a pre-existing condition this feature would otherwise silently inherit/extend, not something this feature caused.", @@ -22273,21 +9145,25 @@ "RTTI_LOCK_WRITE", "job-state-mutation", "race-condition", - "pre-existing", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pre-existing" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "0c1bac44" + ], + "record": ".wolf/bugs/bug-1626.md" }, { "id": "bug-1627", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-23T05:35:00Z", - "error_message": "None yet observed at runtime -- found via code trace during round 3/4 of /kiro-validate-design review of scheduler-dispatch-commit.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "None yet observed at runtime -- found via code trace during round 3/4 of /kiro-validate-design…\n\nNone yet observed at runtime -- found via code trace during round 3/4 of /kiro-validate-design review of scheduler-dispatch-commit.", "file": "src/controller/job.c", "root_cause": "dispatch_a_job_() (job.c:831-963) calls reference(j) at line 854 after successfully looking up the job. On FOUR of its failure/return paths it never calls a matching dereference(j): workermgr key_find() failing (rc<0, line 888-890), the worker not being found (w==NULL, line 892-894), the worker not being in a ready state (E_XCE_BAD_WORKER, line 897-909), and xce_startjob_req() itself failing (rc<0, line 956-960 -- found in a later review pass than the first three). (Contrast: the nc<=0 path at line 869-873 and the get_queue_exec() failure path at line 917-925 both correctly dereference(j) before returning.) Each occurrence permanently leaks one reference on the job object. Separately and NOT included in this bug (flagged as a distinct open question, not a confirmed defect): the function also never calls dereference(j) on its normal SUCCESS return path -- this may be an intentional reference hand-off to the job entering its dispatched/running lifetime rather than a bug, and needs its own investigation rather than being folded in here.", "fix": "FIXED -- scheduler-dispatch-commit task 3.3. dispatch_a_job_() (job.c) now closes all four leak sites as part of the retryable-failure classify/resolve restructuring the bug's own notes anticipated: XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP, _WORKER_NULL, _WORKER_NOT_READY, and _RPC_FAILURE each now lock(j, RTTI_LOCK_WRITE), classify via xce_classify_dispatch_fail(), increment job.dispatch_fail_count under that lock, resolve retryable (within XCE_DISPATCH_FAIL_RETRY_BOUND==3) or terminal (mark_job_fail(), mirroring task 3.2's two sibling terminal sites) as appropriate, unlock(j), then dereference(j) -- closing the leak uniformly across all four. Regression-tested via test_dispatch_fail_retry.c's canary-reference technique (task 3.2's own established pattern) at three of the four sites (WORKER_LOOKUP/WORKER_NULL share one test since a black-box test cannot force which of the two an unregistered worker key actually triggers -- both share the identical resolution code shape). RED confirmed first (all 4 new tests failed against the unmodified four sites, git-stash verified), then GREEN after the fix, full controller_unit_test suite run (1056 tests, filtered only for the pre-existing bug-1429 hang) showed zero new failures beyond the pre-existing bug-1561 pair.", @@ -22297,22 +9173,25 @@ "memory-leak", "refcount", "dispatch_a_job_", - "pre-existing", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pre-existing" ], "related_bugs": [ "bug-1626" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1627.md" }, { "id": "bug-1628", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-08-23T08:03:59", - "error_message": "date: invalid date '-6 days' / date: invalid date '2026-06-29 +5 days' -- relative date arithmetic silently yields EMPTY shell variables, and the empty values then flow into `git log --since=\"\" --until=\"\"`, which git treats as NO time filter (unbounded history) instead of erroring.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "date: invalid date '-6 days' / date: invalid date '2026-06-29 +5 days' -- relative date arithmetic…\n\ndate: invalid date '-6 days' / date: invalid date '2026-06-29 +5 days' -- relative date arithmetic silently yields EMPTY shell variables, and the empty values then flow into `git log --since=\"\" --until=\"\"`, which git treats as NO time filter (unbounded history) instead of erroring.", "file": ".claude/skills/weekly_report/SKILL.md", "root_cause": "`date` first on PATH in this harness is BusyBox v1.36.1 (/nix/store/...busybox-1.36.1/bin/busybox), NOT GNU coreutils, even though /bin/date IS coreutils 9.8 -- the nix profile bin dir precedes /bin. BusyBox date supports absolute parsing (`date -d 2026-06-29 +%A` works, which is what masked the problem during initial probing) but has no GNU relative-expression support ('-7 days', 'last monday', '+5 days'). Because command substitution captures only stdout and BusyBox writes 'invalid date' to stderr, $START/$END come back as empty strings and the failure is invisible to a non -e script. Same BusyBox-shadows-GNU family as bug-1606 (grep -P) and bug-1617 (mktemp template).", "fix": "Do all date arithmetic in python3 (/bin/python3, always present) instead of `date -d`: compute week start via `today - timedelta(days=today.weekday()) - timedelta(weeks=offset)` and emit `START=/END=/SLUG=` lines consumed by `eval \"$(python3 - ... <<'PY' ... PY\n)\"`. General rule: never use `date -d` with a relative expression in this environment; use python3 datetime, or call /bin/date by absolute path if GNU date is genuinely required. Also validate that computed date vars are non-empty before feeding them to `git log --since/--until`, since git silently ignores empty range args rather than failing.", @@ -22325,23 +9204,26 @@ "bash", "git-log", "weekly_report", - "silent-failure", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "silent-failure" ], "related_bugs": [ "bug-1606", "bug-1617" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1628.md" }, { "id": "bug-1629", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-08-23T08:12:20", - "error_message": "AddressSanitizer SEGV in zarray_get_ex (lib/zarray.c:332) when a test-fixture xce_res_account_t.residx was never resolved from -1", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer SEGV in zarray_get_ex (lib/zarray.c:332) when a test-fixture…\n\nAddressSanitizer SEGV in zarray_get_ex (lib/zarray.c:332) when a test-fixture xce_res_account_t.residx was never resolved from -1", "file": "src/scheduler/testsuite/test_scheduler_dispatch_commit.c", "root_cause": "zarray_get_ex() only guards idx >= array->caps before indexing array->data[idx]; it does NOT guard idx < 0. sched_update_res_accounts() (resource.c) resolves xce_res_account_t.residx from -1 to a real index via key_find()/add_vres() against core->resmgr ONLY when the host lookup (sched_get_worker()) and the surrounding call chain actually run to completion; if a test fixture never pre-registers the resource name in core->resmgr, sched_update_account()/sched_update_res_accounts() silently swallow the resolution failure (return values are discarded by sched_update_account()) and residx stays -1. A -1 index then passes the 'idx >= array->caps' bounds check (false, since caps starts at 0 too) and falls through to array->data[-1], an out-of-bounds/underflow read that ASAN reports as a high-address SEGV rather than a clean bounds error.", "fix": "In the test fixture, pre-register the resource with add_vres(&vres, core.resmgr, RES_NAME, 0) BEFORE calling the real charging path, and use the returned (non-negative) index for get_resvals()/put_resvals() calls, instead of relying on the production code's lazy first-use resolution path inside a minimal test fixture that never loads real resource config.", @@ -22352,20 +9234,23 @@ "resource-accounting", "asan", "scheduler-dispatch-commit", - "test-fixture", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-fixture" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1629.md" }, { "id": "bug-1630", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "tooling", + "severity": "unknown", "timestamp": "2026-08-23T08:35:55", - "error_message": "buglog.json integrity: 39 duplicate bug IDs present (bug-875..bug-948). The Stop hook's duplicate-ID validation flags this every turn. bug-908, for example, is BOTH a 2026-06-21 raft_sim.c missing-guard-clause entry AND a 2026-08-16 config-upgrade-snapshot/spec.json refactor note -- two unrelated bugs sharing one id.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "buglog.json integrity: 39 duplicate bug IDs present (bug-875..bug-948)\n\nbuglog.json integrity: 39 duplicate bug IDs present (bug-875..bug-948). The Stop hook's duplicate-ID validation flags this every turn. bug-908, for example, is BOTH a 2026-06-21 raft_sim.c missing-guard-clause entry AND a 2026-08-16 config-upgrade-snapshot/spec.json refactor note -- two unrelated bugs sharing one id.", "file": ".wolf/buglog.json", "root_cause": "A session on 2026-08-15..17 appended new entries using an ID counter that had been restarted (or derived from a stale/remembered max) instead of computed from the file's actual maximum numeric ID. New bugs were therefore written into the already-occupied bug-875..bug-949 range: 8 collisions dated 2026-08-15, 14 dated 2026-08-16, 22 dated 2026-08-17. This is precisely the failure mode the OpenWolf bug-logging rule warns about ('never restart from bug-001; a restarted counter collides with existing history and is unrecoverable without a git-history merge'). Discovered while generating the Aug 16-22 weekly report, where an in-window timestamp query returned an implausible id range of bug-908..bug-1624.", "fix": "Housekeeping only, no code change needed: renumbering commit 755976fc (2026-08-24) already repaired all 39 duplicate IDs. Re-verified 2026-08-30 -- 1077 bug entries, 1077 unique IDs, zero collisions. This entry's own `fix` field was simply never backfilled after the repair landed.", @@ -22376,22 +9261,27 @@ "duplicate-id", "tooling", "needs-git-history-merge", - "fixed", "verified-2026-08-30" ], "related_bugs": [ "bug-1628" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "755976fc" + ], + "record": ".wolf/bugs/bug-1630.md" }, { "id": "bug-1631", "status": "fixed", "status_updated": "2026-09-27", "status_note": "0c1bac44", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-23T19:30:00Z", - "error_message": "New test test_rpc_failure_bound_resets_after_intervening_success (test_dispatch_fail_retry.c) fails: 'Req 4.1 (bug-1631, NOT YET FIXED): a retryable dispatch failure must return the job to PEND -- fails here only because this job previously had a REAL successful dispatch (step 2)'. Expected JOB_STAT_PEND, got JOB_STAT_RUN (and separately, job.allocs stays non-NULL instead of being released).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "New test test_rpc_failure_bound_resets_after_intervening_success (test_dispatch_fail_retry.c)…\n\nNew test test_rpc_failure_bound_resets_after_intervening_success (test_dispatch_fail_retry.c) fails: 'Req 4.1 (bug-1631, NOT YET FIXED): a retryable dispatch failure must return the job to PEND -- fails here only because this job previously had a REAL successful dispatch (step 2)'. Expected JOB_STAT_PEND, got JOB_STAT_RUN (and separately, job.allocs stays non-NULL instead of being released).", "file": "src/controller/job.c", "root_cause": "dispatch_a_job_()'s four retryable-failure resolution branches (XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP/_WORKER_NULL/_WORKER_NOT_READY/_RPC_FAILURE, job.c ~lines 1040-1435, all four added/restructured by scheduler-dispatch-commit task 3.3) never write j->job.state or j->job.allocs on the 'resolved_retryable' (within-bound) path -- only job.dispatch_fail_count is touched. This is CORRECT for a job's very FIRST dispatch attempt: the job was never anything but PEND and allocs was never committed, so 'no reversal needed' (design.md's own 'Controller Dispatch Commit' state-model text: 'on RPC failure, state is untouched -- still PEND, never entered RUN, so no reversal is needed, per the Research Log finding') is vacuously true. That reasoning silently assumes the job has NEVER had a prior successful dispatch. It breaks the moment dispatch_a_job_() is called again on a job that previously succeeded (went RUN, committed allocs) and now fails retryably again -- exactly the scenario Requirement 4.3's own text ('consecutive... dispatch failures with no intervening successful dispatch') and task 3.1's own observable-completion criterion ('a job that fails, succeeds, then fails again is evaluated against the bound as if it had only failed once') explicitly anticipate. In that scenario job.state incorrectly stays RUN and job.allocs incorrectly stays committed across the retry, violating Requirement 4.1 ('the Scheduler and Controller shall return the job to a pending state') and Requirement 4.4 ('the Scheduler and Controller shall each release the resource-allocation and commitment data'). The terminal (bound-exceeded) resolution path is separately confirmed affected the same way: mark_job_fail()/apply_job_fail_transition() (job.c ~2303-2325) unconditionally sets job.state = JOB_STAT_EXIT (so the STATE outcome is correct on the terminal path) but never touches job.allocs either, so Requirement 4.5 is also affected on that path. Every existing retryable-failure test (test_dispatch_fail_retry.c's other three tests, test_rpc_failure_bounded_escalation's own 4-consecutive-failure test) only ever drives a job that starts and stays PEND with allocs always NULL throughout, so none of them could have caught this -- only a real intervening success (constructed by the new fail->succeed->fail scenario, scheduler-dispatch-commit task 5.2) exposes it. This is a design-level gap, not just an implementation oversight: design.md's own Controller Dispatch-Failure Classification / Controller Dispatch Commit sections both assert-by-omission that job.state/job.allocs need no reversal on any of the six failure paths, without qualifying that assertion to 'only true for a job's first-ever dispatch attempt'.", "fix": "NOT YET FIXED. Discovered live while implementing scheduler-dispatch-commit task 5.2 (test-only task; the fix requires modifying src/controller/job.c, out of that task's test-only boundary -- reported as BLOCKED). Candidate fix: in each of the four retryable branches' 'resolved_retryable = true' case (still under the existing lock(j, RTTI_LOCK_WRITE)), add `j->job.state = JOB_STAT_PEND;` and release any committed j->job.allocs (free_JobAlloc_ptr(&j->job.allocs); j->job.allocs = NULL;) before unlock(j) -- mirroring, in reverse, what Controller Dispatch Commit's own success path (job.c ~1343-1372) does when transitioning PEND->RUN. Needs a design-review pass first: design.md's 'Controller Dispatch-Failure Classification' and 'Controller Dispatch Commit' sections both need their state-model text corrected to no longer claim the failure paths need no reversal unconditionally. UPDATE (2026-08-23, task 5.2 completion): a kiro-debug investigation (HIGH confidence, independently re-verified even more exhaustively by task 5.2's own reviewer via a full trace of every pendJobs-reinsertion and controller-side state-mutation call site) confirmed this scenario is genuinely UNREACHABLE via any real caller in the current, fully-implemented system -- the scheduler's picker permanently removes a job from its PEND candidate pool the instant it dispatches it (task 2.1), and nothing already-implemented ever re-adds a successfully-dispatched (RUN) job back to that pool. Only the not-yet-implemented scheduler-preemption spec (confirmed still requirements-generated, no approvals) could someday create that path. Per that finding, the COMMITTED version of test_rpc_failure_bound_resets_after_intervening_success (src/controller/testsuite/test_dispatch_fail_retry.c, commit 16d90319) deliberately does NOT assert the job.state/job.allocs reversal described above -- it keeps only the Req 4.3 counter-reset assertions (which are correct and GREEN). There is no live RED test for this bug today; it remains an intentionally-deferred, documented gap to revisit once scheduler-preemption (or an equivalent mechanism) actually creates a reachable RUN-then-retryable-redispatch path -- not before.\n\nADDENDUM (2026-09-01, worker-rejoin-resync design re-validation): worker-rejoin-resync's design.md now explicitly names this bug as the reason Requirement 4.6 (reconciling an 'unrecognized' RUN job back to PEND, eligible for re-dispatch) must not be shipped without the candidate fix described above. design.md's `record_job_reconcile_pending_fact()` is specified to release `job->job.allocs` at the same point it sets the job PEND -- the exact fix this entry already proposed, now scoped to a concrete call site instead of a hypothetical future one. Still NOT YET FIXED IN CODE (worker-rejoin-resync is design-phase only as of this note) -- this addendum records that the 'unreachable, deliberately deferred' status ends the moment that spec is implemented, and the fix must land in the same change as Requirement 4.6, not as a follow-up.\n\nRESOLVED IN CODE (2026-09-01, worker-rejoin-resync task 2.1): record_job_reconcile_pending_fact() (src/controller/job.c, after mark_job_update) now performs exactly the fix this entry proposed -- free_JobAlloc_ptr(&job->job.allocs) immediately after a successful (JOB_TRANSITION_APPLIED) apply_job_update_transition(job, JOB_STAT_PEND), under the same caller-held write lock, before the durable WAL_JOB_UPDATE fact is appended. This is called when a worker reports a queried job as 'unrecognized' during rejoin reconciliation (Requirement 4.6) -- exactly the RUN-job-returned-to-PEND-and-re-admitted-to-the-dispatch-pool precondition this entry's own root_cause said was needed to make the gap reachable. Regression-tested directly: test_reconcile_pending_applies_releases_allocs_and_notifies (test_worker_resync.c) seeds a real JOB_STAT_RUN job with a real non-NULL allocation and asserts allocs == NULL immediately after the call. Independently reviewed and approved (kiro-review). dispatch_a_job_()'s own four retryable-failure branches (the original bug-1631 site) are still untouched by this fix -- they remain correct for a job's first-ever dispatch attempt, which is still the only reachable case THERE; this fix closes the gap at the new call site this feature introduces, not by changing dispatch_a_job_ itself.", @@ -22405,26 +9295,29 @@ "requirement-4.1", "requirement-4.4", "requirement-4.5", - "design-gap", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "design-gap" ], "related_bugs": [ "bug-1625", "bug-1626", "bug-1627" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "0c1bac44" + ], + "record": ".wolf/bugs/bug-1631.md" }, { "id": "bug-1632", "status": "open", "status_updated": "2026-09-24", "status_note": "test_sim_verify.c stack-local sim read by picker thread (ASAN stack-use-after-return)", + "component": "scheduler", + "severity": "security", "timestamp": "2026-08-23T21:03:00.000Z", - "error_message": "sched_sim_verify_test: AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151), thread T2 (sim_picker_thread_fn, sim_loop.c:456), reading the stack of run_sim_and_capture_stdout's `sim` (xce_sched_sim_driver_t) after that frame returned/was reused. Fired once during a full `/xce-build scheduler` run (scheduler-dispatch-commit task 5.4 regression pass): Ok:50 Fail:2 instead of the documented Ok:51 Fail:1 baseline, extra failure = 'scheduler sim verify test'. Did NOT reproduce on: 3 standalone direct re-runs of sched_sim_verify_test (16 Tests 0 Failures each time) or a second full `meson test --suite scheduler` re-run against the same build (back to Ok:51 Fail:1, only the known test_admin_request.c symlink-escape failure) -- confirmed non-deterministic/order-dependent, not caused by scheduler-dispatch-commit.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "sched_sim_verify_test: AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151)…\n\nsched_sim_verify_test: AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151), thread T2 (sim_picker_thread_fn, sim_loop.c:456), reading the stack of run_sim_and_capture_stdout's `sim` (xce_sched_sim_driver_t) after that frame returned/was reused. Fired once during a full `/xce-build scheduler` run (scheduler-dispatch-commit task 5.4 regression pass): Ok:50 Fail:2 instead of the documented Ok:51 Fail:1 baseline, extra failure = 'scheduler sim verify test'. Did NOT reproduce on: 3 standalone direct re-runs of sched_sim_verify_test (16 Tests 0 Failures each time) or a second full `meson test --suite scheduler` re-run against the same build (back to Ok:51 Fail:1, only the known test_admin_request.c symlink-escape failure) -- confirmed non-deterministic/order-dependent, not caused by scheduler-dispatch-commit.", "file": "src/scheduler/testsuite/test_sim_verify.c", "root_cause": "Same root cause as bug-1598 (already diagnosed, NOT re-derived here): sched_sim_driver_run() (src/scheduler/src/sim_loop.c) spawns sim_picker_thread_fn on a pthread that is, by design, never joined/cancelled -- it loops forever on cbuf_get(&sim->core.wbuf,...) reading `sim` for the life of the process. bug-1598's fix converted 3 stack-local `xce_sched_sim_driver_t sim;` fixtures in test_sim_event_loop.c to `static` storage, but test_sim_verify.c's OWN, separately-defined run_sim_and_capture_stdout() helper (test_sim_verify.c:940-943) still declares `sim` as stack-local -- an unfixed sibling instance of the identical latent bug, never touched by bug-1598's fix. Confirmed via git diff/log that scheduler-dispatch-commit's own commits (which only touch scheduler.c/handler.c per that spec's Requirement 5.2 sim-mode-exclusion) never modified sim_loop.c or test_sim_verify.c -- this is pre-existing latent test infra, exposed non-deterministically by whatever stack-layout/MALLOC_PERTURB_ randomization a given meson test invocation happens to produce, exactly like bug-1598's own trigger mechanism.", "fix": "NOT FIXED -- out of scope for scheduler-dispatch-commit task 5.4 (a validation/regression task, not a scheduler-sim-infra task). Same remediation as bug-1598 would apply if picked up: change run_sim_and_capture_stdout()'s `xce_sched_sim_driver_t sim;` from stack-local to `static` storage (safe under the same reasoning bug-1598 documents: sched_sim_driver_start() memset()s the whole struct first, and this helper is not re-entered while an earlier zombie thread could still be alive within one test binary process -- verify that non-reentrancy holds for test_sim_verify.c's own call sites before applying). Logged so a future session doesn't re-diagnose from scratch.", @@ -22438,23 +9331,25 @@ "flaky", "non-deterministic", "pre-existing", - "sibling-of-bug-1598", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "sibling-of-bug-1598" ], "related_bugs": [ "bug-1598" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1632.md" }, { "id": "bug-1673", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "config", + "severity": "unknown", "timestamp": "2026-08-24T13:40:00Z", - "error_message": "Gitea issue #40 (\"job \u8c03\u5ea6\u4e0d\u6b63\u786e\"): `xcecli hosts list` shows MAX_SLOT=0 for a dynamically-joined worker even though the host's real NCPUS (e.g. 16 or 22) is correctly reported and scheduling correctly dispatches up to that many concurrent jobs. Reproduced live on branch scheduler-framework: single-node deploy, worker joined as 'worker-1', `xcecli hosts list` -> NCPUS=22 MAX_SLOT=0. Submitted 9 single-slot sleep jobs -- all 9 eventually dispatched and ran (no hard cap observed under this test), so the reporter's secondary claim of a hard ~8-job concurrency ceiling did NOT reproduce here (most likely a mid-dispatch snapshot, dispatch is not instantaneous across all pending jobs on this branch; a queue/usergroup slot limit in limits.yaml, e.g. ug1_normal_slots=8, remains a plausible alternate explanation worth asking the original reporter about). Only the MAX_SLOT=0 display value itself is a confirmed, reproducible bug.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Gitea issue #40 (\"job 调度不正确\"): `xcecli hosts list` shows MAX_SLOT=0 for a dynamically-joined…\n\nGitea issue #40 (\"job 调度不正确\"): `xcecli hosts list` shows MAX_SLOT=0 for a dynamically-joined worker even though the host's real NCPUS (e.g. 16 or 22) is correctly reported and scheduling correctly dispatches up to that many concurrent jobs. Reproduced live on branch scheduler-framework: single-node deploy, worker joined as 'worker-1', `xcecli hosts list` -> NCPUS=22 MAX_SLOT=0. Submitted 9 single-slot sleep jobs -- all 9 eventually dispatched and ran (no hard cap observed under this test), so the reporter's secondary claim of a hard ~8-job concurrency ceiling did NOT reproduce here (most likely a mid-dispatch snapshot, dispatch is not instantaneous across all pending jobs on this branch; a queue/usergroup slot limit in limits.yaml, e.g. ug1_normal_slots=8, remains a plausible alternate explanation worth asking the original reporter about). Only the MAX_SLOT=0 display value itself is a confirmed, reproducible bug.", "file": "src/frontend/apiserver/backend/handler.go, src/configs/hosts.yaml", "root_cause": "OnHostConfigUpdated() (handler.go:60-70) only applies config-pushed fields (including MaxSlot) onto a host entry via `store.Hosts.Get(incoming.HostName)` -- an EXACT hostname match. hosts.yaml commonly defines only a wildcard `hostname: default` entry (plus a couple of literally-named ones, e.g. worker2/host3) -- confirmed via grep, no entry named after the actual dynamically-joined worker (e.g. 'worker-1'). So a worker that joins under its own hostname never receives a HostConfig push under that literal name, `store.Hosts.Get()` misses, OnHostConfigUpdated returns early, and MaxSlot is left at Go's zero-value (0) forever -- completely decoupled from the value actually used for scheduling. The C-side path is correct: worker.c's apply_join_state()/find_matched_hostconfig() falls back to the 'default' wildcard entry's mxj/maxcpus for real dispatch decisions, confirmed via a live controller log line showing the correct slots=22 resource value. This is purely a Go apiserver display gap, not a scheduling defect.", "fix": "FIXED (2026-08-24). src/frontend/apiserver/backend/handler.go -- OnHostConfigUpdated now caches the \"default\" pseudo-host's config-sourced fields (defaultHostConfig, package-level, mutex-guarded) instead of discarding them, and retroactively backfills that fallback onto any already-registered host with no explicit (non-default) config of its own; OnHostUpdated applies the same fallback immediately on a host's first-ever registration. The HOSTCONFIG_MXJ_AUTO_DETECT -1 sentinel (bug-1556) is further resolved to the host's real NumCPUs when applied via fallback, mirroring worker.c's C-side `if (nslots<0) nslots = maxcpus`. A host with its own real per-host hosts.yaml entry is never overridden. Added resetHostConfigCache() test hook plus 5 new tests (both arrival orderings, no-override-of-explicit-config, unresolvable-stays-negative) -- all pass, plus all 3 pre-existing OnHostConfigUpdated/OnHostUpdated tests pass unmodified (`make test-backend`). Live-verified: `xcecli hosts list` for the dynamically-joined worker-1 (no hosts.yaml entry) now shows MAX_SLOT=22, matching NCPUS=22, instead of 0.", @@ -22466,24 +9361,27 @@ "hosts-list", "max-slot", "display-bug", - "hostconfig", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "hostconfig" ], "related_bugs": [ "bug-1556" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 40 + "fix_commits": [], + "record": ".wolf/bugs/bug-1673.md", + "gitea_issue": 40, + "gitea_synced_status": "fixed" }, { "id": "bug-1674", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-24T13:40:00Z", - "error_message": "Gitea issue #41 (\"job output timestamp is incorrect\"): reporter's transcript shows `xcecli jobs list -l ` succeeding but omitting submit/start/done timestamps. Reproduced live on branch scheduler-framework, but the actual failure is more severe than reported: `xcecli jobs list -l ` (and bare `xcecli jobs list -l`) fail OUTRIGHT with `query request failed: [invalid parameters: unsupported query field: pending_reason]` -- no output is produced at all, not even a partial listing missing timestamps. Likely a regression introduced after the original issue was filed (this branch is under active scheduler-framework development).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Gitea issue #41 (\"job output timestamp is incorrect\"): reporter's transcript shows `xcecli jobs…\n\nGitea issue #41 (\"job output timestamp is incorrect\"): reporter's transcript shows `xcecli jobs list -l ` succeeding but omitting submit/start/done timestamps. Reproduced live on branch scheduler-framework, but the actual failure is more severe than reported: `xcecli jobs list -l ` (and bare `xcecli jobs list -l`) fail OUTRIGHT with `query request failed: [invalid parameters: unsupported query field: pending_reason]` -- no output is produced at all, not even a partial listing missing timestamps. Likely a regression introduced after the original issue was filed (this branch is under active scheduler-framework development).", "file": "src/frontend/apiserver/controllers/jobs.go:553, src/frontend/apiserver/controllers/jobs.go:901", "root_cause": "Both the `-l`/fields-filtered job-list code paths (jobs.go:553 and the near-duplicate at jobs.go:901) validate the requested field list via `utils.GetSupportedFields(models.JobSubmitRequest{})` -- the job SUBMISSION request struct -- instead of `models.JobInfo{}`, the job query/response struct (the correct pattern, confirmed already used correctly for hosts at hosts.go:105/307 with `models.HostInfo{}`, and for at least one other job code path at jobs.go:1131 with `models.JobInfo{}`). Read-only/response-only fields like `pending_reason`, `start_time`, `end_time`, `pids`, `pgids` don't exist on JobSubmitRequest, so the first such field hit in Go's randomized map-iteration order triggers a hard `unsupported query field` rejection, making `-l` (and any custom -o field list touching these fields) fail unconditionally.", "fix": "FIXED (2026-08-24). src/frontend/apiserver/controllers/jobs.go:553,901 -- both field-validation call sites now use utils.GetSupportedFields(models.JobInfo{}) instead of models.JobSubmitRequest{}, matching the already-correct pattern at jobs.go:1131. Caught during live verification (not source review alone): core_limit/mem_limit exist on JobSubmitRequest but not JobInfo/JobAdditionalFields -- switching wholesale would have newly broken those two previously-valid -o/-l fields. Fixed by adding CoreLimit/MemLimit (CustomLimit) to models.JobAdditionalFields (models/jobs.go), the existing merge-in bucket already used for pids/pgids/interactive_job_url. Added TestJobsListFieldValidation_UsesJobInfoNotSubmitRequest (jobs_test.go) covering all 24 fields in xcecli's real JobFieldsMap vocabulary. Full apiserver/controllers suite passes (CGO_ENABLED=0, this package's pre-existing required mode). Live-verified: `jobs list -l` and `-o \"pending_reason start_time core_limit mem_limit pid pgid\"` all now succeed -- every one previously errored with \"unsupported query field\".", @@ -22494,22 +9392,25 @@ "jobs.go", "jobs-list", "field-validation", - "regression", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "regression" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 41 + "fix_commits": [], + "record": ".wolf/bugs/bug-1674.md", + "gitea_issue": 41, + "gitea_synced_status": "fixed" }, { "id": "bug-1675", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xcecli", + "severity": "unknown", "timestamp": "2026-08-24T13:40:00Z", - "error_message": "Gitea issue #42 (\"\u521a\u542f\u52a8\u7684\u65f6\u5019\u67e5\u8be2\u4f5c\u4e1a\u62a5\u7684\u9519\u8bef\u4e0d\u592a\u53cb\u597d\"): `xcecli jobs list` run in the narrow window right after the apiserver (re)starts, before its listener is up, prints a raw `get the request failed: [connection refused]` instead of a user-friendly message. Reproduced live on branch scheduler-framework: killed and relaunched the apiserver, hammered `xcecli jobs list` in a tight loop from the instant of relaunch -- attempts 1-13 all returned the exact reported text; the listener came up by attempt 14.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Gitea issue #42 (\"刚启动的时候查询作业报的错误不太友好\"): `xcecli jobs list` run in the narrow window right after…\n\nGitea issue #42 (\"刚启动的时候查询作业报的错误不太友好\"): `xcecli jobs list` run in the narrow window right after the apiserver (re)starts, before its listener is up, prints a raw `get the request failed: [connection refused]` instead of a user-friendly message. Reproduced live on branch scheduler-framework: killed and relaunched the apiserver, hammered `xcecli jobs list` in a tight loop from the instant of relaunch -- attempts 1-13 all returned the exact reported text; the listener came up by attempt 14.", "file": "src/frontend/xcecli/common/request/request.go:97-98, src/frontend/xcecli/common/request/request.go:185-188", "root_cause": "ExtractErrorMessage() (request.go:185-188) explicitly special-cases the connection-refused case (`if strings.Contains(errStr, \"connection refused\") { return \"connection refused\" }`) but returns the raw OS-level string verbatim instead of a friendlier message (e.g. explaining the API server may still be starting up). GetRequestFromAPIServer() (request.go:97-98) then wraps it as `\"get the request failed: [%s]\"`, producing exactly the reported output.", "fix": "FIXED (2026-08-24). src/frontend/xcecli/common/request/request.go:187 -- ExtractErrorMessage()'s connection-refused branch now returns \"cannot reach the API server -- it may still be starting up, retry shortly\" instead of the raw \"connection refused\" string. Confirmed no other call site string-matches the old literal. Added TestExtractErrorMessage (request_test.go, table-driven) -- RED before, GREEN after. Full xcecli suite passes. Live-verified: killed+relaunched the apiserver, hit `xcecli jobs list` during the startup window -- new message confirmed both via the fresh test and independently by rebuilding xcecli and re-checking.", @@ -22520,22 +9421,25 @@ "request.go", "error-message", "ux", - "startup-window", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "startup-window" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 42 + "fix_commits": [], + "record": ".wolf/bugs/bug-1675.md", + "gitea_issue": 42, + "gitea_synced_status": "fixed" }, { "id": "bug-1676", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-24T13:40:00Z", - "error_message": "Gitea issue #44 (\"FROM_HOST is -\"): `xcecli jobs list`'s FROM_HOST column always shows `-` instead of the host the job was submitted from. Reproduced live on branch scheduler-framework: submitted a job, checked `xcecli jobs list` ~1 second later (job still PEND) -- FROM_HOST blank; confirmed via the apiserver's own debug log showing `\"from_host\":\"\"` on every job regardless of state (PEND/RUN/DONE), across every job submitted in the session.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Gitea issue #44 (\"FROM_HOST is -\"): `xcecli jobs list`'s FROM_HOST column always shows `-` instead…\n\nGitea issue #44 (\"FROM_HOST is -\"): `xcecli jobs list`'s FROM_HOST column always shows `-` instead of the host the job was submitted from. Reproduced live on branch scheduler-framework: submitted a job, checked `xcecli jobs list` ~1 second later (job still PEND) -- FROM_HOST blank; confirmed via the apiserver's own debug log showing `\"from_host\":\"\"` on every job regardless of state (PEND/RUN/DONE), across every job submitted in the session.", "file": "src/frontend/apiserver/controllers/jobs.go:1483, src/frontend/apiserver/backend/handler.go:18-20, src/schema/controller.capnp", "root_cause": "Three-part chain, fully traced: (1) jobs.go:1483 correctly sets `FromHost: ctx.ClientIP()` at job-CREATION time. (2) controller.capnp's Job/JobSubmitData structs have no from_host field at all (confirmed via grep -- zero matches for fromHost/from_host in the schema), so this value never round-trips to or back from the C controller. (3) handler.go's OnJobUpdated (line 18-20) does a blind `store.Jobs.Put(uint64(job.Unique_JobID), *job)` full overwrite on every controller-sourced job push, unlike OnHostUpdated/OnHostConfigUpdated (handler.go:46-70) which explicitly merge worker-sourced and config-sourced fields onto the existing stored entry. Since a controller-sourced job push never carries from_host (per #2), the very first state-change push after creation -- which happens almost immediately, well before a user can typically observe the initial value -- wipes FromHost back to \"\" via the blind overwrite.", "fix": "FIXED (2026-08-24), full fix. User directive: client should assert its own from-host and the backend should pass it through unchanged, rather than the apiserver deriving it from ctx.ClientIP() (which is an IP not a hostname, and trivially spoofable via X-Forwarded-For since this apiserver never calls gin.Engine.SetTrustedProxies()). Two independent, non-overlapping pieces:\n\n(1) Origin: xcecli's BuildJobRequest() (cmd/jobs/submit.go) now sets JobSubmitRequest.FromHost via os.Hostname(); apiserver's CreateJob (controllers/jobs.go:1483) now does `FromHost: req.FromHost` -- a straight passthrough, ctx.ClientIP() removed entirely from this path. New field added to both xcecli's and apiserver's independent models.JobSubmitRequest structs (separate Go modules, no shared code). New test TestBuildJobRequest_SetsFromHostToLocalHostname (xcecli). Full xcecli suite and apiserver controllers/models suites pass.\n\n(2) Round-trip: src/schema/controller.capnp's JobSubmitData gains `fromHost @10: Text $C.mapname(\"from_host\")` (auto-regenerates controller.capnp.c/h via the existing meson capnpc-c custom_target -- no hand-editing of generated files). src/include/xce/types.h's job_submit_data_t gains `char* from_host`. src/lib/jobj.c's jobsub_init/jobsub_destroy (the hand-written, ref-counted, content-deduped job-submit interning object -- NOT auto-generated) thread from_host through the deep-copy, the dedup identity comparison, and the destroy path; 3 existing tests updated + new test_jobsub_from_host_distinguishes_identity in test_jobj.c/testsuite. Go apiserver/backend/convert.go's modelJobToSubmit/freeJobSubmit (submit direction) and cJobToModel/cJobArrayToModels (receive direction), plus convert_pure.go's rawSubmitData/rawJobToModel (the pure-Go, CGO-free mapping layer) all thread the field through. Did NOT need to touch job.c's build_job_submit_payloads() (bare pointer borrow, flows through automatically) or any WAL persistence code (capnp_encode_job/decode_Job already generically encode/decode the whole xce_job_t including the full .submit sub-struct) or backend/handler.go's OnJobUpdated (its existing blind overwrite is correct by construction once every push genuinely carries the right value).\n\nVerified: C xcelib_test 301/301 (ASAN leak detection on, includes the 3 updated + 1 new jobsub tests, independently re-run and confirmed). Go backend suite: pure-Go tests (CGO_ENABLED=0) and full CGO suite via `make test-backend` against the real regenerated capnp headers, both independently re-run and confirmed (`ok apiserver/backend 14.6s`). Live, independently verified on a freshly-submitted job (not reusing any job the implementer had already submitted): FROM_HOST= visible from the very first query while still PEND, survives the RUN transition, and (implementer's own live check, not independently re-run) survives a full 4-daemon restart for free via existing WAL_JOB_SUBMIT persistence.", @@ -22547,21 +9451,25 @@ "handler.go", "from-host", "capnp-schema", - "data-loss", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "data-loss" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 44 + "fix_commits": [], + "record": ".wolf/bugs/bug-1676.md", + "gitea_issue": 44, + "gitea_synced_status": "fixed" }, { "id": "bug-1677", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "tooling", + "severity": "crash", "timestamp": "2026-08-24T13:40:00Z", - "error_message": "scripts/gitea-sync.py's sync_bugs() would crash (or silently no-op) on any real invocation: `buglog = json.loads(BUGLOG_PATH.read_text())` (line 726) then `for i, bug in enumerate(buglog):` (line 731) treats buglog.json as a bare list, but its actual top-level structure is `{\"version\": 1, \"bugs\": [...]}` -- a dict. `enumerate()` over a dict yields its keys (\"version\", \"bugs\") as `bug`, and the very next line's `bug.get(\"id\", ...)` would raise AttributeError on the string \"version\". Discovered incidentally while checking whether to set the `gitea_issue` cross-reference field on newly-logged bugs (bug-1673..bug-1676) during a Gitea-issue verification pass -- not run live, found via static read of the sync script against the current buglog.json wrapper format.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "scripts/gitea-sync.py's sync_bugs() would crash (or silently no-op) on any real invocation…\n\nscripts/gitea-sync.py's sync_bugs() would crash (or silently no-op) on any real invocation: `buglog = json.loads(BUGLOG_PATH.read_text())` (line 726) then `for i, bug in enumerate(buglog):` (line 731) treats buglog.json as a bare list, but its actual top-level structure is `{\"version\": 1, \"bugs\": [...]}` -- a dict. `enumerate()` over a dict yields its keys (\"version\", \"bugs\") as `bug`, and the very next line's `bug.get(\"id\", ...)` would raise AttributeError on the string \"version\". Discovered incidentally while checking whether to set the `gitea_issue` cross-reference field on newly-logged bugs (bug-1673..bug-1676) during a Gitea-issue verification pass -- not run live, found via static read of the sync script against the current buglog.json wrapper format.", "file": "scripts/gitea-sync.py", "root_cause": "sync_bugs() was apparently written against (or never updated from) an earlier bare-array buglog.json format. Every other buglog reader/writer in this session's own tooling (and the Stop hook's own validation, per .wolf/OPENWOLF.md's 'Always preserve the top-level {\"version\": 1, \"bugs\": [...]} wrapper') expects the wrapper; this script does not, unlike its own spec-sync half (sync_specs et al.) which was presumably tested more recently.", "fix": "FIXED (2026-08-24). scripts/gitea-sync.py's sync_bugs() (~line 726, 799) now unwraps buglog_wrapper[\"bugs\"] on read and re-wraps buglog_wrapper[\"bugs\"] = buglog before _write_json on write, instead of treating the file as a bare list. Verified via a standalone read/write round-trip against a scratch copy of the real buglog.json (not the live file, no Gitea API calls) -- version and all 1019 entries survive intact.", @@ -22571,21 +9479,23 @@ "python", "buglog-format", "latent-bug", - "unrun", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "unrun" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1677.md" }, { "id": "bug-1678", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-24T13:55:00Z", - "error_message": "Gitea issue #43 (\"\u91cd\u542fdaemon\u540e\u4f5c\u4e1a\u8c03\u5ea6/\u4fe1\u606f\u4e0d\u5bf9\"): after restarting ALL daemons (controller, scheduler, worker, apiserver), previously-RUN jobs come back with STATUS=RUN correctly recovered but EXEC_HOST permanently blank (\"-\"), even though the underlying job is genuinely still running. CONFIRMED LIVE on branch scheduler-framework, corrected finding: an earlier verification pass on this same issue (2026-08-24, same session) restarted only controller+scheduler+worker and left the apiserver running, and did NOT reproduce the symptom -- that was a false negative, caused by the still-running apiserver simply continuing to display its stale-but-correct in-memory job cache from before the restart, never refreshed. Repeating the test with the apiserver ALSO restarted (matching the original report's \"restart all daemons\" exactly) reproduces the bug immediately and durably: 3 RUN jobs with EXEC_HOST=1*worker-1 before the restart; killed all 4 daemons; ran start-all.sh; EXEC_HOST was blank on every poll (15 polls over 30s, first poll ~6s after apiserver came back up) while STATUS stayed correctly RUN throughout. Controller log corroborates the mechanism: ctrl-1.log's recovery trace shows apply_job_update_transition logging 'duplicate UPDATE to state 0x2 -- no-op' for every job during WAL replay -- consistent with the WAL_JOB_DISPATCH fact applying SILENTLY first (PEND->RUN, no log line on the APPLIED path), immediately followed by each job's WAL_JOB_START fact (worker-confirmed start, also state=RUN) correctly detecting the now-redundant transition and logging the no-op. Neither replay step ever touches job.allocs.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Gitea issue #43 (\"重启daemon后作业调度/信息不对\"): after restarting ALL daemons (controller, scheduler…\n\nGitea issue #43 (\"重启daemon后作业调度/信息不对\"): after restarting ALL daemons (controller, scheduler, worker, apiserver), previously-RUN jobs come back with STATUS=RUN correctly recovered but EXEC_HOST permanently blank (\"-\"), even though the underlying job is genuinely still running. CONFIRMED LIVE on branch scheduler-framework, corrected finding: an earlier verification pass on this same issue (2026-08-24, same session) restarted only controller+scheduler+worker and left the apiserver running, and did NOT reproduce the symptom -- that was a false negative, caused by the still-running apiserver simply continuing to display its stale-but-correct in-memory job cache from before the restart, never refreshed. Repeating the test with the apiserver ALSO restarted (matching the original report's \"restart all daemons\" exactly) reproduces the bug immediately and durably: 3 RUN jobs with EXEC_HOST=1*worker-1 before the restart; killed all 4 daemons; ran start-all.sh; EXEC_HOST was blank on every poll (15 polls over 30s, first poll ~6s after apiserver came back up) while STATUS stayed correctly RUN throughout. Controller log corroborates the mechanism: ctrl-1.log's recovery trace shows apply_job_update_transition logging 'duplicate UPDATE to state 0x2 -- no-op' for every job during WAL replay -- consistent with the WAL_JOB_DISPATCH fact applying SILENTLY first (PEND->RUN, no log line on the APPLIED path), immediately followed by each job's WAL_JOB_START fact (worker-confirmed start, also state=RUN) correctly detecting the now-redundant transition and logging the no-op. Neither replay step ever touches job.allocs.", "file": "src/controller/job.c:1369-1370, src/controller/persist.c:3708-3757, src/controller/job.c:288-351 (restore_job), src/controller/persist.c:6690-6713 (persist_submit_job_fact)", "root_cause": "dispatch_a_job_() (job.c:1343-1372) sets j->job.allocs directly in memory (lines 1346-1353, the host/resource assignment the scheduler decided) and transitions j->job.state = JOB_STAT_RUN (line 1355), but the durable record of this transition -- persist_submit_job_fact(&c->persist, WAL_JOB_DISPATCH, alloc->jobid, JOB_STAT_RUN) at line 1369-1370 -- only carries {jobid, state} (persist_submit_job_fact()'s own signature, persist.c:6690-6694, has no allocation parameter at all; its payload type wal_jobstate_t is exactly {jobid, state}). The resource allocation itself is never written to the WAL anywhere. On a full-process restart, job objects are reconstructed purely from replay: restore_job() (job.c:288-351) decodes a job's ONLY WAL_JOB_SUBMIT entry (written at initial submission, strictly before dispatch -- confirmed via persist_job_hook_fun's own comment that WAL_JOB_SUBMIT durability is written once, before any job object exists -- so its capnp payload's allocs field is genuinely empty/NULL at encode time). apply_committed_entry()'s WAL_JOB_DISPATCH case (persist.c:3708-3757) then replays the dispatch fact via apply_job_update_transition(), which by explicit design and its own doc comment 'only ever touches state' -- job->job.allocs is never set anywhere in the replay path. Result: after restart, job.state is correctly RUN (both WAL_JOB_DISPATCH and the later WAL_JOB_START fact carry state=RUN and replay correctly) but job.allocs is permanently NULL for the remaining life of that job -- nothing ever re-derives it, since scheduler-dispatch-commit's design treats a job's dispatch as a one-time PEND->RUN event with no re-dispatch/re-reconciliation path for an already-RUN job (see bug-1631's confirmation that no live caller ever re-adds a RUN job back to the scheduler's dispatch pool). EXEC_HOST is rendered by the apiserver from this now-NULL allocation once the recovered, allocation-less job is pushed to it -- which is why restarting the apiserver alongside the controller (forcing a fresh push) reveals the bug, while leaving the apiserver running (as an earlier verification pass on this same issue did) masks it behind stale cached data.", "fix": "FIXED (2026-08-25). No capnp schema change needed -- JobAlloc (xce_job_alloc_t) was already a full top-level capnp struct (the existing Job.allocs field) with generated encode/decode functions already present in the build. New glue only: src/controller/persist.c gains capnp_encode_joballoc()/decode_joballoc() (mirroring the existing capnp_encode_worker()/restore_job() patterns) and persist_submit_job_dispatch_fact(persist_t*, jobid, state, alloc) -- WAL_JOB_DISPATCH's payload is now a wal_jobstate_t header plus trailing capnp-encoded JobAlloc bytes, declared in src/include/persist.h. src/controller/job.c: dispatch_a_job_()'s success path now calls persist_submit_job_dispatch_fact(..., j->job.allocs) instead of persist_submit_job_fact(...), reading j->job.allocs (the freshly-constructed canonical struct) under the same write lock. Both apply seams updated: apply_committed_entry() (primary's own boot replay) gets one added else-if branch inside its existing WAL_JOB_START/WAL_JOB_FINISH stamp-chain (minimal diff, same shared plumbing) that decodes and adopts trailing allocation bytes when present. apply_wal_entry() (standby catch-up / replicated-apply -- also required, since a promoted standby would hit the exact same bug) has WAL_JOB_DISPATCH split into its own case, calling new restore_job_dispatch_state() (job.c, restore_job_state()'s sibling, takes ownership of an optional decoded allocation). A payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_DISPATCH entry written before this fix) decodes no allocation and behaves exactly as before -- full backward compatibility, verified by the 3 pre-existing test_wal_job_dispatch_fact.c tests passing completely unchanged.\n\nPrimary regression test: test_dispatch_commit.c's test_dc_success_appends_wal_job_dispatch_durably (already drove the REAL dispatch_a_job_() production path through a real WAL-write + persist_recover() restart cycle, previously only checking j2->job.state) extended to also assert j2->job.allocs matches what was dispatched (host, resource name, value) -- the user's exact acceptance criterion, 'after recover the job should get the same as it's state before shutdown.' RED->GREEN independently reproduced (not just narrated): git-stashed job.c/persist.c/persist.h with the test extension kept, rebuilt, confirmed FAIL at exactly this assertion; restored, rebuilt, confirmed PASS. Added 3 new tests to test_wal_job_dispatch_fact.c covering the richer payload directly on both apply seams plus a full restart-replay variant -- all pass. Full controller_unit_test suite independently re-run: 1136/1136 pass, zero regressions.\n\nLive-verified at the WAL level via xce_test's waldump: a real dispatch_a_job_() call produced a WAL_JOB_DISPATCH entry with PLEN=176 (vs the old bare wal_jobstate_t shape's PLEN=16), confirming the richer payload is genuinely written on the production path.\n\nIMPORTANT: full xcecli-visible E2E verification (submit -> restart -> check EXEC_HOST live) was blocked by discovering bug-1679 (see that entry) -- a separate, more severe, pre-existing bug where live job-status pushes to the apiserver never fire at all, restart or not. This fix's correctness rests on the unit-test evidence above (which inspects the controller's actual recovered in-memory state directly, with no dependency on the broken apiserver-notification path) and the direct WAL-payload inspection, not on an xcecli-visible live check.", @@ -22600,24 +9510,28 @@ "restart-recovery", "data-loss", "scheduler-dispatch-commit", - "wal-jobdispatch", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "wal-jobdispatch" ], "related_bugs": [ "bug-1631", "bug-1679" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z", - "gitea_issue": 43 + "fix_commits": [], + "record": ".wolf/bugs/bug-1678.md", + "gitea_issue": 43, + "gitea_synced_status": "fixed" }, { "id": "bug-1679", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T03:15:00Z", - "error_message": "A job's live dispatch/start/finish transitions are never pushed to the apiserver -- xcecli jobs list shows a job stuck at PEND forever (or, in one observed case, disappearing from the default query entirely) even though the controller's own WAL and in-memory state correctly progress the job all the way through RUN to EXIT/DONE. Discovered as a side effect while live-verifying bug-1678 (restart-recovery of job.allocs) -- this is a SEPARATE, unrelated, more severe bug: it reproduces with NO restart involved at all, on a single, freshly-submitted job, every time. Confirmed reproducible on the STOCK branch (git-stashed all of bug-1678's changes, rebuilt, redeployed fresh) -- NOT a regression introduced by bug-1678 or any other fix landed this session (bug-1673/1674/1675/1676/1677 touch only Go/apiserver code, a Python script, or job_submit_data_t's from_host field + jobj.c's dedup identity check -- none touch the state-transition/hook-firing logic implicated here).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "A job's live dispatch/start/finish transitions are never pushed to the apiserver -- xcecli jobs…\n\nA job's live dispatch/start/finish transitions are never pushed to the apiserver -- xcecli jobs list shows a job stuck at PEND forever (or, in one observed case, disappearing from the default query entirely) even though the controller's own WAL and in-memory state correctly progress the job all the way through RUN to EXIT/DONE. Discovered as a side effect while live-verifying bug-1678 (restart-recovery of job.allocs) -- this is a SEPARATE, unrelated, more severe bug: it reproduces with NO restart involved at all, on a single, freshly-submitted job, every time. Confirmed reproducible on the STOCK branch (git-stashed all of bug-1678's changes, rebuilt, redeployed fresh) -- NOT a regression introduced by bug-1678 or any other fix landed this session (bug-1673/1674/1675/1676/1677 touch only Go/apiserver code, a Python script, or job_submit_data_t's from_host field + jobj.c's dedup identity check -- none touch the state-transition/hook-firing logic implicated here).", "file": "src/controller/job.c (dispatch_a_job_, on_job_start, and likely other direct-mutation call sites), src/controller/persist.c (apply_committed_entry's WAL_JOB_START/.../DISPATCH shared case, apply_job_update_transition)", "root_cause": "dispatch_a_job_() (job.c, scheduler-dispatch-commit's task 3.1 restructuring) mutates j->job.state DIRECTLY in memory (job.c ~1355, `j->job.state = JOB_STAT_RUN;`) BEFORE calling persist_submit_job_dispatch_fact()/persist_submit_job_fact() to durably record the transition. On a live, single-node, self-committing controller, the commit-then-apply pipeline runs synchronously enough that apply_committed_entry()'s own WAL_JOB_DISPATCH handling (via apply_job_update_transition(j, JOB_STAT_RUN)) sees `cur == state == JOB_STAT_RUN` -- because the state was ALREADY set moments earlier by the same call, under the same lock -- and returns JOB_TRANSITION_NOOP_DUPLICATE. The shared case body's `if (result != JOB_TRANSITION_APPLIED) return 0; /* rejected-backward / no-op-duplicate -- no hook */` then unconditionally skips xce_run_hooks(HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, ...) -- the ONLY mechanism (via apiserver_job_hook_fun/apiserver_push_joblist in apiserver.c, registered against XCE_CONTROLLER_HOOK_JOB) that pushes an updated job record to the apiserver. Live-confirmed via controller log: `apply_job_update_transition: job : duplicate UPDATE to state 0x2 -- no-op` fires for BOTH the WAL_JOB_DISPATCH commit AND the subsequent WAL_JOB_START commit (on_job_start(), job.c) on the exact same job, back to back -- meaning the job NEVER gets a single HOOK_DETAIL_JOB_UPDATE fire through its entire dispatch. apiserver_push_joblist() itself only ever fires ONCE per job, at creation time (apiserver_job_new_fun, subscribed to HOOK_DETAIL_JOB_NEW, itself fired once by apply_committed_entry's WAL_JOB_SUBMIT case) -- there is no periodic/polling full-refresh fallback, confirmed by reading apiserver_push_joblist's only call site. So a job's apiserver-visible record is permanently frozen at whatever it looked like at creation (JOB_STAT_PEND, no exec_host) for its entire lifetime, regardless of what actually happens to it afterward, UNLESS some other code path elsewhere in job.c legitimately fires the hook directly (mutate-then-notify, like mark_job_update()/persist_job_hook_fun's 'still-legitimate append' pattern) rather than relying on the apply-time hook-fire -- not confirmed either way for on_job_exit/mark_job_fail/other terminal-transition call sites; only dispatch_a_job_() and on_job_start were directly traced. This is very likely a structural gap introduced by (or newly exposed by) the raft-apply-decoupling / scheduler-dispatch-commit redesigns moving job-state mutation to happen synchronously ahead of persistence for several call sites, while the apiserver-notification hook-fire mechanism was designed under an assumption (state hasn't already changed by the time apply runs) that held for the OLD architecture but no longer holds for these newer, direct-mutation call sites.", "fix": "FIXED (2026-08-25). Root cause confirmed exactly as filed: dispatch_a_job_() (job.c) mutates job.state/job.allocs directly then persists, firing no hooks at all -- apply_committed_entry()'s later apply of that same WAL_JOB_DISPATCH entry sees state already matches and skips its would-be notify (JOB_TRANSITION_NOOP_DUPLICATE).\n\nNew src/controller/persist.c function persist_notify_job_updated_already_persisted(cluster_t*, job_t*) (declared in src/include/persist.h), called from dispatch_a_job_()'s success branch immediately after unlock(j). Internally guards with the exact same t_apply_seam_notifying flag apply_committed_entry()'s own shared WAL_JOB_START/.../DISPATCH case already uses around its identical hook fire (same guard, same hook -- HOOK_DETAIL_JOB_UPDATE's AFTER_EVENT -- same detail) -- without it, persist_job_hook_fun (also subscribed to this hook) would redundantly re-append the same transition as a second, duplicate WAL_JOB_UPDATE entry. The flag stays file-private to persist.c; job.c reaches it only through this one new function. dispatch_a_job_()'s own mutation/persistence logic is otherwise untouched.\n\nTDD: new test_dc_success_notifies_job_update_hook_exactly_once (test_dispatch_commit.c) -- a spy hook counting HOOK_AFTER_EVENT/HOOK_DETAIL_JOB_UPDATE fires specifically. RED->GREEN independently reproduced: temporarily disabled the new call (production code unchanged otherwise), rebuilt, confirmed FAIL (Expected 1 Was 0); restored, rebuilt, confirmed PASS. Verified the two existing tests that would catch a broken guard (double WAL append): test_jobdispatch_receipt_ack_no_wal_append (asserts exactly one WAL entry for a successful dispatch) and test_handle_jobdispatch_matches_dispatch_a_job_notify_count -- both pass unchanged, and the former's fixture genuinely wires persist.cluster with persist_job_hook_fun registered, so it actually exercises the guard. Full controller_unit_test: 1137/1137 (2 pre-existing test_config_repl.c failures -- the known overlayfs fdatasync errno=5 flake, bug-1561 -- confirmed identical via git-stash on stock code, not a regression).\n\nLive-verified on a fresh single-node deploy (rebuilt controller/scheduler/worker/xcecli): submitted a job, xcecli jobs list showed PEND at t+1s then RUN with EXEC_HOST=1*worker* from t+2s onward continuously -- no restart involved, matching the exact acceptance criterion. Also confirmed record_job_finish_fact's existing notification path needed no fix: on_job_finish fired correctly and xcecli jobs list -d showed DONE with EXEC_HOST intact for both test jobs, exactly as the root-cause analysis predicted (only dispatch_a_job_()'s broken hybrid pattern was ever the problem -- the durable-append-only pattern record_job_start_fact/record_job_finish_fact use was already correct).", @@ -22631,23 +9545,26 @@ "live-status", "data-loss", "high-severity", - "discovered-via-bug-1678", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "discovered-via-bug-1678" ], "related_bugs": [ "bug-1678", "bug-1680" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1679.md" }, { "id": "bug-1680", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "data-loss", "timestamp": "2026-08-25T03:50:00Z", - "error_message": "`xcecli jobs list -a` (\"list run,done,pending,suspended jobs\" -- i.e. show every job regardless of status) returns \"no job found\" even when jobs genuinely exist and are correctly stored with a real status. Discovered as a side effect of live-verifying bug-1679: two jobs that correctly showed PEND->RUN->DONE via the default query and via `-d` (done-only) both come back empty under `-a` specifically. Not a data-loss bug (the jobs ARE correctly stored and queryable through other flags -- `-d`, the default active-jobs view -- this is specifically `-a`'s own query construction that's broken) but a real, reproducible CLI defect: `-a` is documented as the way to see literally everything, and it shows nothing.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "`xcecli jobs list -a` (\"list run,done,pending,suspended jobs\" -- i.e\n\n`xcecli jobs list -a` (\"list run,done,pending,suspended jobs\" -- i.e. show every job regardless of status) returns \"no job found\" even when jobs genuinely exist and are correctly stored with a real status. Discovered as a side effect of live-verifying bug-1679: two jobs that correctly showed PEND->RUN->DONE via the default query and via `-d` (done-only) both come back empty under `-a` specifically. Not a data-loss bug (the jobs ARE correctly stored and queryable through other flags -- `-d`, the default active-jobs view -- this is specifically `-a`'s own query construction that's broken) but a real, reproducible CLI defect: `-a` is documented as the way to see literally everything, and it shows nothing.", "file": "src/frontend/xcecli/filters/job_filter.go:58-60, src/frontend/apiserver/utils/types.go:262-322, src/frontend/apiserver/store/query.go:70-73", "root_cause": "xcecli's `-a` flag (list.go:347, `AllJobInfoFlag`) makes HandleJobFilter (job_filter.go:57-60) build the filter string `{job_status:notnull:}` -- a `notnull` operator on `job_status` with an EMPTY value token (no argument needed for null-checks, by design) -- instead of the normal `{job_status:in:(...)}`/`{job_status:notin:(...)}` shape every other status flag (-p/-r/-d/-s) produces. Server-side, `job_status`/`host_status` get SPECIAL bitmask handling in buildIntCondition (types.go:262-322), but that special-casing only triggers for the `in`/`notin`/`between` operators (the `if filter.Field == \"job_status\" || filter.Field == \"host_status\"` branch sits inside the `if filterOperator == \"in\" || filterOperator == \"notin\" || filterOperator == \"between\"` block, types.go:263-270) -- `notnull` falls through to the generic scalar path (types.go:316-322), which builds a plain `Condition{Column: \"job_status\", Operator: OpNOTNULL, Value: {TypeInt, 0}}` with no bitmask awareness at all. In isolation, store/query.go's own MatchesCondition evaluator (query.go:70-73) implements OpNOTNULL correctly (`fieldVal != nil`), which should trivially match every job (job_status is never actually absent/nil in a stored row) -- meaning the actual break is somewhere between condition construction and evaluation for this specific field/operator combination that was not fully traced (a debugger/print-tracing session would be needed to pin the exact line): plausible candidates are (a) the query engine routing job_status conditions through a bitmask-specific code path unconditionally regardless of operator, silently misinterpreting a non-bitmask NOTNULL Condition as a bitmask EQ/NE against value 0 (which would exclude every real job_status value, all of which are nonzero bitmask flags), or (b) some other job_status-specific special-casing elsewhere in the query pipeline that only handles the `in`/`notin`/`eq`/`ne` shapes types.go's own bitmask branch produces and never anticipated a bare NOTNULL condition on this field at all.", "fix": "FIXED (2026-08-25). Root cause fully pinned down (the prior entry's root_cause left the construction-to-evaluation gap untraced -- now traced exactly): src/frontend/apiserver/store/query.go's FilterRows() (the actual filter-evaluation loop, not something the prior investigation reached) routes ANY condition on job_status/host_status through the bitmask path purely based on IsBitmaskField(cond.Column) -- unconditional on the field NAME alone, never checking the condition's actual Operator. MatchesBitmaskCondition()'s own switch only implements OpEQ/OpNE (the only two shapes buildIntCondition's in/notin bitmask branch ever produces) and silently `return false` for any other operator -- so `{job_status:notnull:}` (Operator: OpNOTNULL, Value: {TypeInt, 0}, per types.go's null/notnull scalar path) gets routed into MatchesBitmaskCondition anyway, matches neither switch case, and returns false for every single row unconditionally. That's the entire bug: field-name-based routing that ignores which operator shape is actually on the condition.\n\nFix: FilterRows's routing condition now requires BOTH IsBitmaskField(cond.Column) AND cond.Operator being OpEQ or OpNE before taking the bitmask path; any other operator (OpNOTNULL, OpISNULL, or anything else that might ever target job_status/host_status) correctly falls through to the generic MatchesCondition evaluator, which already implements OpNOTNULL/OpISNULL correctly (fieldVal != nil / == nil) -- confirmed by the fix requiring zero changes to MatchesCondition or MatchesBitmaskCondition themselves, only the dispatch condition in FilterRows.\n\nNew src/frontend/apiserver/store/query_test.go (no test file existed for this package before -- confirmed via search, matches the prior entry's note): TestFilterRows_JobStatusNotNull_MatchesRealRows reproduces the exact bug (RED before the fix: 0 rows matched instead of 3; independently confirmed) and TestFilterRows_JobStatusBitmaskEQ_StillWorks locks in that the actual bitmask shape (OpEQ/OpNE, what every -p/-r/-d/-s status flag produces) is unaffected -- both pass after the fix. Full apiserver/store, apiserver/controllers, apiserver/models (CGO_ENABLED=0) and apiserver/backend + apiserver/backend/auth (full CGO via make test-backend) suites all pass, zero regressions.\n\nLive-verified: rebuilt and hot-swapped the apiserver binary on a running cluster with 3 pre-existing DONE jobs (previously invisible to `-a`) plus one freshly-submitted job -- `xcecli jobs list -a` now correctly shows all 4 jobs across every status (3x DONE, 1x RUN), immediately, no restart needed.", @@ -22659,22 +9576,25 @@ "query.go", "job-status-filter", "cli-bug", - "discovered-via-bug-1679", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "discovered-via-bug-1679" ], "related_bugs": [ "bug-1679" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1680.md" }, { "id": "bug-1681", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-24T21:30:00Z", - "error_message": "xce-audit-invariants scheduler-driver-equivalence audit finding (not user-reported, no live symptom today): driver.c's normal-mode round (xce_sched_driver_run_normal_mode_round, driver.c:750,802) calls plugin_begin_session(sched)/plugin_end_session(sched) -- iterating every registered plugin and invoking its begin_session_fn/end_session_fn hook, called 'real, load-bearing per-session bookkeeping for EVERY registered plugin' per driver.c's own comment (lines 688-693). sim_loop.c has zero equivalent call anywhere (grep-confirmed across sim*.c/repl*.c). This is structurally the same shape of gap as bug-1613's original root cause (a driver.c-only per-session call with no sim_loop.c mirror), but under the codebase's only currently-supported configuration it is inert: sim mode's sim_bootstrap_picker() (sim_picker.c:60-70) only ever registers builtin_fcfs, which explicitly leaves begin_session_fn/end_session_fn NULL (plugins/builtin_fcfs.c:973-981), and the one plugin that does set a hook (legacy fcfs, plugins/fcfs.c:352, end_session_fn=fcfs_end_session, only sets privdata->new_session=1, no reach into resInUse/sharedInUse/job->accounts) is unreachable in sim mode either way (sim mode refuses any picker name other than builtin_fcfs).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "xce-audit-invariants scheduler-driver-equivalence audit finding (not user-reported, no live…\n\nxce-audit-invariants scheduler-driver-equivalence audit finding (not user-reported, no live symptom today): driver.c's normal-mode round (xce_sched_driver_run_normal_mode_round, driver.c:750,802) calls plugin_begin_session(sched)/plugin_end_session(sched) -- iterating every registered plugin and invoking its begin_session_fn/end_session_fn hook, called 'real, load-bearing per-session bookkeeping for EVERY registered plugin' per driver.c's own comment (lines 688-693). sim_loop.c has zero equivalent call anywhere (grep-confirmed across sim*.c/repl*.c). This is structurally the same shape of gap as bug-1613's original root cause (a driver.c-only per-session call with no sim_loop.c mirror), but under the codebase's only currently-supported configuration it is inert: sim mode's sim_bootstrap_picker() (sim_picker.c:60-70) only ever registers builtin_fcfs, which explicitly leaves begin_session_fn/end_session_fn NULL (plugins/builtin_fcfs.c:973-981), and the one plugin that does set a hook (legacy fcfs, plugins/fcfs.c:352, end_session_fn=fcfs_end_session, only sets privdata->new_session=1, no reach into resInUse/sharedInUse/job->accounts) is unreachable in sim mode either way (sim mode refuses any picker name other than builtin_fcfs).", "file": "src/scheduler/src/sim_loop.c", "root_cause": "plugin_begin_session()/plugin_end_session() (plugin.c:370,407) are a distinct hook category from xce_job_picker_iface_t's begin_session/end_session (which both drivers DO call symmetrically -- scheduler.c:111,131 vs sim_loop.c:467,484). Nothing in sim_loop.c or sim.c acknowledges skipping the plugin-level begin/end_session hooks -- unlike the wire/capnp-only and REPL-only differences the audit also found, which are documented deliberate exclusions.", "fix": "FIXED 2026-08-24. plugin_begin_session()/plugin_end_session() (src/scheduler/src/plugin.c) relocated from an implicit xce_scheduler_t* (void* arg cast to xce_scheduler_t*, only ever touching sched->core.plugins) to a direct xce_sched_core_t* parameter -- mirroring sched_account_jobs()'s identical scheduler-sim-decision-parity task 1.1 relocation, and confirming driver.c's own header comment claim that plugin_begin_session() 'needs the full xce_scheduler_t*' was stale/incorrect (its body never touched anything beyond core->plugins). src/scheduler/src/sim_loop.c's sched_sim_driver_run() round loop now calls plugin_begin_session(&(sim->core)) immediately before sched_account_jobs(&(sim->core)) (same relative order driver.c uses), and plugin_end_session(&(sim->core)) immediately before xce_core_end_session() -- exactly mirroring driver.c's xce_sched_driver_run_normal_mode_round() call sites, which were updated to pass &(sched->core) instead of sched. src/scheduler/src/scheduler.c's legacy sched_begin_session()/sched_end_session() (a third, pre-existing call site not part of the original driver-equivalence audit's scope, found only when the signature change broke its compile) updated the same way. plugin.h's declarations relocated to xce_sched_core_t* (a new forward-declare added earlier in the file so the type is visible at the declaration site). Stale comments describing the old xce_scheduler_t* coupling corrected in driver.c, core.h, and core_session.c (two locations).\n\nTest: extended testsuite/test_sim_session_ordering.c's existing bug-1613 spy-based regression tests (test_sim_replay_calls_account_jobs_once_per_round_within_session_window, test_sim_replay_batch_mode_uses_the_same_round_boundary) with new plugin_begin_session()/plugin_end_session() spies (same call-count + core->session_active-window proof technique as the existing sched_account_jobs spy) -- proving 2 calls/round (recording mode) and 1 call (batch mode), each with session_active==1 at call time. Deliberately does NOT #include the real ../src/plugin.c in this file (unlike test_driver_session_ordering.c, which proves real per-plugin hook firing via a genuinely-registered observer plugin) -- sim.core.plugins does not exist until sched_sim_driver_run()'s own internal sched_sim_driver_start()/sim_bootstrap_picker() calls run, with no hook point for a test to register an observer plugin in between, so this file spies the call site directly instead, plus a verbatim real reimplementation of plugin_register_core() (needed for real by both this file's fixture and sim_picker.c's sim_bootstrap_picker()) and no-op link-satisfying stubs for plugin_register()/plugin_unregister() (both genuinely unreachable in this file's fixtures). Four other white-box test files (test_resval_ownership.c, test_admin_request.c, test_driver_decision_cb.c, test_driver_translate.c) each defined their own plugin_begin_session/plugin_end_session stub against the old void* signature -- all four updated to xce_sched_core_t* to match.\n\nRED/GREEN independently verified (not just narrated): temporarily commented out both new sim_loop.c call sites, rebuilt, confirmed test_sim_replay_calls_account_jobs_once_per_round_within_session_window and test_sim_replay_batch_mode_uses_the_same_round_boundary both FAIL (Expected 2 Was 0 / Expected 1 Was 0) -- proving the assertions genuinely exercise the missing calls, not a tautology. Restored (git diff confirmed byte-identical to the fixed state), rebuilt, reconfirmed both PASS.\n\nFull scheduler suite: 51/52 suites green; the one failure (test_recordstart_symlink_escape_replies_error) is the pre-existing, unrelated bug-1560 sandbox-environment flake (already documented as such in scheduler-core's own task 12.5 verification notes), not a regression from this fix.\n\nDeliberately NOT fixed as part of this change (separate, unrelated pre-existing defect found while touching the same function bodies): plugin_end_session()'s per-plugin guard checks the wrong field (begin_session_fn instead of end_session_fn) -- logged separately as bug-1682.", @@ -22685,22 +9605,26 @@ "plugin", "driver-equivalence", "latent", - "scheduler-driver-equivalence", - "fixed" + "scheduler-driver-equivalence" ], "related_bugs": [ "bug-1611", "bug-1613" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1681.md" }, { "id": "bug-1682", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-24T21:40:00Z", - "error_message": "Found while fixing bug-1681 (source read of plugin_end_session(), src/scheduler/src/plugin.c): the per-plugin iteration guard checks the WRONG field. plugin_end_session()'s loop tests `if (plugin->begin_session_fn != NULL) { plugin->end_session_fn(plugin); }` -- guards on begin_session_fn but calls end_session_fn. Pre-existing, not introduced by bug-1681's fix (untouched by that diff). Previously documented (not fixed) in testsuite/test_driver_session_ordering.c's own file header comment as a known 'CONCERN surfaced during this test's construction' -- flagged there for the reviewer, deliberately left unfixed as out of that task's scope. Confirmed still present in current code.", + "last_seen": "2026-09-19T09:30:00Z", + "occurrences": 1, + "error_message": "Found while fixing bug-1681 (source read of plugin_end_session(), src/scheduler/src/plugin.c): the…\n\nFound while fixing bug-1681 (source read of plugin_end_session(), src/scheduler/src/plugin.c): the per-plugin iteration guard checks the WRONG field. plugin_end_session()'s loop tests `if (plugin->begin_session_fn != NULL) { plugin->end_session_fn(plugin); }` -- guards on begin_session_fn but calls end_session_fn. Pre-existing, not introduced by bug-1681's fix (untouched by that diff). Previously documented (not fixed) in testsuite/test_driver_session_ordering.c's own file header comment as a known 'CONCERN surfaced during this test's construction' -- flagged there for the reviewer, deliberately left unfixed as out of that task's scope. Confirmed still present in current code.", "file": "src/scheduler/src/plugin.c", "root_cause": "Copy-paste from plugin_begin_session()'s loop body (which correctly guards on begin_session_fn before calling it) without updating the guard field for the end_session variant.", "fix": "FIXED (2026-09-19, /xce-audit-invariants scheduler dimension-1 audit). Prior tag-only \"fixed\"/\"closed-by-scheduler-sim-live-allocation\" stamps on this entry were premature/incorrect -- confirmed by this audit's independent re-read of plugin.c that the guard-field bug was still genuinely present in code as of the scheduler-sim-live-allocation feature's completion (that feature touched plugin_job_election/plugin_prepare_allocation/plugin_commit_decision only, never plugin_begin_session/plugin_end_session's bodies). Now actually fixed: changed plugin_end_session()'s guard from plugin->begin_session_fn != NULL to plugin->end_session_fn != NULL (src/scheduler/src/plugin.c). Confirmed via TDD (RED: new test_end_session_invokes_end_session_fn_when_only_it_is_set() fails 84/85 against the pre-fix guard; GREEN: 85/85 after). Duplicate of bug-1492 (same defect, originally filed 2026-08-04, discovered again independently during this bug's own 2026-08-24 investigation without cross-referencing the earlier entry) -- see bug-1492 for the original filing.", @@ -22710,23 +9634,26 @@ "session-hooks", "guard-bug", "latent", - "scheduler-driver-equivalence", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "scheduler-driver-equivalence" ], "related_bugs": [ "bug-1681", "bug-1492" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:30:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1682.md" }, { "id": "bug-1683", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-08-25T00:05:00Z", - "error_message": "xceadmin pytest failure: test_send_frame_writes_exact_known_good_byte_sequence (src/xceadmin/tests/test_wire.py:200). Test sends a 4-byte payload b'xce!' via wire.send_frame() over a real TLS socket to a raw capture handler that reads exactly 12 bytes (8-byte length header + 4-byte payload) directly off the wire, bypassing this module's own recv_frame() decoder. Expected the captured bytes to be struct.pack('=Q', 4) + b'xce!' (header correctly encodes length=4, followed by the literal payload bytes). Actual captured bytes: the 8-byte header is correct (b'\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00', i.e. length=4 little-endian native uint64_t), but the trailing 4 bytes are all zero (b'\\x00\\x00\\x00\\x00') instead of b'xce!' -- the real payload bytes never reached the wire. 1 failed, 266 passed, 3 skipped -- isolated to this one test.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "xceadmin pytest failure: test_send_frame_writes_exact_known_good_byte_sequence…\n\nxceadmin pytest failure: test_send_frame_writes_exact_known_good_byte_sequence (src/xceadmin/tests/test_wire.py:200). Test sends a 4-byte payload b'xce!' via wire.send_frame() over a real TLS socket to a raw capture handler that reads exactly 12 bytes (8-byte length header + 4-byte payload) directly off the wire, bypassing this module's own recv_frame() decoder. Expected the captured bytes to be struct.pack('=Q', 4) + b'xce!' (header correctly encodes length=4, followed by the literal payload bytes). Actual captured bytes: the 8-byte header is correct (b'\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00', i.e. length=4 little-endian native uint64_t), but the trailing 4 bytes are all zero (b'\\x00\\x00\\x00\\x00') instead of b'xce!' -- the real payload bytes never reached the wire. 1 failed, 266 passed, 3 skipped -- isolated to this one test.", "file": "src/xceadmin/src/xcecommon/wire.py", "root_cause": "UNRESOLVED -- not reproducible. send_frame() (wire.py:102-113) is two sequential sock.sendall() calls (header, then payload) -- nothing in it, or in the test's _TlsEchoServer/_capture_handler harness (test_wire.py), or conftest.py's session-scoped tls_certs fixture, could zero out payload bytes; all reviewed and found correct. Confirmed only bare pytest 8.4.2 is installed (no pytest-xdist, no pytest-randomly, no addopts) -- test order is deterministic, ruling out parallel-worker/reordering interference.\n\n45+ reproduction attempts (2026-08-25), zero failures: isolated run x5, full test_wire.py x3, full 270-test suite x3 (ad-hoc PYTHONPATH), full meson-driven build+test pipeline (xce-build.sh xceadmin, the project's real test-running convention) x1 clean (267 passed, 3 skipped), isolated via the meson-built venv x15, full suite via meson-built venv x5, full suite under artificial CPU stress (20x `yes` spinners, load 11+) x5, 8-way concurrent parallel execution of the exact test x8 -- all clean. TCP/TLS cannot spontaneously zero application bytes mid-stream, so a genuine code defect producing this exact symptom (correct 8-byte header, real zeroed 4-byte payload) should be deterministically reproducible, and it isn't.", "fix": "Not a code fix -- treating as a suspected ONE-OFF ENVIRONMENTAL/INFRASTRUCTURE artifact, not a defect in current source. This project is documented (.wolf/cerebrum.md, bug-1604) to run many concurrent automated sessions that race on shared build directories -- e.g. a concurrent xce-build.sh/meson-build.sh invocation's `rm -rf \"$WORK_DIR\"` + `cp -r` mirror step, if raced by another concurrent invocation touching the same build dir, is a plausible analogous corruption mechanism (already confirmed for other components under bug-1604's pattern) -- but no direct evidence this happened for bug-1683 specifically, since it has not recurred. No code changes made. If this recurs, capture `ps aux` (or similar) for a concurrent xce-build.sh/meson-build.sh run racing the same build directory at failure time, per the bug-1604 precedent -- that is the most useful missing evidence right now.", @@ -22737,22 +9664,25 @@ "python", "framing", "test-failure", - "unreproducible", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "unreproducible" ], "related_bugs": [ "bug-1604" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1683.md" }, { "id": "bug-1684", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T00:50:00Z", - "error_message": "xce-audit-invariants controller audit (dimension: error handling) finding: raft_cb_persist() (src/controller/ha.c:2257) always returns 0 to the raft library's on_persist callback contract regardless of whether the term/voted_for durability write to disk actually succeeded -- a disk I/O failure (full disk, permission error, transient I/O error) could let a node grant a vote it has no durable record of, opening a split-vote path if that node then restarts and forgets it voted (Election Safety: 'a server grants at most one vote per term' via durably-persisted votedFor).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-audit-invariants controller audit (dimension: error handling) finding: raft_cb_persist()…\n\nxce-audit-invariants controller audit (dimension: error handling) finding: raft_cb_persist() (src/controller/ha.c:2257) always returns 0 to the raft library's on_persist callback contract regardless of whether the term/voted_for durability write to disk actually succeeded -- a disk I/O failure (full disk, permission error, transient I/O error) could let a node grant a vote it has no durable record of, opening a split-vote path if that node then restarts and forgets it voted (Election Safety: 'a server grants at most one vote per term' via durably-persisted votedFor).", "file": "src/controller/ha.c", "root_cause": "src/lib/raft/raft.c:260-263's raft_on_vote_request Step 7 has an explicit durability-guarantee contract: 'if (s->cbs->on_persist(...) != 0) { /* Persist failed -- deny without updating voted_for */ return RAFT_DENY; }' -- the adapter's on_persist callback (raft_cb_persist) must return non-zero on a genuine persist failure. raft_cb_persist's own doc comment even claimed this contract ('Returns 0 on success, non-zero on error'), but its implementation called ha_persist_term(ha, workdir) and then unconditionally `return 0;` regardless of the outcome. ha_persist_term() itself (ha.c:5174) had a void return with every open()/write()/fsync() failure silently discarded (`if (fd < 0) return; /* best-effort; caller must not rely on this for correctness */`), so the failure was structurally invisible to the caller even before raft_cb_persist's own bug. Compounding: raft_cb_persist mutates ha->voted_for_term/ha->voted_for_name in-memory BEFORE calling ha_persist_term, so even fixing just the return-code propagation would leave the adapter's own in-memory state claiming a vote was cast that the raft library itself denied and never durably recorded -- a second-order inconsistency needing an explicit rollback.", "fix": "FIXED (2026-08-25). ha_persist_term() (ha.c:5174, declared ha.h:1307) relocated from void to int -- returns 0 on success (including the deliberate NULL-workdir test-mode skip) or -errno on a genuine open/write/fsync/rename failure (fsync's own previously-unchecked return value is now checked too). raft_cb_persist() (ha.c:2257) now captures ha_persist_term's return, and on failure: (a) logs a loud ERROR with term/voted_for/rc, (b) rolls back ha->voted_for_term/ha->voted_for_name to their pre-call snapshot (taken under ha->lock before the tentative mutation) so the adapter's in-memory state stays consistent with the raft library's own 'deny without updating voted_for' behavior, (c) returns the negative rc so raft_on_vote_request denies the vote per its own Step 7 contract. ha->term is deliberately NOT rolled back -- confirmed via direct read of raft_on_vote_request's Step 2 (raft.c:191-204) that the raft library itself adopts a higher observed term unconditionally, before and independent of the vote-grant/persist outcome, so the adapter's ha->term correctly mirrors that same unconditional-adoption semantics. Also fixed the adjacent stale doc-comment (src/controller/CLAUDE.md's Protocol Verification Checklist item 1, found by a sibling audit dimension the same session) which incorrectly claimed term+voted_for were written 'atomically under ha->lock' -- term uses a separate atomic store outside the lock.\n\nTest: added 3 tests to testsuite/test_ha_durability.c. (1) test_persist_term_returns_error_on_write_failure -- points workdir at a path whose parent directory doesn't exist (deterministic ENOENT, portable, no permission-bit tricks needed even running as root), asserts ha_persist_term returns <0. (2) test_persist_term_null_workdir_returns_success -- regression guard that the NULL-workdir test-mode skip stays a 0/success return, not folded into the new error path. (3) test_vote_denied_and_state_rolled_back_on_persist_failure -- drives the REAL production path (ha_init_raft wiring raft_cb_persist as the real on_persist callback, then raft_on_vote_request -- the exact function cluster.c's production dispatch calls) against a minimal cluster_t/myconf_t fixture (ha_init_raft(ha, NULL, n) avoids needing a real persist_t, ha->raft_c set directly afterward -- the same test-shim pattern ha.h already documents for raft_cb_become_leader_for_test/raft_cb_become_follower_for_test) with a genuine non-empty candidate name populated so the rollback assertion is meaningful (not trivially satisfied by an already-empty baseline); asserts the vote is RAFT_DENY and ha.voted_for_term/voted_for_name are unchanged from their pre-call baseline.", @@ -22762,20 +9692,23 @@ "raft", "election-safety", "persistence", - "error-handling", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "error-handling" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1684.md" }, { "id": "bug-1685", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-08-25T15:47:00Z", - "error_message": "User-reported live crash: xce_scheduler SIGABRT via glibc malloc_printerr (heap corruption detected during a malloc/free bookkeeping check -- 'double free or corruption'-class abort, not a clean segfault). Core captured from a live 3-node HA deployment run (--peers ctrl-1/ctrl-2/ctrl-3, -vvv). Backtrace: malloc_printerr -> kill_limit (src/lib/xcelimit.c:181) <- limit_update_config (src/scheduler/src/plugins/limit.c:76) <- plugin_config_update (src/scheduler/src/plugin.c:246) <- sched_init_config (src/scheduler/src/scheduler.c:936) <- scheduler_req_handler (src/scheduler/src/handler.c:687) <- vtask_entry (src/lib/xcectx.c:172) <- run_task/vcpu_entry (src/lib/vtaskmgr.c). limit_update_config's own locals at the crash frame: rc=-17 (-EEXIST), nlimits=7, i=0 -- crashes on (or near) the FIRST limit in a 7-limit config update.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported live crash: xce_scheduler SIGABRT via glibc malloc_printerr (heap corruption…\n\nUser-reported live crash: xce_scheduler SIGABRT via glibc malloc_printerr (heap corruption detected during a malloc/free bookkeeping check -- 'double free or corruption'-class abort, not a clean segfault). Core captured from a live 3-node HA deployment run (--peers ctrl-1/ctrl-2/ctrl-3, -vvv). Backtrace: malloc_printerr -> kill_limit (src/lib/xcelimit.c:181) <- limit_update_config (src/scheduler/src/plugins/limit.c:76) <- plugin_config_update (src/scheduler/src/plugin.c:246) <- sched_init_config (src/scheduler/src/scheduler.c:936) <- scheduler_req_handler (src/scheduler/src/handler.c:687) <- vtask_entry (src/lib/xcectx.c:172) <- run_task/vcpu_entry (src/lib/vtaskmgr.c). limit_update_config's own locals at the crash frame: rc=-17 (-EEXIST), nlimits=7, i=0 -- crashes on (or near) the FIRST limit in a 7-limit config update.", "file": "src/scheduler/src/plugins/limit.c", "root_cause": "FIXED (2026-08-25). Confirmed double-free: limit_update_config()'s privdata->limits is a kvmgr created with .datakiller = kill_limit (limit_priv_data_init(), limit.c). Its key_add() call (limit.c, then rc=-17/-EEXIST on the duplicate-name entry -- exactly the crash frame's own captured locals) dispatches to xkvmgr_add() (src/lib/kvmgr.c:107-129): vstore(kvm->data, value) succeeds first, THEN key_add(kvm->kvtab, key, idx) fails (EEXIST for a name already present), so xkvmgr_add rolls back via vclear(kvm->data, idx) -- which dispatches to xzarray_clear() (src/lib/zarray.c:614-637), and since the slot is in-use, fires a->killer(a->data[idx].data), i.e. kill_limit(limit), ONCE, automatically, as part of that rollback. limit_update_config()'s OLD error path then called kill_limit(limit) a SECOND time on the same already-freed pointer -- the double free glibc's malloc_printerr aborts on. Established precedent for this exact ownership-ambiguity shape: src/controller/job.c's restore_job_array() documents the identical asymmetry for a DIFFERENT kvmgr (c->jobmgr, .datakiller = NULL there) -- that kvmgr's own internal rollback is a no-op (killer is NULL), so THAT caller correctly must free the rejected value itself. Whether a caller must free a rejected key_add() value depends entirely on whether THAT specific kvmgr instance was configured with a non-NULL datakiller; privdata->limits has one (kill_limit), so limit_update_config must NOT free it itself. One accepted, documented residual gap (mirrors job.c's own accepted trade-off): if vstore() itself fails (e.g. zarray growth OOM, before ever reaching the kvtab key_add step), xkvmgr_add returns negative WITHOUT having stored or freed `limit` at all, and xkvmgr_add's return code doesn't distinguish this from the common EEXIST case -- removing the kill_limit() call trades a guaranteed crash on the common (duplicate-name) case for a leak in that vanishingly rare OOM-only sub-case. Leak is strictly better than crash; closing this gap cleanly would need a new kvmgr API, out of scope here.", "fix": "FIXED (2026-08-25). Removed the redundant `kill_limit(limit);` call in limit_update_config()'s error path (src/scheduler/src/plugins/limit.c, `if (rc < 0) { ... return rc; }` block) -- single-line removal, replaced with a comment explaining the exact double-free mechanism above so a future reader doesn't reintroduce it 'to be safe'. Nothing else in limit.c/kvmgr.c/zarray.c touched.\n\nTest: added src/scheduler/testsuite/test_limit_plugin.c (this plugin had ZERO test coverage before this fix -- confirmed via `find src/scheduler/testsuite -iname '*limit*'` returning nothing pre-fix), wired into src/scheduler/testsuite/meson.build (new `sched_limit_plugin_test` executable + `scheduler limit plugin test` meson test, suite 'scheduler', ASAN_OPTIONS=detect_leaks=0 matching every sibling test). White-box #include of ../src/plugins/limit.c (same convention as test_plugin_picker_registration.c/test_handlers.c) to reach limit_update_config()/limit_init()/limit_priv_data_init(), all `static`; stubs reset_sched_obj()/get_resvals()/put_resvals()/plugin_register() (scheduler/src functions outside this TU, referenced only by limit.c code paths this test never exercises) purely to satisfy the linker. test_duplicate_limit_name_no_double_free() builds a real xce_scheduler_t fixture (real groupmgr_init()'d core.ugmgr/qgmgr/pgmgr/hgmgr, real limit_init()'d plugin.data), then drives limit_update_config() with an xce_conf4sched_t containing TWO xce_limit_config_t entries sharing the SAME name ('dup_limit') -- the exact duplicate-name collision bug-1685's live crash captured -- and asserts rc == -EEXIST with no crash. test_distinct_limit_names_both_succeed() is the control case (two DISTINCT names, rc == 0), proving the fix didn't just suppress freeing on failure in general.\n\nRED (pre-fix, verified live): built via `nix develop ./src --command bash src/tools/xce-build.sh scheduler asan build-asan --incremental` with the new test present but the buggy kill_limit(limit) call still in place -- sched_limit_plugin_test aborted with a genuine ASAN heap-use-after-free (not a clean assertion failure): 'AddressSanitizer: heap-use-after-free ... in kill_limit ../lib/xcelimit.c:181', READ stack `kill_limit <- limit_update_config (limit.c:76, the buggy call) <- test_duplicate_limit_name_no_double_free`, freed-by stack `kill_limit (xcelimit.c:192) <- xzarray_clear (zarray.c:626) <- xkvmgr_add (kvmgr.c:124) <- limit_update_config (limit.c:72, the key_add call)` -- an exact match to the diagnosed mechanism and to bug-1685's own captured crash signature (malloc_printerr <- kill_limit <- limit_update_config).\nGREEN (post-fix, verified live): rebuilt with the single-line fix applied -- `scheduler limit plugin test OK (0.05s)`. Full scheduler suite re-run: 52/53 OK, 1 pre-existing failure (`test_recordstart_symlink_escape_replies_error`, bug-1560, unrelated flake -- same failure present in the RED run too) -- zero new regressions.", @@ -22787,19 +9720,23 @@ "sigabrt", "live-deployment", "double-free", - "kvmgr-rollback", - "fixed" + "kvmgr-rollback" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1685.md" }, { "id": "bug-1686", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-08-25T16:24:00Z", - "error_message": "User-reported live crash (build.dbg, commit b040dfc0 -- AFTER the bug-1685 double-free fix landed, so this is a distinct, separate crash): xce_scheduler SIGSEGV in fcfs_cursor_enter_queue (src/scheduler/src/plugins/builtin_fcfs.c:673), called from builtin_fcfs_next_job (builtin_fcfs.c:810) <- sched_job_picker (scheduler.c:118) <- picker thread. Crash line 673 is the NULL branch of a ternary expression. queue_item=0x7db7d0000000 -- a suspiciously round address (large power-of-2-aligned), possibly a corrupted/uninitialized pointer rather than a normal heap allocation. Core captured from a live 3-node HA deployment run (--peers ctrl-1/ctrl-2/ctrl-3, -vvv).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported live crash (build.dbg, commit b040dfc0 -- AFTER the bug-1685 double-free fix landed…\n\nUser-reported live crash (build.dbg, commit b040dfc0 -- AFTER the bug-1685 double-free fix landed, so this is a distinct, separate crash): xce_scheduler SIGSEGV in fcfs_cursor_enter_queue (src/scheduler/src/plugins/builtin_fcfs.c:673), called from builtin_fcfs_next_job (builtin_fcfs.c:810) <- sched_job_picker (scheduler.c:118) <- picker thread. Crash line 673 is the NULL branch of a ternary expression. queue_item=0x7db7d0000000 -- a suspiciously round address (large power-of-2-aligned), possibly a corrupted/uninitialized pointer rather than a normal heap allocation. Core captured from a live 3-node HA deployment run (--peers ctrl-1/ctrl-2/ctrl-3, -vvv).", "file": "src/scheduler/src/plugins/builtin_fcfs.c", "root_cause": "FIXED (2026-08-25). fcfs_plugin_data_t.lock (added in the bug-1611-era concurrency fix, task 2.4 -- see builtin_fcfs.c's own file header Concurrency comment) correctly makes every individual hook call (on_queue_update/on_job_state_change/on_job_update/next_job/etc) mutually exclusive at the fcfs_plugin_data_t granularity -- but priv->cursor/priv->job_cursor are raw ordq_item_t* values that PERSIST in fcfs_plugin_data_t across SEPARATE builtin_fcfs_next_job() calls, each with its OWN independent lock acquisition/release. A properly-locked builtin_fcfs_on_queue_update() call for a queue name that already has a registered fcfs_queue_t re-sorts it by removing the queue's existing ordq_item_t node (ordq_get + FREEUP(it)) and inserting a brand-new one (ordq_put, fresh calloc). If priv->cursor is resting on that exact freed node -- which it normally is, since the cursor stays parked on a queue for as long as it keeps yielding ready jobs (sched_queue_is_ready()/sched_job_is_ready() are unconditional `return 1` stubs today in scheduler.c, so a queue's cursor never advances away from it while jobs remain) -- the NEXT next_job() call dereferences priv->cursor->data from freed memory: a real heap-use-after-free, exactly matching this bug's captured live SIGSEGV in fcfs_cursor_enter_queue() (queue_item was a dangling, freed ordq_item_t*, not corrupted/uninitialized memory as originally suspected). The lock prevents CONCURRENT corruption within one call; it does nothing to stop a LATER call from dereferencing a node freed by a call that ran validly in between. The same shape exists for priv->job_cursor via builtin_fcfs_on_job_state_change()'s !JOB_IS_PEND removal branch and fcfs_remove_job_everywhere() (called by on_job_update() and on_dispatch()), both of which ordq_del() a job node job_cursor may be resting on.", "fix": "FIXED (2026-08-25). Three invalidation sites in src/scheduler/src/plugins/builtin_fcfs.c, all following the same peek/compare-then-invalidate-before-free pattern:\n1. builtin_fcfs_on_queue_update()'s reuse branch: before FREEUP(it), if priv->cursor == it, sets both priv->cursor and priv->job_cursor to NULL (in lockstep, matching fcfs_cursor_enter_queue()'s own invariant that job_cursor is only ever meaningful relative to the queue cursor currently names).\n2. builtin_fcfs_on_job_state_change()'s !JOB_IS_PEND removal branch: ordq_peek()s the about-to-be-deleted job node first; if it equals priv->job_cursor, NULLs priv->job_cursor before ordq_del().\n3. fcfs_remove_job_everywhere() (shared by on_job_update() and on_dispatch()): same peek-then-invalidate-then-delete pattern inside its loop over all registered queues.\nAlways safe: next_job() already treats a NULL cursor as \"no more candidates this round\" (pre-existing branch, unrelated to this bug -- see test_picker_fcfs_session.c's no-queues/empty-queue coverage), and begin_session() unconditionally resets both cursors fresh every round, so the worst case of an invalidation is a bounded one-round delay for the affected queue's jobs, never a crash or a permanently lost job.\n\nTest: added src/scheduler/testsuite/test_builtin_fcfs_cursor_uaf.c, wired into src/scheduler/testsuite/meson.build (new sched_builtin_fcfs_cursor_uaf_test executable + 'scheduler builtin fcfs cursor uaf test' meson test, suite 'scheduler', ASAN_OPTIONS=detect_leaks=0, same convention as every sibling scheduler test). White-box #include of ../src/plugins/builtin_fcfs.c (same convention as test_picker_fcfs_session.c) for direct access to fcfs_plugin_data_t's private cursor/job_cursor fields; supplies its own always-ready sched_queue_is_ready()/sched_job_is_ready() test doubles and a plugin_register() stub to satisfy the linker (this TU does not link scheduler.c or plugin.c). The single test drives, on ONE thread with no real concurrency needed: begin_session() -> next_job() (dispatches job 1, leaves priv->cursor parked on queue \"Q1\"'s node with priv->job_cursor mid-walk) -> on_queue_update() again for the SAME queue name with a different priority (genuinely exercises the reuse-and-resort path, freeing the OLD ordq_item_t node priv->cursor is resting on) -> asserts priv->cursor/priv->job_cursor are now NULL (proves invalidation fired) -> next_job() again (the UAF reproduction point; post-fix returns NULL cleanly) -> a fresh begin_session()/next_job() round confirms job 1 and job 2 are BOTH still findable, proving no job is permanently lost, only delayed one round, matching the fix's documented safety property.\n\nRED (pre-fix, verified live): temporarily reverted all three invalidation blocks in builtin_fcfs.c (kept the new test unchanged), rebuilt via `nix develop ./src --command bash src/tools/xce-build.sh scheduler asan build-asan --incremental`. First run: the test's own intermediate `TEST_ASSERT_NULL(priv->cursor)` check (step (e), asserting the invalidation fired) failed as a clean TEST_ASSERT failure -- Unity's assertion longjmp stopped the test function before it ever reached the dereferencing next_job() call, so the real UAF was not yet exercised. Temporarily disabled just that intermediate assertion (`#if 0`) to force execution through to the actual dereference and rebuilt: sched_builtin_fcfs_cursor_uaf_test aborted with a genuine ASAN heap-use-after-free -- 'AddressSanitizer: heap-use-after-free ... READ of size 8 ... #0 builtin_fcfs_next_job ../src/plugins/builtin_fcfs.c:813', freed-by stack 'FREEUP(it) <- builtin_fcfs_on_queue_update builtin_fcfs.c:452 <- test_cursor_invalidated_on_queue_resort_no_uaf test_builtin_fcfs_cursor_uaf.c:181', allocated-by stack 'ordq_put ../lib/ordq.c:48 <- builtin_fcfs_on_queue_update builtin_fcfs.c:457' -- an exact structural match to this bug's diagnosed mechanism and to its own captured live SIGSEGV in fcfs_cursor_enter_queue() <- builtin_fcfs_next_job() <- sched_job_picker(). Restored the temporarily-disabled assertion immediately after capturing this evidence.\nGREEN (post-fix, verified live): restored all three invalidation blocks in builtin_fcfs.c to their applied state (confirmed via `git diff` matching the pre-existing uncommitted fix exactly), rebuilt -- 'scheduler builtin fcfs cursor uaf test OK (0.06s)', 1 Tests 0 Failures. Full scheduler suite re-run via `meson test --suite scheduler`: 53/54 OK, 1 pre-existing failure (test_recordstart_symlink_escape_replies_error, bug-1560, unrelated -- 'Expected 'invalid record file name' Was 'permission denied accessing record file'', same failure present in the RED run too) -- zero new regressions. Note: a full top-level `ninja` build in this shared worktree also fails to link controller_unit_test (undefined references to test_persist_recovery/test_gc_retention/collect_confvars/etc.) due to unrelated, pre-existing uncommitted changes to src/controller/job.c/persist.c/meson.build from concurrent, unrelated work in this worktree (confirmed via `git status`: none of those files were touched by this fix) -- worked around by running `meson test --suite scheduler` directly against the already-built scheduler test binaries rather than requiring a full-tree ninja build to succeed.", @@ -22809,21 +9746,25 @@ "crash", "sigsegv", "live-deployment", - "picker-plugin", - "fixed" + "picker-plugin" ], "related_bugs": [ "bug-1685" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1686.md" }, { "id": "bug-1687", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-08-25T17:30:00Z", - "error_message": "xce-audit-invariants finding: WAL_JOB_START silently drops job fields on a controller restart replay -- the same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), but for the worker-confirmed-start fact. apply_job_start_transition() (job.c) sets job->job.pidinfo (real pids/pgids/exdata a worker reported) and job->job.start_time, but the WAL_JOB_START fact written by record_job_start_fact() only carried wal_jobstate_t{jobid, state} -- pidinfo was never in the log at all, and start_time was re-fabricated with time(NULL) at every apply (both boot replay and standby catch-up) instead of carrying the true worker-reported value. Confirmed live-relevant: user reported StartTime always showing 0 in `xcecli joblist` for both RUN and DONE jobs on a live deployment -- deferred as a distinct follow-up investigation (that deployment's controller predates this fix; whether StartTime=0 specifically, rather than the pre-fix fabricated-nonzero time(NULL), indicates a SEPARATE live-apply-pump gap has not yet been root-caused).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-audit-invariants finding: WAL_JOB_START silently drops job fields on a controller restart…\n\nxce-audit-invariants finding: WAL_JOB_START silently drops job fields on a controller restart replay -- the same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), but for the worker-confirmed-start fact. apply_job_start_transition() (job.c) sets job->job.pidinfo (real pids/pgids/exdata a worker reported) and job->job.start_time, but the WAL_JOB_START fact written by record_job_start_fact() only carried wal_jobstate_t{jobid, state} -- pidinfo was never in the log at all, and start_time was re-fabricated with time(NULL) at every apply (both boot replay and standby catch-up) instead of carrying the true worker-reported value. Confirmed live-relevant: user reported StartTime always showing 0 in `xcecli joblist` for both RUN and DONE jobs on a live deployment -- deferred as a distinct follow-up investigation (that deployment's controller predates this fix; whether StartTime=0 specifically, rather than the pre-fix fabricated-nonzero time(NULL), indicates a SEPARATE live-apply-pump gap has not yet been root-caused).", "file": "src/controller/job.c, src/controller/persist.c, src/include/persist.h, src/include/wal_types.h", "root_cause": "record_job_start_fact() (job.c, on_job_start()'s only caller) called persist_submit_job_fact(&c->persist, WAL_JOB_START, job->job.jobid, JOB_STAT_RUN) -- a thin wrapper whose payload type wal_jobstate_t is exactly {jobid, state, _pad}, discarding the `info` (job_pidinfo_t*) parameter entirely even though it was available at the call site (the worker's real START report). Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/.../DISPATCH fall-through group (persist.c, comment confirms 'these types' payloads are deterministic... start_time... stamped directly below... apply_job_update_transition() only ever touches state') for primary boot replay AND live commit-advance apply, and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) for standby catch-up -- never restored pidinfo, and apply_committed_entry() additionally fabricated start_time = time(NULL) at apply time rather than carrying the receipt-time value, so a restarted/promoted node's start_time drifts from the true worker-reported time even though it is non-NULL.", "fix": "FIXED (2026-08-25). Mirrors bug-1678's WAL_JOB_DISPATCH fix pattern exactly, using the already-existing controller.capnp JobPIDInfo struct (job_pidinfo_t's mapped type -- the existing Job.pidinfo field) with its already-generated encode_JobPIDInfo/decode_JobPIDInfo/free_JobPIDInfo_ptr functions; no schema change needed. src/controller/persist.c gains capnp_encode_pidinfo()/decode_pidinfo() (mirroring capnp_encode_joballoc()/decode_joballoc()) and persist_submit_job_start_fact(persist_t*, jobid, state, start_time, info) -- WAL_JOB_START's payload is now wal_jobstate_t header + a fixed uint64_t start_time (the receipt-time clock, captured ONCE by record_job_start_fact() rather than re-derived independently at each apply seam) + trailing Cap'n Proto-encoded JobPIDInfo bytes, declared in src/include/persist.h. src/controller/job.c's record_job_start_fact() now calls persist_submit_job_start_fact(..., time(NULL), info) instead of persist_submit_job_fact(...). Both apply seams updated: apply_wal_entry() (standby catch-up) splits WAL_JOB_START out of its shared group into its own case (mirroring how bug-1678 split WAL_JOB_DISPATCH out), decoding trailing start_time/pidinfo and calling new restore_job_start_state() (job.c, restore_job_dispatch_state()'s sibling, takes ownership of an optional decoded job_pidinfo_t and a wire-carried start_time -- 0 means 'not carried, leave untouched', matching this seam's pre-fix behavior of never touching start_time for WAL_JOB_START at all). apply_committed_entry() (primary boot replay / live apply pump) stays in its existing shared fall-through group (mirroring how WAL_JOB_DISPATCH joined that same group in bug-1678) but extends its wal_type==WAL_JOB_START branch: a payload with trailing bytes (len >= sizeof(wal_jobstate_t)+sizeof(uint64_t)) decodes and adopts the wire start_time and pidinfo directly; a payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_START entry written before this fix) falls through unchanged to the old fabricated-time(NULL)/pidinfo-untouched behavior -- full backward compatibility, verified by every pre-existing WAL_JOB_START test in test_apply_committed_entry.c/test_apply_reapply_idempotency.c/test_ha_role_lifecycle.c/test_apply_pump.c (all still using the old bare-wal_jobstate_t shape) passing unchanged.\n\nIncidental fix required to reach GREEN: test_commit_barrier_reply.c's cbrp_count_jobstate_entries_ex() (a test-only helper that manually scans the on-disk WAL segment) matched WAL_JOB_START/DISPATCH/etc entries via `hdr.payload_len == sizeof(wal_jobstate_t)` -- an exact-size match that silently undercounted (returned 0 instead of 1) once WAL_JOB_START legitimately grew past that fixed size, in turn making a downstream test in that file poll/spin forever on a count that could never be reached (observed as a genuine deadlock -- all 3 threads futex-blocked, 0 CPU time accruing over 12+ minutes wall clock, reproduced deterministically via `-t test_commit_barrier_reply` and bisected by temporarily inserting/removing an early `return;` in the test group's own RUN_TEST driver to narrow it to one of 8 candidate tests, then to a clean isolated PASS/FAIL once the helper was fixed). Changed the match to `>=` (jobid/state always live at the same fixed leading offset regardless of trailing bytes) -- the general, correct fix, not a workaround; resolves the hang completely (verified: full test_commit_barrier_reply group and the full controller suite both complete in ~38s with zero hangs, both before and after this specific fix was isolated via bisection). Also applied test_commit_barrier_reply.c's cbrp_teardown() the same persist_stop()-before-commit_engine_fini() ordering bug-1452 already established as the fix for this exact TOCTOU-race class (that fix's own note explicitly flagged this file as a same-pattern, not-yet-observed-to-crash follow-up) -- confirmed via bisection that this ordering was NOT the actual cause of this particular hang, but it is a genuine latent race per bug-1452's analysis and the fix is harmless/idempotent, so it was kept as defense-in-depth rather than reverted.\n\nPrimary regression tests: new src/controller/testsuite/test_wal_job_start_fact.c (6 tests, wired into src/controller/meson.build and testsuite/controller_test.c), mirroring test_wal_job_dispatch_fact.c's structure: (1) apply_committed_entry sets pidinfo+start_time from a new-format payload; (2) apply_committed_entry backward-compat with an old-format (bare wal_jobstate_t) payload -- pidinfo stays NULL, start_time still fabricated via time(NULL), unchanged from pre-fix; (3) apply_wal_entry sets pidinfo+start_time, fires no hook; (4) apply_wal_entry backward-compat -- pidinfo stays NULL, start_time left untouched; (5) direct-payload restart-replay (WAL_JOB_SUBMIT + new-format WAL_JOB_START via persist_wal_async + persist_recover()) recovers the exact wire start_time and pidinfo; (6) test_wjs_real_on_job_start_survives_restart -- drives the REAL production on_job_start() path (record_job_start_fact()'s only caller) with a real job_pidinfo_t (3 real pids, 2 real pgids, real exdata string), through a real WAL-write + persist_recover() restart cycle (the exact technique test_dispatch_commit.c's test_dc_success_appends_wal_job_dispatch_durably already established for bug-1678), asserting pidinfo AND start_time (bounded to the real wall-clock window around the on_job_start() call) survive the restart intact -- the audit's exact confirmed root cause and the user's exact acceptance criterion.\n\nRED->GREEN independently verified live (not just narrated): git-stashed job.c+persist.c (keeping the new test file, persist.h, wal_types.h unchanged) via `git stash push -- src/controller/job.c src/controller/persist.c`, rebuilt, ran `./controller_unit_test -t test_wal_job_start_fact` -- 6 Tests 4 Failures 0 Ignored (the 2 backward-compat tests correctly PASS unchanged; all 4 new-behavior tests correctly FAIL with the exact expected mismatches, e.g. 'Expected 1700000001 Was 1787645264' for the fabricated-vs-wire start_time, 'must set job.pidinfo... not just state' for the real on_job_start test). Restored via `git stash pop`, rebuilt, re-ran -- 6 Tests 0 Failures 0 Ignored. Full controller_unit_test suite re-run after all fixes (including the test_commit_barrier_reply.c hang fix and the buglog-id renumber from a concurrent session's own bug-1686 claim): 1146 Tests 2 Failures 0 Ignored, ~38s, zero hangs -- the 2 failures are test_conf_trim_keeps_only_n_versions ('Expected 2 Was 3') and test_conf_get_saved_tag_no_config ('Expected 0 Was ') in test_config_repl.c, an already-logged, pre-existing, unrelated overlayfs sandbox fdatasync artifact (bug-1561, re-observed identically across multiple unrelated sessions with zero diff to test_config_repl.c/config_repl.c each time) -- confirmed by exact signature match, not a new regression.", @@ -22839,24 +9780,27 @@ "data-loss", "apply-seam", "test-infrastructure-hang", - "cbrp-teardown-race", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cbrp-teardown-race" ], "related_bugs": [ "bug-1678", "bug-1452", "bug-1561" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1687.md" }, { "id": "bug-1688", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-25T17:40:00Z", - "error_message": "User-reported issue in the scheduler's interactive simulator REPL (`xce_scheduler -i`/`--sim`): `info worker all` and `info worker ` (e.g. `info worker pc13v`) initially print full per-resource detail (`[sim] resource slots: value=22.00 used=0.00`, `resource mem: value=...`, etc -- ~19 resource lines including slots/mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp/type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks). After stepping the simulation forward through several sessions (`step 5` repeated, crossing RESVALS_DELTA, JOB_PUSH (2 batches, 40+80 jobs), JOB_UPDATE, and WORKERS_PUSH events across sessions 2 through 30), the SAME commands later in the SAME REPL session regress to printing ONLY the header line: `[sim] worker pc13v: status=online` -- zero resource lines, even though the worker still exists and is still 'online' (not removed/dead). Binary under test: `./build.dbg/controller/xce_controller --version` reports `xce v0.0.1-bc3f76b4-5ff97622 built 2026-08-25T08:56:58Z` (a build.dbg build -- note the user ran the CONTROLLER binary's --version, though the symptom itself is in the SCHEDULER's sim REPL; worth confirming which binary actually serves `xce_scheduler -i`/`--sim` and its own version once investigated). A real replay file reproducing the sequence is preserved at /tmp/scheduler-replay-20260825-090251.replay (~1MB, captured 2026-08-25 09:24 UTC) for a future investigation to replay directly instead of needing to reproduce from scratch.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported issue in the scheduler's interactive simulator REPL (`xce_scheduler -i`/`--sim`)…\n\nUser-reported issue in the scheduler's interactive simulator REPL (`xce_scheduler -i`/`--sim`): `info worker all` and `info worker ` (e.g. `info worker pc13v`) initially print full per-resource detail (`[sim] resource slots: value=22.00 used=0.00`, `resource mem: value=...`, etc -- ~19 resource lines including slots/mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp/type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks). After stepping the simulation forward through several sessions (`step 5` repeated, crossing RESVALS_DELTA, JOB_PUSH (2 batches, 40+80 jobs), JOB_UPDATE, and WORKERS_PUSH events across sessions 2 through 30), the SAME commands later in the SAME REPL session regress to printing ONLY the header line: `[sim] worker pc13v: status=online` -- zero resource lines, even though the worker still exists and is still 'online' (not removed/dead). Binary under test: `./build.dbg/controller/xce_controller --version` reports `xce v0.0.1-bc3f76b4-5ff97622 built 2026-08-25T08:56:58Z` (a build.dbg build -- note the user ran the CONTROLLER binary's --version, though the symptom itself is in the SCHEDULER's sim REPL; worth confirming which binary actually serves `xce_scheduler -i`/`--sim` and its own version once investigated). A real replay file reproducing the sequence is preserved at /tmp/scheduler-replay-20260825-090251.replay (~1MB, captured 2026-08-25 09:24 UTC) for a future investigation to replay directly instead of needing to reproduce from scratch.", "file": "src/scheduler/src/repl_info.c (repl_info_worker_one(), ~line 309-388, the `info worker` print path), src/scheduler/src/core_events.c (core_apply_worker_resvals(), ~line 478, the WORKERS_PUSH resval-merge path), src/lib/zarray.c (zarray_put_ex(), ~line 267-303, count-vs-slot bookkeeping)", "root_cause": "FULLY ROOT-CAUSED (2026-08-26), via live TRACE against the same /tmp/scheduler-replay-20260825-090251.replay this bug was originally reported with (build.dbg/scheduler/xce_scheduler --sim -i -vvv, driving the exact `step 5` sequence, plus temporary per-call TRACE instrumentation in core_events.c/core_session.c/lib/zarray.c added and then fully removed once the trigger was pinned down). TWO independent bugs, both now fully understood and fixed:\n(1) core_apply_worker_resvals() (core_events.c) -- the count-doubling anomaly's ACTUAL trigger turned out to be neither an idx-resolution bug nor external memory corruption (both hypotheses from the prior session are RULED OUT: a per-resource `resolved idx=` trace across both pushes showed the SAME resource name always resolves to the SAME r->idx both times, and core_reset_workers()/xce_core_begin_session()/xce_core_end_session() were bracket-traced across many session boundaries with resvals.data's pointer, .count, and slot occupancy all provably untouched -- confirming core_reset_workers() only ever touches w->resInUse, exactly as previously suspected). The REAL mechanism, pinned down via a direct per-entry trace of nv->val itself: a WORKERS_PUSH re-announcing an already-registered worker's resource NAME list can legitimately decode nv->val to NULL for EVERY entry (confirmed: controller.capnp.c's decode_Value_ptr(), line ~2966-2976, sets `*d = NULL` whenever the wire NameValue.val capnp pointer is CAPN_NULL -- a faithful decode of what the controller-side encoder chose to send, not a decode bug). core_apply_worker_resvals() unconditionally MOVE_PTR'd this NULL into `val` and called zarray_put(&worker->resvals, r->idx, NULL) for every one of the 20 resources, silently overwriting each previously-known-good value with NULL. Because zarray_put_ex() (zarray.c:267-303) only increments array->count when the slot being written was NULL beforehand, this FIRST NULL-write left resvals.count unchanged (the snapshot-restored slot was non-NULL, so the 'else count++' branch never fired) while silently zeroing the slot's DATA -- confirmed live via a watched-array TRACE in zarray_put_ex() showing `new_data=(nil)` for all 20 writes on BOTH the first and second WORKERS_PUSH. A THIRD push then found every slot already NULL (from the first push's own corruption) and hit the 'was NULL, count++' branch for all 20, producing the literal 20 -> 40 doubling this bug was originally reported with. Ownership of the 'why does the controller push a repeat announce with no values' question is out of scope for this session (src/controller/ is off limits per this task's boundaries, and the fix does not depend on the answer) -- the defensive scheduler-side fix protects worker resvals regardless of why the wire push omitted a value.\n(2) repl_info_worker_one() (repl_info.c:324, confirmed unchanged from prior session's diagnosis) -- its resource-print loop used `for (i = 0; i < w->resvals.count; i++)`, treating `.count` as a safe index upper bound. `.caps` (resource.c:228's own established convention for this identical structure) is correct; `.count` can legitimately be less than the highest occupied index once entries are sparse -- exactly the state bug (1) produces (9 static resources nulled by a WORKERS_PUSH, 11 dynamic ones kept fresh by later RESVALS_DELTA events, live-confirmed as the actual after-fix-1-absent steady state for this replay: `info worker pc13v` after 30x `step 5` showed only mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp, missing slots/type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks -- a real, reproduced PARTIAL instance of the reported symptom, not the fully empty header-only case, which likely needs a longer replay or a worker with no periodically-refreshed resources at all to reach).", "fix": "FIXED (2026-08-26), TDD RED->GREEN for both root causes, both verified independently and together:\n(1) src/scheduler/src/core_events.c: core_apply_worker_resvals()'s per-resval loop guard extended from `if ((nv == NULL) || (nv->name == NULL))` to `if ((nv == NULL) || (nv->name == NULL) || (nv->val == NULL))` -- a NULL nv->val is now skipped (preserve whatever value is already there), the same defensive 'continue' posture already applied to the other two NULL checks. The pre-existing `if (nv->val != NULL && nv->val->which == Value_u_fvalue)` TRACE guard just below was simplified to `if (nv->val->which == Value_u_fvalue)` since nv->val == NULL can no longer reach that line. New regression test in src/scheduler/testsuite/test_core_events.c: test_workers_push_with_null_val_does_not_clobber_existing_resval -- pushes 2 resources (mem=100.0, r1m=5.0) for worker 'hostA', then pushes the SAME names with nv->val==NULL for both, asserts resvals.count stays 2 and both values survive unchanged; a THIRD identical NULL push re-confirms count stays 2 (would have been 4 under the pre-fix count- doubling behavior). RED confirmed live (git stash of just this file's fix, rebuilt, ran sched_core_events_test: 'Expected Non-NULL' failure at the post-second-push value check, 10/11 tests passing otherwise); GREEN confirmed after git stash pop + rebuild (11 Tests 0 Failures).\n(2) src/scheduler/src/repl_info.c: repl_info_worker_one()'s resource loop bound changed from `w->resvals.count` to `w->resvals.caps`, mirroring resource.c:228's `int num_res = (int)w->resvals.caps;` convention exactly; the loop's existing avail_v==NULL/rname==NULL skip is unchanged and already handles the now-larger, sparser index range safely. New regression test in src/scheduler/testsuite/test_repl_breakpoints_inspection.c: test_info_worker_shows_all_populated_resources_when_count_is_stale -- worker with 2 populated resvals slots (idx0 'mem', idx1 'r1m') but resvals.count forcibly set to 1 (the exact stale/sparse shape bug (1)'s prior failure mode produced), asserts `info worker` output contains BOTH resource lines. RED confirmed live (git stash of just this file's fix, rebuilt sched_repl_breakpoints_inspection_test: 28/29 passing, new test failing with the exact expected message 'a populated slot at an index >= the (stale) .count must still display'); GREEN confirmed after git stash pop + rebuild (29 Tests 0 Failures).\nEnd-to-end reproduction against the ORIGINAL /tmp/scheduler-replay-20260825-090251.replay, driving `info worker pc13v` before any steps, then 30x `step 5`, then `info worker pc13v` again: BEFORE the fix, the final `info worker pc13v` printed only 11 of 20 resource lines (mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp survive via RESVALS_DELTA refresh; slots/type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks silently vanish) -- a live-reproduced instance of the reported regression class. AFTER both fixes, the same sequence's final `info worker pc13v` shows all 20 resource lines identical in shape to the very first (pre-step) dump, with only the naturally-updated dynamic values (mem/r1m/r15m/etc.) differing in magnitude as expected from real RESVALS_DELTA updates over 30 rounds -- status=online, zero missing lines.\nFull scheduler suite (xce-build.sh scheduler, ASAN profile): 53/54 tests OK, 1 failure -- test_recordstart_symlink_escape_replies_error (sched_admin_request_test), the exact pre-existing bug-1560 sandbox artifact (Expected 'invalid record file name' Was 'permission denied accessing record file'), confirmed unrelated (no files this bug touches -- core_events.c/repl_info.c/zarray.c -- are anywhere near record-file-path validation). Zero new failures; both new tests (sched_core_events_test 21/54, sched_repl_breakpoints_inspection_test 52/54) pass in the full suite run.", @@ -22870,21 +9814,25 @@ "zarray-count-vs-caps", "resvals-merge", "worker-push", - "null-value-overwrite", - "fixed" + "null-value-overwrite" ], "related_bugs": [ "bug-1568" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1688.md" }, { "id": "bug-1689", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T18:10:00Z", - "error_message": "xce-audit-invariants finding: WAL_JOB_FINISH silently drops job fields on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs) and bug-1687 (WAL_JOB_START/job.pidinfo+start_time), but for the worker-confirmed-finish fact. apply_job_finish_transition() (job.c) sets job->job.exit_info->exitcode and job->job.finish_time/run_time, but the WAL_JOB_FINISH fact written by record_job_finish_fact() only carried wal_jobstate_t{jobid, state} -- exit_info was never in the log at all (and, worse, apply_job_finish_transition() itself is effectively dead code on the real production apply path -- both apply_committed_entry()'s and apply_wal_entry()'s WAL_JOB_FINISH handling route through apply_job_update_transition() (state-only) instead, per raft-apply-decoupling's deferred-apply design, so exit_info was NEVER populated through the real commit-apply pipeline, restart or not, prior to this fix), and finish_time was re-fabricated with time(NULL) at every apply_committed_entry() apply (and, on apply_wal_entry()'s standby catch-up seam, never touched at all -- an even larger gap than WAL_JOB_START's pre-1687 state) instead of carrying the true worker-reported receipt time.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-audit-invariants finding: WAL_JOB_FINISH silently drops job fields on a controller restart…\n\nxce-audit-invariants finding: WAL_JOB_FINISH silently drops job fields on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs) and bug-1687 (WAL_JOB_START/job.pidinfo+start_time), but for the worker-confirmed-finish fact. apply_job_finish_transition() (job.c) sets job->job.exit_info->exitcode and job->job.finish_time/run_time, but the WAL_JOB_FINISH fact written by record_job_finish_fact() only carried wal_jobstate_t{jobid, state} -- exit_info was never in the log at all (and, worse, apply_job_finish_transition() itself is effectively dead code on the real production apply path -- both apply_committed_entry()'s and apply_wal_entry()'s WAL_JOB_FINISH handling route through apply_job_update_transition() (state-only) instead, per raft-apply-decoupling's deferred-apply design, so exit_info was NEVER populated through the real commit-apply pipeline, restart or not, prior to this fix), and finish_time was re-fabricated with time(NULL) at every apply_committed_entry() apply (and, on apply_wal_entry()'s standby catch-up seam, never touched at all -- an even larger gap than WAL_JOB_START's pre-1687 state) instead of carrying the true worker-reported receipt time.", "file": "src/controller/job.c (apply_job_finish_transition ~L2292, record_job_finish_fact ~L2371, restore_job_finish_state new), src/controller/persist.c (apply_committed_entry's shared job-fact group ~L3890-4080, apply_wal_entry's WAL_JOB_FINISH case, persist_submit_job_finish_fact new, decode_jobexitinfo new), src/include/persist.h, src/include/wal_types.h, src/controller/testsuite/test_wal_job_finish_fact.c (new)", "root_cause": "record_job_finish_fact() (job.c, on_job_finish()'s only caller) called persist_submit_job_fact(&c->persist, WAL_JOB_FINISH, job->job.jobid, state) -- a thin wrapper whose payload type wal_jobstate_t is exactly {jobid, state, _pad}, discarding finish->exit_code entirely even though it was available at the call site (the worker's real FINISH report). Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/FINISH/.../DISPATCH fall-through group (persist.c) for primary boot replay AND live commit-advance apply, and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) for standby catch-up -- never restored exit_info. apply_committed_entry() additionally fabricated finish_time = time(NULL) at apply time (rather than carrying the receipt-time value) and derived run_time from that fabricated finish_time; apply_wal_entry()'s shared group touched neither finish_time nor run_time at all for WAL_JOB_FINISH (a strictly worse gap than WAL_JOB_START's pre-bug-1687 apply_wal_entry behavior, which at least existed as a documented no-op). Confirmed via CodeGraph source read that apply_job_finish_transition() -- the richer core that DOES set exit_info/finish_time/run_time correctly -- is only ever invoked from record_job_finish_fact()'s duplicate-terminal-state peek (a no-op call whose own guard prevents any mutation), i.e. it is effectively dead code on the real production apply path today; the raft-apply-decoupling redesign defers all real mutation to apply_committed_entry()/apply_wal_entry(), which use the leaner apply_job_update_transition() (state-only) instead. job_exitinfo_t (xce/types.h) is a single int (`{int exitcode;}`) with NO existing Cap'n Proto mapping (confirmed -- no JobExitInfo struct in controller.capnp), unlike bug-1687's JobPIDInfo, so no capnp wrapper was invented for it.", "fix": "FIXED (2026-08-25). Mirrors bug-1687's WAL_JOB_START fix pattern exactly, but uses a small NATIVE (non-capnp) trailing payload instead of Cap'n Proto, since job_exitinfo_t has no existing capnp mapping and no variable-length data (unlike JobPIDInfo). src/controller/persist.c gains decode_jobexitinfo() (a native memcpy decode, mirroring decode_pidinfo()/decode_joballoc()'s doc-comment style but without capnp) and persist_submit_job_finish_fact(persist_t*, jobid, state, finish_time, exit_code) -- WAL_JOB_FINISH's payload is now a wal_jobstate_t header + a fixed uint64_t finish_time (receipt-time clock, captured ONCE by record_job_finish_fact() rather than re-derived independently at each apply seam) + a fixed int32_t exit_code, declared in src/include/persist.h. src/controller/job.c's record_job_finish_fact() now calls persist_submit_job_finish_fact(..., time(NULL), finish->exit_code) instead of persist_submit_job_fact(...). Both apply seams updated: apply_wal_entry() (standby catch-up) splits WAL_JOB_FINISH out of its shared group into its own case (mirroring bug-1687's WAL_JOB_START split), decoding trailing finish_time/exit_code and calling new restore_job_finish_state() (job.c, restore_job_start_state()'s sibling -- finish_time==0 means 'not carried, leave finish_time/run_time untouched', matching this seam's pre-fix behavior exactly; exit_info==NULL means 'not carried, leave untouched'; job_exitinfo_t has no nested pointers so it is released with plain FREEUP, not a capnp free_*_ptr helper). apply_committed_entry() stays in its existing shared fall-through group but extends its wal_type==WAL_JOB_FINISH branch: a payload with trailing bytes (len >= sizeof(wal_jobstate_t)+sizeof(uint64_t)) decodes and adopts the wire finish_time and exit_code directly; a payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_FINISH entry written before this fix) falls through unchanged to the old fabricated-time(NULL)/exit_info-untouched behavior. run_time is NEVER carried on the wire in either case -- it is always re-derived as finish_time - start_time at apply time (matching apply_job_finish_transition()'s own formula), since start_time is already correctly recovered via bug-1687's own fix. Full backward compatibility verified by 2 dedicated old-format tests plus every pre-existing WAL_JOB_FINISH test in test_apply_committed_entry.c/test_apply_reapply_idempotency.c/test_apply_pump.c/test_commit_barrier_reply.c/test_commit_barrier_rollout.c (all still using the old bare-wal_jobstate_t shape) passing unchanged.\n\nNew regression tests: src/controller/testsuite/test_wal_job_finish_fact.c (6 tests, mirroring test_wal_job_start_fact.c's structure exactly, wired into src/controller/meson.build and testsuite/controller_test.c): (1) apply_committed_entry sets exit_info+finish_time (+re-derived run_time) from a new-format payload; (2) apply_committed_entry backward-compat with an old-format payload -- exit_info stays NULL, finish_time still fabricated via time(NULL); (3) apply_wal_entry sets exit_info+finish_time+run_time, fires no hook; (4) apply_wal_entry backward-compat -- exit_info/finish_time/run_time all left untouched (sentinel values survive), unlike apply_committed_entry's fabrication; (5) direct-payload restart-replay (WAL_JOB_SUBMIT + WAL_JOB_START + new-format WAL_JOB_FINISH via persist_wal_async + persist_recover()) recovers the exact wire finish_time, exit_code, and a correctly re-derived run_time; (6) test_wjf_real_on_job_finish_survives_restart -- drives the REAL production on_job_finish() path (record_job_finish_fact()'s only caller) with a real xce_jobfinish_t (nonzero exit_code=137), through a real WAL-write + persist_recover() restart cycle, asserting exit_info, finish_time (bounded to the real wall-clock window), and run_time all survive the restart intact -- the audit's exact confirmed root cause and the user's exact acceptance criterion.\n\nRED->GREEN independently verified live (not just narrated): git-stashed job.c+persist.c only (`git stash push -- src/controller/job.c src/controller/persist.c`, keeping the new test file, persist.h, wal_types.h unchanged), rebuilt via the synchronous xce-build.sh script, ran `./controller_unit_test -t test_wal_job_finish_fact` -- 6 Tests 4 Failures 0 Ignored (the 2 backward-compat tests correctly PASS unchanged; all 4 new-behavior tests correctly FAIL with exact expected mismatches, e.g. 'Expected 1700000050 Was 1787651049' for the fabricated-vs-wire finish_time on apply_committed_entry, 'Expected 1700000150 Was 0' for apply_wal_entry never touching finish_time pre-fix, and 'must set job.exit_info... not just state' for the real on_job_finish restart test). Restored via `git stash pop`, rebuilt, re-ran -- 6 Tests 0 Failures 0 Ignored, reproduced 3 consecutive times. Full controller_unit_test suite re-run after the fix: 1152 Tests, 1-2 Failures (flaky between runs) -- both failure signatures ('Expected 2 Was 3' / test_conf_trim_keeps_only_n_versions, and a garbage-uint64 mismatch / test_conf_get_saved_tag_no_config in test_config_repl.c) are the exact, previously-logged bug-1561 overlayfs-fdatasync sandbox artifact, confirmed identical across 2 consecutive re-runs of the unchanged binary (proving the flakiness is a sandbox I/O nondeterminism, not caused by this fix) -- zero new failures. Checked test_commit_barrier_reply.c's cbrp_count_jobstate_entries() (the exact helper bug-1687 had to fix for a payload-size-growth-induced hang) for the same risk: it already uses the generic `hdr.payload_len >= sizeof(wal_jobstate_t)` comparison bug-1687 fixed (not a WAL_JOB_START-specific special case), so WAL_JOB_FINISH's payload growth is already safely handled by that existing fix -- confirmed empirically by the full suite completing in ~38s with zero hangs, matching prior-session timing exactly.", @@ -22900,24 +9848,27 @@ "restart-recovery", "data-loss", "apply-seam", - "native-trailing-payload", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "native-trailing-payload" ], "related_bugs": [ "bug-1678", "bug-1687", "bug-1561" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1689.md" }, { "id": "bug-1690", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T11:00:00Z", - "error_message": "xce-audit-invariants finding: WAL_JOB_RUSAGE silently drops job.run_usage (client-reported per-host CPU/mem usage) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time). update_job_rusages() (job.c) sets job->job.run_usage (xce_job_runusage_t*, an accumulated per-host struct) via update_host_runusage() (misc.c), but the WAL_JOB_RUSAGE fact persist_job_hook_fun (persist.c) wrote only carried wal_jobstate_t{jobid, state} -- run_usage was never in the log at all, so a restart replay left it permanently NULL even though a client actively reported CPU/mem usage. Unlike the three precedents, xce_job_runusage_t had NO existing Cap'n Proto mapping at all (confirmed: grep for xce_job_runusage_t in controller.capnp matched nothing; a similar-looking but unrelated type, xce_job_rusages_t/JobRusages/SchedJobRusages -- the CLIENT's wire update-request shape {jobid, pids[], pgids[], usages[]} -- already existed and is NOT interchangeable), so this fix required a genuine schema addition, not a reuse.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-audit-invariants finding: WAL_JOB_RUSAGE silently drops job.run_usage (client-reported…\n\nxce-audit-invariants finding: WAL_JOB_RUSAGE silently drops job.run_usage (client-reported per-host CPU/mem usage) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time). update_job_rusages() (job.c) sets job->job.run_usage (xce_job_runusage_t*, an accumulated per-host struct) via update_host_runusage() (misc.c), but the WAL_JOB_RUSAGE fact persist_job_hook_fun (persist.c) wrote only carried wal_jobstate_t{jobid, state} -- run_usage was never in the log at all, so a restart replay left it permanently NULL even though a client actively reported CPU/mem usage. Unlike the three precedents, xce_job_runusage_t had NO existing Cap'n Proto mapping at all (confirmed: grep for xce_job_runusage_t in controller.capnp matched nothing; a similar-looking but unrelated type, xce_job_rusages_t/JobRusages/SchedJobRusages -- the CLIENT's wire update-request shape {jobid, pids[], pgids[], usages[]} -- already existed and is NOT interchangeable), so this fix required a genuine schema addition, not a reuse.", "file": "src/schema/controller.capnp (new HostRunusage/JobRunusage structs), src/controller/persist.c (capnp_encode_jobrunusage/decode_jobrunusage ~L327-388, persist_job_hook_fun's HOOK_DETAIL_JOB_RUSAGE_UPDATE case ~L7440-7500, apply_committed_entry's WAL_JOB_RUSAGE branch ~L4116-4144, apply_wal_entry's WAL_JOB_RUSAGE case ~L3712-3740), src/controller/job.c (merge_job_runusage new ~L893-1000, restore_job_rusage_state new), src/include/wal_types.h (WAL_JOB_RUSAGE doc comment), src/controller/testsuite/test_wal_job_rusage_fact.c (new, 8 tests)", "root_cause": "persist_job_hook_fun()'s HOOK_DETAIL_JOB_RUSAGE_UPDATE case (persist.c) built only wal_jobstate_t{jobid, state} for the WAL_JOB_RUSAGE fact, discarding the job's run_usage entirely even though it was available (looked up via the same jobmgr lookup, under the same write lock update_job_rusages() already held). Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/FINISH/.../DISPATCH fall-through group (persist.c) and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) -- never restored run_usage, matching the exact same class of gap bug-1678/1687/1689 already fixed for allocs/pidinfo+start_time/exit_info+finish_time. Confirmed via CodeGraph source read of update_job_rusages() (job.c) and persist_job_hook_fun()'s actual HOOK_DETAIL_JOB_RUSAGE_UPDATE arm (not a direct persist_submit_job_*_fact() call site like DISPATCH/START/FINISH have -- RUSAGE is purely hook-driven from the raw xce_job_rusages_t* payload update_job_rusages() fires, a genuine divergence from the audit's initial reconstructed precedent that was verified against the real code before implementing, per this task's explicit instruction).", "fix": "FIXED (2026-08-25). Added a genuine new Cap'n Proto schema pair to controller.capnp -- struct HostRunusage $C.mapname(\"xce_host_runusage_t\"){hostName, cpuTime, memUsage} and struct JobRunusage $C.mapname(\"xce_job_runusage_t\"){maxMem, avgMem, usedMem, cpuTime, peakCpuTime, peakCpuDuration, memEfficiency, avgCpuEfficiency, peakCpuEfficiency, hostRunusage: List(HostRunusage) $C.maplistcount(\"num_hosts\")} -- following JobPendReason's exact List(NestedStruct)+maplistcount precedent already in the schema, field types verified against types.h's real xce_job_runusage_t/xce_host_runusage_t definitions via CodeGraph before writing. The meson custom_target (src/lib/meson.build) regenerated lib/controller.capnp.c/.h automatically on rebuild -- verified via nm on the built controller_unit_test binary showing the full genuinely-codegen'd symbol set (new_JobRunusage, read/write_JobRunusage, encode/decode_JobRunusage, free_JobRunusage_ptr, get/set per-field accessors, and the equivalent HostRunusage set), never hand-written. persist.c gains capnp_encode_jobrunusage()/decode_jobrunusage() (mirroring capnp_encode_joballoc()/decode_joballoc() exactly). persist_job_hook_fun()'s HOOK_DETAIL_JOB_RUSAGE_UPDATE case now also encodes j->job.run_usage (when non-NULL) as trailing capnp bytes after the wal_jobstate_t header -- inlined into that hook (not a new persist_submit_job_rusage_fact(), unlike bug-1678/1687/1689's direct-call-site pattern, since RUSAGE is genuinely hook-driven). Both apply seams updated: apply_wal_entry() splits WAL_JOB_RUSAGE out of its shared state-only group into its own case, decoding trailing bytes and calling new restore_job_rusage_state() (job.c). apply_committed_entry()'s WAL_JOB_RUSAGE handling stays in its shared job-fact group but -- critically, UNLIKE START/FINISH/DISPATCH -- decodes/adopts run_usage UNCONDITIONALLY, before the JOB_TRANSITION_APPLIED gate: a rusage report almost never changes job.state (the job stays RUN across every repeat report), so gating adoption on a forward state transition the way the other three fixes do would make it dead code for the realistic, common case (every report after the first hits JOB_TRANSITION_NOOP_DUPLICATE and the gated block never runs). This divergence was caught by actually reading apply_job_update_transition()'s guard logic rather than blindly copying the precedent pattern. A payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_RUSAGE entry written before this fix, or one fired while job.run_usage was still NULL at write time) decodes no run_usage on both seams, matching pre-fix behavior exactly -- full backward compatibility.\n\nCRITICAL SAFETY FINDING beyond the precedent's pattern, caught via investigation (not assumed): a naive free-and-replace (mirroring bug-1678/1687/1689's free_X_ptr()-then-adopt exactly) would be a genuine heap-corruption hazard here, specific to RUSAGE. job.run_usage can be the job's LIVE structure -- update_job_rusages()/update_host_runusage() (job.c/misc.c) mutate it synchronously and in place BEFORE the WAL fact is even appended, and host_name is sourced from makeString() (src/lib/xstring.c), an RTTI-managed, refcounted, header-before-data pointer (confirmed via librtti's actual source in the nix store: RTTI_DATA_TO_OBJ subtracts RTTI_OBJ_HEADER_SIZE from the data pointer to reach the real allocation start) -- NOT a plain malloc() allocation, unlike JobAlloc/JobPIDInfo's host/exdata fields (verified to be genuinely capnp-wire-decoded end to end via dispatch_a_job_()'s own alloc parameter, confirming why bug-1678/1687 never hit this). persist_apply_pump_run() (the live online-apply pump, confirmed via its own doc comment to run on the primary itself, not just boot replay) re-applies every committed WAL_JOB_RUSAGE fact via apply_committed_entry() moments after the SAME live mutation already happened -- so calling the capnp-generated free_JobRunusage_ptr() (written for plain malloc'd Text fields) on that live structure would free() a non-malloc'd, RTTI-managed pointer: guaranteed ASAN abort / heap corruption on the very first live rusage report for a new host. Fixed by making adoption a MERGE (new merge_job_runusage(), job.c) instead of a free-and-replace: an already-present host (matched by host_name string comparison, mirroring update_host_runusage()'s own find-existing-or-append semantics) only has its scalar cpu_time/mem_usage overwritten in place -- its host_name pointer, possibly xstring-backed, is NEVER touched or freed; only a genuinely new host's own freshly-decoded (always plain-malloc'd) host_name is adopted by ownership transfer. Applied identically in both restore_job_rusage_state() (apply_wal_entry seam) and apply_committed_entry's own branch, since a demoted-primary-turned-standby could in principle retain live-mutated run_usage data too. Noted (not fixed, genuinely out of scope -- a different file/component/bug class): job_destroy() (src/lib/jobj.c)'s own existing manual FREEUP-based walk over run_usage has this exact same latent host_name/xstring mismatch, independent of and pre-dating this fix.\n\nRegression tests: src/controller/testsuite/test_wal_job_rusage_fact.c (8 tests, mirroring test_wal_job_start_fact.c's structure): (1) apply_committed_entry adopts run_usage when state does NOT change (JOB_TRANSITION_NOOP_DUPLICATE) -- the suite's most important test, proving the unconditional-decode design choice; (2) apply_committed_entry adopts run_usage on a genuine forward transition, hook still fires once; (3) apply_committed_entry backward-compat with an old-format payload; (4) apply_committed_entry merges without freeing a REAL makeString()'d host_name -- builds a live-flavor run_usage by hand, applies an update for that same host plus a new host, and asserts the host_name POINTER IDENTITY survives (proving it was never freed) alongside correct scalar/new-host merging, with careful test-only cleanup (detach + dereference()) so the separate, pre-existing job_destroy() FREEUP issue can't fire inside the test's own teardown; (5) apply_wal_entry adopts run_usage, no hook, absolute state overwrite; (6) apply_wal_entry backward-compat; (7) direct-payload restart-replay (WAL_JOB_SUBMIT + WAL_JOB_RUSAGE via persist_wal_async + persist_recover()); (8) drives the REAL, unmodified persist_job_hook_fun() via the established 'variadic trampoline' pattern (test_commit_barrier_rollout.c/test_persist_hooks.c) through a real WAL-write + persist_recover() restart cycle -- NOT via update_job_rusages() itself, because this same controller_unit_test binary already links test_commit_barrier_reply.c's own non-static stub of update_host_runusage() (a no-op, since misc.c is outside controller_unit_test_src), so calling update_job_rusages() here would silently hit that stub and populate nothing -- documented in the test file's own header comment.\n\nRED->GREEN independently verified live (not narrated): `git stash push -- src/controller/job.c src/controller/persist.c` (keeping the schema change, the new test file, and meson.build/controller_test.c wiring), full clean rebuild via `nix develop ./src --command bash src/tools/xce-build.sh controller asan build-asan` (no --incremental, since this is the schema's first-ever compile) -- 1160 Tests 8 Failures: exactly the 6 new RUSAGE tests (5 with the exact expected 'must set job.run_usage' mismatch, 1 with 'Expected 500 Was 100' for the merge-safety test) plus the 2 pre-existing bug-1561 test_config_repl.c failures; the 2 backward-compat tests correctly passed unchanged. `git stash pop`, incremental rebuild -- 1160 Tests 2 Failures (only the pre-existing bug-1561 pair, exact signature match 'Expected 2 Was 3' / 'Expected 0 Was '), all 8 new tests pass, zero new failures, zero ASAN aborts/heap-corruption findings (explicitly grepped the full test log for AddressSanitizer/heap-buffer-overflow/invalid-free/SEGV/Aborted -- none found). Verified the generated capnp code was genuinely regenerated, not hand-written: `nm` on the built controller_unit_test binary shows the complete real codegen symbol set for both new structs (new_JobRunusage, read/write_JobRunusage, encode/decode_JobRunusage, free_JobRunusage_ptr, per-field get/set accessors, and the equivalent HostRunusage set), with generated-file mtimes strictly after the schema edit's mtime.", @@ -22934,9 +9885,7 @@ "capnp-schema-change", "heap-corruption-hazard", "xstring-rtti-pointer", - "merge-not-replace", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "merge-not-replace" ], "related_bugs": [ "bug-1678", @@ -22944,16 +9893,21 @@ "bug-1689", "bug-1561" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1690.md" }, { "id": "bug-1691", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T12:00:00Z", - "error_message": "xce-audit-invariants finding: WAL_JOB_CKPT silently drops job.ckptinfo (the worker/scheduler-reported checkpoint info string) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), and bug-1690 (WAL_JOB_RUSAGE/job.run_usage). on_job_ckpt() (job.c) sets job->job.ckptinfo via makeString(), but the WAL_JOB_CKPT fact -- purely hook-driven via persist_job_hook_fun's HOOK_DETAIL_JOB_CKPT arm (on_job_ckpt() itself never calls a persist_submit_job_*_fact() function directly, unlike DISPATCH/START/FINISH) -- only ever carried the shared wal_jobstate_t{jobid, state} payload. Neither apply seam restored ckptinfo -- it stayed NULL after restart. job->job.ckptinfo (xce/types.h) is a plain NUL-terminated char*, NOT the richer xce_ckptinfo_t/CheckpointInfo capnp struct used elsewhere for a different purpose (that struct is on_job_ckpt()'s OWN incoming-report parameter type).", - "file": "src/controller/job.c:2999 (on_job_ckpt), src/controller/job.c:1046 (restore_job_ckpt_state, new), src/controller/persist.c:354 (decode_ckptinfo, new), src/controller/persist.c:7579 (persist_job_hook_fun's HOOK_DETAIL_JOB_CKPT case, new), src/controller/persist.c:3757 (apply_wal_entry's WAL_JOB_CKPT case, split out), src/controller/persist.c:4147 (apply_committed_entry's WAL_JOB_CKPT unconditional-adopt block), src/include/wal_types.h (WAL_JOB_CKPT doc comment), src/controller/testsuite/test_wal_job_ckpt_fact.c (new)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-audit-invariants finding: WAL_JOB_CKPT silently drops job.ckptinfo (the…\n\nxce-audit-invariants finding: WAL_JOB_CKPT silently drops job.ckptinfo (the worker/scheduler-reported checkpoint info string) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), and bug-1690 (WAL_JOB_RUSAGE/job.run_usage). on_job_ckpt() (job.c) sets job->job.ckptinfo via makeString(), but the WAL_JOB_CKPT fact -- purely hook-driven via persist_job_hook_fun's HOOK_DETAIL_JOB_CKPT arm (on_job_ckpt() itself never calls a persist_submit_job_*_fact() function directly, unlike DISPATCH/START/FINISH) -- only ever carried the shared wal_jobstate_t{jobid, state} payload. Neither apply seam restored ckptinfo -- it stayed NULL after restart. job->job.ckptinfo (xce/types.h) is a plain NUL-terminated char*, NOT the richer xce_ckptinfo_t/CheckpointInfo capnp struct used elsewhere for a different purpose (that struct is on_job_ckpt()'s OWN incoming-report parameter type).", + "file": "src/controller/job.c:2999 (on_job_ckpt), src/controller/job.c:1046 (restore_job_ckpt_state, new), src/controller/persist.c:354 (decode_ckptinfo, src/controller/persist.c:7579 (persist_job_hook_fun's HOOK_DETAIL_JOB_CKPT case, src/controller/persist.c:3757 (apply_wal_entry's WAL_JOB_CKPT case, split out), src/controller/persist.c:4147 (apply_committed_entry's WAL_JOB_CKPT unconditional-adopt block), src/include/wal_types.h (WAL_JOB_CKPT doc comment), src/controller/testsuite/test_wal_job_ckpt_fact.c (new)", "root_cause": "persist_job_hook_fun()'s job-fact switch (persist.c) routed HOOK_DETAIL_JOB_CKPT through its generic `default:` arm (shared with STOP/RESUME/UPDATE/PREASON), which captures only {jobid, state} from the job_t* -- j->job.ckptinfo, freshly set by on_job_ckpt() moments earlier under the same write lock this AFTER_EVENT fires inside, was never read or encoded. Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/FINISH/.../DISPATCH fall-through group (primary boot replay AND live commit-advance apply) and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) for standby catch-up -- never restored ckptinfo, matching the exact same class of gap bug-1678/1687/1689/1690 already fixed for allocs/pidinfo+start_time/exit_info+finish_time/run_usage. Confirmed via CodeGraph source read that on_job_ckpt() is hook-driven (fires xce_run_hooks only, no direct persist_submit_job_*_fact() call site) -- a genuine divergence from DISPATCH/START/FINISH's direct-call-site pattern, closer to bug-1690's RUSAGE precedent (also purely hook-driven) -- verified against the real code before implementing, per this task's explicit instruction. Also confirmed on_job_ckpt() never changes job.state (only job.ckptinfo), so a naive copy of the START/FINISH/DISPATCH pattern (gating adoption on a forward JOB_TRANSITION_APPLIED) would have made the fix dead code on every real invocation, mirroring bug-1690's exact same discovery for WAL_JOB_RUSAGE.", "fix": "FIXED (2026-08-25). This is the simplest of the five field-parity fixes -- job.ckptinfo is a plain string, so NO capnp struct was needed at all (unlike bug-1678/1687/1690's capnp wrappers, though it does share bug-1689's 'native, non-capnp trailer' style). Mirrors WAL_QUEUE_CTRL's own established [header][name NUL-terminated] wire shape (persist_queue_hook_fun / apply_wal_entry's WAL_QUEUE_CTRL case, read as the reference implementation before designing this fix) exactly: the ckptinfo string is appended as a NUL-terminated string directly after the wal_jobstate_t header, bounded by the WAL entry's own payload_len (no separate length prefix). persist.c gains decode_ckptinfo() (a native memcpy-with-NUL-bound-check decode, mirroring decode_jobexitinfo()'s bug-1689 native-trailer style) and a new HOOK_DETAIL_JOB_CKPT case in persist_job_hook_fun() -- split out of the generic `default:` arm -- that encodes j->job.ckptinfo (when non-NULL) as trailing bytes appended after the wal_jobstate_t header, mirroring the rusage_buf/rusage_len construction bug-1690 already added to this same function for WAL_JOB_RUSAGE (both are hook-driven, unlike DISPATCH/START/FINISH's direct persist_submit_job_*_fact() call sites). src/include/wal_types.h's WAL_JOB_CKPT enum value gains a doc comment documenting the new payload shape and backward-compat contract, mirroring every other WAL_JOB_* entry's existing convention. Both apply seams updated: apply_wal_entry() (standby catch-up) splits WAL_JOB_CKPT out of its shared state-only group into its own case (mirroring bug-1690's WAL_JOB_RUSAGE split), decoding trailing ckptinfo bytes via decode_ckptinfo() and calling new restore_job_ckpt_state() (job.c, restore_job_rusage_state()'s sibling). apply_committed_entry() stays in its existing shared job-fact group but -- mirroring bug-1690's WAL_JOB_RUSAGE design exactly, for the identical reason -- decodes/adopts ckptinfo UNCONDITIONALLY, before the JOB_TRANSITION_APPLIED gate: a checkpoint report never changes job.state, so gating adoption on a forward state transition would make it dead code for the realistic, only-ever-occurring case (every real WAL_JOB_CKPT fact hits JOB_TRANSITION_NOOP_DUPLICATE, since js.state is captured verbatim from the job's own unchanged current state). A payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_CKPT entry written before this fix, or one fired while job.ckptinfo was still NULL at write time) decodes no ckptinfo on both seams, matching pre-fix behavior exactly -- full backward compatibility.\n\nAdoption is a plain free-and-replace (dereference() the prior xstring value, adopt a fresh makeString()'d copy), NOT a merge -- this DIVERGES from bug-1690's WAL_JOB_RUSAGE fix, which required a merge specifically because a naive free-and-replace via the WRONG release function (the capnp-generated free_JobRunusage_ptr(), written for plain malloc'd fields) would free a non-malloc'd, RTTI-managed xstring pointer. job.ckptinfo IS xstring-backed (makeString(), xstring.c) exactly like RUSAGE's host_name was, and CAN be the job's own live value when the primary's own live apply pump re-applies a fact moments after on_job_ckpt() already set it -- but dereference() (the correct RTTI release verb, used here instead of free()/FREEUP) is safe regardless of whether the pointer is 'old' or coincidentally 'the current live one', since it decrements a refcount rather than raw free()ing a non-owned pointer. This distinction was verified by reading xstring.c's actual makeString()/dereference() implementation before choosing the release strategy, not assumed from the RUSAGE precedent.\n\nRegression tests: new src/controller/testsuite/test_wal_job_ckpt_fact.c (7 tests, mirroring test_wal_job_rusage_fact.c's 'unconditional decode independent of transition result' structure plus test_wal_job_finish_fact.c's simpler real-production-call pattern, wired into src/controller/meson.build and testsuite/controller_test.c): (1) apply_committed_entry sets ckptinfo when state does NOT change (JOB_TRANSITION_NOOP_DUPLICATE) -- this suite's most important test, proving the unconditional-decode design choice, hook fires 0 times; (2) apply_committed_entry sets ckptinfo on a genuine forward transition, hook still fires once; (3) apply_committed_entry backward-compat with an old-format payload; (4) apply_wal_entry sets ckptinfo, fires no hook; (5) apply_wal_entry backward-compat; (6) direct-payload restart-replay (WAL_JOB_SUBMIT + new-format WAL_JOB_CKPT via persist_wal_async + persist_recover()); (7) test_wjc_real_on_job_ckpt_survives_restart -- drives the REAL production on_job_ckpt() path (cluster.c's NotifyData_data_ckptinfo dispatch's only callee) with a real xce_ckptinfo_t, through a real WAL-write + persist_recover() restart cycle, asserting ckptinfo survives the restart intact -- the audit's exact confirmed root cause. Test 7 required passing a real cluster_t* (not NULL) as persist_init()'s 3rd argument -- unlike test_wjf_real_on_job_finish_survives_restart's identical-looking fixture (a direct-call-site fix needing no hook registration), on_job_ckpt() is hook-driven and depends entirely on persist_start() registering persist_job_hook_fun on cluster.hooksys (gated on `p->cluster != NULL`) to reach the WAL at all -- caught via a genuine RED-signal debugging pass (the real-path test initially failed even with the fix applied, isolating to this exact fixture gap, not a fix defect).\n\nRED->GREEN independently verified live (not just narrated): `git stash push -- src/controller/job.c src/controller/persist.c` (keeping the new test file, wal_types.h, meson.build/controller_test.c wiring), incremental rebuild via the synchronous xce-build.sh script, ran the full controller_unit_test suite -- 1167 Tests 7 Failures (the 2 pre-existing bug-1561 test_config_repl.c failures plus all 5 new-behavior WAL_JOB_CKPT tests failing with the exact expected 'must set job.ckptinfo' mismatch; the 2 backward-compat tests correctly PASS unchanged). `git stash pop`, incremental rebuild, ran the isolated new test group directly against the binary (`ASAN_OPTIONS=detect_leaks=0 ./controller_unit_test -t test_wal_job_ckpt_fact`, needed because LeakSanitizer's atexit-based leak check bypasses libc's normal exit() stdio-flush path on this binary, silently discarding all buffered PASS/FAIL output when leaks are present -- a pre-existing, suite-wide fixture characteristic unrelated to this fix, confirmed identical on the RUSAGE suite: 78 leaked allocations in isolation, same fixture-leak pattern, not a regression) -- initially 6/7 passed, with test_wjc_real_on_job_ckpt_survives_restart failing; root-caused to the persist_init() NULL-vs-&c fixture gap above, fixed the test, reran -- 7 Tests 0 Failures. Full controller_unit_test suite re-run via xce-build.sh after all fixes: 1167 Tests, 2 Failures -- exact bug-1561 signature match ('Expected 2 Was 3' / test_conf_trim_keeps_only_n_versions, 'Expected 0 Was ' / test_conf_get_saved_tag_no_config in test_config_repl.c), zero new failures, zero ASAN heap-corruption findings (grepped the full test log for heap-buffer-overflow/invalid-free/use-after-free/SEGV/Aborted/double-free -- none found).\n\nENVIRONMENT NOTE: this session's build/test runs were repeatedly disrupted by the /tmp filesystem filling to 100% (0 bytes free), traced to ~15,437 orphaned mkdtemp() scratch directories (xce_ckpt_*/xce_hook_test_*/xce_e2e_wal_*/etc., ~658GB total) accumulated across many prior sessions' test runs whose cleanup never ran (each interrupted mid-flight by the same ENOSPC condition in a compounding cycle). Cleared via `rm -rf /tmp/xce_*` (verified via /proc/*/fd that no live process held any of them open) before the final, valid GREEN run -- unrelated to this fix's correctness, but a genuine environment hazard worth flagging for other sessions sharing this machine.", "tags": [ @@ -22969,9 +9923,7 @@ "native-trailing-payload", "hook-driven-fact", "xstring-release-verb", - "test-fixture-hook-registration-gap", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-fixture-hook-registration-gap" ], "related_bugs": [ "bug-1678", @@ -22980,16 +9932,21 @@ "bug-1690", "bug-1561" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1691.md" }, { "id": "bug-1692", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T13:10:00Z", - "error_message": "xce-audit-invariants finding: WAL_JOB_PREASON silently drops job.pend_reasons (the scheduler-reported host/resource-based pending-job explanation) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), bug-1690 (WAL_JOB_RUSAGE/job.run_usage), and bug-1691 (WAL_JOB_CKPT/job.ckptinfo). set_preason()/do_clr_preason() (job.c) set/clear job->pend_reasons (xce_job_preason_t*: main_reason plus xce_host_reason_t**/xce_res_reason_t** nested arrays), but the WAL_JOB_PREASON fact -- purely hook-driven via persist_job_hook_fun's `default:` arm (shared with STOP/RESUME/UPDATE, same class as bug-1690/1691's hook-driven precedent, NOT a direct persist_submit_job_*_fact() call site like DISPATCH/START/FINISH) -- only ever carried the shared wal_jobstate_t{jobid, state} payload. Neither apply seam restored pend_reasons after a restart. This was the LAST of the five field-parity gaps from the original xce-audit-invariants pass -- closes that invariant class.", - "file": "src/controller/persist.c:206 (capnp_encode_jobpreason, new), src/controller/persist.c:290 (decode_jobpreason, new), src/controller/persist.c:78 (persist_job_hook_fun local vars pend_buf/pend_len/pend_present), src/controller/persist.c (persist_job_hook_fun's new HOOK_DETAIL_SETPREASON/HOOK_DETAIL_CLRPREASON case, split out of default arm), src/controller/persist.c (persist_job_hook_fun's shared-tail pend_present append block), src/controller/persist.c (apply_wal_entry's WAL_JOB_PREASON case, split out of the WAL_JOB_FAIL/STOP/RESUME/UPDATE shared group), src/controller/persist.c (apply_committed_entry's WAL_JOB_PREASON unconditional-adopt block, added after the WAL_JOB_CKPT block, before the JOB_TRANSITION_APPLIED gate), src/controller/job.c (restore_job_preason_state, new, added after restore_job_ckpt_state), src/include/wal_types.h (WAL_JOB_PREASON=10 doc comment, new), src/controller/testsuite/test_wal_job_preason_fact.c (new, 11 tests), src/controller/meson.build, src/controller/testsuite/controller_test.c (test wiring)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-audit-invariants finding: WAL_JOB_PREASON silently drops job.pend_reasons (the…\n\nxce-audit-invariants finding: WAL_JOB_PREASON silently drops job.pend_reasons (the scheduler-reported host/resource-based pending-job explanation) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), bug-1690 (WAL_JOB_RUSAGE/job.run_usage), and bug-1691 (WAL_JOB_CKPT/job.ckptinfo). set_preason()/do_clr_preason() (job.c) set/clear job->pend_reasons (xce_job_preason_t*: main_reason plus xce_host_reason_t**/xce_res_reason_t** nested arrays), but the WAL_JOB_PREASON fact -- purely hook-driven via persist_job_hook_fun's `default:` arm (shared with STOP/RESUME/UPDATE, same class as bug-1690/1691's hook-driven precedent, NOT a direct persist_submit_job_*_fact() call site like DISPATCH/START/FINISH) -- only ever carried the shared wal_jobstate_t{jobid, state} payload. Neither apply seam restored pend_reasons after a restart. This was the LAST of the five field-parity gaps from the original xce-audit-invariants pass -- closes that invariant class.", + "file": "src/controller/persist.c:206 (capnp_encode_jobpreason, new), src/controller/persist.c:290 (decode_jobpreason, src/controller/persist.c:78 (persist_job_hook_fun local vars pend_buf/pend_len/pend_present), src/controller/persist.c (persist_job_hook_fun's new HOOK_DETAIL_SETPREASON/HOOK_DETAIL_CLRPREASON case, split out of default arm), src/controller/persist.c (persist_job_hook_fun's shared-tail pend_present append block), src/controller/persist.c (apply_wal_entry's WAL_JOB_PREASON case, split out of the WAL_JOB_FAIL/STOP/RESUME/UPDATE shared group), src/controller/persist.c (apply_committed_entry's WAL_JOB_PREASON unconditional-adopt block, added after the WAL_JOB_CKPT block, before the JOB_TRANSITION_APPLIED gate), src/controller/job.c (restore_job_preason_state, new, added after restore_job_ckpt_state), src/include/wal_types.h (WAL_JOB_PREASON=10 doc comment, src/controller/testsuite/test_wal_job_preason_fact.c (new, 11 tests), src/controller/meson.build, src/controller/testsuite/controller_test.c (test wiring)", "root_cause": "persist_job_hook_fun()'s job-fact switch (persist.c) routed HOOK_DETAIL_SETPREASON/HOOK_DETAIL_CLRPREASON through its generic `default:` arm (shared with STOP/RESUME/UPDATE), which captures only {jobid, state} from the job_t* -- j->job.pend_reasons, freshly set/cleared by set_preason()/do_clr_preason() moments earlier under the same write lock this AFTER_EVENT fires inside, was never read or encoded. Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/FINISH/.../DISPATCH fall-through group (primary boot replay AND live commit-advance apply) and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) for standby catch-up -- never restored pend_reasons, matching the exact same class of gap bug-1678/1687/1689/1690/1691 already fixed for allocs/pidinfo+start_time/exit_info+finish_time/run_usage/ckptinfo. Confirmed via CodeGraph source read that set_preason()/do_clr_preason() are hook-driven (fire xce_run_hooks only, no direct persist_submit_job_*_fact() call site) -- closer to bug-1690/1691's RUSAGE/CKPT precedent than DISPATCH/START/FINISH's direct-call-site pattern -- verified against the real code before implementing. Also confirmed set_preason()/do_clr_preason() never change job.state (only job.pend_reasons), so a naive copy of the START/FINISH/DISPATCH pattern (gating adoption on a forward JOB_TRANSITION_APPLIED) would have made the fix dead code on every real invocation, mirroring bug-1690/1691's exact same discovery. UNIQUE to this bug (verified, not assumed from precedent): job.pend_reasons has a genuine CLEAR operation (do_clr_preason/HOOK_DETAIL_CLRPREASON) that legitimately resets it to NULL and must survive replay just as durably as the SET side -- bug-1690/1691's own wire convention ('trailing bytes present or absent' == 'field has a new value or is old-format, leave untouched') cannot distinguish that legitimate NULL from a pre-fix payload, so a blind copy of that convention would have silently reintroduced a stale-pend_reasons-after-clear bug on restart.", "fix": "FIXED (2026-08-25). controller.capnp ALREADY had a matching struct -- `struct JobPendReason $C.mapname(\"xce_job_preason_t\")` with `mainReason`/`hostReasons: List(HostBasedReason) $C.maplistcount(\"nhost_reasons\")`/`resReasons: List(ResourceBasedReason) $C.maplistcount(\"nres_reasons\")`, plus sibling `HostBasedReason $C.mapname(\"xce_host_reason_t\")` and `ResourceBasedReason $C.mapname(\"xce_res_reason_t\")` -- verified field-for-field against types.h's real struct layouts, and ALREADY wired into the full Job struct's own encode/decode/free (Job.pendreason field, used by the existing checkpoint/snapshot path) -- confirmed via grep that encode_JobPendReason/decode_JobPendReason/free_JobPendReason/free_JobPendReason_ptr/new_JobPendReason/read_JobPendReason/write_JobPendReason all already exist in the generated controller.capnp.c/.h. NO schema change needed (like bug-1678/1687's JobAlloc/JobPIDInfo reuse, unlike bug-1690's WAL_JOB_RUSAGE which needed a brand-new schema struct).\n\nSAFETY CHECK (bug-1690's RUSAGE lesson, explicitly re-verified rather than assumed): read erase_job_preason() (src/lib/preason.c) -- releases every field via FREEUP() (plain free()), never dereference()/RTTI. Read add_host_preason()/add_res_preason() (same file) -- build host/resource reason entries via strdup()/calloc() throughout, never makeString(). Read the generated decode_HostBasedReason/decode_ResourceBasedReason (controller.capnp.c) -- use STRING_DUP (plain strdup) + calloc(), and free_HostBasedReason/free_ResourceBasedReason/free_JobPendReason/free_JobPendReason_ptr all use plain free(). Also confirmed jobj.c's job_init() ALREADY does a plain MOVE_PTR(dst->job.pend_reasons, src->pend_reasons) when constructing a job from a decoded Job (no merge, no makeString) and job_destroy() releases it via erase_job_preason() (FREEUP-only) -- an existing, working precedent for exactly this adopt pattern elsewhere in the codebase. CONCLUSION: unlike WAL_JOB_RUSAGE's host_name (xstring/RTTI-backed, required a merge to avoid corrupting the codebase's own live in-place-mutated structure), job.pend_reasons and its entire nested tree are plain calloc()/strdup()-backed with NO RTTI involvement -- a straightforward free-old-then-adopt-new via the capnp-generated free_JobPendReason_ptr() is safe, matching bug-1678/1687's JobAlloc/JobPIDInfo pattern, NOT bug-1690's merge-required pattern. set_preason() itself corroborates this: it does erase_job_preason()+FREEUP() then a plain MOVE_PTR of the caller's already-built xce_job_preason_t*, never mutating the SAME live structure in place the way update_host_runusage() does for run_usage -- so there is no live-object-reapplied-via-apply-pump hazard analogous to RUSAGE's here either.\n\npersist.c gains capnp_encode_jobpreason()/decode_jobpreason() (mirroring capnp_encode_joballoc()/decode_joballoc() exactly). persist_job_hook_fun() gains a new HOOK_DETAIL_SETPREASON/HOOK_DETAIL_CLRPREASON case (split out of the generic default arm, mirroring bug-1691's HOOK_DETAIL_JOB_CKPT split) that encodes j->job.pend_reasons when non-NULL.\n\nKEY DESIGN DIVERGENCE from bug-1690/1691's wire convention (the CLEAR-semantics gap identified above): the payload appends an explicit PRESENCE BYTE after the wal_jobstate_t header whenever the fact is SETPREASON or CLRPREASON (not just when a value exists to encode) -- 0 = pend_reasons was genuinely cleared (adopt NULL on replay), 1 = trailing capnp JobPendReason bytes follow (adopt the decoded value). A payload of exactly sizeof(wal_jobstate_t) (no presence byte at all -- any entry written before this fix) leaves pend_reasons untouched on both apply seams, preserving full backward compatibility. Both apply seams updated: apply_wal_entry() splits WAL_JOB_PREASON out of its shared state-only group into its own case, decoding the presence byte (and trailing capnp bytes when present=1) via decode_jobpreason() and calling new restore_job_preason_state() (job.c) -- careful to treat a present-but-failed-decode as 'leave untouched', never as an implicit clear. apply_committed_entry() stays in its shared job-fact group but -- mirroring bug-1690/1691's unconditional-adopt design for the identical reason (set_preason()/do_clr_preason() never change job.state, so gating on JOB_TRANSITION_APPLIED would make it dead code) -- decodes/adopts pend_reasons UNCONDITIONALLY, before the transition gate, using the same presence-byte + decode-failure-safe logic.\n\nRegression tests: new src/controller/testsuite/test_wal_job_preason_fact.c (11 tests, mirroring test_wal_job_ckpt_fact.c's 'unconditional decode independent of transition result' structure, extended with PREASON-specific clear-semantics coverage no CKPT/RUSAGE precedent needed): (1) apply_committed_entry sets pend_reasons (BOTH a host-based AND a resource-based reason, exercising both nested list types) when state does NOT change -- most important test; (2) apply_committed_entry sets pend_reasons on a genuine forward transition; (3) apply_committed_entry backward-compat with an old bare payload; (4) apply_committed_entry CLEARS an existing pend_reasons on a presence-byte=0 payload -- the test that would have caught a naive copy of bug-1690/1691's convention; (5) apply_committed_entry leaves pend_reasons untouched when present=1 but zero trailing bytes follow (decode_jobpreason never even called) -- NOTE: an earlier draft of this test tried to exercise 'corrupt capnp bytes' instead and got a genuine RED signal for the wrong reason (Expected 4 Was 0) -- capnp-c's read path does not validate content, so garbage-but-nonempty bytes decode into a zero-valued struct rather than returning NULL (matching decode_joballoc/decode_jobrunusage's identical pre-existing leniency); corrected to the actually-reachable zero-trailing-bytes case; (6-8) the apply_wal_entry mirrors of (1)/(3)/(4); (9) direct-payload restart replay (WAL_JOB_SUBMIT + WAL_JOB_PREASON with host+resource reasons via persist_wal_async + persist_recover()); (10) test_wjp_real_do_set_preason_survives_restart -- drives the REAL, unmodified do_set_preason() (set_preason() itself is file-static to job.c; do_set_preason() is job.c's only exported entrypoint, matching the actual OPCode_setpreason wire path) through a real WAL-write + persist_recover() restart cycle, requiring a minimal zeroed (fd=-1, no iomgr) endpoint_t since do_set_preason() calls xce_reply() before processing -- mirrors test_commit_barrier_reply.c's established fixture pattern, packets drained via endpoint_deq()/free_packet() to avoid a reachable-leak false positive; (11) test_wjp_real_clear_survives_restart -- PREASON-specific, no CKPT/RUSAGE analog: drives real do_set_preason() THEN real do_clr_preason() before restarting, proving the CLEAR itself durably survives replay (not just the earlier SET) -- the audit's exact confirmed correctness requirement unique to this fact type.\n\nRED->GREEN independently verified live (not just narrated): `git stash push -- src/controller/job.c src/controller/persist.c` (keeping the new test file, wal_types.h doc comment, meson.build/controller_test.c wiring), incremental rebuild via the synchronous xce-build.sh script (`nix develop ./src --command bash src/tools/xce-build.sh controller asan build-asan --incremental`) -- 1178 Tests 9 Failures: the 2 pre-existing bug-1561 test_config_repl.c failures plus exactly 7 of the 11 new tests failing with the exact expected 'must set job.pend_reasons' / 'must reset pend_reasons to NULL' mismatches (tests 1,2,4,6,8,9,10 by suite order); the other 4 new tests (backward-compat x2, the decode-edge-case test, and the clear-survives-restart test) correctly PASS unchanged under the reverted code -- verified this is expected, not a test-quality defect: those 4 scenarios are either genuinely unaffected by the fix (backward-compat) or, for the two restart-replay tests, happen to trivially satisfy their own assertion under BOTH old and new code (old code never encodes pend_reasons at all regardless of set-vs-clear, so a 'must be NULL after clear' check passes vacuously pre-fix too -- acceptable, since the SET-side restart test (10) and the direct-payload test (9) already independently prove the encode/decode mechanism itself, and test (11)'s value is in proving the CLEAR case specifically once mechanism 10 is proven, not in isolating it alone). `git stash pop`, incremental rebuild -- 1178 Tests 2 Failures (only the pre-existing bug-1561 pair, exact signature match 'Expected 2 Was 3' / 'Expected 0 Was 11954236611186853435'), all 11 new tests pass, zero new failures, zero ASAN heap-corruption findings (grepped the full test log for AddressSanitizer/heap-buffer-overflow/invalid-free/double-free/use-after-free/SEGV/Aborted -- none found). Verified no new compiler warnings in persist.c/job.c/the new test file (isolated rebuild + warning grep; the handful of pre-existing format-truncation/comment warnings in persist.c are all at unrelated line numbers, predating this change).\n\nThis was the LAST of the five field-parity gaps identified by the original xce-audit-invariants pass (bug-1678/1687/1689/1690/1691/1692) -- that invariant class (APPLY_CLEAN's entity-count+cursor-position check not proving per-field completeness, persistence.md) is now closed for job.allocs/pidinfo+start_time/exit_info+finish_time/run_usage/ckptinfo/pend_reasons.", "tags": [ @@ -23006,9 +9963,7 @@ "hook-driven-fact", "clear-semantics", "presence-byte-wire-shape", - "test-quality-correction", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-quality-correction" ], "related_bugs": [ "bug-1678", @@ -23018,15 +9973,20 @@ "bug-1691", "bug-1561" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1692.md" }, { "id": "bug-1693", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "hang", "timestamp": "2026-08-25T14:05:00Z", - "error_message": "Invariant audit (Decision Log fidelity dimension) found two violations of the recorded Decision Log entry \"No client reply-wait timeout as a fix for a hung request -- fail loud at the real transport/source instead of masking with a deadline backstop\": (1) OPCode_schedAdmin's forward-and-block-for-reply (cluster.c) blocks up to SCHEDADMIN_TIMEOUT_MS=5000ms via pthread_cond_timedwait, but handle_scheduler_leave() (sched.c) -- the actual scheduler-disconnect handler -- only ever nulled c->sched.ep; it never signalled the pending waiter, so a scheduler disconnect mid-wait could only ever be observed at the full 5s deadline. (2) on_fetchfile's identical proxy pattern (job.c, 10s pthread_cond_timedwait) had the same gap: handle_worker_leave() (worker.c) only ever nulled w->ep, never woke a pending fetchfile waiter proxied to that worker.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Invariant audit (Decision Log fidelity dimension) found two violations of the recorded Decision…\n\nInvariant audit (Decision Log fidelity dimension) found two violations of the recorded Decision Log entry \"No client reply-wait timeout as a fix for a hung request -- fail loud at the real transport/source instead of masking with a deadline backstop\": (1) OPCode_schedAdmin's forward-and-block-for-reply (cluster.c) blocks up to SCHEDADMIN_TIMEOUT_MS=5000ms via pthread_cond_timedwait, but handle_scheduler_leave() (sched.c) -- the actual scheduler-disconnect handler -- only ever nulled c->sched.ep; it never signalled the pending waiter, so a scheduler disconnect mid-wait could only ever be observed at the full 5s deadline. (2) on_fetchfile's identical proxy pattern (job.c, 10s pthread_cond_timedwait) had the same gap: handle_worker_leave() (worker.c) only ever nulled w->ep, never woke a pending fetchfile waiter proxied to that worker.", "file": "src/controller/cluster.c, src/controller/sched.c, src/controller/job.c, src/controller/worker.c, src/controller/include/cluster.h", "root_cause": "Both sites already had a working signal function used on the SUCCESS path (schedadmin_signal() in cluster.c, fetchfile_signal() in job.c, both wake the pending condvar wait when a real reply arrives) -- but the corresponding DISCONNECT handlers (handle_scheduler_leave/handle_worker_leave) were never wired to call them. cluster.c's own OPCode_schedAdmin comment explicitly documents schedAdmin as mirroring on_fetchfile's proxy pattern, confirming this was one shared, repeated design mistake copied into both sites, not two independent bugs. The timeout backstop was therefore the SOLE failure-detection mechanism for a mid-wait disconnect at both sites, exactly what the Decision Log entry (itself the outcome of bug-646/648/650's original fail-loud fix) forbids.", "fix": "Kept the timeout as a defensive backstop (per the Decision Log's exact wording: the rule is about not relying SOLELY on a timeout, not removing timeouts) and added an immediate-wake path on disconnect at both sites, mirroring bug-646/648/650's original fail-loud precedent. schedAdmin site: new int schedadmin_signal_disconnect(void) (cluster.c) -- a thin wrapper calling the existing static schedadmin_signal(-ECONNRESET, NULL) -- declared extern in cluster.h; handle_scheduler_leave() (sched.c) now calls it BEFORE acquiring c->sched.lock (calling it after would deadlock, since the OPCode_schedAdmin dispatch case holds c->sched.lock for its entire forward+wait duration; schedadmin_signal_disconnect() itself never touches c->sched.lock, only g_schedadmin_mu + the pending slot's own mu, so no new lock-ordering hazard). cluster.c's post-wait dispatch gained a new got_status==-ECONNRESET branch relaying a structured SchedAdminReply{error=\"scheduler disconnected\"}, distinguishable from both the immediate \"scheduler not connected\" rejection and the deadline's \"request timeout\". fetchfile site: fetchfile_pending_t (job.c) gained a worker_t* worker field (set in on_fetchfile() while still holding the worker's read lock, before fetchfile_register()); new void fetchfile_signal_worker_disconnect(worker_t*) (job.c, declared extern in cluster.h) walks g_fetchfile_list under g_fetchfile_mu and signals every pending entry matching the disconnected worker with status=-ECONNRESET (unlike fetchfile_signal, which is keyed by a single jobid, more than one fetchfile request can be in flight against the same worker); handle_worker_leave() (worker.c) calls it right after nulling w->ep (no lock-ordering constraint there, since on_fetchfile's wait never holds any worker_t lock). Hidden fan-out cost discovered only at link time (matches this project's established pattern for this class of change): controller_unit_test does not link the real sched.c at all (test_config_repl.c already provides its own no-op handle_scheduler_leave() stub, to avoid an unrelated on_scheduler_join()/notify() link collision with other files in that binary) -- so the primary schedAdmin regression test drives schedadmin_signal_disconnect() directly (the real, unmodified, unstubbed production function in cluster.c) rather than through handle_scheduler_leave(), and a SEPARATE small spy-based test was added in test_sched_worker_change.c (which DOES link the real sched.c) proving handle_scheduler_leave() itself calls schedadmin_signal_disconnect() exactly once, before clearing c->sched.ep. Also had to add a one-line link-satisfying schedadmin_signal_disconnect() stub to test_job_array_sched_push.c and test_sched_worker_change.c (the two standalone binaries that link sched.c without cluster.c), mirroring their existing check_worker_() stub precedent.", @@ -23045,24 +10005,27 @@ "condvar-signal", "reply-wait-timeout", "xce-audit-invariants", - "ha-async-io", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "ha-async-io" ], "related_bugs": [ "bug-646", "bug-648", "bug-650" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1693.md" }, { "id": "bug-1694", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T00:00:00.000Z", - "error_message": "xce-audit-invariants finding (dimension 1, async I/O thread-role): checkpoint_task_fn (src/controller/persist.c) was the last delaymgr-triggered path still calling its full blocking body (persist_write_checkpoint_marker's pwrite+fdatasync, write_snapshot()'s unbounded open/write walk over every job/queue/host frame held under apply_pump_lock, trim_wal_segments, rotate_snapshots' unlink) directly on the delaymgr timer thread -- the same single thread that drives heartbeats and election timeouts cluster-wide (ha-async-io.md thread-role contract: delaymgr must only trigger work, never block). Triggered from the WAL-append hot path via schedule_checkpoint()/delay_task() whenever wal_record_count crosses max_wal_records. Every sibling long-blocking-I/O path (snapshot send, config-push, reconnect) had already been offloaded by raft-drive-async; checkpoint was the one documented remaining gap.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-audit-invariants finding (dimension 1, async I/O thread-role): checkpoint_task_fn…\n\nxce-audit-invariants finding (dimension 1, async I/O thread-role): checkpoint_task_fn (src/controller/persist.c) was the last delaymgr-triggered path still calling its full blocking body (persist_write_checkpoint_marker's pwrite+fdatasync, write_snapshot()'s unbounded open/write walk over every job/queue/host frame held under apply_pump_lock, trim_wal_segments, rotate_snapshots' unlink) directly on the delaymgr timer thread -- the same single thread that drives heartbeats and election timeouts cluster-wide (ha-async-io.md thread-role contract: delaymgr must only trigger work, never block). Triggered from the WAL-append hot path via schedule_checkpoint()/delay_task() whenever wal_record_count crosses max_wal_records. Every sibling long-blocking-I/O path (snapshot send, config-push, reconnect) had already been offloaded by raft-drive-async; checkpoint was the one documented remaining gap.", "file": "src/controller/persist.c, src/include/persist.h, src/controller/testsuite/test_persist_checkpoint_offload.c, src/controller/testsuite/controller_test.c, src/controller/meson.build", "root_cause": "checkpoint_task_fn called persist_checkpoint() inline with no offload mechanism at all -- unlike every other delaymgr-triggered blocking path in the codebase. Investigated whether the obvious fix (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK), the pattern used by persist_apply_pump_kick/commit.c's dispatch_continuation) was safe here: it is NOT -- persist_apply_pump_run() (which ALSO acquires apply_pump_lock, across its entire per-entry walk) is itself dispatched onto that SAME vtaskmgr N-thread pool via persist_apply_pump_kick()'s vtask_enq_classified(..., VTASK_LIVE_BLOCK). Putting checkpoint's genuinely unbounded write_snapshot() (held under apply_pump_lock) on the identical N-thread pool would risk a worker thread parked on apply_pump_lock for the whole snapshot-write duration, starving the pool's liveness-critical capacity (commit continuations, apply-pump drains) -- exactly the failure mode ha-async-io.md's own table entry warns about (\"vtaskmgr must NOT do: long/unbounded blocking I/O (snapshot transfer, config-push) -- starves the pool\"). The correct destination, per the codebase's OWN established distinction (re-verified by reading all 4 reference sites plus grepping every offload flag), is the SLOW LANE (slowlane_submit) -- the same mechanism snapshot-send (ha_snapshot_do_send -> snapshot_lane_fn, g_drive_async_snapshot_offload, default ON) and config-push (configpush_lane_fn, g_drive_async_configpush_offload, default ON) already use, and NOT the vtask pool. Confirms the task brief's suspicion that checkpoint was harder than a simple vtask wrap -- it's a slow-lane fix, not a vtask-pool fix.", "fix": "FIXED. checkpoint_task_fn is now trigger-only: it releases the checkpoint_in_progress CAS slot, then submits the real work (via a new run_checkpoint_body()/checkpoint_lane_fn()) to the cluster's slow lane (slowlane_submit(&persist->cluster->slowlane, key=1, checkpoint_lane_fn, persist)), mirroring snapshot_lane_fn/configpush_lane_fn/catchup_lane_fn/demotion_lane_fn exactly. New feature flag g_persist_checkpoint_offload (persist_set_checkpoint_offload_for_test/persist_checkpoint_offload_enabled, persist.h/persist.c) defaults ON (1) -- unlike the still-experimental catchup/demotion offloads (default OFF pending an unrelated real test-fixture hang, bug-1429), this reuses an ALREADY-PROVEN-SAFE call: persist_checkpoint() already runs on this same slow lane in production today via ha_snapshot_do_send()->snapshot_lane_fn (snapshot offload also default ON), so apply_pump_lock's acquisition from a slow-lane thread is exercised in production every day, not new surface. persist_checkpoint()'s own atomic_exchange guard (checkpoint_in_progress) remains the single-flight barrier preventing double-run with the snapshot-send path, exactly as it already does today. Verified no existing controller_unit_test fixture depends on the delaymgr trigger path completing synchronously (test_ha_core_safety.c deliberately pins max_wal_records=1u<<30 specifically to prevent ever exercising it in a live-cluster fixture) -- ruling out the catchup/demotion-class 'uninitialized slow lane + a synchronous downstream waiter hangs' hazard, since checkpoint's trigger is pure fire-and-forget. Inline-fallback + WARN on genuine slowlane_submit failure only (not -EAGAIN/-EEXIST, both non-failure per slowlane's own documented contract) -- a scheduled checkpoint is never silently dropped. New regression suite test_persist_checkpoint_offload.c (3 tests, mirrors test_ha_liveness.c's reconnect/configpush offload gate-probe pattern): test_checkpoint_offload_trigger_returns_promptly (GREEN: trigger returns <200ms while the real body is observably parked on a slow-lane worker thread, not the caller's), test_checkpoint_inline_blocks_delaymgr_thread_RED (flag OFF reproduces the pre-fix inline-blocking behavior on demand, permanent regression guard), test_checkpoint_no_cluster_runs_inline (defensive NULL-cluster fallback). RED-phase independently confirmed by temporarily forcing checkpoint_task_fn's pre-fix always-inline behavior (a one-line `if (1) ||` probe) and rebuilding: the GREEN test then hung indefinitely (process had to be kill -9'd, PID confirmed via ps as the stuck controller_unit_test binary) -- proving the test genuinely detects the violation, not a tautology. Probe reverted (git diff --stat confirmed clean re-application of the real fix) and the full suite re-run GREEN: 1184 Tests, identical 2 pre-existing failures both runs (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was , test_config_repl.c, both matching bug-1561's well-documented overlayfs fdatasync-EIO artifact signature exactly -- 0 NEW failures. Live HA smoke test (real checkpoint trigger against a running deployed cluster under load) was scoped out given budget already spent on the lock-order investigation, which was this fix's primary risk -- flagged explicitly in the final report rather than silently skipped, as a reasonable follow-up soak.", @@ -23076,22 +10039,25 @@ "apply_pump_lock", "vtask", "xce-audit-invariants", - "raft-drive-async", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "raft-drive-async" ], "related_bugs": [ "bug-1429" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1694.md" }, { "id": "bug-1695", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T15:05:00Z", - "error_message": "Found via the mandatory live HA chaotic soak (xce-audit-invariants controller campaign, Step 4) -- NOT caused by any of the 9 fixes that campaign made (bug-1684/1687/1689/1690/1691/1692/1693/1694), all of which are in src/controller/{ha,persist,job,cluster,sched,worker}.c; this bug is in myapp.c, untouched by any of them. 10-round --chaotic soak: every single round after the first shows 'NO new primary within window (possible election stall)'; ctrl-1's own log shows, on EVERY restart after the first boot, 'myapp_run: leader already known after startup delay (voted_for=ctrl-1, voted_term=1, raft_term=1, primary_fd=-1) -- skipping own election' -- primary_fd=-1 proves there is no live connection to any primary, yet the node skips holding its own election anyway. ctrl-2/ctrl-3 are simultaneously stuck in a permanent ~3s retry loop ('cluster_init_phase2: cluster_config not loaded yet', rc=-22, the bug-1477 self-rescheduling retry firing forever without ever succeeding). The cluster never elects a new, live, functioning primary after the very first election of the cluster's lifetime.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Found via the mandatory live HA chaotic soak (xce-audit-invariants controller campaign, Step 4)…\n\nFound via the mandatory live HA chaotic soak (xce-audit-invariants controller campaign, Step 4) -- NOT caused by any of the 9 fixes that campaign made (bug-1684/1687/1689/1690/1691/1692/1693/1694), all of which are in src/controller/{ha,persist,job,cluster,sched,worker}.c; this bug is in myapp.c, untouched by any of them. 10-round --chaotic soak: every single round after the first shows 'NO new primary within window (possible election stall)'; ctrl-1's own log shows, on EVERY restart after the first boot, 'myapp_run: leader already known after startup delay (voted_for=ctrl-1, voted_term=1, raft_term=1, primary_fd=-1) -- skipping own election' -- primary_fd=-1 proves there is no live connection to any primary, yet the node skips holding its own election anyway. ctrl-2/ctrl-3 are simultaneously stuck in a permanent ~3s retry loop ('cluster_init_phase2: cluster_config not loaded yet', rc=-22, the bug-1477 self-rescheduling retry firing forever without ever succeeding). The cluster never elects a new, live, functioning primary after the very first election of the cluster's lifetime.", "file": "src/controller/myapp.c (myapp_run's bootstrap check, call site only); src/controller/startup_validation.c (myapp_leader_already_known, the extracted+fixed decision function -- placed here, not myapp.c, so it links into controller_unit_test); src/controller/testsuite/test_ha_startup.c (7 new regression tests)", "root_cause": "myapp_run()'s post-stagger 'is a leader already known' bootstrap check (myapp.c ~361-384) has two independent fallback conditions that can each unilaterally set leader_known=1: `c.ha.raft.term > 0` and `c.ha.primary_ctrl_fd >= 0`. The comment's stated intent (lines 345-353) is to catch a narrow race -- 'iomgr was NOT running during the nap ... but the winner's ctrlhello has already established the control channel' -- i.e. genuine NEW raft activity processed during THIS boot's startup-stagger nanosleep. But `ha_restore_term()` (bug-1684's neighbor, unmodified by that fix) restores ha->term from ha-term.bin BEFORE this check ever runs, as part of normal boot -- so `raft.term > 0` is trivially true on EVERY restart after the cluster's first-ever election in its entire lifetime, regardless of whether any NEW raft traffic was processed this boot at all. The check cannot distinguish 'term advanced because I just received a real, fresh vote/heartbeat during this nap' (genuine liveness evidence) from 'term is nonzero because I loaded an old value off disk at startup' (tells us nothing about whether ANY leader is currently reachable). Consequence: after the cluster's first election, EVERY node, on EVERY subsequent restart, unconditionally takes the goto skip_election fast path and never calls ha_start_election_async again -- so no node ever re-asserts leadership through a real raft election, and the cluster permanently loses the ability to elect a NEW primary after any restart, however far in the past the original election was. This is a related-but-distinct gap from bug-1477 (which fixed a stranded node never RETRYING cluster_init_phase2 after one failed attempt -- that retry loop is confirmed still firing correctly here, every ~3s, exactly as bug-1477 designed -- but the retry can never succeed because the underlying cause, no election is ever held to produce a real live primary to push config from, was never addressed).", "fix": "FIXED (2026-08-25). Root cause confirmed exactly as hypothesized: myapp_run()'s raft.term>0 fallback (myapp.c ~372) is trivially true on every restart after the cluster's first-ever election, since ha_restore_term() (called from cluster_init_phase1, strictly before ha_init_raft()/the startup-stagger nap) restores a persisted, monotonically-nondecreasing term from ha-term.bin, and Raft terms never reset.\n\nFix: extracted the inline check into a new, unit-testable function myapp_leader_already_known(ha, my_name, term_at_boot) -- placed in startup_validation.c (NOT myapp.c) purely so it links into controller_unit_test, which meson's controller_unit_test_src deliberately excludes myapp.c/main.c from (myapp_run is an untestable integration entry point requiring a live network socket) but already builds startup_validation.c for the sibling validate_* helpers -- myapp.c now just forward-declares and calls it, mirroring the pre-existing validate_startup_config() precedent exactly. myapp_run() captures `term_at_boot = c.ha.raft.term` immediately after ha_init_raft() seeds ha->raft.term from the just-restored ha->term -- strictly BEFORE the startup-stagger nap -- and the fallback check now requires `ha->raft.term > term_at_boot` (a genuine ADVANCE during THIS boot's nap, i.e. real vote-request/heartbeat traffic actually processed by iomgr via raft_on_vote_request's/raft_on_append_entries_request's UpdateTerm steps) instead of the old bare `> 0`. The voted_for_name/vft==cur check and the primary_ctrl_fd>=0 check are UNTOUCHED, per the task's scope boundary.\n\nVerification of the 5 required questions:\n1. Ordering confirmed via direct source read: ha_restore_term() runs inside cluster_init_phase1 (myapp_run Step 1, ~line 67), strictly before ha_init_raft() (~line 327) and the stagger nanosleep (~line 333-343) -- on every boot including restarts, confirmed via a live log line 'HA: restored term=0 from .../ha-term.bin' printed inside cluster_init_phase1, always preceding 'HA: raft initialized (...term=N...)' from ha_init_raft.\n2. No other path can legitimately advance ha->raft.term between the snapshot and the check in a way the fix would wrongly treat as stale: the only writer after ha_init_raft() is raft_cb_persist, fired only from raft_on_vote_request / raft_on_append_entries_request / raft_start_election -- all driven by genuine incoming or self-initiated raft traffic processed by iomgr during the nap, exactly the signal the check intends to detect (confirmed by reading raft.c's UpdateTerm steps in full).\n3. First-ever-boot preserved: ha_restore_term() leaves term at 0 with no ha-term.bin, so term_at_boot=0, identical operative comparison to the old `raft.term > 0` check on a true first boot -- covered by 2 new regression tests.\n4. No new safety-relevant failure mode: the 'spurious own election while a leader is genuinely live but unobserved' race this fix re-exposes on every restart is the IDENTICAL race the cluster's first-ever boot already ran through before this fix existed (term_at_boot was always 0 pre-first-election) -- not a new class of risk, and Election Safety is unaffected (rests entirely on votedFor + majority inside the untouched raft library, never on this advisory bootstrap check).\n5. voted_for_name/vft==cur deliberately left untouched per scope, but flagged in the code comment as NOT obviously immune to the same class of question (a restored (voted_for_term,term) pair is exactly as internally consistent as a live one). Investigated empirically via the live soak below rather than touched speculatively: in both the baseline and fixed runs, this path never fires for ctrl-2/ctrl-3 in the --chaotic scenario at all, because the driver only kills/restarts the current primary -- ctrl-2/ctrl-3 run continuously and never re-enter myapp_run(), so their eventual takeover goes entirely through the separate, pre-existing, untouched liveness-watchdog/PreVote/RDHUP-failover subsystem (ha_on_primary_ctrl_fd_rdhup / ha_arm_heartbeat_timeout_vtask), not through this bootstrap check. No compounding defect was exercised or found; flagged as a residual, unverified theoretical concern for a scenario this soak doesn't cover (a STANDBY node itself restarting after remembering a now-permanently-dead remote leader).\n\nTDD: RED confirmed by extracting the function with the OLD `raft.term > 0` logic first (marked with a RED/GREEN comment), adding 7 new tests to test_ha_startup.c (test_bug1695_stale_restored_term_does_not_skip_election is the core regression), and rebuilding -- 2 of the 7 genuinely FAILED ('Expected 0 Was 1'), exactly as predicted (the self-vote test also failed under the old logic, since raft.term>0 trivially short-circuited it too -- an incidental but correct extra confirmation of the bug's reach). Applied the one-line semantic fix (`> 0` -> `> term_at_boot`); rebuilt -- all 7 new tests GREEN, full controller suite at 1191 tests with exactly the 2 pre-existing bug-1561 failures (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config -- identical Expected/Was values to bug-1561's own documented signature, 0 new failures).\n\nMANDATORY live soak (6-round --chaotic, fresh ASAN binaries) -- run TWICE for a rigorous A/B: once with the fix, once with myapp.c/startup_validation.c/test_ha_startup.c git-stashed back to the pre-fix original (same binaries otherwise, same scratch-dir mechanics). BASELINE (unfixed): term frozen at 4 for the ENTIRE 6-round run (never advances even once after initial cold-boot settling); 'NO new primary within window' on 6/6 rounds; ctrl-2/ctrl-3's logs show ZERO CTRL_PRIMARY/PreVote-round activity anywhere in their entire lifetime -- a real, direct confirmation of the reported permanent-stall symptom. FIXED: term genuinely progresses 1->2->3->4->7 across the run; ctrl-2 (a DIFFERENT node from the original ctrl-1) achieved a real, log-verified CTRL_PRIMARY transition via a won PreVote round + real election at term=3 -- something that never happens even once anywhere in the baseline's logs. This is direct, unambiguous proof the core bug-1695 symptom (permanent inability to elect a NEW primary after any restart, once the cluster has ever held its first election) is fixed. committed_divergence (walcompare, the authoritative WAL-level check) stayed 0 in BOTH runs.\n\nResidual, honestly-reported finding: the FIXED run's chaotic verdict was `committed_job_loss=6, result=FAIL` (baseline was `committed_job_loss=0, result=PASS` -- matching this task's own prediction that the baseline passes trivially by never actually re-electing anyone). Root-cause investigation (waldump on all 3 nodes' raw WAL files): the SAME single jobid ('0-15-1787671840-0') was reported missing on every affected round, and does NOT appear in ANY of the 3 nodes' on-disk WALs at all -- it traces to the cluster's very first cold-boot settling window (timestamp ~1787671840, well before round 1), during which ctrl-1's FIRST-ever election (with peers still config-less, config_tag=0) produced ambiguous vote-eligibility/config-bootstrap behavior unrelated to this fix (confirmed via ctrl-1's own log: its very first boot conceded to CTRL_STANDBY, never wrote that job to its own WAL). This strongly resembles a pre-existing, already-documented false-positive class (bug-1487: 'reachable primary reports job missing... post-election own-term commit-barrier lag while a peer is still catching up' -- verify_classify's own in-code comment names this exact class), newly and more frequently exposed because this fix, for the first time, lets the chaotic driver's overlapping-kill design exercise GENUINE repeated elections at all (the baseline's frozen-forever cluster never gave bug-1487's window a chance to occur more than once, if ever). NOT independently re-fixed in this session (out of this fix's myapp.c-only scope, and .claude/skills/xce-test-ha/run-ha-soak.sh is test-harness tooling, not controller code) -- flagged here as a genuine, needs-owner follow-up: either extend bug-1487's verify_classify retry margin further, or investigate whether the initial cold-boot config-bootstrap vote-eligibility path can genuinely lose a pre-config-push submission. Confidence in the CORE fix remains high (direct log proof of a new node winning a real election; walcompare clean in both runs); confidence in the chaotic verdict's exact job-loss root cause is medium (waldump-verified absence + a highly plausible, precedented explanation, but not traced to a specific line the way the core fix was).", @@ -23105,7 +10071,6 @@ "myapp", "live-deployment", "found-via-soak", - "fixed", "skip_election", "term_at_boot", "live-soak-verified", @@ -23116,15 +10081,20 @@ "bug-1684", "bug-1487" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1695.md" }, { "id": "bug-1696", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T16:20:00Z", - "error_message": "Chaotic HA soak (post-bug-1695-fix) reports committed_job_loss on every round; waldump on ctrl-1's live data dir shows the ENTIRE on-disk WAL truncated to a single NOOP entry (entries: 1 submits: 0) despite 21+ jobs previously verified committed; no snapshot exists; log shows ha_compute_safe_committed_lsn=0 immediately before raft_cb_become_leader promotes ctrl-1 to CTRL_PRIMARY at term=2 on its SECOND boot.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "Chaotic HA soak (post-bug-1695-fix) reports committed_job_loss on every round; waldump on ctrl-1's…\n\nChaotic HA soak (post-bug-1695-fix) reports committed_job_loss on every round; waldump on ctrl-1's live data dir shows the ENTIRE on-disk WAL truncated to a single NOOP entry (entries: 1 submits: 0) despite 21+ jobs previously verified committed; no snapshot exists; log shows ha_compute_safe_committed_lsn=0 immediately before raft_cb_become_leader promotes ctrl-1 to CTRL_PRIMARY at term=2 on its SECOND boot.", "file": "src/controller/ha.c", "root_cause": "raft_cb_become_leader() computes the promotion WAL-truncation floor (line ~1658-1662: trunc_floor = ha_promotion_truncate_floor(safe_lsn, persist_durable_floor(&c->persist)); persist_truncate_wal_above(&c->persist, trunc_floor)) and evaluates the whole promotion-eligibility gate (durable_floor/truncated/apply_verdict, ~line 1699-1823) BEFORE cluster_init_phase2() -- the ONLY call that invokes persist_recover() for this process's first-ever boot -- which does not run until is_first_boot handling at line ~1873-1899, further down in the SAME function. persist_durable_floor()/persist_recovery_truncated()/persist_apply_verdict() all read fields persist_recover() is responsible for populating from the real on-disk WAL; before that call, a freshly persist_init()'d struct reports durable_lsn=0 (bug-1445's comment at line 1901 already flagged this exact 'gate evaluated blind before recovery' hazard for ONE flag -- persist_apply_resource_fenced -- with a narrow re-check-after-phase2 patch, but the far more consequential durable_floor/truncation-floor computation was never covered by that fix). With durable_floor=0 (bogus, pre-recovery) and safe_lsn=0 (every voter, including self, ALSO advertised its LSN via persist_durable_floor() before its own recovery ran -- myapp.c:324-329, ha_candidate_advertise_lsn), ha_promotion_truncate_floor(0,0)=0 and persist_truncate_wal_above(p, 0) physically ftruncates/unlinks the real on-disk WAL segments down to LSN 0/1 -- destroying committed job data that persist_recover(), had it run FIRST, would have correctly reported as durable and protected via the bug-1060 floor guard. This scenario (a node RESTARTS with real durable data on disk, then immediately WINS its own promotion on that same boot, before its own recovery has run) was unreachable before bug-1695's fix -- previously a restarted node's stale-but-nonzero persisted term made it always skip its own election and defer to a peer, so it never hit is_first_boot-during-become_leader with real data at stake. bug-1695 correctly restored real re-election on restart and, in doing so, newly exposed this pre-existing boot-ordering flaw. A genuine Leader Completeness / bug-1060-guard-defeat, not a test-harness artifact: confirmed via direct waldump on-disk evidence plus source-level trace of the read-before-recover ordering.", "fix": "FIXED. Reordered raft_cb_become_leader() (ha.c) so cluster_init_phase2()/persist_recover() (is_first_boot path) runs BEFORE the promotion-eligibility gate and the promotion-truncation-floor computation, so persist_durable_floor()/persist_recovery_truncated()/persist_apply_verdict() always reflect this boot's REAL recovered WAL state before any eligibility/truncation decision reads them. Split the pure vote-derived safe_committed_lsn computation (reads only ha->elect_ctx, safe pre-recovery) from the truncation-floor computation (needs post-recovery persist state) -- the former stays early, the latter moved to run after phase2. Original RED test hypothesis (physical WAL destruction via truncate) was disproven by source tracing (persist_truncate_wal_above's own no-op guard plus persist_init's current_lsn=1 reset make that specific mechanism impossible); the REAL, provable defect was the promotion-eligibility gate wrongly judging a genuinely corrupt/divergent node ELIGIBLE because it read persist_recovery_truncated()/persist_apply_verdict() before recovery ran. New regression test test_first_boot_promotion_refuses_when_own_first_recovery_finds_corruption (CRC-corrupts a real on-disk WAL entry, restarts, promotes) confirmed RED against unfixed code (wrongly became CTRL_PRIMARY despite RECOVER_TRUNCATED) and GREEN against the fix (correctly refused). Full controller unit suite: 1193 tests, 2 pre-existing bug-1561 failures only, 0 new. Live proof: full 10-round --chaotic soak under ASAN (build-asan-1696repro) PASSED (VERDICT mode=chaotic rounds=10 jobs=33 committed_divergence=0 committed_job_loss=0 result=PASS) and full 10-round --sequential soak PASSED (PASS: 10 rounds, 33 jobs submitted, zero job loss; all 3 nodes led (ctrl-1,ctrl-2,ctrl-3)) -- both legs of the audit skill's Step 4 mandate satisfied. Note: the fix's own verification agent died mid-A/B-soak leaving the fix orphaned in a git stash for a period before being discovered and restored; a subsequent independent full soak (this entry) is what actually closes it out.", @@ -23143,7 +10113,6 @@ "found-via-soak", "critical", "bug-1695-adjacent", - "fixed", "live-soak-verified", "asan-verified" ], @@ -23154,15 +10123,20 @@ "bug-828", "bug-1487" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1696.md" }, { "id": "bug-1697", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-25T20:22:00Z", - "error_message": "Job start_time is always 0 for every job, live, no restart needed (observed via xce_test REPL joblist during an HA soak). run_time (finish_time - start_time) is consequently also wrong for every finished job.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Job start_time is always 0 for every job, live, no restart needed (observed via xce_test REPL…\n\nJob start_time is always 0 for every job, live, no restart needed (observed via xce_test REPL joblist during an HA soak). run_time (finish_time - start_time) is consequently also wrong for every finished job.", "file": "src/controller/job.c, src/controller/persist.c", "root_cause": "dispatch_a_job_() (job.c:1686, scheduler-dispatch-commit task 3.1) synchronously sets j->job.state = JOB_STAT_RUN at DISPATCH time, before the worker ever reports back -- this is a deliberate, already-made semantic change: the controller decides RUN at dispatch, it no longer waits for a worker ack. When the worker's LATER on_job_start()/WAL_JOB_START fact commits and apply_committed_entry()'s live apply pump (persist.c, shared WAL_JOB_START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH case group starting ~persist.c:4430) processes it, apply_job_update_transition(j, JOB_STAT_RUN) (job.c:2779) sees job->job.state already == JOB_STAT_RUN (set by dispatch, moments/seconds earlier) and returns JOB_TRANSITION_NOOP_DUPLICATE (job.c:2795-2799), never JOB_TRANSITION_APPLIED. The wire-carried start_time/pidinfo decode/adopt block bug-1687 added (persist.c ~4602-4637) is gated on `result == JOB_TRANSITION_APPLIED` (persist.c:4594), so for WAL_JOB_START it can now NEVER run, for any job -- start_time and pidinfo (pids/pgids/exdata) both silently stay at their zero-initialized defaults forever. This is the exact same class of bug already fixed for WAL_JOB_RUSAGE/WAL_JOB_CKPT/WAL_JOB_PREASON in this identical function (their own comments describe it verbatim: 'gating this adoption on result==JOB_TRANSITION_APPLIED ... would make it dead code ... decode/adopt unconditionally') -- WAL_JOB_START was never given the same treatment because bug-1687 (which added the start_time/pidinfo wire fields) predates/was unaware of the dispatch-time RUN semantic collision. bug-1679's own comment (job.c:1709-1719) already independently documented and worked around the SAME root collision for a different symptom (the apiserver update-notification hook silently not firing on this same NOOP_DUPLICATE path) -- the two symptoms were never connected as the same root cause until now.", "fix": "FIXED -- start_time moved to dispatch-owned, pidinfo adoption made unconditional, exactly the confirmed scope. (1) job.c dispatch_a_job_() (job.c:1696-1708) stamps j->job.start_time = time(NULL) synchronously alongside j->job.state = JOB_STAT_RUN, under the same write lock, before j->job.allocs/persist_submit_job_dispatch_fact(). (2) wal_types.h adds WAL_JOB_DISPATCH_HAS_START_TIME_FLAG (new _pad bit, mirroring WAL_JOB_FAIL_STAGE_FLAG/WAL_JOB_UPDATE_POSTDONE_MARK). persist_submit_job_dispatch_fact() (persist.c:7670, persist.h:1465) gained a `uint64_t start_time` parameter; payload is now [wal_jobstate_t header, _pad=FLAG][uint64_t start_time][JobAlloc bytes] -- flag-gated, not length-based, since JobAlloc's own variable length would make a length heuristic ambiguous against the old header+alloc-only shape. (3) Both apply seams' WAL_JOB_DISPATCH cases (apply_committed_entry persist.c:4716 branch; apply_wal_entry persist.c:4183 case; restore_job_dispatch_state job.c:801) decode/adopt start_time when the flag is set, else fall through to the pre-fix alloc-only decode untouched. (4) apply_committed_entry's WAL_JOB_START handling (persist.c:4627-4660): start_time touching removed entirely (dead-code branch and its time(NULL) fallback deleted); pidinfo decode/adopt moved OUT of the `result == JOB_TRANSITION_APPLIED` gate to run unconditionally, mirroring the RUSAGE/CKPT/PREASON blocks in the same function. apply_wal_entry's WAL_JOB_START case (persist.c:4105) and restore_job_start_state() (job.c:840, dropped the start_time parameter) likewise stop touching start_time; pidinfo there was already unconditional (this seam never calls apply_job_update_transition, so it never had the NOOP-gating hazard). persist_submit_job_start_fact()'s wire format is UNCHANGED (still writes start_time for on-disk layout continuity) -- only the READ side stopped consuming it. Pre-fix WAL_JOB_DISPATCH/WAL_JOB_START entries (_pad flag unset) fall back to the exact pre-fix behavior, an accepted one-time migration-boundary gap consistent with every other fix in this campaign. TDD: RED confirmed by temporarily reverting just the two defect lines (job.c's start_time stamp; persist.c's pidinfo gate) and running the full suite -- 5 genuine new failures (test_dc_success_stamps_start_time_synchronously, test_dc_success_appends_wal_job_dispatch_durably, test_wjs_apply_committed_entry_adopts_pidinfo_when_already_run_from_dispatch, test_wjs_restart_replay_recovers_dispatch_start_time_and_started_pidinfo, test_ace_job_start_sets_start_time) plus the pre-existing baseline (2 known bug-1561 test_config_repl.c failures + 1 concurrent bug-1696 test_job_array_replay_fence.c failure, both left untouched). GREEN after restoring the fix: rebuilt clean (0 compile errors) in an isolated dir (avoided the live HA soak's build-asan), full controller_unit_test run 3x direct (ASAN_OPTIONS=detect_leaks=0, matching this binary's own meson.build env) -- consistently \"1198 Tests 3 Failures 0 Ignored\", exactly the known baseline, 0 new failures. Live manual verification: single-node controller+scheduler+xcecd_virtual worker deployed via deploy-ha.sh (isolated port 19991, isolated workdir, no interference with the concurrent bug-1696 soak); 4 real xce_test jobsub->dispatch->finish cycles via the REPL all showed nonzero StartTime in `joblist` immediately (e.g. jobid 0-14-...-55018: StartTime 1787718938, matching SubmitTime). pidinfo's live collision path (jstart is a worker-mode-only REPL command, racy/awkward to trigger against a job a real worker is also concurrently finishing) was left to the deterministic unit tests above rather than a flaky live race -- covered exhaustively there at the exact apply_committed_entry()/on_job_start() call sites. Files: src/controller/job.c, src/controller/persist.c, src/include/persist.h, src/include/wal_types.h, src/controller/testsuite/{test_wal_job_dispatch_fact.c,test_wal_job_start_fact.c,test_dispatch_commit.c,test_apply_committed_entry.c,test_repl_apply.c (weak stub signature only)}.", @@ -23180,8 +10154,7 @@ "bug-1679-adjacent", "bug-1687-adjacent", "bug-1678-adjacent", - "critical", - "fixed" + "critical" ], "related_bugs": [ "bug-1679", @@ -23191,15 +10164,20 @@ "bug-1691", "bug-1692" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1697.md" }, { "id": "bug-1698", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T07:20:00Z", - "error_message": "User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster, build.dbg): after ./scripts/start-all.sh reported 'Controller cluster healthy (primary elected)', ctrl-1's log showed it perpetually retrying ha_connect_finish_peer() against both peer[1] (localhost:9902) and peer[2] (localhost:9903) every ~3s for 8+ minutes straight (120+ attempts, INFO-level 'connecting outbound to peer[N]' logged each time -- the SUCCESS-path log line per ha_connect_finish_peer's own code, not a failure), never once completing a working peer connection. Meanwhile ctrl-2/ctrl-3 were actively, healthily replicating with each other (real AppendEntries traffic, LSN steadily advancing) -- they formed a working 2-of-3 quorum excluding ctrl-1 entirely. `xceadmin scheduler record status` failed with 'no primary available: timed out contacting localhost:9901' (ctrl-1's own client port) even though `xce_test`'s REPL joblist worked fine against the real 2-node quorum.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster, build.dbg): after…\n\nUser-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster, build.dbg): after ./scripts/start-all.sh reported 'Controller cluster healthy (primary elected)', ctrl-1's log showed it perpetually retrying ha_connect_finish_peer() against both peer[1] (localhost:9902) and peer[2] (localhost:9903) every ~3s for 8+ minutes straight (120+ attempts, INFO-level 'connecting outbound to peer[N]' logged each time -- the SUCCESS-path log line per ha_connect_finish_peer's own code, not a failure), never once completing a working peer connection. Meanwhile ctrl-2/ctrl-3 were actively, healthily replicating with each other (real AppendEntries traffic, LSN steadily advancing) -- they formed a working 2-of-3 quorum excluding ctrl-1 entirely. `xceadmin scheduler record status` failed with 'no primary available: timed out contacting localhost:9901' (ctrl-1's own client port) even though `xce_test`'s REPL joblist worked fine against the real 2-node quorum.", "file": "src/controller/ha.c (ha_connect_finish_peer, reconnect_lane_fn, ha_connect_to_peer)", "root_cause": "PARTIALLY CONFIRMED, NOT FULLY ROOT-CAUSED. Ruled out as a test-harness/environment artifact: independently confirmed via a completely fresh, isolated 3-node deploy (~/tmp/xce-ha-fresh-repro, same build.dbg binaries, same run-ha-soak.sh topology) that the SAME symptom (impossible term jump, standby permanently failing cluster_init_phase2) reproduced ONCE due to a genuine same-session zombie-process port collision (see cerebrum.md's 2026-08-26 Do-Not-Repeat entry) -- but that specific mechanism does NOT apply to /tmp/xce-ha-deploy itself: confirmed via `ss -tlnp` that no live process holds ports 9901-9903 at investigation time, and no OTHER unrelated cluster shares those specific ports (9991-9993 was the colliding zombie's port range, unrelated). Direct on-disk evidence via `waldump` on the STOPPED cluster's data dirs CONFIRMS a real, non-artifactual divergence: ctrl-1's WAL has only 138 entries (lsn 1..138, all term=2, CRC chain intact, no gaps) while ctrl-2 has 2648 entries and ctrl-3 has 2641 entries (both showing term=3 activity in ha-term.bin, one of which -- ctrl-3 -- legitimately recorded voted_for_name='ctrl-2', a NORMAL vote-grant record, not corruption). This proves ctrl-1 was genuinely functioning (as primary, at term=2, given the job data the user's own joblist output showed) up through WAL entry 138, then something caused it to lose contact with both peers, a new election (term=3) happened among ctrl-2/ctrl-3 without it, and ctrl-1's own RECONNECT mechanism (ha_connect_finish_peer, dispatched via the SlowLane per raft-drive-async task 2.1's offload design) never once succeeded in re-establishing a working connection for the remainder of the run, despite logging its own SUCCESS path (endpoint_connect()+iomgr_monitor() both returning >=0) on every single attempt. This strongly suggests the bug is NOT in the initial connect() call itself (which appears to succeed at the socket/iomgr-registration level every time) but in whatever HIGHER-LEVEL protocol step is supposed to follow a successful low-level connect (completing a vote/ctrlhello handshake, registering the peer in standby_endpoints, or some per-peer state that should be cleared/retried differently after a genuine peer-loss) -- that step silently never completes or never gets attempted, so the SAME low-level connect()+iomgr_monitor() cycle repeats every ~3s forever without progress. Could NOT be reproduced live/interactively -- no debugger/ptrace access to the (already-stopped) real deployment's processes was available; this analysis is from static log/WAL evidence only, not a live capture of the actual failure in progress. FURTHER REPRO ATTEMPTS (2026-08-26, all failed to reproduce): (a) fresh 3-node deploy, kill+restart both standbys simultaneously while primary stays alive -- clean reconnect both times; (b) same, with continuous background jobsub load throughout -- clean reconnect (one transient ECONNREFUSED-then-retry-succeeds cycle observed, not a stuck state); (c) user-supplied exact recipe (submit a mixed array of jobs giving RUN/PEND/DONE states, then xceadmin scheduler record status) repeated 10x against a genuinely healthy full-component deploy (3x controller + worker + scheduler + xceadmin) -- succeeded cleanly every time; (d) 8x fresh-boot-only attempts (no artificial kills at all) immediately running reslist/workerlist/jobsub/joblist in sequence within the first ~5s of boot, matching the user's report that the real failure occurred within MINUTES of start-all.sh with no explicit kill -- all 8 clean, no crash, no elevated reconnect-retry counts. Total: 20+ reproduction attempts across 4 distinct scenario shapes, zero reproductions. This is a genuinely rare, timing-sensitive condition, not reliably triggerable by any tested scenario shape so far.", "fix": "NOT YET ATTEMPTED. Needs a live reproduction with process-level tracing (gdb/strace on a debug build) or targeted new logging in ha_connect_finish_peer/reconnect_lane_fn's post-connect handshake path to see exactly which step silently fails to progress after a genuine peer-loss (RDHUP) event, as distinct from initial cold-boot connection establishment (which DOES work correctly, per the fresh-repro soak's clean PASS with normal kill/restart/reconverge cycles). Next investigation step: reproduce a genuine mid-run peer-loss (not a driver-initiated kill, which already reconverges fine per the fresh-repro soak's PASS) -- e.g. a transient network blip or a peer restart landing in an unlucky window -- under gdb/strace, and trace ha_connect_finish_peer's full call chain including whatever runs after endpoint_connect()/iomgr_monitor() succeed (the ctrlhello/vote-channel handshake) to find where it silently stalls.", @@ -23214,25 +10192,27 @@ "partially-root-caused", "needs-live-repro", "needs-owner", - "unfixed", - "not-a-zombie-process-artifact", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "not-a-zombie-process-artifact" ], "related_bugs": [ "bug-1696", "bug-854", "bug-1487" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1698.md" }, { "id": "bug-1699", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-08-26T07:35:00Z", - "error_message": "ctrl-1 segfaulted (SIGSEGV, core dumped -- confirmed via bash's own accurate job-control message, not a misread) within ~1.5s of a genuinely fresh boot, immediately after processing its first client 'ctrlstatus' probe request, during a fresh 10-round --chaotic run-ha-soak.sh --chaotic invocation (build.dbg, no ASAN) launched to verify bug-1696. Driver reported 'FAIL: no primary elected within ~90s' as a consequence (ctrl-1 never came back; ctrl-2 was subsequently Killed by cleanup).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ctrl-1 segfaulted (SIGSEGV, core dumped -- confirmed via bash's own accurate job-control message…\n\nctrl-1 segfaulted (SIGSEGV, core dumped -- confirmed via bash's own accurate job-control message, not a misread) within ~1.5s of a genuinely fresh boot, immediately after processing its first client 'ctrlstatus' probe request, during a fresh 10-round --chaotic run-ha-soak.sh --chaotic invocation (build.dbg, no ASAN) launched to verify bug-1696. Driver reported 'FAIL: no primary elected within ~90s' as a consequence (ctrl-1 never came back; ctrl-2 was subsequently Killed by cleanup).", "file": "src/controller/ha.c or src/controller/cluster.c (exact function not identified -- crash occurred somewhere in the ctrlstatus request-handling path shortly after raft_cb_become_follower/first_boot=1)", "root_cause": "NOT ROOT-CAUSED -- occurred exactly ONCE and could not be reproduced despite two dedicated repro attempts: (1) re-running the identical build.dbg chaotic soak under run-ha-soak.sh's built-in GDB_WRAP=ctrl-1 debug seam caught only a benign SIGPIPE (gdb's own default stop-on-SIGPIPE behavior firing on a normal peer-disconnect-during-TLS-write event that the real code already ignores via signal(SIGPIPE, SIG_IGN) at netio.c:1250 -- confirmed this was a gdb artifact, not the real crash, since production code never sees this signal); (2) a manual gdb session with `handle SIGPIPE nostop noprint pass` set before `run` (correctly letting SIGPIPE pass through), driving 15+ rapid ctrlstatus probes against a freshly-booted ctrl-1 via xce_test's REPL (mirroring the exact probe pattern run-ha-soak.sh's own wait-primary/discover_primary functions use), did NOT reproduce any crash. (3) A full, from-scratch 10-round --chaotic soak (33 jobs, 10 kill/elect cycles) run under a fresh ASAN rebuild (src/build-asan-1696repro) completed with a clean PASS (committed_divergence=0 committed_job_loss=0) and no SIGSEGV/ASAN report at all. This strongly suggests either a genuinely rare, narrow-window race (possibly specific to cold dynamic-linker/first-request timing that a debug or ASAN build's different memory layout/timing perturbs away) or an environment-specific fluke from that one specific boot -- NOT a deterministic, reliably-triggerable defect. No core file was accessible for postmortem gdb analysis (systemd-coredump is configured as the core_pattern handler but coredumpctl is not installed/reachable in this sandbox). FURTHER NOTE (2026-08-26): did not recur across the same 20+ reproduction attempts made while investigating bug-1698 (fresh boots, standby kill/restart cycles, load, exact user recipe replay) -- still zero occurrences since the single original sighting. FURTHER NOTE (2026-08-26, cont.): USER-SUPPLIED core dumps from the real deployment (4 core.xce_controller.*.zst files today) confirm this is a real, recurring heap corruption (glibc malloc_printerr: \"unaligned tcache chunk detected\", detected inside endpoint_get()/calloc for a 312-byte endpoint_t -- the abort site is downstream of the actual corruption, not the cause). Confirmed the crash DID occur at least once in this session too (attempt 7 of an earlier 8-attempt loop) but was MISSED by an earlier flawed stderr-grep detection method (setsid nohup ... & disown fully detaches a process from the shells job table, so bash never prints a Aborted/Segmentation fault line for it -- a real methodology gap, not evidence of rarity). Redone with PROPER job-control-based exit-code detection: 15 attempts under a fresh ASAN build (build-asan-1696repro) = 0 reproductions; 15 more attempts with all 3 nodes wrapped in a properly-SIGPIPE-configured gdb session against build.dbg (where it previously reproduced) = 0 reproductions. Total 30 further attempts, zero catches. Hypothesis: ASANs different allocator/timing may suppress the race entirely (consistent with 0/15 under ASAN); build.dbg without a live gdb/gdb overhead may still need many more attempts or a specific concurrent-connection pattern not yet isolated. bug-1698s stuck-reconnect symptom and this heap corruption are plausibly the SAME root cause (both point at endpoint_t lifecycle -- corrupted/UAFd endpoint state could equally explain a connection that never completes ITS handshake). Next step if revisited: run MANY more attempts under build.dbg with proper exit-code detection (not ASAN, which appears to suppress it), or request the user run the real deployment under a debug build with core dumps enabled and share the NEXT core file for backtrace analysis the same way this session did.", "fix": "NOT ATTEMPTED -- cannot fix an unreproduced crash responsibly. If this recurs, capture a live gdb session (with SIGPIPE explicitly passed through, per this investigation's corrected GDB_WRAP-equivalent technique) or install/access a working coredumpctl-equivalent BEFORE the crashing process is cleaned up, to get a real backtrace with symbols. Also worth checking core_pattern/ulimit -c in the actual execution environment ahead of time so a future occurrence leaves a usable core file automatically.", @@ -23246,24 +10226,26 @@ "found-via-soak", "needs-live-repro", "needs-owner", - "unfixed", - "core-dump-inaccessible", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "core-dump-inaccessible" ], "related_bugs": [ "bug-1696", "bug-1698" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1699.md" }, { "id": "bug-1700", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T16:35:00Z", - "error_message": "User-reported: `xceadmin scheduler record status` (and by the same mechanism, `record start`/`record stop`) fails with `Error: controller returned status -10014` whenever the `--controller`/default address happens to point at a controller that is currently a STANDBY, not the primary. User's expectation: 'xceadmin/libxce should find the correct primary to talk to it' instead of surfacing the raw rejection.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "User-reported: `xceadmin scheduler record status` (and by the same mechanism, `record…\n\nUser-reported: `xceadmin scheduler record status` (and by the same mechanism, `record start`/`record stop`) fails with `Error: controller returned status -10014` whenever the `--controller`/default address happens to point at a controller that is currently a STANDBY, not the primary. User's expectation: 'xceadmin/libxce should find the correct primary to talk to it' instead of surfacing the raw rejection.", "file": "src/controller/cluster.c (myreq_handler, ~line 406-414)", "root_cause": "FULLY ROOT-CAUSED, gap is server-side only. `-10014` is `E_XCE_STANDBY` (errnox.h, XCE_ERRNO_BASE+13) -- ha_dispatch_gate() (ha.c:4745-4775) correctly rejects OPCode_schedAdmin on a CTRL_STANDBY node (it is absent from that role's opcode allow-list, by design -- schedAdmin is a primary-only forward-to-scheduler operation). The gate's generic caller in myreq_handler (cluster.c:406-414) sends the rejection reply with `reply.status = rc; reply.data_which = RequestReply_data_nodata;` and NEVER calls fill_redirect_reply() (join.c:27-31, the exact same one-line helper OPCode_join's handler already uses successfully) -- so redirectHost/redirectPort are left at their zero-value defaults on EVERY dispatch-gate rejection, for EVERY opcode, not just schedAdmin (OPCode_upgradeConfig has the identical, already-documented gap -- see test_e2e_live_ha_push.py's module docstring, confirmed 2026-07-30-era). Critically, this is NOT a client-side deficiency: xceadmin's redirect-following client logic is ALREADY fully implemented, generic/opcode-agnostic, and well-tested (`xceadmin.discovery.follow_redirects()`, with a full test suite covering 1-hop/3-hop/too-many-hops/timeout cases; `xceadmin.scheduler._send_admin_request()` already calls it for every record_start/stop/status call). `_redirect_target()` (discovery.py:80-89) correctly requires BOTH redirectHost and redirectPort to be non-empty/non-zero to treat a reply as a redirect -- given the gate-rejection reply supplies neither, the client correctly concludes there is nothing to follow and surfaces the bare rejection status as-is. The client is not 'giving up' -- it is working exactly per its documented contract given the reply it receives; the whole path already works end-to-end against a properly-behaving server (test_cli_scheduler_record_start_follows_redirect_to_primary already exercises the client side against a mock that DOES send a redirect).", "fix": "PARTIALLY FIXED (2026-08-26), live verification FAILS -- see bug-1701 for the blocking root cause, do not close this bug until that is resolved. IMPLEMENTED, unit-tested, and CORRECT exactly as scoped: myreq_handler's generic ha_dispatch_gate rejection path (cluster.c ~410-433) now calls fill_redirect_reply(&reply, &c->ha) when, and only when, rc == -XCE_ERR_STANDBY -- confirmed via ha_dispatch_gate's full body (ha.c:4573-4781) that -XCE_ERR_STANDBY (CTRL_STANDBY's default-case rejection, ha.c:4774) and -XCE_ERR_NOT_READY (CTRL_INIT/CTRL_ELECTING, and CTRL_PRIMARY's config-serve-fence / upgrade-materialization-fence / no-write-quorum branches) are the ONLY two nonzero codes the gate can return -- no third code path exists. -XCE_ERR_NOT_READY is deliberately excluded (no settled primary to redirect to in those states). RED/GREEN proof: added test_schedadmin_standby_with_known_primary_redirects (FAILED pre-fix with 'Expected 10.0.1.7 Was ''', PASSED post-fix) and test_schedadmin_standby_no_known_primary_no_redirect (regression companion, passes in both states) to test_sched_admin_dispatch.c, driving the real myreq_handler via myreq_handler_for_test with OPCode_schedAdmin + CTRL_STANDBY role. Full controller_unit_test: 1200 Tests, 2 Failures both times (pre- and post-fix) -- exactly the pre-existing bug-1561 overlayfs pair (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), zero new failures. Encode-path NULL-safety CONFIRMED (not assumed): ha_state_t.primary_host is a fixed char[XCE_MAXNAME_LEN] buffer embedded in ha_state_t (ha.h:161), never a pointer, so its address is never NULL; encode_RequestReply (controller.capnp.c:13777-13786) gates strictly on `s->redirect_host != NULL` and falls back to an explicit empty string/0 otherwise; decode_RequestReply (controller.capnp.c:13841) strdup()s the wire text unconditionally (so a client-side redirect_host is always a real, non-NULL, possibly-empty C string, never NULL) -- an unknown primary therefore safely round-trips as redirectHost='' / redirectPort=0, which xceadmin's discovery.py _redirect_target() (host and port both required truthy) already correctly treats as 'no redirect, use this reply as-is'. No NULL-pointer/crash risk anywhere in this path, confirmed by code inspection of every hop, not assumed. LIVE VERIFICATION FAILS, however: deployed a fresh 3-node cluster twice (deploy-ha.sh, controller+worker+scheduler via scripts/start-all.sh) -- once on a binary built from this fix (git stash of cluster.c to produce a clean before-binary, git stash pop to restore it for the after-binary; xce_test ctrlstatus confirmed ctrl-1 elected primary, ctrl-2/ctrl-3 standby, in both deploys). `xceadmin scheduler record status --controller localhost:9992` (a standby) against BOTH the pre-fix and the post-fix binary produced the IDENTICAL result: 'Error: controller returned status -10014', exit 1, no 'redirected: ...' line on stderr either time -- i.e. this fix has ZERO observable effect on the user-reported symptom in a real deployment. Root cause of the no-op: ha_state_t.primary_host/primary_port (ha.h:161-162) are NEVER WRITTEN by any production code path on a standby -- confirmed by grepping every .c file in src/ for an assignment site (none exists outside test fixtures, which set it manually) and by ctrlstatus.c's own pre-existing doc comment (line 38/95: 'the vestigial, never-written ha->primary_host') -- so fill_redirect_reply() always sees an empty string / 0 on a real standby, in every state, permanently, not just during the 'hasn't learned yet' transient window this bug's own original root-cause writeup anticipated and asked to be verified. This is independently corroborated by the ALREADY-CLOSED bug-854 (this bug's own related_bugs entry), whose root_cause section states the identical fact ('primary_host is vestigial/never-written') and whose fix explicitly says 'Deliberately NOT adding server-side primaryName source -- it is only a latency optimization; the client rotation fix alone resolves the hang' for the DIFFERENT ctrlstatus/client-rotation code path. New implication surfaced by this session (see bug-1701): since fill_redirect_reply() is fed by the same permanently-empty field for EVERY caller, OPCode_join's pre-existing, supposedly-successful use of the exact same helper (join.c:277) is, by the same logic, equally non-functional against a live standby in production today -- this was not previously flagged anywhere in the bug log. Per this bug's own instructions, NOT papering over this with a workaround guessed in this session -- stopping for review. The gate-rejection code-path change above is correct, tested, and worth keeping (it is necessary infrastructure and fixes the OPCode_upgradeConfig-adjacent gap's code shape identically to join.c's precedent), but closing the user-visible bug additionally requires a second, larger, currently-unscoped change: something must actually populate primary_host/primary_port on a standby from live heartbeat/AppendEntries/ctrlhello traffic (touches ha.c, which has other work in flight this session was told not to touch), OR xceadmin needs a bug-854-style client-side master-ring rotation fallback for schedAdmin/upgradeConfig (mirroring xcecli_ctrlstatus_primary's fix) so an empty-redirect standby reply drives a retry against the next configured master instead of surfacing the bare rejection. See bug-1701 for the dedicated tracking of the underlying data-gap finding.\n\nUPDATE (2026-08-26): bug-1701 is now FIXED (local peers_cfg lookup in handle_ctrlhello, src/controller/config_repl.c:2170-2246 -- see bug-1701's own entry for the full fix writeup) and LIVE-VERIFIED end to end, closing the gap that made this bug's fix a live no-op. Re-ran the exact failing scenario from this bug's own live-verification section: fresh 3-node cluster, `xceadmin scheduler record status --controller ` now prints `redirected: -> ` on stderr and the real recording status on stdout, exit 0 -- confirmed against both standbys in the cluster and again after a re-election changed which node was primary. This bug's own code change (myreq_handler's fill_redirect_reply() call on -XCE_ERR_STANDBY, cluster.c ~410-433) required NO further modification -- it was correct all along; it only needed a real, populated ha->primary_host/primary_port to redirect to, which bug-1701 now supplies. Status: FIXED, LIVE-VERIFIED.\n\nOriginal PENDING fix text (superseded by the above, kept for history): PENDING. Minimal, single-call-site fix: in myreq_handler's generic ha_dispatch_gate rejection path (cluster.c:406-414), when the rejection reason is specifically -XCE_ERR_STANDBY (i.e. this node knows or may know who the primary is) call fill_redirect_reply(&reply, &c->ha) before xce_reply(), exactly mirroring OPCode_join's existing handle_worker_join()-adjacent usage (join.c:277). This fixes OPCode_schedAdmin AND the already-documented OPCode_upgradeConfig gap simultaneously, with ZERO xceadmin/client-side changes needed -- the existing follow_redirects() mechanism will start working immediately once the server populates the fields. Must verify: (a) ha_state_t.primary_host/primary_port are populated/valid on a CTRL_STANDBY node that has learned of a live primary (the common case) vs. a node that genuinely does not know any primary yet (mid-election, or has never heard from one) -- fill_redirect_reply() unconditionally aliases these fields with no NULL/empty guard, so confirm what a standby's primary_host looks like before any primary is ever discovered (likely NULL/empty, in which case the reply's redirectHost would encode as empty/absent and the client's existing `if host and port` guard already handles this correctly as 'no redirect' -- but verify, don't assume); (b) add a regression test proving a schedAdmin request to a standby-with-known-primary now receives a populated redirect reply (mirroring test_join_redirect.c's existing pattern, `test_redirect_reply_carries_primary_address`) and a live xceadmin CLI test (extending test_e2e_live_ha_push.py's pattern) proving `xceadmin scheduler record status` against ANY of the 3 nodes now transparently reaches the primary and succeeds.", @@ -23279,7 +10261,6 @@ "root-caused", "fix-scoped", "low-risk", - "fixed", "code-fix-landed", "live-verified", "unblocked-by-bug-1701", @@ -23290,15 +10271,20 @@ "bug-854", "bug-1701" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1700.md" }, { "id": "bug-1701", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T00:00:00Z", - "error_message": "Discovered while implementing/live-verifying bug-1700's fix (myreq_handler's generic dispatch-gate rejection now calls fill_redirect_reply() on -XCE_ERR_STANDBY, cluster.c ~410-433): the fix has ZERO effect in a real deployment. `xceadmin scheduler record status --controller ` against a live 3-node cluster still fails with 'Error: controller returned status -10014' identically before and after the fix, with no 'redirected: ...' hop ever printed.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Discovered while implementing/live-verifying bug-1700's fix (myreq_handler's generic dispatch-gate…\n\nDiscovered while implementing/live-verifying bug-1700's fix (myreq_handler's generic dispatch-gate rejection now calls fill_redirect_reply() on -XCE_ERR_STANDBY, cluster.c ~410-433): the fix has ZERO effect in a real deployment. `xceadmin scheduler record status --controller ` against a live 3-node cluster still fails with 'Error: controller returned status -10014' identically before and after the fix, with no 'redirected: ...' hop ever printed.", "file": "src/controller/ha.h:161-162 (ha_state_t.primary_host/primary_port), src/controller/join.c:27-31 (fill_redirect_reply), src/controller/ctrlstatus.c:38 (pre-existing doc comment already naming this)", "root_cause": "ha_state_t.primary_host/primary_port are NEVER WRITTEN by any production code path on a standby -- confirmed by grepping every assignment site across the entire src/ tree (`grep -rn primary_host src/ --include=*.c --include=*.h`): the only writers are test fixtures (test_join_redirect.c, test_p1_integration.c, test_p3_hardening.c, test_ha_redirect.c, test_ha_role_lifecycle.c, this bug's own new test_sched_admin_dispatch.c tests) manually poking the field for isolated unit tests; no heartbeat/AppendEntries/ctrlhello handler in ha.c ever copies the leader's identity into it on the receiving standby. ctrlstatus.c's own field-sourcing doc comment (lines 37-38, 93-101) already names this explicitly ('the vestigial, never-written ha->primary_host') and documents that ctrlstatus_fill() deliberately derives primaryName from role + voted_for_name instead, NEVER from primary_host, for exactly this reason -- so the gap was known and worked around for ctrlstatus, but not for the redirect-reply mechanism (fill_redirect_reply/redirect_host/redirect_port), which has no equivalent workaround and is the ONLY consumer of this permanently-empty field. bug-854 (2026-06-21, CLOSED/FIXED) independently documented the identical fact while root-causing a related but distinct bug (positional client-side primary discovery for ctrlstatus) and explicitly chose NOT to add a server-side primaryName/primary_host source, fixing that bug entirely client-side (xcecli_ctrlstatus_primary master-ring rotation) instead -- a reasonable call for THAT bug, but it leaves fill_redirect_reply() as a piece of server-side machinery that looks functional (it has a real caller, a real unit test proving its own field-copying logic, and a real capnp wire encode/decode path) yet is fed a field that is permanently empty in every real deployment. Direct consequence: OPCode_join's existing use of fill_redirect_reply() (join.c:277, the very precedent bug-1700's fix was told to mirror) is, by the same reasoning, ALSO non-functional against a live standby in production today -- a worker/scheduler/controller join redirect to the primary silently never happens; nothing in the existing test suite (test_join_redirect.c) catches this because those tests call fill_redirect_reply() directly with a manually-populated ha_state_t, never through a live standby that only knows what production heartbeat/election code actually wrote into it.", "fix": "FIXED (2026-08-26) via option (a): a pure LOCAL lookup in handle_ctrlhello() (src/controller/config_repl.c:2170-2246, inside the existing primary_ctrl_fd==-1 idempotency-guarded block, ~2185-2233), NOT a new wire-protocol field -- every node already statically knows every candidate's host:port from boot (c->myconf->peers_cfg, parsed once from --peers). At the exact moment a standby first establishes primary_ctrl_fd (the same instant it 'discovers' its primary), it now matches req->from->name (populated for EVERY incoming request at src/lib/request.c:379, `req->from = ep->peer;` -- the same name= identity used in --peers name=host:port) against c->myconf->peers_cfg.candidates[] by name (loop pattern mirrors the existing voted_for_name-by-index lookup in ha_init_raft, ha.c:7822-7829) and, on a match, copies that candidate's hostname/port into ha->primary_host/primary_port. No match (real config mismatch) or a NULL req->from/name leaves primary_host/primary_port at their previous value (never wiped) and logs a WARN identifying the unmatched sender name. Locking: ha.h's `lock` field doc explicitly says 'Protects primary_host/port + voted_for_term/name'; the existing writers of voted_for_name (the other field under the same lock, ha.c:2304-2332 and ha.c:5559-5593) take c->ha.lock around the write, so the new write here does too (pthread_mutex_lock/unlock around the strncpy+port assignment only -- the name-matching loop itself runs outside the lock, matching the narrow critical-section style of the precedent). Types verified by direct read, not assumed: ha_state_t.primary_host is `char primary_host[XCE_MAXNAME_LEN]` (ha.h:161, XCE_MAXNAME_LEN=256, config.h:24) and primary_port is `uint32_t` (ha.h:162) -- NOT matching the task brief's assumption that primary_port's type already lined up with xce_candidate_t.port (`int`, cluster.h:22); the int->uint32_t assignment is a safe implicit widening for any real port value (0-65535), cast added for clarity, no risky truncation. No clear-on-primary-loss needed: ha_on_primary_ctrl_fd_rdhup (ha.c, read-only verified, not modified) resets primary_ctrl_fd back to -1 on genuine primary loss, so the next real ctrlhello (from the same or a new primary) re-satisfies the same idempotency guard and naturally re-runs this lookup -- confirmed by reading ha.c:8594-8680, no explicit clear required. RED/GREEN proof: added test_handle_ctrlhello_populates_primary_host_from_known_peer (FAILED pre-fix: \"Expected '10.0.1.7' Was ''\"; PASSED post-fix) and test_handle_ctrlhello_unknown_peer_leaves_primary_host_unchanged (edge case: unmatched sender name -- no crash, primary_host/primary_port stay at fresh-node empty/zero, primary_ctrl_fd bookkeeping still proceeds) to src/controller/testsuite/test_ha_connection_reuse.c (same file as the pre-existing test_handle_ctrlhello_sets_primary_ctrl_fd, which continues to pass unchanged, proving the idempotency behavior for primary_ctrl_fd itself is untouched). Full controller_unit_test: 1202 Tests, 2 Failures both pre- and post-fix -- exactly the pre-existing bug-1561 overlayfs pair (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), zero new failures. LIVE END-TO-END VERIFIED -- the exact scenario bug-1700 failed on: deployed two independent fresh 3-node clusters (deploy-ha.sh) on git-stash-isolated before/after controller binaries (stashed only config_repl.c + its test file, leaving other in-flight ha.c/job.c/persist.c work untouched). BEFORE (pre-fix binary): `xceadmin scheduler record status --controller localhost:29992` (a standby) -> `Error: controller returned status -10014`, exit 1, no redirect line -- reproduces bug-1700's original live-verification failure exactly. AFTER (post-fix binary, fresh deploy + also re-verified after clearing controller data to force a different node to win election): `xceadmin scheduler record status --controller ` -> stderr prints `redirected: localhost:19991 -> localhost:19992` (confirmed via separate stdout/stderr capture: the redirect line is genuinely on stderr, the status body on stdout, matching xceadmin.discovery.follow_redirects()'s documented behavior), stdout prints the real recording status (`recording: no`, etc.), exit 0 -- verified against BOTH standbys in the cluster and re-verified after a re-election flipped which node was primary (ctrl-2 elected instead of ctrl-1), proving the fix is dynamic/general, not coincidentally tied to one node. This unblocks bug-1700's fix, which was otherwise correct and complete but fed by this permanently-empty field -- see bug-1700's own entry for its updated live-verified confirmation. OPCode_join's pre-existing fill_redirect_reply() usage (join.c:277) is also now unblocked by the same fix (not separately re-verified live in this session, but shares the identical previously-broken data source).", @@ -23314,7 +10300,6 @@ "fill_redirect_reply", "found-live", "root-caused", - "fixed", "live-verified", "cross-cutting" ], @@ -23322,16 +10307,20 @@ "bug-1700", "bug-854" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1701.md" }, { "id": "bug-1702", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test jobrun -m host strncpy into 256B stack buffer unbounded (src/harness/test.c)", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_jobrun's `-m :` parsing copies the host-name substring into a fixed 256-byte stack buffer with an unbounded strncpy (length = bytes before ':' in the user-supplied argument, never clamped to sizeof(buf)). A -m argument with a hostname longer than 256 bytes before the ':' overflows the stack buffer.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_jobrun's `-m :` parsing copies the host-name substring into a fixed 256-byte…\n\ncmd_jobrun's `-m :` parsing copies the host-name substring into a fixed 256-byte stack buffer with an unbounded strncpy (length = bytes before ':' in the user-supplied argument, never clamped to sizeof(buf)). A -m argument with a hostname longer than 256 bytes before the ':' overflows the stack buffer.", "file": "src/controller/testsuite/test.c:2130-2136 (cmd_jobrun)", "root_cause": "`strncpy(buf, optarg, (int)(p - optarg))` uses the attacker/user-controlled substring length directly with no `MIN(..., sizeof(buf)-1)` clamp, unlike sibling code elsewhere in the file that does bound similar copies.", "fix": "NOT YET FIXED. Recommended: clamp the copy length to sizeof(buf)-1 (or reject a host-name segment longer than the buffer with a usage error) before strncpy.", @@ -23342,24 +10331,25 @@ "cmd_jobrun", "buffer-overflow", "memory-safety", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [ "bug-1703" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1702.md" }, { "id": "bug-1703", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test jobsub -n min strncpy into 64B stack buffer unbounded (src/harness/test.c)", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_jobsub's `-n ,` parsing copies the min-value substring into a fixed 64-byte stack buffer with an unbounded strncpy (length = bytes before ',' in the user-supplied argument, never clamped to sizeof(buf)). A -n argument with 64+ characters before the comma overflows the stack buffer.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_jobsub's `-n ,` parsing copies the min-value substring into a fixed 64-byte stack…\n\ncmd_jobsub's `-n ,` parsing copies the min-value substring into a fixed 64-byte stack buffer with an unbounded strncpy (length = bytes before ',' in the user-supplied argument, never clamped to sizeof(buf)). A -n argument with 64+ characters before the comma overflows the stack buffer.", "file": "src/controller/testsuite/test.c:2246-2254 (cmd_jobsub)", "root_cause": "Same pattern as bug-1702: `strncpy(buf, optarg, (int)(p - optarg))` with no bound check against sizeof(buf).", "fix": "NOT YET FIXED. Recommended: clamp the copy length to sizeof(buf)-1 (or reject) before strncpy, same fix shape as bug-1702.", @@ -23370,23 +10360,25 @@ "cmd_jobsub", "buffer-overflow", "memory-safety", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [ "bug-1702" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1703.md" }, { "id": "bug-1704", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_qctrl declares `int action;` with no initializer and only assigns it inside the -a/-i/-o/-c getopt cases; there is no post-loop check that one of them was given before `queuectrl.action = action` is sent over the wire. Running `qctrl -q normal` with no action flag (syntactically valid per USAGE_QCTRL, which lists all four as optional) transmits indeterminate stack memory as the queue action.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_qctrl declares `int action;` with no initializer and only assigns it inside the -a/-i/-o/-c…\n\ncmd_qctrl declares `int action;` with no initializer and only assigns it inside the -a/-i/-o/-c getopt cases; there is no post-loop check that one of them was given before `queuectrl.action = action` is sent over the wire. Running `qctrl -q normal` with no action flag (syntactically valid per USAGE_QCTRL, which lists all four as optional) transmits indeterminate stack memory as the queue action.", "file": "src/controller/testsuite/test.c:2663 (declaration), :2668-2688 (getopt loop), :2695 (queuectrl.action = action)", "root_cause": "Missing initializer + missing post-loop validation, unlike the very next sibling command cmd_hostctrl which initializes action=HOST_DO_NONE and explicitly rejects that sentinel.", "fix": "FIXED (2026-09-09, xce-test-cli-ux task 3.1, flagspec migration). cmd_qctrl now initializes action = QUEUE_DO_NONE (xce/types.h pre-existing sentinel, same enum family as HOST_DO_NONE) before the getopt_long() loop, and rejects the invocation post-loop if action is still QUEUE_DO_NONE after parsing -- mirroring cmd_hostctrl's exact HOST_DO_NONE pattern. Landed as part of migrating cmd_qctrl (and 16 other flag-accepting commands) from getopt() to the shared flagspec_t + getopt_long() mechanism (flagspec.c/.h, tasks 1.1/1.2). Regression-locked by test_qctrl_missing_action_flag_rejected and test_qctrl_flag_forms_short_and_long_both_accepted in src/controller/testsuite/test_test_cli.c.", @@ -23397,20 +10389,23 @@ "cmd_qctrl", "uninitialized-variable", "undefined-behavior", - "cli-ux-audit", - "fixed" + "cli-ux-audit" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1704.md" }, { "id": "bug-1705", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test jobsub -x exdata overwritten by build_job_exdata and leaked", + "component": "controller", + "severity": "leak", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_jobsub's `-x ` flag is documented (USAGE_JOBSUB) as overriding the built-in PATH-only exdata JSON, but build_job_exdata()/build_job_envs() unconditionally overwrite submit.start_info->exdata with the built-in PATH JSON regardless of whether -x already set a value -- the exact opposite of the documented behavior. The original -x value is also leaked (never freed before being replaced).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_jobsub's `-x ` flag is documented (USAGE_JOBSUB) as overriding the built-in PATH-only…\n\ncmd_jobsub's `-x ` flag is documented (USAGE_JOBSUB) as overriding the built-in PATH-only exdata JSON, but build_job_exdata()/build_job_envs() unconditionally overwrite submit.start_info->exdata with the built-in PATH JSON regardless of whether -x already set a value -- the exact opposite of the documented behavior. The original -x value is also leaked (never freed before being replaced).", "file": "src/controller/testsuite/test.c:2243-2245 (-x parsing), :2181-2195 (build_job_envs unconditional overwrite), :2336 (unconditional call site), USAGE_JOBSUB at :484-485", "root_cause": "build_job_exdata()/build_job_envs() have no guard for an already-non-NULL exdata field before overwriting it.", "fix": "NOT YET FIXED. Recommended: skip build_job_exdata() when submit.start_info->exdata is already non-NULL (from -x), and free the old pointer before any legitimate reassignment.", @@ -23422,23 +10417,23 @@ "dead-flag", "silent-data-loss", "memory-leak", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1705.md" }, { "id": "bug-1706", "status": "duplicate", "status_updated": "2026-09-24", "status_note": "same defect as bug-1574 (set_resval() has no string case)", - "duplicate_of": "bug-1574", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_resupdate's own documented example (`resupdate -n mylabel -v foo -t string`) and any string-typed resource (e.g. `type`, `model`) corrupts the submitted resource value: set_resval() has no `case Value_u_svalue:` in its switch, so for a string-typed resource, vals[idx].name is left as uninitialized stack memory and the wire type stays the calloc'd default (ivalue32) instead of the string the user typed.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_resupdate's own documented example (`resupdate -n mylabel -v foo -t string`) and any…\n\ncmd_resupdate's own documented example (`resupdate -n mylabel -v foo -t string`) and any string-typed resource (e.g. `type`, `model`) corrupts the submitted resource value: set_resval() has no `case Value_u_svalue:` in its switch, so for a string-typed resource, vals[idx].name is left as uninitialized stack memory and the wire type stays the calloc'd default (ivalue32) instead of the string the user typed.", "file": "src/controller/testsuite/test.c:3323-3364 (set_resval, missing Value_u_svalue case), :3483 (vals[] never zeroed), :363-364/375 (USAGE_RESUPDATE's own -t string example)", "root_cause": "set_resval()'s switch only handles fvalue/ivalue32/ivalue64/bvalue; resval_resolve_type() can resolve to Value_u_svalue (type/model, or any name with -t string) but nothing handles that case. A comment at test.c:3365-3372 acknowledges the gap as intentionally deferred by a prior task.", "fix": "NOT YET FIXED. Recommended: add a Value_u_svalue case to set_resval() that sets name/val->u.svalue correctly, and zero-init vals[] in cmd_resupdate as defense in depth.", @@ -23449,25 +10444,27 @@ "cmd_resupdate", "uninitialized-variable", "silent-data-corruption", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [ "bug-1707", "bug-1712" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1706.md", + "duplicate_of": "bug-1574" }, { "id": "bug-1707", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test resupdate trailing -n without -v silently dropped", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_resupdate: `resupdate -n slots -v 4 -n mem` silently drops the trailing `mem` resource with zero diagnostic (the final pending -n/-v pair is only flushed if a value was seen), yet the command still prints \"resource update ok!\" as if everything the user typed succeeded. A mid-list missing -v does print a warning; only the trailing one is silent.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_resupdate: `resupdate -n slots -v 4 -n mem` silently drops the trailing `mem` resource with…\n\ncmd_resupdate: `resupdate -n slots -v 4 -n mem` silently drops the trailing `mem` resource with zero diagnostic (the final pending -n/-v pair is only flushed if a value was seen), yet the command still prints \"resource update ok!\" as if everything the user typed succeeded. A mid-list missing -v does print a warning; only the trailing one is silent.", "file": "src/controller/testsuite/test.c:3551-3561 (trailing-pair flush, comment admits the drop), :3502-3507 (mid-list warning that trailing case lacks), :3567 (unconditional success message)", "root_cause": "Trailing pending -n/-v pair flush logic only runs when resval_str != NULL, with no fallback warning path like the mid-list case has.", "fix": "NOT YET FIXED. Recommended: apply the same \"no value specified, default will be used\" warning (or reject outright) to the trailing pair instead of a silent drop.", @@ -23477,24 +10474,25 @@ "xce_test", "cmd_resupdate", "silent-data-loss", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [ "bug-1706" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1707.md" }, { "id": "bug-1708", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test jobkill -s non-numeric -> signal 0, still reports success", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_jobkill's `-s ` is parsed via unchecked atoi(); a non-numeric value (e.g. a plausible typo of the symbolic name `SIGKILL` instead of `9`) silently becomes signo=0 (POSIX null signal, delivers nothing), yet the command still prints \"Job signaled\" as if it succeeded.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_jobkill's `-s ` is parsed via unchecked atoi(); a non-numeric value (e.g\n\ncmd_jobkill's `-s ` is parsed via unchecked atoi(); a non-numeric value (e.g. a plausible typo of the symbolic name `SIGKILL` instead of `9`) silently becomes signo=0 (POSIX null signal, delivers nothing), yet the command still prints \"Job signaled\" as if it succeeded.", "file": "src/controller/testsuite/test.c:2572-2574 (signo = atoi(optarg)), :2601 (unconditional success message)", "root_cause": "No validation that -s's argument is numeric/nonzero before using it and reporting success.", "fix": "NOT YET FIXED. Recommended: reject non-numeric/unparseable -s values with a clear error instead of silently defaulting to signal 0.", @@ -23504,22 +10502,23 @@ "xce_test", "cmd_jobkill", "silent-failure", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1708.md" }, { "id": "bug-1709", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test jobinfo returns 0 on malformed/missing jobid", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_jobinfo always returns 0 (success) even when its -j jobid is malformed (parse_jobid's return code is discarded), when no -j is given at all, or when xcecli_jobinfo fails for every -j given -- breaking the tool's own scripted-mode exit-code-verdict contract for this command specifically.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_jobinfo always returns 0 (success) even when its -j jobid is malformed (parse_jobid's return…\n\ncmd_jobinfo always returns 0 (success) even when its -j jobid is malformed (parse_jobid's return code is discarded), when no -j is given at all, or when xcecli_jobinfo fails for every -j given -- breaking the tool's own scripted-mode exit-code-verdict contract for this command specifically.", "file": "src/controller/testsuite/test.c:2029-2058 (cmd_jobinfo)", "root_cause": "No error/found flag tracked across the -j loop; the function unconditionally `return 0`s regardless of whether any valid lookup succeeded.", "fix": "NOT YET FIXED. Recommended: track an error/found flag across the loop and return nonzero (with a usage message) when no valid -j was given or every lookup failed.", @@ -23530,22 +10529,23 @@ "cmd_jobinfo", "silent-failure", "exit-code", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1709.md" }, { "id": "bug-1710", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test hello -c <=0 reports vacuous success", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_hello with -c 0, a negative count, or a non-numeric value (all yield count<=0 from unchecked atoi()) sends zero hellos (the while(count>0) loop body never runs) but still returns the still-zero-initialized rc, reporting success even though nothing was sent.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_hello with -c 0, a negative count, or a non-numeric value (all yield count<=0 from unchecked…\n\ncmd_hello with -c 0, a negative count, or a non-numeric value (all yield count<=0 from unchecked atoi()) sends zero hellos (the while(count>0) loop body never runs) but still returns the still-zero-initialized rc, reporting success even though nothing was sent.", "file": "src/controller/testsuite/test.c:1341 (rc=0 init), :1351 (count=atoi(optarg), unchecked), :1367 (while count>0 loop), :1373 (return rc)", "root_cause": "No validation that count is >=1 before entering the loop; the initial success value is never overwritten when the loop body never executes.", "fix": "NOT YET FIXED. Recommended: reject count<=0 with an explicit error instead of falling through to a silent vacuous success.", @@ -23555,22 +10555,23 @@ "xce_test", "cmd_hello", "silent-failure", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1710.md" }, { "id": "bug-1711", "status": "fixed", "status_updated": "2026-09-24", "status_note": "a8154acb", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "xce_test: several command-specific typos and output-formatting inconsistencies found during a /cli-ux command-by-command audit: worker_status_str returns the misspelled \"UNKOWN\" for an unrecognized worker status bit (visible in every workerlist row hitting that path); cmd_help prints the malformed \"No commands matchs `%s'\" on an unknown help topic; cmd_runtasks's default launch_type for a single task is misspelled \"sequentail\" (display/data-only, not behavior-altering -- worker only special-cases \"openmpi\"/\"mpich\"); cmd_reslist right-justifies its Name/Description text columns (inconsistent with cmd_workerlist's left-justified equivalent columns) and prints the Flags column as unlabeled %x hex (ambiguous vs decimal, unlike this file's own \"0x%x\"-prefixed state_raw precedent); cmd_joblist prints SubmitTime/StartTime/FinishTime as raw Unix epoch seconds instead of using this file's own ctrlstatus_format_time_ms-style formatting, and labels a hidden mintasks,maxtasks pair as a single \"NTasks\" column; cmd_joblist's job_state_str has no raw-state fallback for an ambiguous RUN+EXIT state the way print_jobinfo already guards against two functions away; getconf's User Groups/Host Groups/Configs sections always print their header followed by nothing (print_grouplist/print_configlist are no-op stubs), indistinguishable from a genuinely empty cluster.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_test: several command-specific typos and output-formatting inconsistencies found during a…\n\nxce_test: several command-specific typos and output-formatting inconsistencies found during a /cli-ux command-by-command audit: worker_status_str returns the misspelled \"UNKOWN\" for an unrecognized worker status bit (visible in every workerlist row hitting that path); cmd_help prints the malformed \"No commands matchs `%s'\" on an unknown help topic; cmd_runtasks's default launch_type for a single task is misspelled \"sequentail\" (display/data-only, not behavior-altering -- worker only special-cases \"openmpi\"/\"mpich\"); cmd_reslist right-justifies its Name/Description text columns (inconsistent with cmd_workerlist's left-justified equivalent columns) and prints the Flags column as unlabeled %x hex (ambiguous vs decimal, unlike this file's own \"0x%x\"-prefixed state_raw precedent); cmd_joblist prints SubmitTime/StartTime/FinishTime as raw Unix epoch seconds instead of using this file's own ctrlstatus_format_time_ms-style formatting, and labels a hidden mintasks,maxtasks pair as a single \"NTasks\" column; cmd_joblist's job_state_str has no raw-state fallback for an ambiguous RUN+EXIT state the way print_jobinfo already guards against two functions away; getconf's User Groups/Host Groups/Configs sections always print their header followed by nothing (print_grouplist/print_configlist are no-op stubs), indistinguishable from a genuinely empty cluster.", "file": "src/controller/testsuite/test.c:1435 (UNKOWN), :1069 (matchs), :2939 (sequentail), :1401/1404 (reslist formatting), :1454/1473 (workerlist formatting for contrast), :1551-1572 (joblist raw epoch + NTasks label), :1486 (job_state_str), :1972-1992 (print_jobinfo's raw-state fallback for contrast), :3632/3634 (print_grouplist/print_configlist stubs), :3659-3666 (getconf headers)", "root_cause": "Accumulated small inconsistencies/typos/unimplemented stubs found via a systematic per-command audit; none individually severe, several visible in routine tool usage.", "fix": "NOT YET FIXED. Recommended: fix the three typos directly; left-justify reslist's text columns and label/prefix its Flags column as hex to match workerlist/state_raw conventions; format joblist's three time columns like ctrlstatus does and relabel/split the NTasks column; either implement print_grouplist/print_configlist from the already-fetched data or print \"(not implemented)\" instead of silent nothing.", @@ -23580,22 +10581,25 @@ "xce_test", "typo", "output-formatting", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "a8154acb" + ], + "record": ".wolf/bugs/bug-1711.md" }, { "id": "bug-1712", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test resupdate duplicate -v gives misleading message", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T09:32:18Z", - "error_message": "cmd_resupdate: a duplicate -v for the same pending resource (`resupdate -n slots -v 4 -v 8`) prints the misleading message \"ignore res value `%s' as no res name found\" even though a resource name WAS found -- the real problem is a value is already pending for it, not a missing name.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_resupdate: a duplicate -v for the same pending resource (`resupdate -n slots -v 4 -v 8`)…\n\ncmd_resupdate: a duplicate -v for the same pending resource (`resupdate -n slots -v 4 -v 8`) prints the misleading message \"ignore res value `%s' as no res name found\" even though a resource name WAS found -- the real problem is a value is already pending for it, not a missing name.", "file": "src/controller/testsuite/test.c:3525-3531 (case 'v' else-branch)", "root_cause": "The else-branch of the -v case handles two distinct situations (no preceding -n at all, vs. a duplicate -v for an already-named pending resource) with one shared, incorrectly-worded message.", "fix": "NOT YET FIXED. Recommended: distinguish the two cases with an accurate message for the duplicate-value case, e.g. \"ignoring duplicate value '%s' for res '%s' (already has a pending value)\".", @@ -23605,24 +10609,26 @@ "xce_test", "cmd_resupdate", "misleading-error-message", - "cli-ux-audit", - "unfixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "cli-ux-audit" ], "related_bugs": [ "bug-1706", "bug-1707" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1712.md" }, { "id": "bug-1713", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-08-26T17:15:00Z", - "error_message": "User-reported, live 3-node cluster, right after bug-1700/bug-1701's redirect fix landed and was live-tested: `xceadmin scheduler record status --controller ` now correctly redirects ('redirected: localhost:9903 -> localhost:9901'), but the SAME command against the SAME redirect target intermittently fails with `Error: controller returned status -10013` (sometimes succeeds with real recording status, sometimes fails on an immediate repeat) instead of always succeeding.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported, live 3-node cluster, right after bug-1700/bug-1701's redirect fix landed and was…\n\nUser-reported, live 3-node cluster, right after bug-1700/bug-1701's redirect fix landed and was live-tested: `xceadmin scheduler record status --controller ` now correctly redirects ('redirected: localhost:9903 -> localhost:9901'), but the SAME command against the SAME redirect target intermittently fails with `Error: controller returned status -10013` (sometimes succeeds with real recording status, sometimes fails on an immediate repeat) instead of always succeeding.", "file": "src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request), possibly xceadmin/discovery.py (follow_redirects)", "root_cause": "FULLY ROOT-CAUSED. -10013 is E_XCE_NOT_READY (errnox.h: XCE_ERRNO_BASE+12, 'Controller in CTRL_INIT or CTRL_ELECTING state'). The redirect from bug-1701's fix correctly points at whichever node was primary as of its last ctrlhello, but that node can be transiently mid-(re-)election by the time the redirected request actually arrives -- expected/normal on a freshly-booted cluster (election churn during the first several seconds), or any time a real failover is in flight. ha_dispatch_gate() returns -XCE_ERR_NOT_READY for exactly this transient window, with NO redirect populated (by design -- a CTRL_INIT/CTRL_ELECTING node genuinely has no settled primary to point to yet). The sibling code E_XCE_NOMASTER (errnox.h:22) carries an explicit design-intent comment: 'Standby has no known primary -- client should retry' -- but `_send_admin_request()` (scheduler.py:119-161) treats EVERY nonzero reply.status, including both E_XCE_NOT_READY and E_XCE_NOMASTER, as an immediate hard SchedAdminError with no retry at all. This is a documented client-side gap, not a server bug: the server-side behavior (refuse with a specific, distinguishable transient-state code) is correct and intentional; the client simply never implements the retry the errno's own comment already promises.", "fix": "FIXED and LIVE-VERIFIED (2026-08-26). `_send_admin_request()` (src/xceadmin/src/xceadmin/scheduler.py:119-231) now wraps its whole request/redirect/decode cycle in a bounded retry loop, re-running `follow_redirects()` FROM SCRATCH each attempt (not a bare resend to the same host:port) -- a fresh attempt can discover a newly-settled or different primary once an election concludes. Retries EXACTLY `reply.status in (_E_XCE_NOMASTER, _E_XCE_NOT_READY)` (module-level constants at scheduler.py:80-111, values -10011/-10013 -- NEGATIVE, matching the real wire value confirmed via join.c:284 `rc = -E_XCE_NOMASTER` / commit.c:259 and controller.capnp:608's signed `Int32 status` field). `E_XCE_STANDBY` (-10014) is deliberately excluded -- untouched, still resolved purely by `follow_redirects()`'s existing redirect-following (bug-1701); a standby reply that DOES carry a redirect never even reaches this status check (follow_redirects consumes it internally), and one that does NOT carry a redirect is a genuine non-transient rejection that must keep failing immediately. No shared Python-side mirror of errnox.h's enum exists anywhere in xcecommon/xceadmin (confirmed by grepping the whole package) -- followed the one existing precedent instead of inventing a new convention: test_e2e_live_ha_push.py's own local `_E_XCE_STANDBY = 10014` constant with an errnox.h-citing comment. Retry budget: `_MAX_TRANSIENT_ATTEMPTS=4` total attempts (1 initial + 3 retries), `_TRANSIENT_RETRY_DELAY_S=1.5` fixed delay between attempts (`time.sleep`) -- sized at the same scale as this project's existing bounded-retry shapes (follow_redirects' own `_MAX_REDIRECT_HOPS=3`; the HA soak driver's verify_classify retry loop) since election-churn settling is single-digit-seconds, not a long outage; no exponential backoff needed at this scale. Retry-budget exhaustion still raises the existing `SchedAdminError` (no new exception type needed) but with a message appending '(still transient after N attempts over ~Xs -- giving up)' for clear distinguishability from an immediate non-retried hard failure. RED/GREEN proof (src/xceadmin/tests/test_scheduler.py): added `_SequentialMockController` (a finite, exact-connection-count multi-TLS-connection mock server, since the retry loop reconnects from scratch each attempt -- distinct from the file's existing one-shot `_MockController`) plus 5 new tests -- `test_record_status_retries_on_transient_status_then_succeeds` (parametrized NOT_READY/NOMASTER, 2 failures then success -> eventual success, FAILED pre-fix at collection time since the referenced constants didn't exist yet, and confirmed behaviorally via an ad hoc script that the pre-fix code raises immediately on a single transient reply with only 1 connection attempt), `test_record_status_retry_exhaustion_raises_distinguishable_error` (always-transient mock, `scheduler.time.sleep` monkeypatched so the test does not actually wait; asserts exact attempt count + distinguishable message), `test_record_status_standby_without_redirect_not_retried` and `test_record_status_standby_with_redirect_follows_redirect_not_retried` (regression locks: E_XCE_STANDBY is never retried, with/without a populated redirect, zero sleep calls in either case) -- all 5 PASS post-fix in 0.20s total (no real sleeping). Full xceadmin suite: 274 passed, 1 failed -- the failure (`test_live_config_push_against_standby_returns_standby_rejection_not_redirect`, test_e2e_live_ha_push.py) is PRE-EXISTING and unrelated: confirmed via `git stash push -- ` that it reproduces IDENTICALLY on the unmodified codebase -- it is a live e2e test whose own docstring already anticipated going stale once bug-1700/1701's generic dispatch-gate redirect fix (already landed this session) generalizes to `OPCode_upgradeConfig`, which it now has; zero new failures caused by this fix. LIVE VERIFICATION (real 3-node cluster, deploy-ha.sh + xce.build's real controller binary, editable-install xceadmin venv pointed at scheduler.py so before/after needed no rebuild, only a file swap): a plain fresh-boot tight loop immediately after start-all.sh did NOT naturally reproduce -10013 in this sandbox (election settles with highly variable timing, sometimes near-instantly, since `ha_has_write_quorum()`'s standby_endpoints are populated synchronously inside `raft_cb_become_leader` and are typically already satisfied by the time start-all.sh's 1s-granularity health-check first observes CTRL_PRIMARY) -- but a REAL FAILOVER (kill -9 the live primary, then hammer the two survivors with zero sleep, no health-check wait) reliably reproduced the exact user-reported symptom, which this bug's own root-cause writeup already named as an equally valid trigger ('...or any time a real failover is in flight'): pre-fix, the client surfaced a bare 'Error: controller returned status -10013' to the user at t+8.60s post-kill (exit 1); post-fix, an identical failover on a fresh cluster produced ZERO -10013/-10011 failures across the equivalent ~15s post-kill window (52 requests; every non-success was either a legitimate pre-election-completion 'connection refused' to the just-killed primary's now-dead listener, or a legitimate post-gate 'scheduler not connected' business-level SchedAdminError from this sandbox having no real xce_scheduler process deployed -- neither is a transient status code, and correctly reaches the caller unretried in both runs, proving the fix does not overreach). `push.py`'s `push_config()`/`probe_config()` (src/xceadmin/src/xceadmin/push.py:92-179) have the IDENTICAL gap -- same two transient codes, same immediate-return-as-data with zero retry -- confirmed by direct code read; NOT fixed here (out of this bug's Python-scheduler-admin-path scope, and push.py's error-vs-return convention differs from scheduler.py's raise-based one, so a proper fix there is more than a trivial shared-helper change) -- flagged as a sibling finding for a future bug, not silently left undocumented.", @@ -23639,7 +10645,6 @@ "found-live", "root-caused", "fix-scoped", - "fixed", "live-verified", "push.py-sibling-gap-flagged" ], @@ -23647,19 +10652,20 @@ "bug-1700", "bug-1701" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1713.md" }, { "id": "bug-1714", - "gitea_synced_status": "cannot-reproduce", - "gitea_issue": 265, - "repro_attempts": "0/2 runs, 2026-09-25: documented recipe (3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' = 2400 array elements each) on b63bb581 build.dbg and on d87b78dc-era ASAN build -- no crash, scheduler alive throughout, 0 redelivered array pushes (0 'sched_add_job failed' duplicates; the original repro saw up to 18x redelivery, the recipe's precondition)", "status": "cannot-reproduce", "status_updated": "2026-09-25", "status_note": "recipe no longer reproduces on this machine: its trigger (array pushes redelivered to the scheduler) did not occur on either build, so it was already gone before 2026-09-25's fixes; ASAN run clean. Plausibly related fixes since: bug-1864 (per-endpoint message reordering in the vtask dispatcher), bug-1769 (ALTER heap overflow write). Reopen to investigating if a scheduler segfault recurs.", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-08-26T18:35:00Z", - "error_message": "User-reported live deployment: xce_scheduler process died and restarted unexpectedly (PID changed from 3964918 to 4137745 in scheduler.log with a ~79s gap, no manual restart by the user) during a live 3-node cluster test session; xceadmin briefly reported 'Error: scheduler disconnected' (the already-correct bug-1693 disconnect-detection path in cluster.c, working as designed -- not itself a bug) then recovered on the next attempt once the scheduler rejoined. User confirmed via `sudo dmesg -T` a REAL kernel-level segfault: `xce_scheduler[2815660]: segfault at 30 ip 0000601569e1fb35 sp 00007db7db1f6e00 error 4 in xce_scheduler[23b35,601569e01000+de000] likely on CPU 5 (core 16, socket 0)` -- though this specific dmesg entry is dated Aug 25 19:21:39, a DIFFERENT, earlier occurrence than today's specific restart (Aug 26 ~18:00-18:02), not necessarily the direct cause of THIS restart. No core dump exists for xce_scheduler anywhere in /var/lib/systemd/coredump/ (only xce_controller and python entries present, despite the scheduler segfault being kernel-confirmed) -- a separate, notable gap in this environment's crash-capture coverage for the scheduler process specifically.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 8, + "error_message": "User-reported live deployment: xce_scheduler process died and restarted unexpectedly (PID changed…\n\nUser-reported live deployment: xce_scheduler process died and restarted unexpectedly (PID changed from 3964918 to 4137745 in scheduler.log with a ~79s gap, no manual restart by the user) during a live 3-node cluster test session; xceadmin briefly reported 'Error: scheduler disconnected' (the already-correct bug-1693 disconnect-detection path in cluster.c, working as designed -- not itself a bug) then recovered on the next attempt once the scheduler rejoined. User confirmed via `sudo dmesg -T` a REAL kernel-level segfault: `xce_scheduler[2815660]: segfault at 30 ip 0000601569e1fb35 sp 00007db7db1f6e00 error 4 in xce_scheduler[23b35,601569e01000+de000] likely on CPU 5 (core 16, socket 0)` -- though this specific dmesg entry is dated Aug 25 19:21:39, a DIFFERENT, earlier occurrence than today's specific restart (Aug 26 ~18:00-18:02), not necessarily the direct cause of THIS restart. No core dump exists for xce_scheduler anywhere in /var/lib/systemd/coredump/ (only xce_controller and python entries present, despite the scheduler segfault being kernel-confirmed) -- a separate, notable gap in this environment's crash-capture coverage for the scheduler process specifically.", "file": "src/scheduler/ (exact crash site unknown -- no core dump, no backtrace available for either occurrence)", "root_cause": "NOT ROOT-CAUSED -- confirmed real (kernel dmesg segfault line, error code 4 = user-mode read fault, not a random/unrelated event) but no core dump or backtrace exists for any xce_scheduler crash observed so far, including the Aug 25 19:21:39 kernel-confirmed occurrence and today's (Aug 26, ~18:00-18:02) restart (PID 3964918 -> 4137745), which may or may not be the SAME underlying bug -- the two occurrences have not been correlated beyond both being unexplained scheduler deaths. Notable secondary finding: systemd-coredump is confirmed working for xce_controller (4 real core.xce_controller.*.zst files captured and used earlier this session, see bug-1699) but has captured ZERO xce_scheduler crashes despite at least one kernel-confirmed scheduler segfault -- needs investigation on the user's real machine (check core_pattern/ulimit -c/systemd-coredump size or process-name filtering specifically for the scheduler binary; also worth checking whether the scheduler's own launch wrapper -- start-scheduler.sh, a bare unwrapped invocation with no core-dump-enabling ulimit call -- inherits a different ulimit -c than however the controller happens to get its unlimited setting). FOLLOW-UP (2026-08-26, disambiguating today's restart from the Aug-25 segfault): checked dmesg for today's exact restart window (17:45-18:15) -- zero xce_scheduler entries (the ONLY dmesg segfault hit remains the single Aug 25 19:21:39 occurrence). journalctl for the same window -- zero xce_scheduler entries at all. dmesg confirms OOM killer was never actually invoked against any process in this window (only periodic OOM-killer enable/disable toggling, unrelated). ulimit -c is unlimited and core_pattern correctly routes to systemd-coredump (confirmed working for xce_controller, 4 real cores captured this session). The combination of (a) unlimited core-dump capacity + correct core_pattern, (b) zero kernel fault-signal record, (c) zero OOM record, (d) zero journalctl record for today's specific window strongly indicates today's PID change (3964918 -> 4137745) was NOT a crash of any kind -- a genuine fault signal (SIGSEGV/SIGABRT/SIGBUS/SIGILL/SIGFPE) is unconditionally kernel-logged regardless of which process triggers it, and none was. The scheduler is launched as a bare backgrounded process (start-scheduler.sh, no systemd unit), so an ordinary kill/pkill sent by an unrelated concurrent process on this shared machine would leave NO trace in dmesg or journalctl -- most likely explanation for today's specific restart, DECOUPLED from this bug going forward. This buglog entry now tracks ONLY the confirmed, kernel-verified Aug 25 19:21:39 xce_scheduler segfault (still unreproduced, still no core available, still worth investigating on recurrence) -- not today's restart, which is environmental noise, not a code bug.\n\nUPDATE (2026-08-29, bug-1731 live gdb-wrapped repro session): CRASH CAPTURED LIVE, first-ever backtrace for this crash family. Root cause of the earlier no-core-dump gap circumvented by launching xce_scheduler wrapped in `gdb -batch -ex run` FROM PROCESS START (not attach-after-the-fact) during a heavy burst-array workload (6 concurrent `jobsub -a 50` array submissions, 300 near-instant /bin/true elements, on a live 3-node HA cluster, debug build profile). Reproduced at T+150s into the burst. Full backtrace:\nSIGSEGV in sched_get_project(core, name=0x41d0000000000000) at scheduler/src/core.c:412, called from core_build_job() at scheduler/src/core_events.c:221, called from xce_core_push_events() at scheduler/src/core_events.c:725 (crashed on the 3rd of a 6-event batch), called from xce_sched_driver_translate_pending_events() (driver.c:555) <- xce_sched_driver_run_normal_mode_round() (driver.c:749) <- main() (main.c:465). Registers: rax=rdx=rsi=0x41d0000000000000 (corrupted `name` arg present in 3 registers), rdi=0x555555695da0 (valid `core` arg). Single-thread fault in the main scheduling-round thread (all 9 other threads idle in pthread_cond_wait/epoll_wait) -- not a cross-thread race at the fault site itself, though the corruption's origin is not yet located.\ncore_build_job() (core_events.c:221) calls sched_get_project(core, j->submit->project_name). The two preceding calls in the same function (sched_get_user, sched_get_queue, reading earlier fields of the same job_submit_data_t*) succeeded, so j->submit itself is a valid pointer -- but its project_name field held 0x41d0000000000000, not NULL and not a plausible string pointer. clone_job_submit_data() (handler.c:102-150) calloc()s the destination and only strdup()s project_name when the source is non-NULL, so a legitimately-absent project name would be NULL (which sched_get_project's own guard at core.c:412 handles safely) -- this crash requires the field to have been overwritten with garbage AFTER allocation. Points to memory corruption or a stale/reused pointer in the array-job event-translation/batching path under heavy concurrent load, not a missing-null-check. NOT YET further isolated -- the corruption's write site is still unknown. Fault class (SIGSEGV, invalid-pointer read via strlen()) matches the original kernel dmesg description (`error 4` = user-mode read fault) closely enough that this is plausibly the same crash family, now finally attributable to an exact function/line/call-chain instead of an unattributed dmesg line.\n\nUPDATE (2026-08-29, post-fix live-repro session -- bug-1730/1731/1733 fixes now all landed and included): two-phase reproduction attempt, both phases succeeded.\n\nPHASE 1 (ASAN, fresh clean builds of controller/scheduler/worker/xceadmin including commits 42eeaa1e[bug-1733 vtaskmgr UAF fix], b1323f8d[bug-1730], ebe691c3+67009815[bug-1731]): deployed fresh 3-node HA cluster (/tmp/xce-deploy-bug1714-asan2), launched xce_scheduler directly under ASAN (ASAN_OPTIONS=detect_leaks=0:halt_on_error=1:abort_on_error=1), ran a 10x50-element concurrent array-job burst (500 elements, wave 1 of a planned two-wave/1000-element burst). CRASH REPRODUCED WITHIN WAVE 1 ITSELF -- before all 10 clients' submissions even completed, after only 2 of 10 array pushes had been expanded by the scheduler (100 of the intended 500 elements). This is FASTER than the original bug-1731-session capture (T+150s) and definitively proves the bug is STILL PRESENT after all three recent fixes, not masked or perturbed by ASAN instrumentation. Verbatim ASAN report:\n AddressSanitizer:DEADLYSIGNAL\n ==562488==ERROR: AddressSanitizer: SEGV on unknown address (pc 0x64b0be11076e ...)\n ==562488==The signal is caused by a READ memory access.\n #0 sched_get_project ../scheduler/src/core.c:412\n #1 core_build_job ../scheduler/src/core_events.c:221\n #2 xce_core_push_events ../scheduler/src/core_events.c:725\n #3 xce_sched_driver_translate_pending_events ../scheduler/src/driver.c:555\n #4 xce_sched_driver_run_normal_mode_round ../scheduler/src/driver.c:749\n #5 main ../scheduler/src/main.c:465\n Register values: rax=rdx approx corrupted; rax = 0x41d0000000000000 (still exactly 2^30=1073741824.0 as an IEEE-754 double, matching the original capture bit-for-bit) -- IDENTICAL crash signature, file, line, and call chain to the original bug-1731-session capture. SUMMARY: AddressSanitizer: SEGV core.c:412 in sched_get_project.\n Diagnostically important: ASAN's own heap-buffer-overflow and use-after-free shadow-memory detectors did NOT fire -- only the raw SIGSEGV/DEADLYSIGNAL handler caught this, exactly as a non-ASAN build would. This RULES OUT a classic redzone-violating heap overflow into an adjacent allocation (ASAN would have caught that at the WRITE) and argues for either (a) a wrong-offset/type-confusion write within a still-valid allocation, or (b) a use-after-free where the stale read happens only after the freed chunk has already left ASAN's quarantine and been legitimately reallocated for a new, unrelated live object -- which Phase 2 then confirmed directly.\n\nPHASE 2 (gdb scripted watchpoint, debug -O0 -g3 profile, same fix set, fresh cluster /tmp/xce-deploy-bug1714-dbg2): launched xce_scheduler under `gdb -batch -x scheduler-watchpoint.gdb`, breaking on clone_job_submit_data() (handler.c:102, called once per array element from make_array_element), sampling every 31st call and arming a hardware watchpoint on the returned job_submit_data_t's ->project_name field (via `finish` + cast of $rax to the known x86-64 System V pointer-return-value register), capped at 3 simultaneous watchpoints to stay within HW watchpoint budget. Ran the same burst pattern. ALL THREE armed watchpoints fired within the first burst wave. Full verbatim capture in this session's transcript; key facts:\n - All three watched job_submit_data_t->project_name fields transitioned from a plausible-looking heap string pointer (e.g. 'Old value = 0x7fffe000bc60 \"0\\275\"') to exactly NULL (0x0) -- i.e., we caught a real WRITE, but a NULL-out, not the resource-double corruption pattern itself.\n - Root cause of that write, nailed by backtrace: the write instruction was __memset_avx2_unaligned_erms, called from libCapnC_Runtime.so's capn_init_mem()/init_fp(), called from xce_decode() (lib/request.c:103) -- i.e. Cap'n Proto's own message-decode-buffer zero-initialization for a BRAND NEW INCOMING NETWORK REQUEST, executing on a COMPLETELY DIFFERENT thread (LWP 580496, decoding its own client's array submission) than the thread that had originally cloned the watched job_submit_data_t (LWP 580499, decoding a DIFFERENT client's array submission, job_array pointer 0x7fffe0000cf0 vs 0x7fffe4002fe0).\n - This is DEFINITIVE PROOF of the corruption mechanism: a job_submit_data_t's heap chunk, after being freed, gets immediately recycled by glibc malloc to serve an UNRELATED incoming request's Cap'n Proto decode buffer on a different thread. Any stale pointer still referencing the old (freed) job_submit_data_t will, from that point on, read whatever the CURRENT occupant of that memory happens to contain -- first zeros (as caught here), but on a LATER reuse cycle for something that stores a raw IEEE-754 double (e.g. an xce_alloc_t's `value` field, src/include/xce/types.h:270), it would read exactly the 2^30/1GiB bit pattern seen in both the original and this session's crash.\n - IMPORTANT CAVEAT / what remains UNRESOLVED: the free() itself, in this specific capture, was almost certainly LEGITIMATE, not a bug by itself. Verified by static reading of driver.c's xce_sched_driver_translate_pending_events(): within one drain/translate call, xce_core_push_events(core, events, n) (driver.c:555, which runs every core_build_job() call for the whole batch) unconditionally executes BEFORE the cleanup loop that calls release_translated_payload()->free_Job_ptr()->free_JobSubmitData_ptr() (driver.c:585-588) -- ownership/free ordering within a single batch is correct by inspection. sched_on_job_array() (handler.c) also does not push a client's array to the scheduler until its ENTIRE 50-element clone loop completes, so the specific object we watched (armed at clone call #31, i.e. mid-loop) could only have been freed AFTER its owning thread's full loop finished, was submitted as one combined XCE_SCHED_OP_UPDATE_JOB request, and was drained+processed+freed by an EARLIER, already-completed scheduling round -- i.e. a normal, successful lifecycle, not an early/premature free caught in the act. The actual bug is therefore NOT in this ownership-transfer/free ordering itself, but in some OTHER, not-yet-identified code path that reads a job_submit_data_t (or the containing xce_job_t) a SECOND time, AFTER it has already been correctly processed and freed once. Did not have time this session to isolate that second/stale reader. Two concrete leads for the next session: (1) whether a redelivered/retried array push can race sched_on_job_array()'s per-element `sched_get_job(&(sched->core), jobid) != NULL` dedup check (handler.c ~212) against the window before a just-submitted job is actually added to sched->jobs (which only happens later, inside core_build_job()/sched_add_job(), not at submit time) -- if so, two independent xce_job_t/job_submit_data_t clones could exist for the same jobid, and it's worth checking whether anything then conflates or double-references them; (2) core_build_schedobj() (core_events.c:174-197, called from core_build_job() at core_events.c:253 with the ORIGINAL `j` still valid) does `obj->project = makeString(j->submit->project_name)` -- a SECOND independent read of j->submit->project_name beyond the sched_get_project() call at core_events.c:221 that actually crashes; worth double-checking core_build_schedobj/its 3 tag-dispatch helpers for any raw pointer-aliasing (vs. deep copy) into the resulting long-lived xce_sched_job_t/xce_sched_obj_t that could keep referencing freed submit-owned memory after `j` itself is gone.\n\nBoth clusters deployed this session (/tmp/xce-deploy-bug1714-asan2, /tmp/xce-deploy-bug1714-dbg2) were torn down cleanly after their captures; the crash/watchpoint logs remain on disk at /tmp/xce-deploy-bug1714-asan2/logs/scheduler_asan.log and /tmp/xce-deploy-bug1714-dbg2/logs/scheduler_gdb_watchpoint.log for reference (not committed, local /tmp only -- copy out if this environment gets reset before a follow-up session).\n\nUPDATE (2026-08-30, Part 1/Part 2 investigation session): PART 1 (static-only, XceData_data_schedjobs dataset->rows aliasing) -- RULED OUT, not a bug. Traced the full decode chain for the schedjobs/jobs table (src/lib/xcectx.c:139 xce_decode() -> src/lib/request.c:1786 extract_data() -> src/lib/controller.capnp.c decode_Job_list()/decode_Job_ptr() (line 6348/6369) -> decode_JobSubmitData_ptr() (line 12427) -> decode_JobSubmitData() (line 12223)). Confirmed every xce_job_t (calloc, decode_Job_ptr/decode_Job_list) and every job_submit_data_t (calloc, decode_JobSubmitData_ptr) is an independent heap allocation, and every job_submit_data_t string field uses STRING_DUP (= strdup, controller.capnp.h:9) -- structurally identical to clone_job_submit_data()/decode_pidinfo(). extract_data() (request.c:1786-1815) only memcpy's the already-decoded xce_job_t* POINTER VALUES into dataset->rows (then memsets the source array to prevent double-free) -- it never aliases into the capnp segment/request buffer, and this happens entirely INSIDE xce_decode(), before capn_free(&c) at request.c:281 destroys the wire buffer. So sched_handle_dataset()'s XceData_data_schedjobs case handing dataset->rows to sched_on_jobs() for deferred driver-thread processing is a correct, safe zero-copy ownership transfer of already-independent objects, not a second UAF class. The XceData_data_jobArray case's contrasting comment ('we never alias... unlike the list-shaped cases above') is about a different concern (N-way sharing of one spec across array elements), not capnp-buffer lifetime -- both cases are equally safe on that specific axis.\n\nPART 2 (live pointer-lifecycle tracking, main task) -- did NOT capture a fresh crash this session despite extensive effort (see below), but surfaced a significant, previously-undocumented, independently-reproducible finding that materially narrows the investigation and directly answers this entry's own Question 3 lead about a redelivery/retry mechanism. Full method: added temporary mutex-guarded fprintf/fflush diagnostic logging (address, jobid, tid, timestamp) at three sites -- make_array_element() (handler.c, ALLOC), every free_Job_ptr()/free_JobSubmitData_ptr() call site in driver.c's release_translated_payload() (FREE), and core_build_job() (core_events.c) immediately before the historically-crashing sched_get_project() line (READ). The shared logging implementation was placed in src/lib/utils.c (a genuine, non-generated libxce.a source file) rather than controller.capnp.c, after discovering mid-session that src/lib/controller.capnp.c is NOT actually compiled -- meson's custom_target regenerates lib/controller.capnp.c/.h from src/schema/controller.capnp fresh into the BUILD directory on every build, silently discarding any manual source-tree edit to that file (confirmed empirically: an initial edit there vanished from the compiled object with no error). Built (debug profile), deployed multiple fresh 3-node HA clusters, launched xce_scheduler under `gdb -batch -ex run -ex \"thread apply all bt full\"`, and drove very heavy concurrent `jobsub -a 50` array-job bursts -- far exceeding the original recipe's intensity: 6-15 concurrent submissions per wave, 20+ waves across several redeploys (both -v and -vvv controller verbosity), well over 100 total concurrent array submissions / 5000+ array elements pushed. The scheduler did NOT crash in any of these attempts (longest single continuous run: ~16 minutes / 5 waves without a restart), a notably higher bar than the original session's 150s/6-concurrent capture -- see 'What changed / open question' below.\n\nKEY FINDING (from the diagnostic log, cross-referenced against the scheduler's own pre-existing Bug-A WARN logging): every array-job jobid gets independently, repeatedly RE-CONSTRUCTED and pushed to the scheduler via the ORDINARY (non-array) XceData_data_schedjobs/JOBS_PUSH bulk path -- confirmed by cross-referencing FREE events against ALLOC events by jobid: every array-path jobid showed MORE free events than array-path (make_array_element) allocs, each with a DISTINCT heap address (never the same address freed twice -- ruling out a literal double-free), e.g. jobid 4658596961282883584 was array-allocated once but freed 18 times from 18 distinct addresses over a 16-minute run. Each of these extra frees is preceded by a matching WARN in scheduler.log: 'xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid ... (rc=-17), discarding duplicate/orphaned job' -- i.e. Bug A's fix (this same buglog entry, prior session) is correctly catching and safely discarding every one of these redelivered duplicates before they reach the picker. CRITICALLY, this was reproduced in a run with ZERO client-side submission failures/retries (confirmed: grep for 'jobsubmit failed'/'retry' across all burst client output files in the clean final run returned zero matches, yet the SAME redelivery pattern still occurred, up to 18x per jobid) -- this RULES OUT the client-retry-on-timeout hypothesis (XCECLI_FAILOVER_CALL's retry loop, client_internal.h) as the trigger; the redelivery is NOT a symptom of client-observed transport failure, it happens under clean, fully-acknowledged submissions too. Static reading of the two most likely controller-side mechanisms did not find an obvious culprit: (1) persist_apply_pump_run() (persist.c:5521) advances last_applied_lsn strictly one entry at a time under apply_pump_lock and only ever processes next=last_applied_lsn+1, so a genuine re-walk of an already-applied LSN range looks structurally impossible by construction; (2) apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case (persist.c:4391-4461) explicitly fires HOOK_DETAIL_JOB_NEW exactly once per successful apply, with an EEXIST short-circuit for an idempotent whole-record re-apply. Neither explains a genuinely repeated, single-job-shaped (l->count==1, routed through xcesched_push_joblist() per sched.c:326-327's dispatch, not xcesched_push_jobarray()) HOOK_DETAIL_JOB_NEW fire for jobids that were originally created via the array path. bug-1419's in-flight submission registry (job.c:71-140, tracks [base_jobid, base_jobid+array_size) ranges) is the most likely place a gap could exist for this specific pattern but was not dynamically traced this session -- NEXT STEP for a follow-up session: either a live gdb breakpoint on sched_job_new_fun()/xcesched_push_joblist() (sched.c) capturing the call's l->count and full backtrace, or controller-side instrumentation logging out_new_jobs.count at every HOOK_DETAIL_JOB_NEW fire site, to determine whether this is a genuine second independent WAL commit for an overlapping jobid range (a real gap in bug-1419's registry or jobid_range_exists()) or some other apply-side effect not yet identified by static reading.\n\n(a) vs (b) QUESTION -- NOT SETTLED this session (no fresh crash captured to inspect). The redelivery-storm finding above establishes a plausible AMPLIFYING mechanism (far more job_submit_data_t alloc/free churn under array-job load than previously suspected, all safely discarded by Bug A but still consuming and freeing large volumes of same-shaped heap chunks under concurrent capnp-decode pressure on other threads) but does not by itself prove a stale-pointer read, since every redelivered object observed in this session's log was a freshly-decoded, valid object at its own READ time (not a stale/freed one) -- the two prior sessions' watchpoint-proven UAF mechanism (freed job_submit_data_t memory reused by malloc for an unrelated concurrent capnp decode buffer) still requires a genuinely stale SECOND reference somewhere, which this session did not catch live.\n\nWhat changed / open question: this session's burst intensity substantially exceeded the recipe that reliably crashed within 150s-2.5min in the two prior sessions (same debug build profile, same gdb wrapper, same jobsub -a 50 command), yet produced zero crashes across ~20+ waves / 100+ concurrent submissions / 5000+ array elements over multiple redeploys. Possible explanations, none confirmed: (1) the two prior sessions' side-bug fixes (bug-1730's core_events.c fix, bug-1731's HOOK_DETAIL_JOB_FAIL fix, and this entry's own Bug A/Bug B, all landed 2026-08-29 per git log -- 'fix(scheduler): don't hand an orphaned duplicate job to the picker' at commit 38437e57 among them) may have shifted timing enough to reduce (not eliminate) the race window; (2) this session's burst driver script (piped `xce_test --norc` REPL commands, one process per concurrent submission) may differ subtly in timing/pacing from whatever exact mechanics the original crash-capturing sessions used; (3) simple bad luck across a large but still finite number of attempts for a genuinely rare race. All temporary diagnostic instrumentation (handler.c, driver.c, core_events.c, src/lib/utils.c) was fully reverted before this session ended -- confirmed via `git diff` returning clean/identical-to-HEAD for all four tracked files, plus a manual re-check of the untracked/regenerated src/lib/controller.capnp.c (git-invisible, confirmed no leftover bug1714_diag_log references anywhere under src/). Full scheduler ASAN test suite re-run after revert: 54/55 pass, the one failure (test_recordstart_symlink_escape_replies_error) is the already-documented pre-existing sandbox filesystem-permissions flake, unrelated to this session. All deployed clusters (3x) torn down; zero xce_controller/xce_scheduler/xcecd_virtual/gdb processes left running. NEXT STEP for a follow-up session: (a) pursue the redelivery-storm's exact trigger per the NEXT STEP above (this is now the most concrete, evidence-backed lead); (b) if reproduction is attempted again, consider a substantially longer single continuous run (30+ min) rather than many shorter redeploys, since this session's longest unbroken run was only ~16 minutes and the original captures may have benefited from cumulative state a fresh redeploy resets each time.\n\nUPDATE (2026-08-30, later same day, large-ASAN-quarantine live-repro session): NEW APPROACH TRIED, NOT YET DOCUMENTED IN THIS ENTRY -- launched the ALREADY-BUILT ASAN xce_scheduler (src/xce.build/scheduler/xce_scheduler, built 2026-08-30 11:36, already including every fix through commit 22aa16bf/38437e57 -- no rebuild needed since ASAN_OPTIONS is a runtime env var, not compile-time) with a deliberately oversized quarantine/redzone (ASAN_OPTIONS=quarantine_size_mb=256:redzone=256:max_redzone=2048:detect_leaks=0:abort_on_error=1:halt_on_error=1) specifically to test whether the documented \"freed block reused too fast for ASAN's default quarantine\" explanation for plain-ASAN's prior silence (see the 2026-08-29 post-fix update above) would resolve with a much larger quarantine. Deployed a controller-only + scheduler 3-node HA cluster (NO worker -- confirmed by static reading of the crash call chain, in this same entry, that sched_get_project()/core_build_job()/xce_core_push_events() all fire at JOBS_PUSH submission-ingestion time, before any picker/allocator/worker involvement, so a worker is not required to trigger this specific crash; xcecd_virtual was also unavailable to build this session, see environmental note below, making this a load-bearing simplification, not just a shortcut). Drove a sustained, unthrottled burst-submission loop (8 concurrent `client`+`jobsub -u root -q normal -c hostname -a 50` xce_test invocations per wave, back-to-back waves, ~1-8s apart) for ~8 minutes continuous wall clock, reaching 64 waves = 25,600 array elements pushed (verified via a final controller `joblist` count of 25,756 PEND jobs, consistent with no jobs lost at the controller level) -- substantially higher volume and, without a worker providing any execution-time backpressure, likely higher submission RATE than any prior session's recipe (the original crash-capturing sessions used 6 concurrent x 50 = 300 elements in 150s; this session pushed 25,600 in ~480s, roughly an order of magnitude more elements per unit time). RESULT: the scheduler did NOT crash and produced ZERO AddressSanitizer reports of any kind (grep -ci AddressSanitizer scheduler.log == 0) across the full run. ADDITIONAL NEGATIVE FINDING: the 2026-08-30-morning session's \"redelivery-storm\" (array-job jobids redelivered via the ordinary JOBS_PUSH path, discarded by Bug A with a \"discarding duplicate/orphaned job\" WARN, observed up to 18x per jobid that session) did NOT reproduce AT ALL this run either -- zero occurrences of that WARN string anywhere in 94,794 scheduler.log lines, despite comparable-or-higher load. This is a second, independent \"previously-reliable symptom now absent\" data point in addition to bug-1714's own crash, on the same day, suggesting SOMETHING about either the exact submission-driver mechanics (this session used direct `client`+`jobsub` xce_test invocations; the morning session used a different piped-REPL driver script whose exact mechanics were not preserved) or genuine environment/timing variance materially changes whether either symptom fires -- not just bad luck on the crash alone. CAVEAT (important, do not skip): a controller build from an EARLIER point in this same work session (commit 38437e57 onward, built 2026-08-30 03:21) and a scheduler build from 11:36 the same day were reused as-is (not rebuilt) for this attempt -- both already post-date every fix this entry and its siblings (bug-1730/1731/1733) describe, so this is a like-for-like continuation of the \"post-fix\" investigation, not a regression to older code.\n\nENVIRONMENTAL INCIDENT (important for future sessions on this sandbox): this session began by finding the root filesystem at 0 bytes available (936.8G/936.8G used), which blocked ALL work (builds, deploys, even log writes) until resolved. `nix-collect-garbage` (no -d) was run to reclaim space, which worked (freed ~790GB, root cause was almost certainly months of accumulated `nix develop` closures across many past sessions' worth of ASAN toolchains, never garbage-collected) but had a serious side effect: it evicted this flake's own cached bootstrap-toolchain closure (gnum4/bison/gmp/mpfr/isl/libmpc/autoconf/a full multi-stage GCC bootstrap), forcing EVERY subsequent `nix develop ./src` invocation on this box -- including a DIFFERENT, concurrently-running agent's own controller build for an unrelated bug (bug-1735) -- into a full from-source toolchain rebuild. This cost roughly 15-20 minutes of wall clock before ANY build (mine or the other agent's) could proceed, and materially ate into this session's time budget. LESSON for next time: if disk space must be reclaimed on this box again, prefer `nix-store --gc --print-dead` / a bounded `nix-collect-garbage --max-freed ` first to gauge what will be evicted, since this flake's own dev-shell closure is apparently NOT protected by a persistent gcroot between sessions and is fully re-buildable-from-scratch (not just re-fetchable) when evicted -- a full unbounded `nix-collect-garbage` is disproportionately expensive here. Once resolved, this session worked around not having a fresh full build by (a) reusing the ALREADY-BUILT src/xce.build/{controller,scheduler} binaries directly (their /nix/store runtime deps were, fortunately, NOT evicted -- `ldd` still resolved cleanly) with the new ASAN_OPTIONS applied purely at launch time, and (b) building ONLY the missing `xceadmin` component fresh (into a separate dir, src/xce.build.bug1714xa) since a cluster.bin config cannot be produced without it -- this minimal-rebuild strategy is worth reusing directly if this situation recurs, rather than a full `xce-build.sh all`.\n\nbug-1734 was NOT reproduced this session: its repro recipe needs a worker to actually run jobs to DONE/EXIT so the scheduler-vs-controller view can be compared; no `xcecd_virtual` binary exists in any build dir on this box currently (checked src/xce.build, src/xce.build.validate, src/build-asan, src/build-asan-1696repro -- only worker *test* binaries exist, no `xcecd_virtual` itself), and this session's bounded time budget (already consumed by the disk/nix incident above) did not allow building `worker` fresh on top of the already-spent time. See bug-1734's own entry for a same-day cross-reference note.", "fix": "STILL NOT FIXED -- but no longer blocked on evidence collection; see root_cause UPDATE above for the first captured backtrace. NEXT STEP: find what writes garbage into job_submit_data_t->project_name after clone_job_submit_data() allocates it -- likely somewhere in the array-job batching/event-translation path (xce_core_push_events / xce_sched_driver_translate_pending_events) given the crash hit specifically while processing event index 2 of a 6-event batch under heavy concurrent array-job load. Reproduction recipe confirmed reliable enough to use again: debug-profile build, xce_scheduler launched via `gdb -batch -ex run -ex \"thread apply all bt full\"` from process start (script preserved at /tmp/xce-deploy-bug1731/scripts/start-scheduler-gdb.sh), 6 concurrent 50-element array submissions against a live 3-node HA cluster -- crashed within 150s this run.\n\nUPDATE (2026-08-29): STILL NOT FIXED. This session's two-phase live-repro (ASAN + gdb watchpoint, see root_cause UPDATE) upgrades the diagnosis from 'read-site backtrace only' to 'corruption mechanism proven' (freed job_submit_data_t memory gets recycled by malloc for unrelated Cap'n Proto decode buffers on other threads; any surviving stale pointer to the old object then reads garbage matching whatever now occupies that memory) but the actual SOURCE of the stale pointer/second-reader is still not isolated. NEXT STEP, sharpened from this session's two leads: (1) audit sched_on_job_array()'s per-element dedup check (handler.c ~212) for a race window against a redelivered/retried array push (job not yet added to sched->jobs at submit time, only later inside core_build_job()/sched_add_job()); (2) audit core_build_schedobj() and its 3 tag-dispatch helpers (core_events.c:110-197) for any raw pointer stored into the long-lived xce_sched_obj_t/xce_sched_job_t that aliases (rather than deep-copies via makeString()) memory owned by the short-lived original xce_job_t/job_submit_data_t, which release_translated_payload() frees shortly after core_build_job() returns.\n\nUPDATE (2026-08-29, two side-bugs found during this investigation FIXED -- bug-1714's OWN root cause remains UNFIXED, see this session's root_cause UPDATE above and the still-open leads there): while chasing this bug's stale-pointer UAF, static code reading (not live repro) surfaced two SEPARATE, independently-confirmed defects, real bugs on their own merits, not proven to BE this bug's own root cause. Both now fixed with TDD (RED->GREEN) and full regression evidence.\n\nBug A -- xce_core_push_events()'s JOBS_PUSH case (src/scheduler/src/core_events.c, the XCE_CORE_EVENT_JOBS_PUSH switch case) never checked sched_add_job()'s return value. sched_add_job() (core.c:219) is a thin wrapper around key_add(core->jobs, jobid, job), which returns a negative errno (duplicate-jobid class failure) if the jobid already exists -- previously silently discarded, so a freshly-built xce_sched_job_t that FAILED to register in core->jobs was still handed to picker->picker_iface->on_job_push() anyway: an orphaned job object the picker now tracks but that sched_get_job()/any other core->jobs-keyed lookup can never find again. FIX: check sched_add_job()'s return value; on failure, skip the on_job_push() call, log a WARN (matching this case's existing WARN convention for the job==NULL branch), and free the orphaned job at the same granularity core_build_job()'s own early-failure paths already use (FREEUP(job->resreq); FREEUP(job->schedobj); FREEUP(job);) -- job->user/queue/project deliberately left alone (kvmgr-owned get-or-create lookups, not owned by this job instance); job->allocs/accounts deliberately left alone too (out of scope -- xce_sched_job_t has no complete destructor anywhere in this codebase; kill_sched_job() in schedobj.c only frees job->rusages, a genuine pre-existing gap this fix does not attempt to close). TDD: RED -- added test_jobs_push_duplicate_jobid_does_not_call_on_job_push_or_leak to src/scheduler/testsuite/test_core_events.c (pushes one job successfully, baseline on_job_push count 1, then pushes a second JOBS_PUSH batch with the SAME jobid); on unfixed code this failed with 'Expected 1 Was 2' (on_job_push fired a second time for the orphaned duplicate), confirming the bug. GREEN -- after the fix, the same test passes: on_job_push count stays at 1, and sched_get_job() for that jobid still returns the ORIGINAL job object, not the duplicate.\n\nBug B -- sched_on_job_array()'s per-element dedup check (src/scheduler/src/handler.c ~line 179-226) has a TOCTOU race window: it runs on the network request-handling thread and checks sched_get_job() against core->jobs, but a job is only actually inserted into core->jobs later, when the separate DRIVER thread drains the pending-update queue and calls xce_core_push_events() -> JOBS_PUSH -> sched_add_job(). A redelivered (e.g. network-retried) copy of the same array push arriving on the request thread during that window is not caught by this check, so a second, independent set of xce_job_t/job_submit_data_t clones for the same jobids gets constructed and enqueued too. Per this session's analysis, eliminating the race itself (e.g. a new synchronized pending-jobids set) was deliberately NOT attempted -- out of scope, bigger architectural change, own risk -- because it is not actually necessary: once Bug A is fixed, this race is SAFE rather than dangerous, since the redelivered duplicate is now correctly and safely rejected at the authoritative uniqueness boundary (sched_add_job()'s key_add()), freed, and never reaches the picker. FIX: documentation only, no logic change -- added a comment on the dedup check explaining the race window, that it is a best-effort optimization (not a correctness boundary), and that sched_add_job()'s key_add() (now correctly handled per Bug A) is the actual authoritative dedup boundary a redelivered duplicate ultimately hits. No test needed (no behavior change); covered indirectly by Bug A's test, which models exactly the duplicate-jobid scenario this race can produce.\n\nFull regression (nix develop ./src --command bash src/tools/xce-build.sh scheduler asan --incremental): 54/55 scheduler test suites pass. The one remaining failure, test_recordstart_symlink_escape_replies_error in test_admin_request.c (expected 'invalid record file name', got 'permission denied accessing record file'), is the already-known, pre-existing bug-1561-adjacent sandbox filesystem-permissions flake, unrelated to and unaffected by this change -- present identically before these fixes were applied (also seen in the RED run). IMPORTANT: these two fixes are scoped, standalone correctness fixes for real, independently-confirmed defects -- they do NOT resolve bug-1714's own root cause (the stale-pointer UAF into a freed job_submit_data_t, see root_cause above). That remains open; the two still-open leads listed in this entry's prior fix UPDATE (2026-08-29) are what a future session should pursue next, now narrowed since lead (1) [sched_on_job_array()'s dedup race] has been read closely enough to conclude it is not itself hazardous post-Bug-A -- lead (2) [core_build_schedobj()'s makeString()-vs-alias handling] remains the live suspect.\n\nUPDATE (2026-08-30): STILL NOT FIXED -- no fix attempted this session (per this entry's own 'do not attempt a speculative fix without a confirmed root cause' guidance). This session's Part 1 static audit ruled out the XceData_data_schedjobs dataset->rows aliasing as a second UAF class (see root_cause UPDATE). Part 2's live pointer-lifecycle instrumentation did not capture a fresh crash despite ~20+ burst waves substantially exceeding the historically-successful recipe's intensity, but did surface a new, real, independently-reproducible finding: array-job jobids get redelivered to the scheduler via the ordinary single-job XceData_data_schedjobs/JOBS_PUSH path many times (up to 18x observed) after their original array-path submission, confirmed unrelated to client-side retries (reproduced with zero client submission failures). Bug A (this same entry, prior session) correctly discards every one of these before they reach the picker, so this redelivery is not itself a proven crash cause, but it is a strong candidate AMPLIFYING factor for the previously-proven malloc-reuse race (more job_submit_data_t churn = more opportunities to hit the race) and a concrete, scoped lead for a follow-up session (see root_cause UPDATE's 'NEXT STEP'). All temporary diagnostic instrumentation was fully reverted (verified via git diff + full re-run of the scheduler ASAN test suite: 54/55, same pre-existing flake as before). No code changes remain in the working tree from this session.\n\nUPDATE (2026-08-30, later same day): STILL NOT FIXED, STILL NOT REPRODUCED. Tried the large-ASAN-quarantine idea this entry's own root_cause section flagged as untried (quarantine_size_mb=256:redzone=256:max_redzone=2048, launched via already-built binaries, no rebuild) against a controller+scheduler-only cluster (no worker -- confirmed unnecessary for this specific crash's call chain, which fires entirely at JOBS_PUSH submission-ingestion time). 64 waves / 25,600 array elements over ~8 minutes, an order of magnitude more throughput than the original 150s/300-element capture: zero crash, zero ASAN reports of any kind, and (new, separate negative finding) zero recurrence of the morning session's reliably-observed redelivery-storm WARN either. Ruling nothing new out beyond what's already documented -- this is a THIRD consecutive session-day attempt (after the 08-29 post-fix session and the 08-30-morning session) where heavy live load did not reproduce a symptom that DID reproduce reliably in the two original capturing sessions (bug-1731's live-repro session and this entry's own 08-29 post-fix Phase-1/Phase-2 session), now across three different techniques (plain heavy load, gdb-wrapped heavy load with diagnostic instrumentation, and now large-quarantine ASAN). NEXT STEP suggestion, sharpened by three failed-to-reproduce sessions in a row: the two ORIGINAL successful captures both happened on 2026-08-29, before commit 38437e57 (Bug A) and its sibling Bug B documentation landed the same day, and EVERY session since (including this one) has been AFTER those fixes landed. While Bug A/B were carefully reasoned to be independent of bug-1714's own root cause (see this entry's history), the fact that reproduction rate appears to have dropped sharply coincident with their landing -- both the crash itself AND the previously-reliable redelivery-storm symptom now failing to reproduce under comparable or heavier load -- is a pattern worth taking seriously rather than dismissing as noise. Concrete suggestion for a follow-up session: (a) try reverting JUST commit 38437e57 (Bug A's fix) in a throwaway branch/build and re-run this exact large-quarantine-ASAN recipe against pre-Bug-A code, to directly test whether Bug A's fix (or the TOCTOU race it interacts with) was itself part of what made the race window practically hittable, rather than assuming independence holds; (b) if pre-Bug-A code DOES still crash under this recipe while post-Bug-A does not, that would suggest Bug A's fix (checking sched_add_job()'s return and discarding the duplicate instead of silently handing it to the picker) also incidentally narrows or closes bug-1714's own race window as a side effect worth understanding precisely rather than just accepting as an unexplained improvement. Both this session's build artifacts (src/xce.build.bug1714xa, a from-scratch xceadmin build with wheel+venv) were left in place (not cleaned up, unlike the temp deploy dir and processes) since they represent real, reusable build work with no live-process footprint.\n\nUPDATE (2026-09-04, live-repro session -- CRASH REPRODUCED, major new causal-chain finding, still no confident fix): Built an isolated debug profile (controller/scheduler/worker/xceadmin) into src/xce.build.bug1714repro specifically to avoid a concurrently-running interactive session also building/testing in this same worktree (tmux 'scheduler-1') -- deployed a fresh 3-node cluster on ports 9971-9973 (/tmp/xce-deploy-1714), launched xce_scheduler under `gdb -batch -ex run -ex \"thread apply all bt full\"` from process start, then drove the same recipe as the two original successful captures: 6 concurrent `jobsub -a 50` array submissions (300 elements) via xce_test against the live cluster.\n\nCRASH REPRODUCED, ~12s after the burst started (faster than either prior successful capture). Identical signature to every previous occurrence, byte-for-byte: SIGSEGV in sched_get_project (core.c:412), name=0x41d0000000000000 (same corrupted-double bit pattern), called from core_build_job (core_events.c:222) <- xce_core_push_events (core_events.c:754, this time on event index 16 of a 67-event batch, vs the original capture's index 2 of 6) <- xce_sched_driver_translate_pending_events (driver.c:555) <- xce_sched_driver_run_normal_mode_round (driver.c:749) <- main (main.c:465). All 9 other threads idle in a blocking syscall at fault time (same as every prior capture) -- confirms again this is a stale READ of already-corrupted memory, not a live cross-thread write race at the fault site itself.\n\nMAJOR NEW FINDING (not identified or looked for in any of the 6 prior sessions): the scheduler's connection to the primary controller GENUINELY, REPEATEDLY DROPS AND RECONNECTS under this exact burst load -- confirmed independently on BOTH sides of the connection, not a hypothesis:\n - SCHEDULER side: this session's build included the bug-1760 partial fix (sched_check_pending_rejoin() called every round from main.c, logs on success/failure). During the ~12s burst window, it logged 'pending rejoin succeeded, cluster membership restored' SEVEN times (03:38:03-03:38:07), each preceded by scheduler.log's own `xcecli_retry_next: xcecli failover: attempt N -> localhost:99xx` lines showing the client library actively rotating through all 3 candidates.\n - CONTROLLER side (ctrl-1, the primary throughout -- confirmed via `RAFT[p0] term=2 event=...` staying at term=2 the whole burst, i.e. this is NOT a controller-cluster election/failover, ctrl-1 never lost leadership): sched.c's handle_scheduler_leave() ('scheduler leave') fires repeatedly, each one immediately followed by a fresh `server sock 4 accepted 13` (a brand-new TCP accept, confirmed via a different priv-data/endpoint pointer each time) -- i.e. genuinely new connections, not the same one persisting.\n - LIKELY TRIGGER, evidenced but not yet proven with byte-level certainty: ctrl-1's own persist_submit_job_array_async() logged 'lsn=6/7/8/9 appended but commit_submit_async registration failed (rc=-10013)' (E_XCE_NOT_READY -- see bug-1713's own decoding of this same errno) at 03:37:56.346, ~300ms before the scheduler's first observed failover attempt (03:37:56.632) -- i.e. the PRIMARY CONTROLLER itself was under registration/commit backpressure from the burst of 5 near-simultaneous 50-element array submissions at that exact moment. The most likely mechanism: ctrl-1, busy with WAL/commit backpressure, delays some response the scheduler's client library is waiting for past its own retry/timeout threshold; XCECLI_FAILOVER_CALL's client-side timeout-and-rotate logic (a deliberate, correct behavior in isolation) then treats this as 'candidate unresponsive' and fails over -- to OTHER, non-primary candidates, which correctly redirect it back, so it rejoins ctrl-1 again shortly after. This is a legitimate committed-backpressure symptom being misread as a connection failure, not memory corruption in itself.\n\nWHY THIS MATTERS FOR THIS BUG: every one of these reconnects fires on_scheduler_join() (sched.c:742) controller-side, which unconditionally re-sends a FULL xcesched_push_jobs() resync of every committed job via the ordinary (non-array) XceData_data_schedjobs wire path -- the EXACT mechanism the 2026-08-30 session's 'redelivery-storm' finding observed (array-job jobids redelivered via the single-job JOBS_PUSH path up to 18x per jobid) but could not explain the source of. This session's finding directly explains it: 7 reconnects in 12s here would scale to easily 18+ over a longer/heavier session. Each full resync constructs an entirely fresh set of job_submit_data_t/xce_job_t clones (decode_JobSubmitData_ptr(), independent heap allocations per PART 1's own prior static-analysis finding) for EVERY committed job, all safely discarded by Bug A's sched_add_job() dedup on the scheduler side -- but this churns a much higher volume of same-shaped job_submit_data_t alloc/free cycles under concurrent capnp-decode pressure than array-job load alone would produce, which is exactly the amplifying condition the 2026-08-29 watchpoint session proved is necessary for the malloc-reuse UAF (a freed job_submit_data_t chunk reused by an unrelated concurrent decode on another thread) to actually get hit.\n\nWHAT STILL IS NOT FOUND: the actual second, stale reader -- i.e. which specific code path retains a pointer to a job_submit_data_t (or its containing xce_job_t) PAST its own correct one-time processing+free, and reads it again later. This session did not have time to also capture a fresh gdb watchpoint trace (PHASE 2 style, 2026-08-29 precedent) to nail that down given how much of the session went into establishing the reconnect-storm causal chain above; the batch gdb run (-ex run -ex 'thread apply all bt full' -ex quit) auto-quit after the crash, so no live interactive follow-up was possible against this specific capture.\n\nNO FIX ATTEMPTED this session (per this entry's own established 'do not attempt a speculative fix without a confirmed root cause' guidance, followed again here) -- the reconnect-storm mechanism is a strong, well-evidenced AMPLIFYING factor, not yet the root cause of the UAF itself.\n\nNEXT STEPS, sharpened and now much more concrete than before:\n(1) The reconnect-storm itself may be its own separate, worthwhile fix regardless of bug-1714: investigate why ctrl-1's commit/registration backpressure under a burst of concurrent array submissions is severe enough to trip the scheduler client's failover timeout even though ctrl-1 never actually loses leadership -- possibly a client-side timeout tuned too aggressively for this load pattern, or a controller-side responsiveness gap under WAL/commit pressure worth its own investigation (cross-reference bug-1713's E_XCE_NOT_READY history). Fixing or tuning this would likely reduce (not necessarily eliminate) bug-1714's own reproduction rate as a side effect, the same way Bug A/B's landing seemed to reduce it, per the 2026-08-30 session's own observation.\n(2) To find the actual stale-pointer read site: repeat this session's exact recipe (now confirmed reliable again, ~12s to crash) but with a scripted gdb watchpoint pass (PHASE 2 precedent, 2026-08-29 session) armed BEFORE the burst starts, on several job_submit_data_t->project_name fields sampled from make_array_element() calls, to catch the actual write-after-free this time under the now-better-understood load conditions.\n(3) Given the crash this session hit event index 16 of a 67-event batch (previous captures: index 2 of 6, and index unspecified in the 2026-08-29 session) -- worth checking whether crash position scales with reconnect-storm-driven batch size (more concurrent JOBS_PUSH full-resync traffic -> larger translate_pending_events batches -> more job_submit_data_t churn per batch -> proportionally higher crash odds within a batch), which would further support the amplification theory quantitatively.\n\nCleanup: isolated build dir (src/xce.build.bug1714repro) and deploy dir (/tmp/xce-deploy-1714) left in place (reusable, no live-process footprint) -- all started processes (3x xce_controller, 1x xcecd_virtual, xce_scheduler + its gdb wrapper) confirmed killed via ps aux before this session ended.\n\nUPDATE (2026-09-04, static-only session, ~30 min): Pursued a new, previously-unchecked lead -- traced full ownership of j->submit->resreq (the raw wire string) through parse_resreq() -> parse_resreq_primary() (resreq.c:307) -> parse_section() (splits select[]/rusage[]/span[]/same[]/affinity[] substrings, MOVE_PTR'd into a local xce_resreq_data_t) -> parse_select() (resreq.c:205, builds the actual xce_select_t AST attached to job->resreq long-term) -> the select-expression PEG parser itself (xce_select.peg/xce_select.c). Hypothesis: if any hop stores a RAW pointer into the short-lived job_submit_data_t/its derived substrings instead of an owned copy, job->resreq (long-lived, read on every subsequent scheduling round) would hold a dangling pointer once job_submit_data_t is freed post-push -- a different, more delayed UAF trigger than the push-time-only theories every prior session chased.\n\nRULED OUT, with exact citations. (1) Every SEL_STR string-literal node in the select AST is makeString()-interned (xstring-refcounted, heap-independent) at the exact grammar action that parses it, BEFORE being stored -- xce_select.peg:161 (\"char* s = makeString(buf);\") and xce_select.peg:261 (\"char* s = makeString((char*)$0);\" -- $0 is the PEG grammar's own captured-token metavariable, not a shell variable), confirmed by makeSelectNode()'s own SEL_STR handling (xce_select.c) explicitly calling dereference()/reference() on op1, i.e. treating it as an xstring, not a raw pointer. (2) Every attribute/resource-name reference (e.g. the `type` in `type == \"x86_64\"`) resolves via resolve_resname() (xce_select.peg:17-55), which does key_find(auxil->resmgr, (void*)name, ...) -- the raw parsed token is used ONLY as a transient lookup key; the node stores the LOOKUP RESULT, a record owned by resmgr itself (long-lived, cluster-wide, unconditionally outlives any single job), never the job's own substring. (3) parse_resreq_primary() (resreq.c:307-349) only frees the intermediate section-split strings (kill_resreq_data(&data), line 346) AFTER parse_select()/parse_rusage() have already synchronously consumed them within the same function call -- no cross-call aliasing window exists there either. Net: job->resreq's entire AST is fully self-contained/independently-owned by the time core_build_job() returns; this lead does not explain bug-1714's crash and should not be re-chased.\n\nNo fix attempted (none needed -- hypothesis ruled out, not confirmed). No live repro attempted this session (static tracing alone was sufficient to close this specific lead; the actual stale-second-reader for j->submit itself, as opposed to any of its parsed derivatives, remains unfound -- see this entry's existing NEXT STEP notes, still the sharpest open leads).\n\nUPDATE (2026-09-04, later same session): a second static lead was checked and RULED OUT -- whether the SAME xce_job_t* object (not just a duplicate jobid via independent clones, already covered by Bug A) could be drained/processed twice via the pend_updates dataq double-buffer swap (driver.c's xce_sched_driver_translate_pending_events(), handler.c's enq_pend_update()/sched_on_jobs(), lib/dataq.c). Traced dataq_enq()/dataq_deq() (mylock.c-style single mutex per queue, dataq.c) and the pend_updates[0]/[1] swap-under-core->lock pattern line by line: dataq_deq_head() unconditionally unlinks the returned item (b->next = NULL, q->head advanced) before returning it, so a given enqueued data_item_t can structurally never be dequeued twice without an intervening fresh dataq_enq() of that same pointer -- and no such re-enqueue site exists (sched_on_jobs()/sched_handle_admin_request() both calloc() a fresh xce_sched_update_request_t per call, enq_pend_update() is their only caller). The swap itself (pend_updates[0] <-> [1]) and the concurrent producer-side enq_pend_update() both hold core->lock, so there is no window for the drain loop to observe a queue the producer is still mutating. This path is clean; does not explain bug-1714.\n\nRunning tally of candidates now RULED OUT across all sessions (do not re-chase without new evidence): (1) XceData_data_schedjobs dataset->rows aliasing (2026-08-30). (2) core_build_schedobj()'s user/queue/project handling -- confirmed makeString() deep copies (2026-09-04). (3) resreq/select-expression string ownership through parse_resreq() -- confirmed fully independent AST, makeString()-interned literals, resmgr-owned resname lookups (2026-09-04). (4) pend_updates dataq double-enqueue/double-drain (this update). Sharpest remaining unexplored angle: the controller-side HOOK_DETAIL_JOB_NEW/xcesched_push_joblist() live trace (2026-08-30's own NEXT STEP, never actually run live) and the connection-flap-driven resync amplification mechanism (found 2026-09-04, this session's earlier fork) -- both point at the CONTROLLER side re-pushing job data the scheduler already has, not at anything in the scheduler's own translate/build/free pipeline (now thoroughly checked and clean). A future session should pursue the controller side next, not re-audit scheduler.c/core_events.c/driver.c string or pointer ownership again.\n\nUPDATE (2026-09-04, controller-side dual-push-path check): a third static lead was checked and RULED OUT before committing to another live gdb session (the main session's own codegraph read already showed the key evidence; independently re-confirmed by re-reading src/controller/sched.c directly) -- whether the controller's two independent job-push paths could race and deliver the same jobid via two concurrent wire messages: (1) on_scheduler_join() (sched.c:742-752), which unconditionally calls xcesched_push_jobs() (sched.c:214, a FULL resync of every currently-committed job via XceData_data_schedjobs/collect_jobs_committed()) on EVERY scheduler join or rejoin -- including a reconnect-triggered rejoin after a connection flap; and (2) sched_job_new_fun() (sched.c:305-337), the XCE_CONTROLLER_HOOK_JOB hook installed exactly once per controller process lifetime (on_scheduler_join()'s own static hook_installed guard), which fires incrementally on every new job commit via xcesched_push_joblist()/xcesched_push_jobarray(). RULED OUT: both functions hold c->sched.lock for their ENTIRE body (mylock_lock(&c->sched.lock) wraps all of on_scheduler_join()'s pushes including xcesched_push_jobs(); the same lock wraps sched_job_new_fun()'s push call too) -- a single mutex held for the full duration of each makes concurrent/interleaved execution of the two paths structurally impossible. No live capture could show otherwise; this is provable by code reading alone.\n\nThis DOES fully explain (without needing a race) the still-unexplained 'redelivery storm' the 2026-08-30 session found (array-job jobids redelivered via the ordinary single-job XceData_data_schedjobs path, up to 18x per jobid, safely discarded by Bug A's sched_add_job() dedup): since the two paths are strictly SEQUENTIAL rather than concurrent, an already-committed job is pushed once via the incremental commit-hook, then pushed AGAIN in full every single time the scheduler's connection flaps and rejoins -- on_scheduler_join() re-pushes the ENTIRE committed job set on every join, not just the first. Combined with this session's earlier finding that the scheduler's connection genuinely flaps under controller-side commit-path backpressure (E_XCE_NOT_READY) during heavy array-job load, a burst with N flaps produces N redundant full resyncs of every already-known job -- sufficient on its own to explain 'up to 18x per jobid' with zero race involved. Expected (if wasteful) behavior, not a crash cause.\n\nRunning tally of candidates now RULED OUT across all sessions (do not re-chase without new evidence): (1) XceData_data_schedjobs dataset->rows aliasing. (2) core_build_schedobj()'s user/queue/project handling. (3) resreq/select-expression string ownership through parse_resreq(). (4) pend_updates dataq double-enqueue/double-drain. (5) controller-side dual job-push-path race (this update) -- serialized by c->sched.lock, and its sequential nature fully explains the redelivery-storm volume without a race. All scheduler-side memory-ownership candidates AND the controller-side push-path race are now exhausted. The actual stale-second-reader for j->submit remains unfound after 9 sessions total. No further cheap/static leads are known at this time -- a future session likely needs either (a) a fresh live gdb-watchpoint campaign specifically instrumenting core->jobs's OWN lifecycle (does anything else hold a raw xce_sched_job_t*/job_submit_data_t* reference outside core->jobs and outside the short window core_build_job() itself runs in? e.g. snapshot/recording code paths, REPL inspection commands, or a picker plugin caching a job reference across sessions) which has NOT been checked by any prior session, or (b) accepting this as a rare, low-frequency-under-extreme-load production risk to monitor rather than a bug worth further dedicated live-debugging investment at this time -- a decision for the project owner, not something to keep re-attempting speculatively.", @@ -23673,7 +10679,6 @@ "found-live", "needs-live-repro", "needs-owner", - "unfixed", "kernel-confirmed", "reproduced-post-fix", "asan-confirmed", @@ -23695,23 +10700,29 @@ "resreq-select-ast-ruled-out-2026-09-04", "dataq-path-ruled-out", "controller-dual-push-race-ruled-out", - "redelivery-storm-fully-explained", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "redelivery-storm-fully-explained" ], "related_bugs": [ "bug-1699", "bug-1731" ], - "occurrences": 8, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1714.md", + "repro_attempts": "0/2 runs, 2026-09-25: documented recipe (3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' = 2400 array elements each) on b63bb581 build.dbg and on d87b78dc-era ASAN build -- no crash, scheduler alive throughout, 0 redelivered array pushes (0 'sched_add_job failed' duplicates; the original repro saw up to 18x redelivery, the recipe's precondition)", + "gitea_issue": 265, + "gitea_synced_status": "cannot-reproduce" }, { "id": "bug-1715", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "crash", "timestamp": "2026-08-26T18:45:00Z", - "error_message": "User-reported, live 3-node cluster: `xceadmin scheduler record status --controller localhost:9901` failed once with `Error: scheduler disconnected`, then succeeded on the very next identical invocation moments later. Root-caused to a full cluster redeploy (user confirmed: 'This is a full rebuilt and re-deployed cluster') restarting all 5 processes (3 controllers + worker + scheduler) together -- PIDs across ctrl-1/ctrl-2/ctrl-3/scheduler/worker logs are all tightly clustered (4137667/4137684/4137696/4137745/4138046), confirming a coordinated restart, not an isolated scheduler crash. The request happened to land in the brief, normal window between the old scheduler disconnecting and the new one reconnecting.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported, live 3-node cluster: `xceadmin scheduler record status --controller localhost:9901`…\n\nUser-reported, live 3-node cluster: `xceadmin scheduler record status --controller localhost:9901` failed once with `Error: scheduler disconnected`, then succeeded on the very next identical invocation moments later. Root-caused to a full cluster redeploy (user confirmed: 'This is a full rebuilt and re-deployed cluster') restarting all 5 processes (3 controllers + worker + scheduler) together -- PIDs across ctrl-1/ctrl-2/ctrl-3/scheduler/worker logs are all tightly clustered (4137667/4137684/4137696/4137745/4138046), confirming a coordinated restart, not an isolated scheduler crash. The request happened to land in the brief, normal window between the old scheduler disconnecting and the new one reconnecting.", "file": "src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request)", "root_cause": "FULLY ROOT-CAUSED, same family as bug-1713 but NOT covered by that fix. 'scheduler disconnected' is delivered via a DIFFERENT signaling path than bug-1713's two numeric status codes: cluster.c's bug-1693 disconnect-detection mechanism (cluster.c:1052-1067) sends `reply.status = 0` (success at the RPC level) with `schedadminreplymsg{ok=false, error=\"scheduler disconnected\"}` -- a normal-status reply carrying a domain-level failure, not a nonzero `reply.status` the way E_XCE_NOT_READY/E_XCE_NOMASTER are. `_send_admin_request()` (scheduler.py:145-154) treats `admin_reply.ok == False` as an unconditional hard SchedAdminError for EVERY reason string, with no distinction between genuinely transient conditions ('scheduler disconnected', 'scheduler not connected', 'request timeout' -- all connection/timing related, expected to self-resolve within seconds, especially during a redeploy/restart cycle) and genuinely permanent domain errors ('not recording', 'already recording', 'invalid record file name' -- real error states that must NOT be retried, e.g. calling record-stop on an already-stopped recorder is a correct, meaningful rejection, not a transient hiccup).", "fix": "FIXED and LIVE-VERIFIED (2026-08-26). Extended bug-1713's retry loop in `_send_admin_request()` (src/xceadmin/src/xceadmin/scheduler.py:158-249) to a SECOND, differently-signaled transient family: after decoding `admin_reply = reply.data.schedadminreplymsg` and finding `admin_reply.ok == False`, the code now checks `admin_reply.error in _TRANSIENT_ADMIN_REASONS` (new module-level frozenset, scheduler.py:114-160) and, if so and `attempt < _MAX_TRANSIENT_ATTEMPTS`, sleeps `_TRANSIENT_RETRY_DELAY_S` and `continue`s the SAME `for attempt in range(1, _MAX_TRANSIENT_ATTEMPTS+1)` loop bug-1713 already added -- no second retry loop, same budget (4 attempts, 1.5s fixed delay), same re-run-`follow_redirects()`-from-scratch shape (a fresh attempt can discover a newly-reconnected scheduler, exactly like a fresh attempt can discover a newly-settled primary in bug-1713's case). On exhaustion, raises the same `SchedAdminError` shape with the reason string annotated '(still transient after N attempts over ~Xs -- giving up)', mirroring bug-1713's own distinguishable-message convention exactly. AUTHORITATIVE error-string list (grepped every literal `sched_admin_reply_t`/`SchedAdminReply` `.error = \"...\"` assignment across src/controller/ and src/scheduler/, then read each site's surrounding code, not just the string text): transient (all 4 now in `_TRANSIENT_ADMIN_REASONS`) -- \"scheduler disconnected\" (cluster.c:1062, bug-1693's disconnect-detection path), \"scheduler not connected\" (cluster.c:958, `c->sched.ep==NULL`), \"request timeout\" (cluster.c:1078, SCHEDADMIN_TIMEOUT_MS wait expired), and \"server busy, try again\" (cluster.c:937, `vtaskmgr_blocking_acquire()` admission-reservation exhaustion -- a 4th transient reason NOT in this bug's original three-string hypothesis, added after reading the admission-gate's own 'try again' design comment: resource contention that self-resolves as soon as another admitted vtask op completes, not a domain rejection). Permanent (left untouched, still raise on the very first attempt): \"already recording\"/\"not recording\" (driver.c:390/426, rc==-EALREADY) and validation_error_text()'s six filename/path-rejection strings (driver.c:298-316: \"invalid record file name\", \"record directory not found\", \"record directory path is not a directory\", \"permission denied accessing record file\", \"a non-file entry with this name already exists\", \"record file path too long\"), plus a dynamic (not a fixed literal, so it could never join an exact-match retry set anyway) \"cannot write to %s: %s\" I/O-failure message (driver.c:398-404). RED/GREEN proof (src/xceadmin/tests/test_scheduler.py): confirmed RED by temporarily reverting only the two bug-1715 edits back to bug-1713's exact shipped state (not via git stash, since bug-1713's own fix was itself still an uncommitted working-tree change sharing the same file -- a bare `git stash` would have reverted bug-1713 too) -- the new parametrized `test_record_status_retries_on_transient_admin_reason_then_succeeds`/`..._retry_exhaustion_on_transient_admin_reason_...` tests failed at COLLECTION time (`AttributeError: module 'xceadmin.scheduler' has no attribute '_TRANSIENT_ADMIN_REASONS'`), and a standalone ad hoc script against that same pre-fix code confirmed BEHAVIORAL red too: a real TLS mock returning `ok=false, error='scheduler disconnected'` once raised `SchedAdminError` immediately after exactly 1 connection, no retry. Reapplied the fix -> GREEN: all 41 tests in test_scheduler.py pass, including 4 new `_TRANSIENT_ADMIN_REASONS`-parametrized retry-then-succeed tests, 4 new retry-exhaustion tests (one per confirmed-transient reason, each asserting exact connection_count == _MAX_TRANSIENT_ATTEMPTS and a distinguishable exhaustion message), and a new `test_record_stop_not_recording_never_retried` non-regression test (asserts connection_count==1, zero sleep calls, for the permanent 'not recording' reason). The pre-existing `test_record_start_error_codes_raise_sched_admin_error` parametrize list was edited (not left untouched): \"scheduler not connected\"/\"request timeout\" were removed (they are now confirmed-transient and retried, not raised immediately -- their coverage moved to the new retry tests) and replaced with two more confirmed-permanent validation_error_text() strings (\"record directory not found\", \"permission denied accessing record file\") alongside the original \"invalid record file name\", for stronger permanent-error coverage. bug-1713's own 5 tests (transient-status retry/exhaustion, both E_XCE_STANDBY regression locks) were run UNMODIFIED and all still pass -- no shared-loop refactor was needed since bug-1713's `for attempt in range(...)` loop and its `reply.status != 0` branch were reused as-is; only a second `if`/retry check was added inside the existing `if not admin_reply.ok:` branch. Full xceadmin suite: 283 passed, 1 failed -- the failure (`test_live_config_push_against_standby_returns_standby_rejection_not_redirect`) is the SAME pre-existing, already-flagged-stale failure named in bug-1713's own report (test_e2e_live_ha_push.py, stale since bug-1700/1701's redirect-gap fix generalized to OPCode_upgradeConfig) -- zero new failures caused by this fix (284 total tests vs bug-1713's reported 275, consistent with the 9 new tests added here). LIVE VERIFICATION (real 3-node cluster, deploy-ha.sh + start-all.sh, build.dbg's real xce_controller/xce_scheduler/xcecd_virtual binaries -- no C rebuild needed since this fix is Python-only and none of the C-side cluster.c/driver.c code changed): hammered `xceadmin.scheduler.record_status('localhost:9991')` (the elected primary) in a tight zero-delay loop against the fixed client for 20s while concurrently `kill -9`-ing the live xce_scheduler process mid-run and restarting it via `start-scheduler.sh` (a genuine scheduler-process-death-and-restart, not a mock, not a controller failover). Result: 153 of 154 requests during the window succeeded, most transparently absorbing the scheduler-down period with zero visible failure to the caller; ONE request genuinely hit the real 'scheduler not connected' condition, went through the full retry budget (logged exactly as designed: 'scheduler not connected (still transient after 4 attempts over ~4.5s -- giving up)'), and still surfaced a SchedAdminError because this particular sandbox's scheduler restart (relaunched via a fresh `nix develop` subshell each time, which carries its own multi-second evaluation overhead on top of the scheduler's own re-join handshake) took longer to fully rejoin than the shared 4-attempt/1.5s (~4.5s total) budget -- the very next request after that succeeded again. This is honestly reported as PARTIAL absorption, not a zero-failure live result like bug-1713's controller-failover scenario got: the retry mechanism is proven to engage correctly on a genuine, non-mocked signal end-to-end (real controller, real scheduler restart, real wire reply, real client-side retry/give-up message), and it converted what would have been 100% visible failure for the whole outage window (pre-fix) into 1 failure out of 154 requests (post-fix) -- but this sandbox's nix-develop-relaunch overhead for the scheduler restart is likely slower than a real production supervisor restart would be, so the single exhaustion here should not be read as proof the shared 4.5s budget is too short in general. Kept the identical bug-1713 budget as instructed (no concrete evidence specific to THIS class of condition that it's wrong -- the one exhaustion observed is plausibly a sandbox-restart-mechanism artifact, not a signal about the real-world scheduler-restart timescale) rather than inventing a different cadence from one ambiguous data point. `push.py`'s `push_config()`/`probe_config()` gap flagged by bug-1713 as a sibling finding remains unfixed and out of this bug's scope (unchanged from bug-1713's report).", @@ -23726,22 +10737,26 @@ "found-live", "root-caused", "fix-scoped", - "fixed", "live-verified" ], "related_bugs": [ "bug-1713", "bug-1693" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1715.md" }, { "id": "bug-1716", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T14:25:00Z", - "error_message": "test_rcfile_load_directory_path_warns_and_returns_0:FAIL: Expected TRUE Was FALSE (stderr capture was empty -- no warning printed) when rcfile_load() opened a directory path", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_rcfile_load_directory_path_warns_and_returns_0:FAIL: Expected TRUE Was FALSE (stderr capture…\n\ntest_rcfile_load_directory_path_warns_and_returns_0:FAIL: Expected TRUE Was FALSE (stderr capture was empty -- no warning printed) when rcfile_load() opened a directory path", "file": "src/controller/testsuite/rcfile.c", "root_cause": "On this platform (glibc/Linux), fopen(path, \"r\") SUCCEEDS for a directory path -- it does not return NULL/set errno the way the design doc's error-handling section (and a naive reading of 'the file could not be opened for another reason') implied. The open failure only surfaces on the FIRST READ: getline() returns -1 with errno==EISDIR and ferror(f) set (feof(f) stays 0). rcfile_load()'s original implementation only checked `if (!f)` right after fopen() and had no post-loop check on the stream's error state, so a directory path silently produced rc=0 with zero lines processed and NO warning ever printed -- functionally 'safe' (no crash, no runner call) but silently violating the design's documented 'present but unreadable: print one stderr warning' contract for exactly this case.", "fix": "Added a check after the getline() loop in rcfile_load(): `if (rc == 0 && ferror(f)) { fprintf(stderr, ...); }` -- prints the same warning message used for the fopen()-failure branch, keyed off ferror(f) rather than assuming the failure is only detectable at fopen() time. Confirmed live via a small standalone C repro (fopen(\"/tmp\",\"r\") then fread/getline) that fopen() returns a valid FILE* for a directory and the read call is what fails with EISDIR + ferror set.", @@ -23753,20 +10768,23 @@ "getline", "xce-test-rc-file", "controller-testsuite", - "stderr-warning", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "stderr-warning" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1716.md" }, { "id": "bug-1717", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-08-26T00:00:00Z", - "error_message": "ASAN heap-use-after-free abort in controller_unit_test: commit_resolve_pending (commit.c:807) racing fsync_worker (persist.c:867) against tap_teardown (test_apply_pump.c:243)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ASAN heap-use-after-free abort in controller_unit_test: commit_resolve_pending (commit.c:807)…\n\nASAN heap-use-after-free abort in controller_unit_test: commit_resolve_pending (commit.c:807) racing fsync_worker (persist.c:867) against tap_teardown (test_apply_pump.c:243)", "file": "src/controller/commit.c", "root_cause": "Background fsync_worker thread appears to still be resolving a pending commit via commit_resolve_pending() while a test teardown (tap_teardown in test_apply_pump.c) is freeing/tearing down state concurrently -- a shutdown-ordering race between the async fsync/commit-resolution thread and per-test teardown. Observed once during xce-test-rc-file task 1.2 review (ad-hoc standalone controller_unit_test run); did NOT reproduce on a second standalone run or in the canonical xce-build.sh run (both completed cleanly with 1213 Tests 2 Failures 0 Ignored, the 2 failures being the unrelated tracked bug-1561). Not caused by and unrelated to rcfile.c/test_rcfile.c (xce-test-rc-file spec) -- flagged as a pre-existing flake surfaced incidentally during that task-local review.", "fix": "Not yet root-caused or fixed -- logged for tracking since it did not reproduce reliably enough to investigate further in that review pass. Next occurrence should capture the full ASAN report (not just the top frames) and check whether test_apply_pump.c's teardown waits for fsync_worker to fully quiesce before freeing shared state.", @@ -23780,20 +10798,23 @@ "fsync_worker", "test_apply_pump.c", "teardown-race", - "unreproduced", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "unreproduced" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1717.md" }, { "id": "bug-1718", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T16:23:00Z", - "error_message": "A python3 pty-driver embedded via a heredoc on a command that ALSO receives piped stdin silently discards the piped input", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "A python3 pty-driver embedded via a heredoc on a command that ALSO receives piped stdin silently…\n\nA python3 pty-driver embedded via a heredoc on a command that ALSO receives piped stdin silently discards the piped input", "file": "src/controller/testsuite/test_rcfile_integration.sh", "root_cause": "First draft of run_interactive_pty() did: printf '%s\\n' \"$script\" | python3 - \"$XCE_TEST\" ... <\"$PTY_DRIVER\" < does not consume stdin to obtain its own source.", @@ -23804,20 +10825,23 @@ "heredoc", "stdin", "xce-test-rc-file", - "task-3.1", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-3.1" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1718.md" }, { "id": "bug-1719", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-26T17:14:00Z", - "error_message": "resource-value-verification skill's deployment failed: 'controller binary missing' immediately after a scheduler-only xce-build.sh run that had just succeeded", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "resource-value-verification skill's deployment failed: 'controller binary missing' immediately…\n\nresource-value-verification skill's deployment failed: 'controller binary missing' immediately after a scheduler-only xce-build.sh run that had just succeeded", "file": "src/tools/xce-build.sh", "root_cause": "During xce-test-rc-file task 3.3 (compatibility verification, no product-code change), ran `nix develop ./src --command bash src/tools/xce-build.sh scheduler` (no --incremental) to fill a gap where xce.build/scheduler/xce_scheduler had never been linked. xce-build.sh's documented default behavior (see its own top-of-file comment) does a full `rm -rf \"$BUILD_DIR\"` + fresh `meson setup` on every invocation unless --incremental is passed, then `meson compile` only rebuilds the ONE requested component -- so the plain `scheduler` invocation silently wiped xce.build/controller/xce_controller and xce.build/controller/xce_test (built moments earlier by task 3.1/3.2's controller build, containing the rc-file feature under test) even though I never touched controller source. A prior attempt in the same session had additionally passed an explicit build_dir arg (`src/xce.build`) which -- combined with xce-build.sh's own internal cd into src/ -- produced a bogus nested `src/src/xce.build` directory instead of reusing the real one (harmless, fully gitignored via meson's own auto-generated per-builddir `.gitignore: *`, but wasted a build cycle and caused brief confusion investigating why `git status`/`git check-ignore` showed nothing for it).", "fix": "Rebuilt controller incrementally (`xce-build.sh controller asan xce.build --incremental`) to restore xce_controller/xce_test alongside the already-built scheduler, without wiping it back out. Confirmed both controller_unit_test (1213 Tests 0 Failures -- even bug-1561's usually-flaky 2 tests passed clean this run) and the scheduler suite (53/54, only the pre-existing tracked bug-1560 symlink-escape failure) came back healthy. General lesson for any task that needs a SECOND component built into an ALREADY-built xce.build dir: always pass --incremental, and never pass an explicit build_dir positional unless it already matches xce-build.sh's own default relative-path resolution (omit the 3rd arg and let it default to 'xce.build' resolved from src/tools/xce-build.sh's own cwd, don't hand it a repo-root-relative 'src/xce.build' path).", @@ -23828,20 +10852,23 @@ "scheduler", "controller", "xce-test-rc-file", - "task-3.3", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-3.3" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1719.md" }, { "id": "bug-1720", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-08-26T17:17:00Z", - "error_message": "xce-test-resources run-resource-verify.sh: apiserver layer UNREACHABLE for all 9 injected resources; apiserver.log showed 'API Server fails to start: listen tcp :8080: bind: address already in use'", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-test-resources run-resource-verify.sh: apiserver layer UNREACHABLE for all 9 injected…\n\nxce-test-resources run-resource-verify.sh: apiserver layer UNREACHABLE for all 9 injected resources; apiserver.log showed 'API Server fails to start: listen tcp :8080: bind: address already in use'", "file": "environment (stale orphaned process, not a code defect)", "root_cause": "A leftover /tmp/apiserver process (pid 1458981, launched via /tmp/xce-deploy-bug1701/scripts/start-apiserver.sh) from an unrelated, already-concluded prior session's bug-1701 investigation was still running and bound to :8080 -- 7.5 hours old (started 09:44, found at 17:18) with its own deploy directory (/tmp/xce-deploy-bug1701) already deleted, i.e. genuinely abandoned rather than an active concurrent session. This is a general sandbox hygiene gap: xce-test-resources' own trap-based cleanup only covers processes IT started: nothing in this sandbox periodically reaps orphaned daemons left behind by an unrelated script/session that itself never got to run its own cleanup (e.g. was killed externally, or its subagent's context ended before the trap fired).", "fix": "Identified via `lsof -i :8080` / `ss -ltnp`, confirmed genuinely orphaned (dead deploy dir, long uptime, no relation to this session's own work), then `kill -9` on both the wrapper script and the apiserver binary PID. Re-ran run-resource-verify.sh immediately after and got a clean PASS (9 resources, 3 layers, zero mismatches/unreachable/rejected). General lesson: an apiserver-layer UNREACHABLE result in this skill should prompt checking `lsof -i :8080` / `pgrep -af apiserver` for a stale bind-conflict BEFORE suspecting a propagation regression -- the skill's own FAILURE/UNREACHABLE line doesn't distinguish 'never started due to port conflict' from 'started but never logged the trace line', both surface identically as UNREACHABLE.", @@ -23852,21 +10879,23 @@ "orphaned-process", "sandbox-hygiene", "xce-test-rc-file", - "task-3.3", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-3.3" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1720.md" }, { "id": "bug-1721", "status": "open", "status_updated": "2026-09-24", "status_note": "clean_failover_preservation.sh default ha/data dir no longer exists", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-26T00:00:00Z", - "error_message": "clean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "clean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite)\n\nclean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite)", "file": "src/controller/testsuite/clean_failover_preservation.sh", "root_cause": "The script hardcodes its DATADIR precondition default to $REPO_ROOT/ha/data (a pre-generated cluster.bin + TLS cert template directory it expects to already exist), but ha/data/ was deliberately deleted from the repo by an unrelated commit (ee8473bd, admin-config-tool: drop ha/data/ entirely, match real client TLS). The script was never updated afterward, so its own documented precondition check now fails unconditionally in any fresh checkout/worktree that has not manually recreated that directory out-of-band. Discovered while adding --norc to the script (xce-test-rc-file spec follow-up) and independently verifying the edit did not break anything -- confirmed via git history that the failure reproduces identically on the pre-edit script too, so it is unrelated to that change.", "fix": "Not fixed -- out of scope for the xce-test-rc-file spec. Needs either: (a) DATADIR default updated to point at a still-valid TLS/config template location consistent with admin-config-tool's TLS changes, or (b) the script updated to self-generate its own template via deploy-ha.sh/xce-deploy-ha before the precondition check, matching how run-ha-soak.sh and run-resource-verify.sh already self-deploy their own scratch clusters instead of depending on a fixed pre-existing directory.", @@ -23876,20 +10905,23 @@ "clean_failover_preservation.sh", "ha-term-safety", "pre-existing", - "admin-config-tool-regression", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "admin-config-tool-regression" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1721.md" }, { "id": "bug-1722", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-08-27T00:00:00Z", - "error_message": "AddressSanitizer heap-buffer-overflow in make_job_account() (src/lib/xcealloc.c:188), WRITE of size 8 -- found while verifying viability of a proposed scheduler snapshot-allocation-capture fix, not user-reported.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer heap-buffer-overflow in make_job_account() (src/lib/xcealloc.c:188), WRITE of…\n\nAddressSanitizer heap-buffer-overflow in make_job_account() (src/lib/xcealloc.c:188), WRITE of size 8 -- found while verifying viability of a proposed scheduler snapshot-allocation-capture fix, not user-reported.", "file": "src/lib/xcealloc.c", "root_cause": "make_job_account() allocates account->accounts via calloc(1, sizeof(xce_host_account_t*)) -- room for exactly 1 host pointer -- but its own loop writes account->accounts[i] for i in [0, jalloc->nhosts), overflowing the heap allocation for any job spanning more than one host. Already reachable from live production dispatch (core_events.c:262, schedobj.c:207, scheduler.c:1444) for any multi-host job, not introduced by this session. Sibling function make_host_allocs() (xcealloc.c:134) has the identical bug shape (calloc(1, ...) vs a loop writing hallocs[i] up to nhosts-1), though every current call site happens to pass nhosts=1 so it has never actually overflowed in production -- still a live footgun for any future 2+-host caller of a public, exported helper.", "fix": "FIXED, TDD RED->GREEN. New test_xcealloc.c (src/lib/testsuite/, wired into libxce_test_src in meson.build + test.c) drives a 2-host xce_job_alloc_t through make_job_account(), reproducing the exact ASAN heap-buffer-overflow at xcealloc.c:188 pre-fix. Fix: both calloc(1, ...) call sites changed to calloc((size_t)nhosts, ...) / calloc((size_t)jalloc->nhosts, ...) -- xcealloc.c:134 (make_host_allocs) and xcealloc.c:180-181 (make_job_account). Also added a covering test for make_host_allocs multi-host correctness. Verified: full libxce test suite 303/303 (was 301, +2 new), scheduler suite 53/54 (1 pre-existing unrelated bug-1560 flake, test_recordstart_symlink_escape_replies_error), controller suite 1211/1213 (2 pre-existing unrelated bug-1561 overlayfs flakes, test_conf_trim_keeps_only_n_versions/test_conf_get_saved_tag_no_config) -- zero regressions, zero new failures.", @@ -23900,22 +10932,25 @@ "multi-host-job", "make_job_account", "make_host_allocs", - "pre-existing", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pre-existing" ], "related_bugs": [ "bug-1621" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1722.md" }, { "id": "bug-1723", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-08-27T00:00:00Z", - "error_message": "LeakSanitizer (xcelib_test's ASAN default detect_leaks=1, per bug-1437's established precedent that this is a separate pass/fail axis from Unity PASS/FAIL) aborts the whole xcelib_test binary at exit with SIGABRT even on a completely clean run (all Unity tests, including the new scheduler-snapshot-job-allocation task 1.1 dup_job_alloc() suite, report 0 Failures) -- 456 byte(s) leaked in 29 allocation(s), confirmed present identically at HEAD before task 1.1's changes (baseline A/B comparison: stashed task 1.1's 3 changed files, rebuilt, reran standalone binary -- same 456 bytes/29 allocations, same two leak sources, same SIGABRT).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "LeakSanitizer (xcelib_test's ASAN default detect_leaks=1, per bug-1437's established precedent…\n\nLeakSanitizer (xcelib_test's ASAN default detect_leaks=1, per bug-1437's established precedent that this is a separate pass/fail axis from Unity PASS/FAIL) aborts the whole xcelib_test binary at exit with SIGABRT even on a completely clean run (all Unity tests, including the new scheduler-snapshot-job-allocation task 1.1 dup_job_alloc() suite, report 0 Failures) -- 456 byte(s) leaked in 29 allocation(s), confirmed present identically at HEAD before task 1.1's changes (baseline A/B comparison: stashed task 1.1's 3 changed files, rebuilt, reran standalone binary -- same 456 bytes/29 allocations, same two leak sources, same SIGABRT).", "file": "src/lib/testsuite/test_xcealloc.c", "root_cause": "test_make_job_account_multi_host_no_overflow() (added in bug-1722's fix) calls free_job_account(account) but never frees its own directly-built fixture tree (jalloc/hallocs/h0/h1/a0/a1 -- make_job_alloc()'s calloc at xcealloc.c:156 and the two make_alloc()-internal strdup()s it roots). test_make_host_allocs_multi_host_no_overflow() (same commit) frees nothing at all (make_host_allocs()'s calloc at xcealloc.c:134 and its two hosts/allocs). Neither test calls free_JobAlloc_ptr() or any other teardown on its own locally-constructed xce_job_alloc_t/xce_host_alloc_t** tree. Pre-existing since bug-1722's commit (00207503), not introduced by task 1.1 -- confirmed via git-stash A/B rebuild showing byte-for-byte identical leak signature with task 1.1's 3 files reverted.", "fix": "FIXED (2026-09-08, follow-up session). Applied exactly the fix this entry already specified: added free_JobAlloc_ptr(&jalloc); at the end of test_make_job_account_multi_host_no_overflow() (frees jalloc/hallocs/h0/h1/a0/a1's whole tree, after free_job_account(account) already frees the account side) and free_HostAlloc_list(2, hallocs); at the end of test_make_host_allocs_multi_host_no_overflow() (which previously freed nothing at all). This reduced the leak from 456 bytes/29 allocations to 12 bytes/2 allocations -- the residual was a DIFFERENT, deeper pre-existing production bug (free_host_account() never freeing acct->host), already independently logged as bug-1621 and fixed in the same pass; see that entry. Verified: standalone xcelib_test binary now 1/1 Ok with zero LeakSanitizer reports (was: SIGABRT on every run, even with all Unity tests passing); full xce-build.sh all asan --incremental: 94/94 Ok (controller/scheduler/lib/worker/apiserver C suites + all Go test packages), zero regressions.", @@ -23927,23 +10962,27 @@ "test-fixture-teardown", "scheduler-snapshot-job-allocation", "pre-existing", - "out-of-scope", - "fixed" + "out-of-scope" ], "related_bugs": [ "bug-1722", "bug-1621", "bug-1437" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1723.md" }, { "id": "bug-1724", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-08-28T00:00:00Z", - "error_message": "3 xceadmin live E2E tests (test_e2e_live_push.py::test_live_config_push_against_real_single_node_controller, test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path, test_e2e_live_ha_push.py::test_live_config_push_against_standby_returns_standby_rejection_not_redirect) always SKIPPED under the meson-driven build path (`/xce-build xceadmin`, `nix build .#xcecore`) with 'no xce_controller binary found under xce.build/build.release/build.dbg', regardless of whether a real xce_controller/xceadmin binary actually existed anywhere in the checkout.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "3 xceadmin live E2E tests…\n\n3 xceadmin live E2E tests (test_e2e_live_push.py::test_live_config_push_against_real_single_node_controller, test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path, test_e2e_live_ha_push.py::test_live_config_push_against_standby_returns_standby_rejection_not_redirect) always SKIPPED under the meson-driven build path (`/xce-build xceadmin`, `nix build .#xcecore`) with 'no xce_controller binary found under xce.build/build.release/build.dbg', regardless of whether a real xce_controller/xceadmin binary actually existed anywhere in the checkout.", "file": "src/xceadmin/meson-build.sh", "root_cause": "tests/_live_e2e_common.py's CONTROLLER_CANDIDATES/XCEADMIN_CANDIDATES/DEPLOY_HA_SH resolve paths relative to the test file's own on-disk location (Path(__file__).resolve().parents[2]). That works when pytest runs directly from the real checkout, but meson-build.sh copies the whole xceadmin test tree into an isolated out-of-tree repo-mirror before invoking pytest (to keep the meson build pure), and only ever symlinked ha/, src/configs, src/schema into that mirror -- never the sibling xce.build/build.release/build.dbg controller-binary directories, nor src/tools/ (deploy-ha.sh). So the mirror's own SRC_DIR (repo-mirror/src/) never had a controller subdir, a deploy-ha.sh, or a matching xceadmin binary at the expected relative path, no matter what was actually built in the real repo -- the skip was structural, not a signal about the real build state.", "fix": "Two changes to meson-build.sh: (1) symlink each profile's `controller` subdirectory (xce.build/controller, build.release/controller, build.dbg/controller -- only the controller subdir, never the whole profile-root, since the profile-root can be an ancestor of the mirror itself when xceadmin is built in that same profile, which would create a self-referential symlink loop) into the mirror so CONTROLLER_CANDIDATES can find a real xce_controller wherever it was built; (2) symlink src/tools (checked-in, not a build-output dir, so a plain safe symlink like configs/schema) so DEPLOY_HA_SH resolves. For xceadmin's OWN binary, which genuinely is self-referential (the mirror lives inside that very build's own xceadmin output), routed around it via the existing XCEADMIN_BIN env-var override (resolve_binary_candidates in _live_e2e_common.py), set to $VENV_DIR/bin/xceadmin -- the binary this very meson-build.sh invocation just pip-installed -- before invoking pytest. Verified end-to-end via a real `xce-build.sh xceadmin debug build.dbg --incremental` run: all 3 tests now execute for real against a live controller/cluster instead of skipping.", @@ -23957,19 +10996,23 @@ "build-infra", "found-live", "root-caused", - "fixed", "live-verified" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1724.md" }, { "id": "bug-1725", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-08-28T00:00:00Z", - "error_message": "After bug-1724's fix made test_live_config_push_against_standby_returns_standby_rejection_not_redirect actually execute (previously always skipped), it FAILED for real: pushing config to a standby now redirects to the primary and succeeds, contradicting the test's own hard-coded 'standby flatly rejects, no redirect' assertions. Separately, the rewritten success-path test's reused _assert_primary_committed() helper intermittently failed its 4th assertion ('config activated from committed marker tag=...' not found in the primary's log immediately after the push CLI returned success).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "After bug-1724's fix made…\n\nAfter bug-1724's fix made test_live_config_push_against_standby_returns_standby_rejection_not_redirect actually execute (previously always skipped), it FAILED for real: pushing config to a standby now redirects to the primary and succeeds, contradicting the test's own hard-coded 'standby flatly rejects, no redirect' assertions. Separately, the rewritten success-path test's reused _assert_primary_committed() helper intermittently failed its 4th assertion ('config activated from committed marker tag=...' not found in the primary's log immediately after the push CLI returned success).", "file": "src/xceadmin/tests/test_e2e_live_ha_push.py", "root_cause": "(1) The test's assumption was stale, not wrong test infra: bug-1700/1701 (fixed 2026-08-26) closed the exact standby-redirect gap this test was written to regression-lock, at the generic ha_dispatch_gate() rejection level in cluster.c -- the test's own docstring had explicitly predicted this exact failure as the correct signal to replace it with a success-path assertion. design.md ('Boundary Commitments'/'Revalidation Triggers') and roadmap.md still described the gap as open. (2) config_activate_committed() on the primary runs off the WAL apply-pump dispatch (persist.c ~5111, the bug-1469 fix), asynchronously relative to the majority-ack-gated client reply (commitIndex vs lastApplied are distinct steps) -- the CLI can legitimately return success before the primary's own local apply of that marker has logged its activation line, a latent race in the existing single-shot log read shared by the direct-to-primary test too.", "fix": "Renamed and rewrote the test as test_live_config_push_against_standby_redirects_and_succeeds, asserting the redirect line IS present and the push succeeds (same success bar as the direct-to-primary test: positive ackedCount, exit 0, primary commit evidence). Updated the module docstring, design.md (Boundary Commitments, Revalidation Triggers, sequence-diagram note, redirect-implementation rationale, Testing Strategy), and roadmap.md's corresponding item to describe the gap as closed instead of open -- also corrected an adjacent, independently-verified stale claim (bug-1468, fixed 2026-07-25, already resolved but never reflected in design.md) that no controller role could reach the upgradeConfig handler at all. For the activation race, changed _assert_primary_committed() from a single-shot log_text argument to a Path-based helper that polls (up to 5s, 0.2s interval) for the activation line instead of assuming synchronous ordering, and updated both call sites. Verified end-to-end: full xce-build.sh xceadmin run now passes 288/288 with zero skips/failures.", @@ -23985,7 +11028,6 @@ "config_activate_committed", "found-live", "root-caused", - "fixed", "live-verified" ], "related_bugs": [ @@ -23994,17 +11036,20 @@ "bug-1701", "bug-1468" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1725.md" }, { "id": "bug-1726", "status": "cannot-reproduce", "status_updated": "2026-09-24", "status_note": "single occurrence, no surviving logs", - "repro_attempts": "5 immediate retries, 0 reproduced", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-08-28T08:15:00Z", - "error_message": "test_e2e_live_ha_push.py::test_live_config_push_against_standby_redirects_and_succeeds ERRORed at live_ha_cluster fixture setup (single occurrence, user-run via `nix develop ./src` + direct pytest against the built build.dbg mirror, not via any Claude-driven build): _wait_for_primary(nodes, 30.0) raised TimeoutError -- none of the 3 freshly-spawned xce_controller processes ever printed the CTRL_PRIMARY election-complete marker within the 30s timeout (normal election completes ~10.2s per this file's own docstring, keyed to HA_DEFAULT_HEARTBEAT_TIMEOUT_MS=10000).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_e2e_live_ha_push.py::test_live_config_push_against_standby_redirects_and_succeeds ERRORed at…\n\ntest_e2e_live_ha_push.py::test_live_config_push_against_standby_redirects_and_succeeds ERRORed at live_ha_cluster fixture setup (single occurrence, user-run via `nix develop ./src` + direct pytest against the built build.dbg mirror, not via any Claude-driven build): _wait_for_primary(nodes, 30.0) raised TimeoutError -- none of the 3 freshly-spawned xce_controller processes ever printed the CTRL_PRIMARY election-complete marker within the 30s timeout (normal election completes ~10.2s per this file's own docstring, keyed to HA_DEFAULT_HEARTBEAT_TIMEOUT_MS=10000).", "file": "src/xceadmin/tests/test_e2e_live_ha_push.py (live_ha_cluster fixture, _wait_for_primary)", "root_cause": "NOT ISOLATED -- deliberately distinct from bug-1485 (do not merge into it): bug-1485's signature is election COMPLETING (all 3 markers print) then the freshly-promoted primary going /unreachable shortly after; this occurrence is election never visibly settling AT ALL within 3x the normal budget, a different symptom with a different plausible cause. XCE_KEEP_DEPLOY_DIR was not set for this run, so the 3 nodes' stdout/stderr logs from the actual failure were deleted by the fixture's own cleanup before this could be forensically investigated -- no diagnostic evidence survives from this specific occurrence. Immediate reruns (5/5, by Claude directly after the report, same build.dbg binaries, same test, quiet system: load avg ~0.6-1.5/22 cores, no concurrent xce_controller/meson/nix processes found) all passed cleanly in ~14-15s each, well inside the 30s budget -- consistent with (but not proof of) transient CPU/IO contention from another concurrent session on this shared multi-agent sandbox (the established pattern documented at length in bug-1486 and this same file's bug-1485 investigation) delaying all 3 nodes' heartbeat timers past the window, rather than a deterministic protocol liveness bug. Not ruled out: a genuine, rare election-liveness edge case (e.g. a PreVote/RequestVote split-vote condition) that happens not to leave a marker before some other transient condition times it out.", "fix": "NOT FIXED -- single occurrence, unreproduced on 5 immediate retries, no surviving log evidence to root-cause further. Logged per this project's own established convention (bug-1485) of tracking an inconclusive live-cluster-election flake rather than silently dismissing it. Next step if this recurs: re-run with XCE_KEEP_DEPLOY_DIR=1 (already supported by the fixture, added for exactly this class of investigation per bug-1485's own remediation) to preserve deploy_dir/logs/*.log and *.stderr.log for real forensic inspection instead of losing them to the fixture's own cleanup.", @@ -24017,30 +11062,33 @@ "unreproduced", "low-priority", "distinct-from-bug-1485", - "found-live", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "found-live" ], "related_bugs": [ "bug-1485", "bug-1486" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1726.md", + "repro_attempts": "5 immediate retries, 0 reproduced" }, { "id": "bug-1727", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-30T22:00:00Z", - "error_message": "User-reported (live REPL, real cluster-captured replay /tmp/scheduler-replay-20260828-085356.replay, 40 sessions, 60 jobs): after `xce_scheduler -i -s ` + `step`, `info worker all` shows `resource slots: value=44.00 used=22.00` for worker pc13v, but the worker's real installed capacity is 22 slots (confirmed by the same worker's own `resource maxcpus: value=22.00` line) and all 22 running jobs are 1-slot-each, so the correct display is `value=22.00 used=22.00` (fully packed, 0 free) -- `used=22.00` is correct, `value=44.00` is double the true installed capacity.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "User-reported (live REPL, real cluster-captured replay…\n\nUser-reported (live REPL, real cluster-captured replay /tmp/scheduler-replay-20260828-085356.replay, 40 sessions, 60 jobs): after `xce_scheduler -i -s ` + `step`, `info worker all` shows `resource slots: value=44.00 used=22.00` for worker pc13v, but the worker's real installed capacity is 22 slots (confirmed by the same worker's own `resource maxcpus: value=22.00` line) and all 22 running jobs are 1-slot-each, so the correct display is `value=22.00 used=22.00` (fully packed, 0 free) -- `used=22.00` is correct, `value=44.00` is double the true installed capacity.", "file": "src/scheduler/src/repl_info.c (repl_info_worker_one, ~line 281-390)", "root_cause": "NOT FIXED -- recorded for future investigation, but well-narrowed: repl_info_worker_one()'s own header comment (repl_info.c:281-291) asserts w->resvals holds 'currently-AVAILABLE' capacity that shrinks as jobs are charged, and that 'value=' should be printed as avail+inUse ('a conservation invariant this codebase's own resource accounting already relies on'), implemented at repl_info.c:376-390's Value_u_fvalue branch: `avail_v->u.fvalue + used` printed as value=. Traced the actual charge path (sched_account_jobs -> sched_charge_account, resource.c:586 -> sched_update_account, resource.c:559-583 -> sched_update_res_accounts, resource.c:401-470) and confirmed it ONLY ever writes to w->resInUse (`sched_update_res_accounts(core, &(w->resInUse), 0, haccount->naccount, haccount->accounts, charge)`, resource.c:580-581) -- it never touches w->resvals. Exhaustively grepped every `zarray_put(&(w->resvals), ...)`/`zarray_put(&(worker->resvals), ...)` call site in the scheduler (core_events.c:500, core_events.c:848, snapshot_load.c:333, schedobj.c:311): all four are WORKERS_PUSH / worker-update / snapshot-restore INGESTION paths only -- i.e. w->resvals is populated purely from what the worker daemon self-reports (its own configured/installed capacity, e.g. 'slots: 22'), and is architecturally NEVER decremented when a job is dispatched or charged. So for a live worker's CONSUMABLE resource like 'slots', w->resvals already IS the total installed capacity (not a live 'remaining free' counter), making repl_info.c's own documented 'avail+used = value, a conservation invariant' assumption appear to be FALSE in practice -- the comment's claimed invariant does not match how w->resvals is actually populated for this resource type. This makes `value = avail + used` double-count: true installed capacity (22) gets added to itself once resInUse also reads 22, yielding 44. CONFIRMED PRE-EXISTING, NOT caused by today's scheduler-snapshot-job-allocation task 4.6 fix (the sched_account_jobs()-at-load-time change earlier this session): this exact 44/22 pattern already appeared in the user's ORIGINAL first-ever paste this session, produced via the normal `step` -> sched_account_jobs() path (not the load-time path) -- task 4.6 only changed WHEN the display first goes wrong (now visible immediately at load, previously only after the first step/session, since sched_account_jobs() previously never ran until then). Two competing hypotheses for the real fix, not yet distinguished: (a) repl_info.c's display formula is simply wrong for how resvals is actually populated -- should print avail_v directly as value= (already the total) with used= from resInUse, no addition; or (b) w->resvals was INTENDED to be a live 'remaining free' counter (matching the comment's claimed invariant) and something is missing a `w->resvals -= charged_amount` step at dispatch/charge time that was never implemented -- but no evidence of (b) was found anywhere in resource.c's charge/uncharge functions, so (a) is the better-supported hypothesis pending closer investigation of what a live worker daemon (xcecd) actually reports for a 'slots' resval on each heartbeat (constant install size vs. something already load-adjusted).\n\n--- CONFIRMED + FIXED 2026-08-30 (scheduler-framework session, user's live Symptom 2 report) ---\n\nHypothesis (a) from the prior investigation is CONFIRMED correct, and hypothesis (b) is definitively ruled out. Live-reproduced end to end: built a fresh scheduler+controller+worker (schema change for a separate bug bundled in the same build, unrelated to this fix), deployed a real 3-node HA cluster + xcecd_virtual worker (22 real cores, matching the user's original pc13v report), submitted a 50-element array job (`sleep 60 && hostname`, overlapping runtime) against the 22-slot worker, captured a scheduler recording mid-run (`step 100`, 19 jobs genuinely RUN), and inspected `info worker worker-1` on the UNPATCHED binary first: `resource slots: value=41.00 used=19.00` -- NOT even a clean 2x (44), because avail_v (=22, the worker's real installed capacity, confirmed never decremented -- grepped every zarray_put(&w->resvals,...) call site: all four (core_events.c:500/848, snapshot_load.c:333, schedobj.c:311) are WORKERS_PUSH/RESVALS_DELTA/snapshot-restore INGESTION only) plus used_v (=19, however many jobs happen to be running at inspection time) are ADDED together by repl_info_worker_one()'s own display formula -- so the displayed 'value' grows in lockstep with 'used' as more jobs run, not a fixed doubling. The ORIGINAL user report's exact 22+22=44 was simply the specific case of a fully-packed worker (used == the worker's own real capacity), which is why it read as a clean 2x. Cross-checked against the actual admission-control formula that this exact w->resvals field feeds -- resource.c's unsatisfy_(): `total[i] < inuse[i] + uncomitted[i] + wanted[i]` -- which treats get_resvals(&w->resvals,...) as `total` directly, already inclusive of whatever is `inuse`. This is airtight proof w->resvals[idx] is the STATIC TOTAL, not a live 'remaining available' counter that would need `used` added back to reconstruct the total (hypothesis (b), now ruled out -- no such 'w->resvals -= charged_amount' step exists anywhere, and none should, since the admission check already subtracts inuse separately). The real scheduling/admission decisions are UNAFFECTED by this bug -- they read w->resvals and w->resInUse directly, not through this display formula -- so job placement correctness was never at risk; this is a pure `info worker`/`info resource` REPL display defect. IMPORTANT DISAMBIGUATION: this display bug is entirely SEPARATE from Symptom 1 (phantom RUN jobs exceeding real worker capacity, logged as bug-1739) -- that symptom is a genuine allocation-bookkeeping gap in bug-1734's JOBS_PUSH resync-merge path, unrelated to this printf-time double-count.\n\nRebuilt with the SAME binary against the SAME live-captured replay file: `resource slots: value=22.00 used=19.00` -- matches `resource maxcpus: value=22.00` exactly, confirming the fix.", "fix": "FIXED (2026-08-30). src/scheduler/src/repl_info.c: repl_info_worker_one()'s per-worker CONSUMABLE-resource branch and repl_info_resource_one()'s identical GLOBAL/SHARED-resource branch both changed from printing `avail + used` as `value=` to printing `avail` alone (used= is unchanged, still read from w->resInUse / core->sharedInUse) -- avail_v already IS the total capacity, so adding `used` on top double-counted it. Corrected the header comment above repl_info_worker_one() (previously asserted a false 'avail+used is a conservation invariant' claim) to document the real semantics, with a citation to resource.c's unsatisfy_() as the proof. TDD: updated the 2 existing tests that had baked in the wrong expected value (test_repl_breakpoints_inspection.c: test_info_worker_reports_consumable_value_and_used, expected string changed from 'value=7.00 used=2.00' to 'value=5.00 used=2.00' for avail=5/used=2; test_info_resource_reports_consumable_value_and_used, 'value=11.00 used=3.00' to 'value=8.00 used=3.00' for avail=8/used=3) -- both now correctly assert the raw total, not the double-counted sum. The two tests in test_sim_e2e_original_motivation.c asserting 'value=32.00 used=0.00' / 'value=15.00 used=0.00' needed no change (used=0 makes the old and new formulas identical). Full scheduler + controller suite: 61/61 Ok after the fix, zero regressions. Live re-verified against the real captured replay: value now matches maxcpus exactly at every inspection point (mid-run and post-completion).", "tags": [ "confirmed-live", "display-bug", - "fixed", "found-live", "not-caused-by-task-4.6", "pre-existing", @@ -24048,23 +11096,26 @@ "resInUse", "resource-accounting", "resvals", - "scheduler", - "unfixed" + "scheduler" ], "related_bugs": [ "bug-1568", "bug-1590" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1727.md" }, { "id": "bug-1728", "status": "fixed", "status_updated": "2026-09-27", "status_note": "33c4f28c: failed join barrier unbinds w->ep (was dangling); idx -1 teardown now WARN", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-28T10:25:14Z", - "error_message": "Controller logs `ERROR [handle_endpoint_error] incorrect worker idx -1` in all 3 nodes of a live HA cluster (ctrl-1.log, ctrl-2.log, ctrl-3.log), always immediately after the same two-line sequence: `WARN [commit_submit] commit: WAIT deadline expired without commit (lsn=X committed_lsn=Y writable_lsn=1 timeout_ms=500)` then `WARN [handle_worker_join] handle_worker_join commit barrier failed for pc13v, rc -110`. All three occurrences in the sampled log correlate exactly with an in-progress PreVote/failover window (liveness poll timeout -> primary_ctrl_fd RDHUP -> PreVote begin, immediately preceding each ERROR).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "Controller logs `ERROR [handle_endpoint_error] incorrect worker idx -1` in all 3 nodes of a live…\n\nController logs `ERROR [handle_endpoint_error] incorrect worker idx -1` in all 3 nodes of a live HA cluster (ctrl-1.log, ctrl-2.log, ctrl-3.log), always immediately after the same two-line sequence: `WARN [commit_submit] commit: WAIT deadline expired without commit (lsn=X committed_lsn=Y writable_lsn=1 timeout_ms=500)` then `WARN [handle_worker_join] handle_worker_join commit barrier failed for pc13v, rc -110`. All three occurrences in the sampled log correlate exactly with an in-progress PreVote/failover window (liveness poll timeout -> primary_ctrl_fd RDHUP -> PreVote begin, immediately preceding each ERROR).", "file": "src/controller/cluster.c (handle_endpoint_error, ~line 52-77); src/controller/worker.c or join.c (handle_worker_join, commit-barrier caller -- not yet located)", "root_cause": "NOT FIXED -- recorded for future investigation. Read handle_endpoint_error() (cluster.c:52-119): on any endpoint teardown/error, it inspects priv->role; for HostRole_worker it branches on priv->u.idx -- if >=0, calls handle_worker_leave(c, idx) (the normal, silent teardown path); if <0 (the sentinel for 'never finished joining'), it logs ERROR('incorrect worker idx %d', priv->u.idx) instead (cluster.c:75). Working hypothesis (not yet confirmed by reading handle_worker_join() itself): a worker's endpoint gets `priv->u.idx` assigned only once its join fully completes; the commit barrier inside handle_worker_join() is what actually blocks that completion, and it just failed (rc -110 = -ETIMEDOUT) because quorum was mid-PreVote-failover and couldn't ack the WAL entry within the fixed 500ms commit_submit deadline (already WARNed one line above, correctly). So the worker's join never finishes, its endpoint's idx sentinel never advances off -1, and when that half-joined endpoint then errors out (e.g. the join's own failure path tearing the connection down, or the worker retrying), handle_endpoint_error() sees role==worker && idx==-1 and logs it as a hard, unexplained ERROR -- even though it is the direct, already-WARNed consequence of the commit-barrier timeout two lines above, not a new/distinct fault. Same shape as bug-1727's session sibling finding for scheduler sched_init_resources() and the worker kill-job ERROR-logging issue: an expected, self-healing HA-transition condition (worker retries the join once quorum stabilizes -- confirmed later in the same logs by 'HA quorum restored') surfacing as an unexplained hard ERROR instead of being correlated with the WARN that actually explains it. NOT YET CONFIRMED: have not read handle_worker_join()'s own body (location not pinned down -- likely worker.c or join.c) to verify priv->u.idx really stays -1 specifically because of THIS commit-barrier failure path (vs. some other reason), nor traced exactly what tears the endpoint down afterward to trigger handle_endpoint_error().", "fix": "NOT FIXED -- recorded for future investigation only, no code change made. Next steps: (1) locate and read handle_worker_join()'s full body to confirm priv->u.idx is left at -1 specifically on the commit-barrier-timeout path (rc -110) and identify what closes/errors the endpoint afterward; (2) if confirmed, either (a) downgrade cluster.c:75's ERROR to WARN with a message that explicitly references the join-in-progress/commit-barrier-timeout context (avoids the unexplained-hard-ERROR read), or (b) have the commit-barrier-timeout failure path in handle_worker_join() itself suppress/mark the endpoint so handle_endpoint_error() can recognize 'this was a known, already-logged join failure' and skip the redundant ERROR entirely; (3) verify against a live 3-node HA soak (xce-test-ha) that downgrading doesn't hide a genuinely-different idx<0 cause (e.g. a real accounting bug elsewhere assigning idx incorrectly) -- keep ERROR (or WARN-with-detail) for any idx<-1 or any case NOT immediately preceded by a commit-barrier-timeout WARN for the same worker, so a genuinely new failure mode doesn't get silently swallowed by too broad a downgrade.", @@ -24075,24 +11126,28 @@ "false-error", "worker-join", "commit-barrier", - "unfixed", "found-live", - "session-sibling-of-bug-1727", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "session-sibling-of-bug-1727" ], "related_bugs": [ "bug-1727" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "33c4f28c" + ], + "record": ".wolf/bugs/bug-1728.md" }, { "id": "bug-1729", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "worker", + "severity": "unknown", "timestamp": "2026-08-28T10:25:14Z", - "error_message": "xcecd_worker.log logs `[ERROR] Job exited with code 137: Killed` for an ordinary, deliberate job kill/cancel (exit code 137 = 128+SIGKILL) -- user-reported as misleading, since 'kill job' is a normal admin operation, not a failure.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xcecd_worker.log logs `[ERROR] Job exited with code 137: Killed` for an ordinary…\n\nxcecd_worker.log logs `[ERROR] Job exited with code 137: Killed` for an ordinary, deliberate job kill/cancel (exit code 137 = 128+SIGKILL) -- user-reported as misleading, since 'kill job' is a normal admin operation, not a failure.", "file": "src/worker/src/uv_ipc.c (ipc_worker_send_status, JOB_EXITED branch)", "root_cause": "CONFIRMED: process_exit_callback() (xcecd_worker.c:467-502) already treats a signal-terminated job exit as informational -- `XCECD_WORKER_LOG(XCE_LOG_INFO, \"Child %d killed by signal %d\\n\", pid, term_signal)` at line 497 -- immediately before calling ipc_worker_send_status(&job_runtime_info, \"Killed\"). But ipc_worker_send_status() (uv_ipc.c, JOB_EXITED branch) re-derived its own severity purely from job_status, hardcoded to XCE_LOG_ERROR for every JOB_EXITED regardless of cause -- so the SAME kill event logged INFO two lines earlier and ERROR two lines later, in the same call chain. There is no JOB_KILLED status distinct from JOB_EXITED, so a deliberate xcecli kill (worker_signal_callback(), xcecd_worker.c:1118, itself logs INFO: 'Terminating job %s with signal %d') produced an ERROR-level status log downstream regardless of intent. worker_signal_callback() only ever sends SIGTERM or SIGKILL for a normal cancel/force-kill (xcecd_worker.c:1116-1125) -- exit_code is always exactly 128+term_signal (xcecd_worker.c:493), so exit_code==143 (SIGTERM) or 137 (SIGKILL) reliably identifies a deliberate worker-initiated termination, distinct from a genuine crash signal (SIGSEGV=139, SIGBUS=135, SIGABRT=134, etc.).", "fix": "FIXED and unit-tested (TDD red->green, negative-control verified: temporarily forced job_exit_log_level() to always return XCE_LOG_ERROR, confirmed 2/4 new tests failed as expected, then restored the real fix and confirmed all 4 pass). Extracted the severity decision into a small pure function, `static LogLevel job_exit_log_level(int exit_code)` (uv_ipc.c, just above ipc_worker_send_status()): returns XCE_LOG_INFO for exit_code == 128+SIGTERM or 128+SIGKILL (matches process_exit_callback()'s own INFO judgment for the identical event), XCE_LOG_ERROR otherwise (preserves ERROR for genuine crash signals and any other/unrecognized exit code). Wired into the JOB_EXITED branch: `XCECD_WORKER_LOG(job_exit_log_level(job_info->exit_code), \"Job %s exited with code %d: %s\", ...)`. Added #include for SIGTERM/SIGKILL/SIGSEGV/etc. New test worker/testsuite/test_uv_ipc_job_exit_severity.c (4 cases: deliberate SIGTERM, deliberate SIGKILL, genuine crash signals SIGSEGV/SIGABRT/SIGBUS, unrecognized/plain exit codes) white-box #includes uv_ipc.c directly (existing suite convention) and exercises only the pure helper -- no libuv/IPC machinery needed. Two link-only stub functions added (find_worker_by_pid, xcecd_job_remote_start) to satisfy the linker for code paths this test never calls, same stubbing idiom test_resource_sync.c already establishes. Full worker test suite (5 binaries, 25 tests total) green after the fix; real xcecd_worker binary rebuilds cleanly.", @@ -24101,7 +11156,6 @@ "logging-severity", "false-error", "job-kill", - "fixed", "tdd", "found-live", "session-sibling-of-bug-1727", @@ -24111,15 +11165,20 @@ "bug-1727", "bug-1728" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1729.md" }, { "id": "bug-1730", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-28T13:39:36Z", - "error_message": "User-reported (live 3-node HA cluster, real workload): a finished job's array elements sometimes remain visible as RUN forever in the scheduler's own state (confirmed both live and via a scheduler session recording's .replay snapshot -- xce_scheduler -i -s showing several 0-12-1787907177-0[N] elements as RUN when the controller's joblist/xcecli already shows them DONE). User's own log correlation: scheduler-side handler.c:414 'sched_handle_sched_update_job updated' fires once per job (the dispatch-time notification) but a second fire for the finish notification never arrives for the affected jobs, while the worker log independently proves those jobs genuinely completed (uv_ipc.c:1042 'Job completed successfully', milliseconds after dispatch -- these are near-instant /bin/true-style array elements).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (live 3-node HA cluster, real workload): a finished job's array elements sometimes…\n\nUser-reported (live 3-node HA cluster, real workload): a finished job's array elements sometimes remain visible as RUN forever in the scheduler's own state (confirmed both live and via a scheduler session recording's .replay snapshot -- xce_scheduler -i -s showing several 0-12-1787907177-0[N] elements as RUN when the controller's joblist/xcecli already shows them DONE). User's own log correlation: scheduler-side handler.c:414 'sched_handle_sched_update_job updated' fires once per job (the dispatch-time notification) but a second fire for the finish notification never arrives for the affected jobs, while the worker log independently proves those jobs genuinely completed (uv_ipc.c:1042 'Job completed successfully', milliseconds after dispatch -- these are near-instant /bin/true-style array elements).", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_UPDATE case), src/controller/sched.c (sched_job_exit_fun/sched_job_start_fun/sched_job_fail_fun), src/controller/job.c (HOOK_DETAIL_JOB_EXIT/START/FAIL fire sites)", "root_cause": "TWO CONFIRMED FINDINGS from this investigation, one FIXED here, one still OPEN (see bug-1731, its own entry, for the still-open half -- kept separate since the fixed half does not explain the original reported symptom by itself, see below).\n\nFINDING 1 (FIXED by this entry): sched_job_exit_fun()/sched_job_start_fun()/sched_job_fail_fun() (controller/sched.c) -- the controller-side notify handlers wired to route a job exit/start/fail specifically to the scheduler's accounts/allocs-cleaning sched_job_state_change() (via NotifyData_data_schedExitJob etc. -> XCE_SCHED_OP_JSTATE_CHANGE) -- are PERMANENTLY DEAD CODE: each only acts on `situation == HOOK_AFTER_EVENT` (default: break for HOOK_BEFORE_EVENT), but HOOK_DETAIL_JOB_EXIT/JOB_START/JOB_FAIL are ONLY EVER fired as HOOK_BEFORE_EVENT anywhere in the controller (job.c:2515/2640/2744/2749/2783, all inside the durable-append-only record_job_*_fact() receipt-time functions -- confirmed via exhaustive grep, zero HOOK_AFTER_EVENT fires of these three details exist). The actual state mutation + notification for a job finish/start/fail happens later, at deferred-apply time (apply_committed_entry()'s shared WAL_JOB_START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH 'group' case, persist.c:4464), which fires a DIFFERENT, generic detail -- HOOK_DETAIL_JOB_UPDATE -- at HOOK_AFTER_EVENT, routing instead through sched_job_update_fun() -> NotifyData_data_schedUpdateJob -> scheduler's sched_handle_sched_update_job() (handler.c:404) -> XCE_SCHED_OP_UPDATE_JOB_FIELDS -> (in the currently-live core_events.c-based driver path) XCE_CORE_EVENT_JOB_UPDATE (core_events.c:933). That handler correctly sets job->state, but -- unlike its sibling XCE_CORE_EVENT_JOB_STATE_CHANGE case right above it in the same file, and unlike scheduler.c's own (also now-dead) sched_job_state_change() -- it never released job->accounts/job->allocs on a terminal transition, so every job finishing via this (the only live) path leaked its resource-allocation/accounting objects for the rest of the scheduler process's life. Almost certainly a regression from the raft-apply-decoupling refactor, which collapsed START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH's deferred-apply hook fire into one generic HOOK_DETAIL_JOB_UPDATE without ever rewiring (or retiring) the scheduler-notification handlers that were built for the old, type-specific hook details.\n\nConfirmed this FINDING alone does NOT explain the user's original reported symptom (job->state itself staying RUN): job->state is set correctly by XCE_CORE_EVENT_JOB_UPDATE's existing code regardless of this leak -- the leak only means job->accounts/allocs are never released, a distinct, confirmed-real defect. Extensive live investigation of the actual state-staying-wrong symptom is tracked separately in bug-1731 (open).", "fix": "FIXED and TDD-verified (RED->GREEN) for FINDING 1 only. src/scheduler/src/core_events.c's XCE_CORE_EVENT_JOB_UPDATE case: added the identical `if (!JOB_IS_RUN(updated->state)) { free job->accounts via free_job_account(), job->accounts = NULL; free job->allocs via free_JobAlloc_ptr() }` block XCE_CORE_EVENT_JOB_STATE_CHANGE's case already has, right after `job->state = updated->state;`. New test test_job_update_to_terminal_state_frees_accounts_and_allocs (scheduler/testsuite/test_core_events.c): builds a real RUNNING job with a genuine allocation via production make_alloc()/make_host_alloc()/make_job_alloc()/make_job_account() (src/lib/xcealloc.c), pushes a JOB_UPDATE transitioning it to DONE, asserts job->accounts/job->allocs are both NULL afterward. RED confirmed (FAIL: Expected NULL) before the fix; GREEN after. Full scheduler test suite: 52/52 with the new test, only the pre-existing, unrelated symlink-escape-error-message mismatch (sched_admin_request_test) failing before and after -- zero regressions. Real xce_scheduler binary rebuilds cleanly.", @@ -24132,7 +11191,6 @@ "resource-leak", "accounts-allocs", "raft-apply-decoupling-regression", - "fixed", "tdd", "found-live", "session-sibling-of-bug-1731" @@ -24142,15 +11200,20 @@ "bug-1679", "bug-1731" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1730.md" }, { "id": "bug-1731", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-28T13:39:36Z", - "error_message": "STILL OPEN half of the bug-1730 investigation: the user's originally-reported symptom itself -- a finished job's job->state staying stuck as RUN in the scheduler's own view (live and in a .replay snapshot) despite the controller and the worker both having genuinely completed it -- was NOT conclusively reproduced live despite extensive targeted testing, so its exact trigger mechanism remains unconfirmed.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "STILL OPEN half of the bug-1730 investigation: the user's originally-reported symptom itself -- a…\n\nSTILL OPEN half of the bug-1730 investigation: the user's originally-reported symptom itself -- a finished job's job->state staying stuck as RUN in the scheduler's own view (live and in a .replay snapshot) despite the controller and the worker both having genuinely completed it -- was NOT conclusively reproduced live despite extensive targeted testing, so its exact trigger mechanism remains unconfirmed.", "file": "src/controller/persist.c (apply_committed_entry, WAL_JOB_FINISH/START/DISPATCH shared case), src/controller/sched.c (sched_job_update_fun)", "root_cause": "NOT ISOLATED. Investigation trail: traced the full controller-to-scheduler-to-replay-recording pipeline for a job finish end-to-end via static code reading (every hop -- live apply, hook fire, record-encode in recording_events.c, replay-decode in sim_loop.c, replay-apply -- checked individually and found correctly wired for the ordinary case) and found no defect that would explain a PERMANENT loss by itself. Added temporary diagnostic WARN logging at apply_committed_entry()'s two previously-100%-silent early-return points (job not found in jobmgr; apply_job_update_transition() returning REJECTED_BACKWARD/NOOP_DUPLICATE) and reproduced a burst-array-job workload (matching the user's original scenario -- a 30+ element array of near-instant jobs) against both a single-node and a real 3-node HA deploy of this exact worktree's build. Findings from live reproduction: (a) WAL_JOB_FINISH consistently showed result=APPLIED (hook fires) in every observed instance -- did not catch a live silent-drop for FINISH specifically; (b) DID catch WAL_JOB_START correctly no-op'ing as a harmless duplicate when WAL_JOB_DISPATCH already set RUN moments earlier in the same batch (expected, not a bug); (c) a genuinely concerning, SEPARATE, NOT-YET-DIAGNOSED finding: under a very heavy burst (4 concurrent 50-element arrays on a live 3-node cluster), the xce_scheduler process itself DIED entirely mid-run with zero error/fatal/assert output in its own log -- no backtrace captured yet (a subsequent gdb-attached rerun of a similarly-sized burst did not reproduce the crash within the session's remaining time budget); (d) a separate apparent scheduling-throughput stall under a large PEND backlog (200+ near-instant jobs against a 22-slot worker progressing far slower than expected) -- not distinguished from ordinary fork/exec overhead of spawning hundreds of real OS processes on one worker, so not confirmed as a genuine scheduler defect either. Any of (c)/(d) could plausibly be entangled with, or be a completely independent symptom alongside, the original state-staying-wrong report -- not established either way.\n\nUPDATE (2026-08-29, live gdb-wrapped burst repro): two concrete new findings, neither yet closing this bug's original symptom (job state itself stuck at RUN).\n(1) LIVE-CAPTURED NOTIFY-DROP, SELF-HEALING IN THE OBSERVED CASE: the DEBUG3 diagnostic trace added earlier this session (sched_job_update_fun, sched.c) fired for real under this burst -- e.g. `DIAG sched_job_update_fun: jobid=4618064451492872238 state=0x2 -- c->sched.ep is NULL, notify SKIPPED` during a ~2s scheduler reconnect blip (handle_scheduler_leave then a rejoin then a second, fatal leave at the crash). Across the run: 173 RUN notifies attempted, 178 DONE notifies delivered, 53 SKIPPED (dropped because c->sched.ep was NULL at fire time). The one traced job above still received its terminal DONE notify ~300ms later and self-healed -- but this is a live, reproducible demonstration of the exact drop mechanism this bug has theorized: if a job's TERMINAL notify (not just an intermediate RUN one) were ever dropped in the same way, with no subsequent notify to self-heal it, the scheduler's view would stay stuck at whatever state it last received -- matching the original reported symptom exactly. Not yet caught happening to a terminal notify specifically, so still not a full reproduction, but the mechanism is now demonstrated live rather than theoretical.\n(2) THE SCHEDULER CRASH (open finding (c) above) WAS FINALLY CAPTURED WITH A FULL BACKTRACE -- see bug-1714 for the complete crash detail (SIGSEGV in sched_get_project via a corrupted job_submit_data_t->project_name pointer, core_events.c:221). Tracked as bug-1714 rather than duplicated here since it's a distinct symptom (crash vs. stuck-state) that may or may not share a root cause with this bug's own open half -- both occur under the same heavy concurrent-array-batching conditions, so they remain plausibly entangled, not yet proven either way.\n(3) NO THROUGHPUT STALL observed this run (open finding (d) above) -- DIAG counts climbed steadily and continuously right up to the crash; of 300 submitted array elements only 178 reached DONE before the scheduler died at T+150s, but that is a crash artifact (elements cut off mid-flight), not evidence of an independent stall. Finding (d) remains unconfirmed either way, not further investigated this run.", "fix": "FIXED (2026-08-29). Root cause (confirmed, not the same theory this entry's prior root_cause investigation was chasing -- see note at the end): apply_committed_entry() (src/controller/persist.c, the shared WAL_JOB_START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH case) fired exactly ONE generic HOOK_DETAIL_JOB_UPDATE at HOOK_AFTER_EVENT for every job-state fact -- correct and still relied on by sched_job_update_fun/apiserver_job_update_fun -- but NEVER the type-specific HOOK_DETAIL_JOB_START/HOOK_DETAIL_JOB_EXIT, even though live handler pairs for both (sched_job_start_fun/sched_job_exit_fun in sched.c, apiserver_job_start_fun/apiserver_job_exit_fun in apiserver.c, all wired via xce_add_hook) have existed the whole time, subscribed at HOOK_AFTER_EVENT. Those handlers were permanently dead code: job.c only ever fires HOOK_DETAIL_JOB_START/JOB_EXIT/JOB_FAIL at HOOK_BEFORE_EVENT (receipt-time, for WAL-append purposes), never AFTER_EVENT -- see persist.h's apply_committed_entry doc comment, 'Deviation from a literal per-type hook fan-out', for the original documented rationale (the old wal_jobstate_t {jobid, state} wire format could not supply the richer job_pidinfo_t/xce_jobfinish_t payloads those handlers require). That blocker is now stale for START and FINISH specifically: bug-1697 (WAL_JOB_START) and bug-1689 (WAL_JOB_FINISH) already enriched this exact wire format with trailing job_pidinfo_t / job_exitinfo_t+finish_time bytes, decoded every apply into j->job.pidinfo / j->job.exit_info -- just never forwarded to the dead handlers. On the apiserver side this meant DefaultHandler.OnJobFinished (src/frontend/apiserver/backend/handler.go) -- itself correct and fully wired via handleNotifyData's NotifyData_data_jobFinishInfo case (clib.go) -- was permanently starved of data: a job could finish for real (worker and controller both correct) and the apiserver's store.Jobs entry would simply never transition off its pre-finish status.\n\nFIX (src/controller/persist.c, apply_committed_entry's shared job-state-fact case): (1) WAL_JOB_FINISH -- inside the existing `result == JOB_TRANSITION_APPLIED` block (the same guard HOOK_DETAIL_JOB_UPDATE already used), also fire HOOK_DETAIL_JOB_EXIT with a stack-local xce_jobfinish_t{jobid=js->jobid, exit_code=(j->job.exit_info ? j->job.exit_info->exitcode : 0)} -- this is the PRIMARY fix. (2) WAL_JOB_START -- a new call-scoped `bool pidinfo_just_decoded` flag, set ONLY where the pre-existing pidinfo decode-and-adopt block (bug-1697) actually decodes a NEW job_pidinfo_t this call (deliberately NOT `j->job.pidinfo != NULL`, which could already be true from an earlier apply and would misfire on an unrelated later transition for the same job); also belt-and-suspenders stamps `decoded->jobid = js->jobid` explicitly right where it is adopted. HOOK_DETAIL_JOB_START now fires whenever pidinfo_just_decoded is true, INDEPENDENT of `result` -- required because bug-1697 already established that every real job's WAL_JOB_START fact applies as a confirmed NOOP_DUPLICATE in production (dispatch_a_job_() sets state=RUN synchronously before WAL_JOB_START ever applies), so gating on result==APPLIED the way JOB_UPDATE is gated would make the new fire dead code forever, live, exactly the class of bug being fixed. The early-return gate right after unlock(j) was widened from a flat `if (result != JOB_TRANSITION_APPLIED) return 0;` to `if (result != JOB_TRANSITION_APPLIED && !pidinfo_just_decoded) return 0;` so a genuinely-just-decoded START pidinfo still reaches the single t_apply_seam_notifying-guarded hook-fire bracket even on a NOOP_DUPLICATE transition; all three fires (new JOB_START, new JOB_EXIT, pre-existing JOB_UPDATE) stay inside that ONE bracket, not duplicated or narrowed. NULL-safety: sched_job_start_fun/sched_job_exit_fun (sched.c) dereference info->jobid unconditionally with no NULL guard (confirmed via codegraph read of both) -- pidinfo_just_decoded's call-scoped design and the belt-and-suspenders jobid stamp exist specifically to make firing with a NULL/stale payload structurally impossible. Confirmed via codegraph/grep that persist_job_hook_fun (the third XCE_CONTROLLER_HOOK_JOB subscriber, registered via ha_register_persist_hooks) does have live HOOK_DETAIL_JOB_START/EXIT cases (unlike the doc comment's characterization of it as 'HOOK_BEFORE_EVENT only for these details') -- but it early-returns immediately on `t_apply_seam_notifying` (Hazard-1 guard, already true for every fire inside apply_committed_entry's bracket) before ever reaching those cases, so no unintended WAL re-append happens; this nuance is a buglog correction to the doc-comment's own framing, not a defect.\n\nTDD evidence: added test_wjf_apply_committed_entry_fires_job_exit_hook_exactly_once and test_wjf_apply_committed_entry_reapply_finish_fires_no_hook (test_wal_job_finish_fact.c), test_wjs_apply_committed_entry_fires_job_start_hook_with_pidinfo and test_wjs_apply_committed_entry_old_format_no_job_start_hook (test_wal_job_start_fact.c), using a new detail-aware spy hook per file (distinguishes HOOK_DETAIL_JOB_EXIT/JOB_START from JOB_UPDATE and captures the fired payload, unlike the existing blunt 'any AFTER_EVENT' counters already in those files). RED confirmed on unfixed code (persist.c edits temporarily reverted, test files kept): exactly the 2 new primary-path assertions failed (test_wjs_..._fires_job_start_hook_with_pidinfo: Expected 1 Was 0; test_wjf_..._fires_job_exit_hook_exactly_once: Expected 1 Was 0), plus the pre-existing bug-1561 test_config_repl.c flake pair -- 4 failures total, matching expectations exactly. The two negative-guard tests (old-format-no-pidinfo, reapply-fires-no-hook) already passed RED (correctly -- pre-fix code never fires these hooks at all, so 'must not fire' trivially holds), confirming they are genuine regression guards, not accidentally-tautological.\n\nGREEN evidence: after restoring the fix, four PRE-EXISTING tests broke because their assertions hard-coded the old 'exactly one notification' invariant that this fix deliberately changes (more hooks now fire, by design) -- all four updated in place with corrected expected counts plus an explanatory bug-1731 comment (not weakened, not deleted): test_wjs_apply_committed_entry_sets_pidinfo_but_not_start_time (1 -> 2: JOB_UPDATE + new JOB_START both fire on this genuine PEND->RUN transition with pidinfo), test_wjs_apply_committed_entry_adopts_pidinfo_when_already_run_from_dispatch (0 -> 1: this IS bug-1697's exact NOOP_DUPLICATE production scenario -- the new JOB_START fire is the whole point of the fix here, so asserting 0 would mean re-breaking the bug), test_wjf_apply_committed_entry_sets_finish_time_and_exit_info (1 -> 2: JOB_UPDATE + new JOB_EXIT), test_tari_finish_then_stale_start_preserves_terminal_state_via_dispatch in test_apply_reapply_idempotency.c (2 -> 3 after FINISH, stays 3 after the stale-START no-op -- confirms idempotency: the stale START carries no pidinfo trailing bytes, so pidinfo_just_decoded stays false and the already-guard-rejected transition fires nothing new). Full controller regression after all fixes: 1217 tests, 2 failures -- exactly the known, pre-existing, unrelated bug-1561 flake pair (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config) and nothing else. scheduler/worker/xceadmin builds also completed clean (scheduler build surfaced one unrelated, pre-existing test failure -- test_recordstart_symlink_escape_replies_error, in record-file-path validation, zero relation to job-state hooks or any file this fix touched -- not investigated further, out of scope).\n\nLIVE VERIFICATION (single-node cluster, src/tools/deploy-ha.sh to /tmp/xce-deploy-1731, controller+worker+scheduler+apiserver all built from this fix and started via start-all.sh; two real jobs submitted via xce_test jobsub -u root -q normal -c /bin/true, both ran to completion):\n Scheduler side -- scheduler.log shows, for jobid 0-7-1787994329-448775 (and identically for a second job, 0-14-1787994109-0): `event = schedStartJob` immediately followed by `sched_job_state_change 0-7-1787994329-448775 state 2` (handler.c:388, inside sched_job_state_change, handler.c:381 -- state 2 = JOB_STAT_RUN, the NEW HOOK_DETAIL_JOB_START fire), then later `event = schedExitJob` followed by `sched_job_state_change 0-7-1787994329-448775 state 4` (state 4 = JOB_STAT_EXIT, the NEW HOOK_DETAIL_JOB_EXIT fire) -- with `sched_handle_sched_update_job ... updated` (the pre-existing JOB_UPDATE path) still firing on both sides of it, confirming the new fires are additive, not a replacement. Final `joblist` output confirms the job's state in the scheduler-driven view reached DONE, not stuck at RUN.\n Apiserver side (the part specifically asked to be verified) -- added permanent Info-level logging to DefaultHandler.OnJobFinished (handler.go, mirroring this file's existing zap-field logging convention) since it had none before; rebuilt /tmp/apiserver, restarted it against the live cluster, submitted a fresh job. apiserver.log recorded: `OnJobFinished: job status updated {\\\"jobid\\\": 33400045333567751, \\\"exit_code\\\": 0, \\\"job_status\\\": 8}` -- job_status 8 = models.JOBSTAT[\\\"DONE\\\"], exit_code 0 matching /bin/true's real exit code, jobid matching the submitted job -- direct proof OnJobFinished, previously never invoked in production, now fires correctly end-to-end. All 4 processes (controller, worker, scheduler, apiserver) remained healthy throughout with zero ASAN/crash output in any log.\n\nKept out of scope, as directed: WAL_JOB_FAIL's own dead HOOK_DETAIL_JOB_FAIL handler pair (apiserver_job_fail_fun -> jobFailInfo -> OnJobFinished too) was NOT touched -- it lives in a separate, more complex `case` (dispatches on js->_pad, apply_job_fail_transition vs. apply_job_update_transition) a bit further down persist.c's apply_committed_entry, and is a distinct follow-up fix, not covered here.\n\nFiles touched: src/controller/persist.c (the fix), src/controller/testsuite/test_wal_job_finish_fact.c and test_wal_job_start_fact.c (new regression tests + corrected pre-existing assertions), src/controller/testsuite/test_apply_reapply_idempotency.c (corrected one pre-existing assertion), src/frontend/apiserver/backend/handler.go (permanent OnJobFinished logging, kept -- not reverted, judged a reasonable permanent addition matching this file's existing logging conventions).\n\nCorrection to this entry's own prior root_cause note: the earlier investigation (2026-08-28/29 updates above) was chasing a DIFFERENT theory -- a transient sched.ep==NULL notify-drop window in sched_job_update_fun -- as the suspected mechanism for the original stuck-at-RUN symptom, and left bug-1714 (a scheduler crash found during that chase) as a possibly-entangled open thread. This fix does not address that notify-drop theory or bug-1714 at all; it closes a DIFFERENT, independently confirmed defect (the dead JOB_START/JOB_EXIT hook pair) that was diagnosed and fixed in this session. Both could in principle contribute to real-world stuck-job reports; this entry now tracks the JOB_START/JOB_EXIT hook fix as FIXED, with bug-1714's scheduler-crash thread and any residual sched.ep notify-drop risk left open elsewhere (bug-1714's own entry) rather than reopening speculation here.\n\nFOLLOW-UP (2026-08-29, same session): the WAL_JOB_FAIL HOOK_DETAIL_JOB_FAIL gap explicitly left out of scope above (\"Kept out of scope, as directed\") is now ALSO fixed. HOOK_DETAIL_JOB_FAIL was fired only at HOOK_BEFORE_EVENT (job.c's mark_job_fail and record_job_fail_fact, at receipt time, for WAL-append purposes) -- never at HOOK_AFTER_EVENT -- leaving two already-written, already-correct handlers permanently dead: sched_job_fail_fun (sched.c, routes to the scheduler via NotifyData_data_schedFailJob) and apiserver_job_fail_fun (apiserver.c, routes to the Go apiserver's NotifyData_data_jobFailInfo case in clib.go -> DefaultHandler.OnJobFinished). A job that FAILED (as opposed to finished cleanly, which this bug's primary fix already covers via WAL_JOB_FINISH/HOOK_DETAIL_JOB_EXIT) never updated the apiserver's job store.\n\nFIX (src/controller/persist.c, apply_committed_entry's `case WAL_JOB_FAIL:` block): hoisted the case-local `job_failinfo_t failinfo` to case scope (it needs to survive past `unlock(j)`) with a new `bool have_failinfo` flag, set true only in the `js->_pad & WAL_JOB_FAIL_STAGE_FLAG` branch (the only branch that constructs a real job_failinfo_t -- the legacy `_pad == 0` absolute-overwrite branch, job_ctrl's synchronous mark_job_fail path, has no fail-specific data to report and must NOT fabricate one, exactly mirroring how this bug's own WAL_JOB_START fix only fires HOOK_DETAIL_JOB_START when pidinfo was genuinely decoded). Inside the existing `t_apply_seam_notifying`-guarded bracket, added `if (have_failinfo) xce_run_hooks(..., HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_FAIL, c, &failinfo, j);` immediately before the pre-existing HOOK_DETAIL_JOB_UPDATE fire -- both fires stay inside the SAME bracket, additive not replacing. Confirmed via reading apply_job_fail_transition() (job.c) that unlike WAL_JOB_START's dispatch-time-optimization no-op'ing, this transition has no internal guard at all -- it unconditionally mutates and returns JOB_TRANSITION_APPLIED for every stage -- so the existing `if (result != JOB_TRANSITION_APPLIED) return 0;` early-return gate did NOT need widening the way WAL_JOB_START's did; a rejectable/no-op outcome for this case can only come from the legacy _pad==0 branch's apply_job_update_transition() guard (job_state_is_backward / duplicate-state checks), which was already correctly gated before this follow-up.\n\nTDD evidence: added 3 tests to test_apply_committed_entry.c -- test_ace_job_fail_stage_flag_fires_faildetail_and_update (seeds job 605 RUN|EXIT, applies WAL_JOB_FAIL with _pad=WAL_JOB_FAIL_STAGE_FLAG|JOB_EXEC_STAGE_POST_EXEC, asserts JOB_STAT_PERR merged AND both hooks fire with the correct jobid in the captured job_failinfo_t), test_ace_job_fail_legacy_pad_zero_fires_update_only (regression guard: _pad==0 branch fires JOB_UPDATE only, never fabricates JOB_FAIL), test_ace_job_fail_guard_rejected_fires_no_hook (a backward transition via the legacy branch, mirroring test_ace_job_state_transition_rejects_backward_suppresses_notify, fires NEITHER hook). New detail-aware spy hook tace_faildetail_spy_hook added (modeled on the file's existing tace_array_spy_hook), with va_arg order (cluster_t*, job_failinfo_t*, job_t*) matching job.c's own HOOK_BEFORE_EVENT fire sites for this detail; g_tace_last_fail_jobid reset to -1 in tace_setup() alongside the existing g_tace_hook_fires reset.\n\nRED confirmed on unfixed code (persist.c fix temporarily git-stashed, test file kept): exactly test_ace_job_fail_stage_flag_fires_faildetail_and_update failed (Expected 2 Was 1 -- HOOK_DETAIL_JOB_FAIL did not fire), plus the pre-existing bug-1561 test_config_repl.c flake pair (2 failures) -- 3 failures total, matching expectations (the other two new tests are regression guards that correctly already passed pre-fix, since the unfixed code never fires HOOK_DETAIL_JOB_FAIL at all).\n\nGREEN confirmed after restoring the fix: all 3 new tests pass. Full controller regression: 1220 tests, 2 failures -- exactly the known, pre-existing, unrelated bug-1561 flake pair (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config) and nothing else.\n\nLIVE VERIFICATION: attempted against a fresh single-node deploy (controller+worker(xcecd_virtual)+scheduler+apiserver, src/tools/deploy-ha.sh to /tmp/xce-deploy, all built from this fix). Confirmed via code reading (xcecd_job_finished, worker/src/controller_connector.c) that a worker-side nonzero exit code routes through xcecd_notify_job_fail (WAL_JOB_FAIL with STAGE_FLAG), never WAL_JOB_FINISH -- so a genuinely-failed job is the right live trigger. Submitted a `sleep 60` job (chosen so the virtual worker's simulated run-time, parsed from the literal command string in VIRTUAL_CLUSTER builds, stays RUN long enough to signal) and sent jobkill -s 9 while it was RUNNING; the controller/worker IPC path correctly delivered the signal and set worker->killed_signal=9 (confirmed in xcecd-worker-1.log). However this sandbox has no D-Bus system bus (no systemd manager reachable), and xcecd_dbus_manager.c's dbus_workers_update() -- unchanged by this fix, in src/worker/, not src/controller/ -- calls get_systemd_job_exit_code() every poll even under VIRTUAL_CLUSTER; that call fails to connect to D-Bus and returns -1, which routes into the `exit_code < 0` (dbus-unreachable) branch of the VIRTUAL_CLUSTER dispatch -- a branch which, on inspection, only checks check_worker_sleep_time_elapsed() and never checks worker->killed_signal (that check exists only in the sibling `exit_code == 0` branch, unreachable in this sandbox since dbus never succeeds). Net effect: in THIS sandboxed environment specifically, xcecd_virtual can only ever report a job as succeeding (exit_code=0) once its simulated sleep time elapses, or hang indefinitely if killed before then -- a genuine nonzero-exit/killed-job WAL_JOB_FAIL could not be produced live here. This is a pre-existing environment/worker-code limitation unrelated to and untouched by this fix (no file outside the fix's own scope was modified to work around it, per this task's constraints) -- an honestly-reported limitation, not a gap in the fix itself. The unit-test coverage above (RED->GREEN on the exact apply-time dispatch code) is the primary proof of correctness; the same live-verification shape this bug's primary fix already used successfully for WAL_JOB_FINISH/WAL_JOB_START (a real dbus-independent xce_test jobsub) does not exist for a genuine WAL_JOB_FAIL trigger in this environment.\n\nFiles touched by this follow-up: src/controller/persist.c (the fix), src/controller/testsuite/test_apply_committed_entry.c (3 new tests + 1 new spy hook + 1 new reset line).", @@ -24164,23 +11227,26 @@ "found-live", "possible-crash", "possible-stall", - "session-sibling-of-bug-1730", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "session-sibling-of-bug-1730" ], "related_bugs": [ "bug-1714", "bug-1730" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1731.md" }, { "id": "bug-1732", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-08-29T05:21:00Z", - "error_message": "src/tools/xce-build.sh's `worker` component case never builds `xcecd_virtual` (only `xcecd xcecd_worker xrun controller_connector_test`), even though deploy-ha.sh requires xcecd_virtual whenever no real systemd manager is present (the common case in sandboxes) -- `/xce-build worker` alone leaves a deploy unable to auto-detect/generate start-worker.sh for the virtual worker.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "src/tools/xce-build.sh's `worker` component case never builds `xcecd_virtual` (only `xcecd…\n\nsrc/tools/xce-build.sh's `worker` component case never builds `xcecd_virtual` (only `xcecd xcecd_worker xrun controller_connector_test`), even though deploy-ha.sh requires xcecd_virtual whenever no real systemd manager is present (the common case in sandboxes) -- `/xce-build worker` alone leaves a deploy unable to auto-detect/generate start-worker.sh for the virtual worker.", "file": "src/tools/xce-build.sh (worker component case)", "root_cause": "The `worker)` case's explicit meson target list omits `xcecd_virtual`. Confirmed while dispatching a debug-profile build for a bug-1731 live repro: `/xce-build worker debug --incremental` completed successfully but did not produce `xcecd_virtual` in the build dir, requiring a manual `meson compile -C xcecd_virtual` before deploy-ha.sh's own worker-binary auto-detection (Step 3, per the xce-deploy-ha skill) would find it and generate start-worker.sh instead of soft-skipping it.", "fix": "FIXED. Added `xcecd_virtual` to the `worker)` case's meson compile target list in src/tools/xce-build.sh (alongside the existing xcecd xcecd_worker xrun controller_connector_test), with a bug-1732-referencing comment. Verified: `xce-build.sh worker asan` now builds xcecd_virtual with a fresh mtime, and the full worker test suite (5/5, including test_uv_ipc_job_exit_severity) still passes with zero regressions.", @@ -24189,20 +11255,23 @@ "xce-build.sh", "worker", "xcecd_virtual", - "low-priority", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "low-priority" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1732.md" }, { "id": "bug-1733", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-08-29T06:30:00Z", - "error_message": "AddressSanitizer: heap-use-after-free WRITE of size 8 at vtask_enq (src/lib/vtaskmgr.c:517) -- crashed 2 of 3 controllers in a live 3-node HA cluster under a heavy burst-array-job workload (6 concurrent `jobsub -a 50` submissions, 300 near-instant job elements, driving high-frequency WAL-replication-ACK traffic). Found while chasing bug-1714/bug-1731's scheduler crash under ASAN -- the scheduler itself did not crash in this run, but two controllers did, independently, ~7 and ~14 minutes into the run, with byte-for-byte identical ASAN stacks (different PIDs/addresses).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "AddressSanitizer: heap-use-after-free WRITE of size 8 at vtask_enq (src/lib/vtaskmgr.c:517)…\n\nAddressSanitizer: heap-use-after-free WRITE of size 8 at vtask_enq (src/lib/vtaskmgr.c:517) -- crashed 2 of 3 controllers in a live 3-node HA cluster under a heavy burst-array-job workload (6 concurrent `jobsub -a 50` submissions, 300 near-instant job elements, driving high-frequency WAL-replication-ACK traffic). Found while chasing bug-1714/bug-1731's scheduler crash under ASAN -- the scheduler itself did not crash in this run, but two controllers did, independently, ~7 and ~14 minutes into the run, with byte-for-byte identical ASAN stacks (different PIDs/addresses).", "file": "src/lib/vtaskmgr.c (vtask_enq, line 517), src/lib/dataq.c (dataq_enq)", "root_cause": "vtask_enq() (vtaskmgr.c:497-526) calls dataq_enq(&(vtaskmgr->taskq), &(task->link)) at line 514, which PUBLISHES the task onto the shared taskq (under q->lock, released on return) -- immediately making it visible to any idle vcpu worker thread. Only AFTER that publish does line 517 write `task->owner = vtaskmgr;`. A concurrent vcpu thread can dequeue the just-published task, run it to completion (run_task -> the task's own callback, here peer_ctrl_vtask_entry), and vcpu_entry (vtaskmgr.c:106-109) can then call vtask_fini(t)+free(t) -- all BEFORE the producer thread reaches line 517 -- so the producer's write lands on already-freed memory. ASAN's two captured stacks confirm exactly this: thread T6 allocates+enqueues the task (persist_apply_pump_kick -> vtask_alloc/vtask_enq, called via the replication-ACK path repl_ack_decode_and_dispatch -> repl_on_ack -> commit_on_ack -> commit_resolve_pending -> persist_apply_pump_kick), while thread T5 has ALREADY freed the same 48-byte vtask_t region inside vcpu_entry by the time T6's own `task->owner = vtaskmgr` write executes. This is a publish-before-init race -- the task is made visible to concurrent consumers before its own initialization (the `owner` field) completes. Notably, the code immediately above (lines 508-512, `endpoint_ref(ep)`) already gets this right, per an existing bug-1298 comment explicitly calling out 'acquire BEFORE dataq_enq publishes the task to the dispatcher' -- line 517's `owner` assignment simply doesn't follow that same established discipline. Reproduces reliably under this project's burst-array live-HA workload (both controller instances that received the assignment crashed on it during a single ~30-minute run); did not reproduce in the test suite's pool-less fixtures (matches the same class of gap bug-1415 documents for a different vtaskmgr race -- production n_vcpus>1 pool concurrency needed to manifest, not exercised by tests that never call commit_set_pool).", "fix": "FIXED and TDD-verified (RED attempted, GREEN confirmed). Root cause and fix exactly as diagnosed: vtask_enq() (src/lib/vtaskmgr.c) now sets `task->owner = vtaskmgr;` BEFORE calling dataq_enq(), mirroring the existing bug-1298 endpoint_ref-before-publish discipline immediately above it in the same function; the post-enqueue success block now only calls vtaskmgr_wakeup(vtaskmgr) (the owner write no longer straddles the publish that makes the task visible to concurrent vcpu workers).\n\nNew stress test test_vtask_enq_race_stress (src/lib/testsuite/test_vtask_enq_race.c, wired into libxce_test_src in src/lib/meson.build and testsuite/test.c's extern/TEST() list as test_vtask_enq_race, right after test_vtask_admission): a real vtaskmgr_t pool (8 vcpus) with 16 producer threads each calling vtask_enq_classified(mgr, task, VTASK_LIVENESS) in a tight loop (8000 iterations each, 128,000 tasks total) with true no-op entry callbacks (atomic increment only, matching test_vtask_admission.c's live_task_entry) -- maximizing the odds a consumer vcpu dequeues, runs, and frees a task before the producer's own next instruction lands, exactly the bug-1733 race window.\n\nRED ATTEMPT: this is a genuine data race confined to the two-line window between dataq_enq()'s publish and the owner write, which cannot be forced deterministically from a test without modifying production code (unlike test_vtask_admission.c's fully deterministic cbuf-gated design). Extensive tuning was attempted against the UNMODIFIED vtaskmgr.c before applying the fix -- over a dozen configurations across 60+ manual runs: the default stress shape (6 vcpus/10 producers/4000 iters), bumped concurrency (16v/24p/6000iters, 32v/48p/20000iters), taskset-pinned to 4 cores and to a single core, background CPU-load saturation (22-44 concurrent `yes` processes churning all 22 host cores to force involuntary scheduler preemption right after dataq_enq() returns), and a few-producers/many-idle-vcpus variant (4p/16v/100000iters) meant to keep the dispatcher in a tight dequeue-polling loop. None reproduced the ASAN heap-use-after-free within these bounded attempts, including the official `nix develop ./src --command bash src/tools/xce-build.sh lib asan` run against the unmodified file (the new test itself PASSED cleanly there; the run's only SIGABRT came from a pre-existing, unrelated LeakSanitizer report in test_xcealloc.c -- 456 bytes/29 allocations via strdup in make_alloc/make_host_alloc -- confirmed present byte-for-byte identically both before and after this fix, so not a regression from this change). This is consistent with this entry's own root-cause note that the race needed real multi-node HA production load (WAL-replication-ACK bursts under `jobsub -a 50` burst-array-job load) to manifest, and that it also did not reproduce in prior pool-less test fixtures (same class of gap bug-1415 documents). The original ASAN report captured in this entry (heap-use-after-free WRITE of size 8 at vtask_enq ../lib/vtaskmgr.c:517, freed inside vcpu_entry, matching stacks on 2 of 3 live controllers) remains the authoritative confirmation the bug is real; a stress test that does not happen to hit a hard-to-trigger race in bounded iterations is a weaker regression guard than one that does, but per this fix's own instructions is not a reason to withhold the fix.\n\nGREEN: after the fix, the new stress test passes cleanly and repeatably -- 8 consecutive standalone runs, zero ASAN reports -- at both the isolated and full-suite (`xce-build.sh lib asan --incremental`) level.\n\nFull controller-suite regression (`xce-build.sh controller asan --incremental`, the real consumer of vtask_enq_classified via persist_apply_pump_kick/commit.c): 1213 tests, 2 failures -- both the pre-existing, already-logged bug-1561 timing-collision flake in test_config_repl.c (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), unrelated to this change. The specific real vtask_enq_classified callers this fix touches indirectly -- test_ha_dispatch_integration.c, test_ha_vep_teardown.c, test_apply_pump.c, test_apply_pump_integration.c, test_commit_admission_pool.c, test_commit_resolve.c -- all pass with zero failures. No regressions.", @@ -24214,9 +11283,7 @@ "concurrency", "high-severity", "live-crash", - "found-via-bug-1731-repro", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "found-via-bug-1731-repro" ], "related_bugs": [ "bug-1298", @@ -24224,15 +11291,20 @@ "bug-1714", "bug-1731" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1733.md" }, { "id": "bug-1734", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "security", "timestamp": "2026-08-30T10:15:00Z", - "error_message": "User-reported live deployment: scheduler's view of a job array's state (`[sim] jobs` / xce_scheduler -i -s) diverges sharply from the controller's own joblist for the SAME array, well after every element genuinely finished. Array 0-4-1788084587-0 (50 elements, command `hostname`): controller's `joblist` shows ALL 50 elements as DONE with real, monotonically-increasing StartTime/FinishTime timestamps (1788084587..1788084602) -- the array is genuinely, fully complete. The scheduler's own concurrent view of the identical array shows a scrambled mix: [0-4]=RUN, [5-16]=DONE, [17]=EXIT, [18-19]=DONE, [20]=EXIT, [21]=DONE, [22-24]=EXIT, [25]=DONE, [26-29]=EXIT, [30-46]=RUN, [47-49]=PEND. Two things about this pattern are themselves suspicious, not just 'the scheduler is behind': (a) elements [30-46] show RUN and [47-49] show PEND, i.e. the scheduler believes 20 elements haven't even STARTED yet, even though the controller's timestamps show the whole array finished by 1788084602 -- this is not a small lag, the scheduler's view is drastically behind reality for over a third of the array; (b) EXIT appears interspersed with DONE for a uniform array of a single reliably-successful command (`hostname`, not `/bin/false` or anything failure-prone) -- elements 17/20/22-24/26-29 show EXIT while their immediate neighbors show DONE, which is not what a simple 'scheduler hasn't caught up yet' lag would produce (a lagging-but-otherwise-correct view should show a clean PEND/RUN prefix behind a DONE suffix, not DONE and EXIT interleaved for what should be uniform outcomes). CORRELATED evidence from the SAME run: xce_scheduler's own log shows 10 consecutive WARN lines at the very same timeframe -- `xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid (rc=-17), discarding duplicate/orphaned job` for jobid 4631575359520768000 through 4631575359520768009 -- which decode (JobID bit layout: tag/cluster/apiserver/timestamp/array_idx) to EXACTLY array elements 0-4-1788084587-0[0] through [9] of this SAME array. This WARN is emitted by the bug-1714-adjacent fix landed earlier this session (commit 38437e57, 'fix(scheduler): don't hand an orphaned duplicate job to the picker') -- confirms live, in this exact reproduction, that this array's first 10 elements were pushed to the scheduler TWICE (the TOCTOU dedup race in sched_on_job_array()/handler.c that commit documented as theoretically possible is firing for real here), and the second copy is now being correctly discarded rather than corrupting state or crashing (that part of the fix is working as designed). User also separately requested (logged here as part of the same report, lower priority, not a correctness bug): xce_core_push_events's WARN line above prints the raw numeric jobid (`4631575359520768000`) instead of this project's human-readable fmt_jobid() form (`0-4-1788084587-0[0]`) that every other job-related log line in this codebase uses -- a logging-usability gap in that specific WARN, worth a one-line fix.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "User-reported live deployment: scheduler's view of a job array's state (`[sim] jobs` /…\n\nUser-reported live deployment: scheduler's view of a job array's state (`[sim] jobs` / xce_scheduler -i -s) diverges sharply from the controller's own joblist for the SAME array, well after every element genuinely finished. Array 0-4-1788084587-0 (50 elements, command `hostname`): controller's `joblist` shows ALL 50 elements as DONE with real, monotonically-increasing StartTime/FinishTime timestamps (1788084587..1788084602) -- the array is genuinely, fully complete. The scheduler's own concurrent view of the identical array shows a scrambled mix: [0-4]=RUN, [5-16]=DONE, [17]=EXIT, [18-19]=DONE, [20]=EXIT, [21]=DONE, [22-24]=EXIT, [25]=DONE, [26-29]=EXIT, [30-46]=RUN, [47-49]=PEND. Two things about this pattern are themselves suspicious, not just 'the scheduler is behind': (a) elements [30-46] show RUN and [47-49] show PEND, i.e. the scheduler believes 20 elements haven't even STARTED yet, even though the controller's timestamps show the whole array finished by 1788084602 -- this is not a small lag, the scheduler's view is drastically behind reality for over a third of the array; (b) EXIT appears interspersed with DONE for a uniform array of a single reliably-successful command (`hostname`, not `/bin/false` or anything failure-prone) -- elements 17/20/22-24/26-29 show EXIT while their immediate neighbors show DONE, which is not what a simple 'scheduler hasn't caught up yet' lag would produce (a lagging-but-otherwise-correct view should show a clean PEND/RUN prefix behind a DONE suffix, not DONE and EXIT interleaved for what should be uniform outcomes). CORRELATED evidence from the SAME run: xce_scheduler's own log shows 10 consecutive WARN lines at the very same timeframe -- `xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid (rc=-17), discarding duplicate/orphaned job` for jobid 4631575359520768000 through 4631575359520768009 -- which decode (JobID bit layout: tag/cluster/apiserver/timestamp/array_idx) to EXACTLY array elements 0-4-1788084587-0[0] through [9] of this SAME array. This WARN is emitted by the bug-1714-adjacent fix landed earlier this session (commit 38437e57, 'fix(scheduler): don't hand an orphaned duplicate job to the picker') -- confirms live, in this exact reproduction, that this array's first 10 elements were pushed to the scheduler TWICE (the TOCTOU dedup race in sched_on_job_array()/handler.c that commit documented as theoretically possible is firing for real here), and the second copy is now being correctly discarded rather than corrupting state or crashing (that part of the fix is working as designed). User also separately requested (logged here as part of the same report, lower priority, not a correctness bug): xce_core_push_events's WARN line above prints the raw numeric jobid (`4631575359520768000`) instead of this project's human-readable fmt_jobid() form (`0-4-1788084587-0[0]`) that every other job-related log line in this codebase uses -- a logging-usability gap in that specific WARN, worth a one-line fix.", "file": "src/scheduler/src/handler.c (sched_on_job_array's dedup-race, already documented as a known TOCTOU race by commit 38437e57), src/scheduler/src/core_events.c (xce_core_push_events, WARN log site + the JOBS_PUSH duplicate-discard path from commit 38437e57), src/scheduler/src/driver.c / src/controller/sched.c / src/controller/persist.c (the job-update notify pipeline this session's bug-1731/bug-1714 investigations already covers)", "root_cause": "NOT YET ROOT-CAUSED -- logged for follow-up investigation per user request, not investigated live this session (a separate, still-running background investigation was mid-flight on bug-1714's own stale-reader when this was reported). Working hypothesis, NOT confirmed: this looks like more than one mechanism compounding. (1) The already-known, already-partially-mitigated sched_on_job_array() dedup race (see bug-1714's own root_cause history and commit 38437e57's message) is CONFIRMED firing live here (the WARN evidence decodes to exactly this array's first 10 elements) -- but discarding a genuine duplicate PUSH should be harmless on its own (both copies start from identical fresh JOB_STAT_PEND state; sched_add_job()'s key_add() correctly keeps the first copy), so this alone should not explain elements staying stuck at RUN/PEND when the controller confirms DONE. (2) The DONE/EXIT interleaving for a uniform, reliably-successful command (hostname) suggests either an ordering bug in how terminal-state notifications get applied (an older notification overwriting a newer one, or vice versa, for different elements) -- which would tie into bug-1731's own notify-drop/redelivery investigation and its live-captured evidence of scheduler-endpoint-NULL notify drops -- or a distinct bug in how JOB_STATE_CHANGE vs JOB_UPDATE detail routing interacts for array elements specifically (both hook-detail paths were touched this session by bug-1731's fixes; worth checking whether an array element's terminal notification can race between the two). (3) The ~20 elements the scheduler shows as never having started (RUN/PEND for [30-49]) despite controller-confirmed completion by 1788084602 is the most severe and least explained part of this report -- if genuinely not just a stale UI snapshot, this suggests SOME elements' notifications are being lost entirely, not just delayed, distinct from anything bug-1731's fix addressed (bug-1731 fixed the CONTROLLER never firing the right hook; this could be a SEPARATE loss further downstream, e.g. in delivery to the scheduler, or in how the scheduler's REPL/inspect view is rendered from its own internal state). NOT RULED OUT: this could also be nothing more than a snapshot-timing artifact if `[sim] jobs`/`joblist` were captured at genuinely different moments and the scheduler was still catching up -- but the user's own framing ('still have some issues') and the DONE/EXIT interleaving for a uniform command argue against pure timing lag being the whole explanation. Needs live reproduction with synchronized capture (query both views at the same instant, ideally with sequence numbers/timestamps on the query itself) to distinguish 'genuinely stuck' from 'was mid-catch-up when sampled'.\n\nADDITIONAL CONFIRMATION (same report, user follow-up): xce_test itself (the submitting client) also reports every job in this array finished successfully -- i.e. all three independent vantage points agree the work genuinely completed (worker execution, controller's own joblist/WAL state, and the submitting client's own completion tracking). This rules out worker-side execution failure and controller-side completion-tracking failure as explanations, narrowing this specifically to the scheduler-side notification delivery/application path (or, less likely, the scheduler's own REPL/inspect rendering of otherwise-correct internal state) -- consistent with, but not identical to, bug-1731's already-fixed dead-hook gap and its still-open notify-drop-on-disconnect finding.\n\nUPDATE (2026-08-30, later same session, FULLY ROOT-CAUSED via live reproduction): Built controller+scheduler+worker+xceadmin fresh into src/xce.build.bug1734 (asan), deployed a real 3-node HA cluster (localhost:19991-19993) + xcecd_virtual worker + xce_scheduler (-vvv), submitted a 50-element `hostname` array job via xce_test, and captured synchronized evidence: controller `joblist` (ground truth, all 50 DONE with monotonic timestamps) vs the SAME scheduler process's own live session recording (`xceadmin scheduler record start/stop`) replayed via `xce_scheduler -i -s ` + `continue` (runs the ENTIRE recorded session set to completion, ruling out lag) + `info jobs`. REPRODUCED EXACTLY: final replayed state showed 23 of 50 elements wrong (3 EXIT, 13 RUN, 7 PEND) despite `continue` reaching 'replay complete' -- confirms GENUINE PERMANENT DESYNC, not lag. \n\nFull causal chain (all three links independently confirmed in the SAME live run's logs): (1) The scheduler's own OUTBOUND xcecli connection to the primary controller repeatedly failed over and reconnected under the array's load -- scheduler.log shows `xcecli failover: attempt N -> localhost:1999X` bouncing across all 3 candidates, 8 extra times in an ~8s window (exact trigger for THIS layer not further isolated -- plausibly -vvv DEBUG3 log volume + concurrent WAL/replication load causing a client-side request timeout to fire spuriously; flagged as a follow-up, see below). (2) Every one of those 8 reconnects made ctrl-1.log log `handle_scheduler_join` again, which runs on_scheduler_join() (controller/sched.c) -- a FULL resync (xcesched_push_configs + xcesched_push_workers + xcesched_push_jobs) on every join, no incrementality. xcesched_push_jobs()'s XceData_data_schedjobs wire path re-sends EVERY currently-committed job (not just the array), hitting the scheduler's ordinary (non-array) JOBS_PUSH handling with zero pre-existence filtering. Confirmed live: 700 `sched_add_job failed ... discarding duplicate` WARNs across the session (100 distinct jobids x up to 8 resyncs each), vs only ONE `job array push received and expanded` INFO line -- proving these duplicates come from the ordinary resync path, NOT bug-1714's documented sched_on_job_array() TOCTOU race (that race is real but was a red herring for THIS bug's data-loss mechanism -- it only ever double-submits freshly-PEND array elements, which key_add() correctly and harmlessly rejects, exactly as bug-1714/commit 38437e57 already established). (3) During the gap between disconnect and each reconnect, c->sched.ep is NULL, and any real job-finish notify due in that window is SILENTLY SKIPPED -- ctrl-1.log's pre-existing bug-1731-era DEBUG3 diagnostic `DIAG sched_job_update_fun: jobid=... state=... -- c->sched.ep is NULL, notify SKIPPED` fired 36 times in this run, for jobids matching the array. The very next full resync (link 2) is that job's ONLY remaining chance to reach the scheduler -- but XCE_CORE_EVENT_JOBS_PUSH's handling of an already-tracked jobid (core_events.c) discarded the incoming (correct, newer) job object UNCONDITIONALLY on sched_add_job()->key_add() EEXIST, NEVER updating the existing tracked xce_sched_job_t's state -- so a job whose terminal state only ever reached the scheduler via this resync path was permanently frozen at whatever PEND/RUN state it had before, exactly matching the reported DONE/EXIT-interleaved-with-stuck-RUN/PEND symptom. CONFIDENCE: HIGH -- root cause reproduced live, instrumented with the project's own existing diagnostics (no new printf needed), and closed with a RED->GREEN regression test that fails against the pre-fix code and passes after. \n\nNOT FIXED / follow-up (deliberately out of scope for this session, logged for a future bug): WHY the scheduler's own outbound xcecli connection to the primary keeps failing over under load (causal link 1 above) is not isolated to a specific timeout value or code site -- worth its own investigation (possibly a too-aggressive client-side request timeout in the shared xcecli/xce_request layer under DEBUG3 logging + concurrent WAL replication load). Reducing that flapping would reduce how often on_scheduler_join()'s full resync fires at all, but this session's fix (below) makes the resync SAFE regardless of how often it fires, which is the more load-bearing property (a resync will always be needed on genuine scheduler restart/rejoin, not just spurious flapping).", "fix": "FIXED (2026-08-30). Root cause: src/scheduler/src/core_events.c's XCE_CORE_EVENT_JOBS_PUSH case treated sched_add_job()'s -EEXIST (job already tracked) as always a harmless no-op duplicate (per bug-1714/commit 38437e57's Bug A fix) and unconditionally freed the incoming xce_job_t without ever touching the existing tracked job. That assumption held for the array-push redelivery bug-1714 covers (both copies start fresh JOB_STAT_PEND) but not for on_scheduler_join()'s full-resync redelivery (controller/sched.c's xcesched_push_jobs(), fired on every scheduler (re)join), which can legitimately carry a job's CURRENT state -- including a terminal state the scheduler never saw because the matching real-time notify was dropped while c->sched.ep was NULL during the disconnect gap (bug-1731's still-open 'notify SKIPPED' diagnostic). Discarding that redelivery unconditionally silently and PERMANENTLY froze the scheduler's own view of the job at its pre-disconnect state, exactly reproducing this entry's reported controller-vs-scheduler divergence live. \n\nFIX (src/scheduler/src/core_events.c, XCE_CORE_EVENT_JOBS_PUSH case): on sched_add_job() EEXIST, look up the existing tracked xce_sched_job_t via sched_get_job() and, if its state differs from the redelivered copy's state, apply the SAME terminal-transition accounts/allocs cleanup XCE_CORE_EVENT_JOB_STATE_CHANGE/JOB_UPDATE already perform, update existing->state in place, and fire picker->picker_iface->on_job_state_change() so the picker's own bookkeeping stays consistent -- instead of silently dropping the newer information. The redelivered xce_job_t clone itself is still freed afterward (it was never registered in core->jobs and never will be) -- only its STATE is salvaged into the object future sched_get_job() lookups actually resolve to. Deliberately scoped to state only (not queue/resreq/other fields) -- documented as an explicit scope decision in the fix's own comment, same convention persistence.md's field-completeness guidance uses elsewhere. \n\nTDD: new regression test test_jobs_push_duplicate_jobid_with_newer_state_updates_existing_job (src/scheduler/testsuite/test_core_events.c) -- pushes jobid 56 as JOB_STAT_PEND, then redelivers the SAME jobid via a second JOBS_PUSH batch carrying JOB_STAT_DONE (simulating on_scheduler_join()'s resync racing a dropped terminal notify), asserts sched_get_job() still resolves to the SAME xce_sched_job_t (no orphaned duplicate, no key_add() churn) AND that its state is now JOB_STAT_DONE, not frozen at JOB_STAT_PEND. RED confirmed first (`Expected 8 Was 1` -- JOB_STAT_DONE vs JOB_STAT_PEND); GREEN after the fix. Full scheduler suite: 55/55 passing after the fix, zero regressions (bug-1714's own test_jobs_push_duplicate_jobid_does_not_call_on_job_push_or_leak still passes unchanged -- the two duplicate-handling paths, state-differs vs state-identical, are both exercised and both correct). \n\nLIVE RE-VERIFICATION: rebuilt xce_scheduler with the fix, restarted it against the SAME live 3-node cluster (still under the same reconnect-storm conditions -- 700 duplicate-discard WARNs still fired, confirming the fix does not suppress or depend on suppressing the underlying reconnect flapping), submitted a second 50-element array job, captured a fresh session recording, and replayed it via `xce_scheduler -i -s ` + `continue` + `info jobs`: all 100 jobs across both arrays (the pre-fix array replayed through the fixed binary this time, plus the new array) now show DONE, matching the controller's joblist exactly -- zero divergence, confirming the fix closes the reported gap under the exact live conditions that originally exposed it.", @@ -24243,7 +11315,6 @@ "state-desync", "found-live", "array-job", - "fixed", "root-caused", "reconnect-storm", "notify-drop-on-disconnect", @@ -24259,16 +11330,22 @@ "bug-1731", "bug-1733" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "38437e57" + ], + "record": ".wolf/bugs/bug-1734.md" }, { "id": "bug-1735", "status": "fixed", "status_updated": "2026-09-24", "status_note": "d8a37fa2", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-30T10:30:00Z", - "error_message": "PRIORITY: HIGH -- affects data the scheduler needs for resource-usage-aware scheduling decisions (see priority note below). Split 2026-08-30 from a broader initial report into this (core usage: memory + CPU time) and bug-1736 (display-only statistics: swap, task count, peak/efficiency percentages) at user request, since the two have very different urgency and the broader one should not block this one.\n\nUser-reported: job memory usage always shows 0 in the scheduler's simulation record file (.replay), despite the worker daemon presumably measuring real resource usage. User asked to check whether the worker even reports this data at all -- CONFIRMED: it does, but the value never survives to the controller due to a plain resource-name string mismatch, plus a second, independent gap in how the controller aggregates whatever it does receive.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "affects data the scheduler needs for resource-usage-aware scheduling decisions (see priority note…\n\nPRIORITY: HIGH -- affects data the scheduler needs for resource-usage-aware scheduling decisions (see priority note below). Split 2026-08-30 from a broader initial report into this (core usage: memory + CPU time) and bug-1736 (display-only statistics: swap, task count, peak/efficiency percentages) at user request, since the two have very different urgency and the broader one should not block this one.\n\nUser-reported: job memory usage always shows 0 in the scheduler's simulation record file (.replay), despite the worker daemon presumably measuring real resource usage. User asked to check whether the worker even reports this data at all -- CONFIRMED: it does, but the value never survives to the controller due to a plain resource-name string mismatch, plus a second, independent gap in how the controller aggregates whatever it does receive.", "file": "src/worker/src/xcecd_cgroups.c (collect_job_rusages, line ~1418-1436 -- the worker-side cgroup-based measurement and resname assignment), src/controller/misc.c (update_host_runusage, line ~128-138 -- the controller-side parser), src/controller/job.c (update_job_rusages, line ~3200 -- pre-existing 'FIXME update other fields of job rusage later' comment), src/scheduler/include/sched/types.h:115 (xce_sched_job_t.rusages -- confirmed the scheduler stores/forwards the RAW xce_job_rusages_t wire report unmodified, does not itself re-parse or aggregate it), src/scheduler/src/recording_events.c:142 (wire.body.job_rusage = ev->u.job_rusage.rusages -- confirms the .replay file records whatever raw resname/value pairs it received, unchanged)", "root_cause": "TWO CONFIRMED, INDEPENDENT GAPS -- either alone would produce 'memory usage always 0'; both are real. Neither was live-verified this session (no real cgroup-backed xcecd deployment was exercised -- see the environment-limitation note below), but both are confirmed by direct code reading with exact citations, and there is NO translation/mapping layer anywhere between the two mismatched naming conventions (grepped the whole worker/controller/lib/scheduler source tree for both string sets -- only the two sites below reference them at all).\n\nFINDING 1 (the primary, most likely explanation): resource-name string mismatch between producer and consumer. collect_job_rusages() (xcecd_cgroups.c, the REAL, cgroup/systemd-backed worker's rusage collector -- queries a job's own 'xcecd-worker-.service' systemd unit via sd_bus_open_system()) reports exactly 4 resnames: 'cpu_time_sec', 'memory_rss', 'memory_swap', 'tasks_current'. update_host_runusage() (misc.c, the controller-side parser that receives this over the wire) only recognizes the LITERAL strings 'cputime' and 'mem' -- neither of which matches ANY of the 4 names the worker actually sends. Every rusage report from the real worker therefore falls through both `if (strcmp(...))` checks doing nothing, silently, for every single field -- host_runusage[j]->cpu_time and ->mem_usage stay at their calloc-zeroed default FOREVER, regardless of what the worker's cgroup query actually measured. No error, no log, no indication anything was dropped -- this is why it reads as 'always 0' rather than an obvious failure.\n\nFINDING 2 (independent, compounding): even if Finding 1 were fixed and host_runusage[]'s PER-HOST fields started populating correctly, the TOP-LEVEL AGGREGATE fields of xce_job_runusage_t (max_mem, avg_mem, used_mem, cpu_time, peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency) are NEVER computed from the per-host breakdown anywhere -- update_job_rusages() (job.c) calls update_host_runusage() (which only touches host_runusage[]) then has an explicit, still-open '/* FIXME update other fields of job rusage later */' comment immediately after, with no code implementing that FIXME anywhere in this codebase (grepped). Any downstream consumer that reads the AGGREGATE fields (rather than iterating host_runusage[] itself) would see 0 regardless of Finding 1's fix.\n\nSCHEDULER-SIDE STRUCTURE NOTE (relevant context, not itself a bug): the scheduler does NOT receive or store the aggregated xce_job_runusage_t at all -- xce_sched_job_t.rusages (sched/types.h:115) is typed xce_job_rusages_t*, the SAME flat wire-report struct the worker originally sent (jobid/npids/pids/count/usages[] of raw {resname,value} pairs) -- forwarded/stored as-is, and recording_events.c's capnp encode for the .replay file writes that same raw struct unchanged. So the .replay file the user is inspecting genuinely contains whatever resname/value pairs the worker sent, byte for byte -- if Finding 1 is fixed (or the worker already sent matching names), the RAW data WOULD flow all the way into the recording faithfully. Grepped src/scheduler/src/repl*.c for any resname-aware rendering of memory usage in the interactive REPL/inspect commands and found NONE -- could not determine within this session's time budget whether the specific tool/view the user used to inspect the .replay file (unclear from the report whether it's the live REPL's own display, a separate dump/analysis tool, or something else) has ITS OWN resname expectation that might differ yet again from both 'mem' and 'memory_rss' -- flagged as an open question for whoever picks this up, not confirmed either way.\n\nENVIRONMENT LIMITATION NOTE: this session's own live burst-testing (bug-1714 investigation) exclusively used xcecd_virtual (the no-root virtual worker), never the real, cgroup/systemd-backed xcecd binary collect_job_rusages() lives in -- this sandbox has no D-Bus/systemd (already confirmed in this session's bug-1731 follow-up work: get_systemd_job_exit_code() always returns -1 here), so sd_bus_open_system() at collect_job_rusages()'s own first real step would itself fail immediately in this environment (returns NULL before ever reaching the resname assignment code) -- this bug could not have been, and was not, live-reproduced in this sandbox; it is a pure static-code finding, applicable specifically to the user's own real deployment (which does have real cgroups/systemd, matching the user's own 'live deployment' framing throughout this investigation).\n\nPRIORITY NOTE (2026-08-30, user request): this half of the original finding -- memory (memory_rss/'mem') and CPU time (cpu_time_sec/'cputime') specifically -- is the HIGH-PRIORITY half. These are exactly the two metrics xce_host_runusage_t already has dedicated fields for (cpu_time, mem_usage) and the two the controller's parser already has matching (if misspelled) if-branches for -- clearly the originally-intended 'core' usage metrics, as opposed to the swap/task-count/peak/efficiency fields split out to bug-1736, which have no controller-side field or computation logic at all today. IMPORTANT CAVEAT, checked directly this session: grepped the full scheduler source tree for any consumer of rusage/usage data in scheduling/picker/allocation logic and found NONE -- xce_sched_job_t.rusages (sched/types.h:115) is stored and forwarded into recordings but not read by any picker plugin or allocation code today. So this fix corrects the DATA PIPELINE (worker measurement -> controller aggregation -> scheduler storage/recording), the necessary prerequisite for any future usage-aware scheduling policy (e.g. backfill sizing, over-commit detection, preemption on real memory pressure) -- it does not itself wire usage data into a scheduling decision, since no such consumer exists yet. Marked high priority on that basis (unblocks correct data for future scheduling work, and is also simply a correctness bug in its own right for any operator-facing usage reporting), not because it changes scheduling behavior today.", "fix": "NOT FIXED -- logged for follow-up, scope narrowed 2026-08-30 to memory + CPU time only (see bug-1736 for the split-off lower-priority statistics). RECOMMENDED FIX, Finding 1 (resname mismatch): align 'cpu_time_sec'/'memory_rss' (worker, xcecd_cgroups.c) with 'cputime'/'mem' (controller, misc.c) -- pick a direction (see the two options in the original finding, not repeated here) and apply consistently. RECOMMENDED FIX, Finding 2 (aggregation): implement update_job_rusages()'s (job.c) still-open FIXME specifically for max_mem/avg_mem/used_mem/cpu_time -- derive from host_runusage[]'s per-host entries each time a report merges in (max/avg/sum per field semantics). Both need TDD (unit tests, not just a live check) plus live verification against a REAL cgroup-backed xcecd deployment -- this session's sandbox has no D-Bus/systemd and cannot provide that verification itself (see the original environment-limitation note).\n\nUPDATE (2026-09-03, verification session): CONFIRMED ALREADY FIXED. Both recommended fixes were implemented and committed 2026-08-30 (commit d8a37fa27 for this entry's resname-mismatch Finding 1; commit 5c2c6664e for bug-1736's sibling swap/task-count fields) but this entry was never marked fixed. update_host_runusage() (misc.c) now matches the worker's real resnames ('cpu_time_sec'/'memory_rss'), and aggregate_job_runusage() (misc.c) computes max_mem/avg_mem/used_mem/cpu_time/peak_cpu_time from the per-host breakdown, called from update_job_rusages() (job.c:3502) -- replacing the old FIXME. TDD unit tests exist (src/controller/testsuite/test_host_runusage.c, 7 tests covering resname matching, multi-host update-in-place, swap/task-count parsing, and aggregation single/multi-host/running-max-peak/null-safety). Re-verified this session: fresh /xce-build controller asan, all 6 controller test suites pass including 'host runusage test' (test_host_runusage.c). No code change was needed -- this session's action was verification + closing out the bug-log entry.", @@ -24291,23 +11368,28 @@ "priority-high", "split-2026-08-30", "core-usage-metrics-only", - "fixed", "verified-2026-09-03" ], "related_bugs": [ "bug-1734", "bug-1736" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "d8a37fa2" + ], + "record": ".wolf/bugs/bug-1735.md" }, { "id": "bug-1736", "status": "partial", "status_updated": "2026-09-24", "status_note": "swap/tasks/peak_cpu_time fixed 5c2c6664; peak_cpu_duration/efficiency still not computed", + "component": "controller", + "severity": "unknown", "timestamp": "2026-08-30T10:45:00Z", - "error_message": "PRIORITY: LOW -- display/reporting only, not consumed by any scheduling logic. Split 2026-08-30 from bug-1735 at user request: the 'statistics-only' half of that finding -- resource usage fields with no scheduling relevance, purely for operator-facing job accounting/reporting (bjobs -l/bhist-style views), as opposed to bug-1735's memory+CPU-time core, which is a prerequisite for future scheduling decisions. Worker (collect_job_rusages(), xcecd_cgroups.c) already measures and sends 'memory_swap' and 'tasks_current' per job, but the controller has NO field to receive them at all (xce_host_runusage_t only has cpu_time/mem_usage -- no swap or task-count field exists) -- these two are silently dropped entirely, not just misnamed. Separately, xce_job_runusage_t's peak/efficiency fields (peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency) have no computation logic anywhere in the codebase (grepped) -- nothing populates them, ever, regardless of bug-1735's fix.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "display/reporting only, not consumed by any scheduling logic\n\nPRIORITY: LOW -- display/reporting only, not consumed by any scheduling logic. Split 2026-08-30 from bug-1735 at user request: the 'statistics-only' half of that finding -- resource usage fields with no scheduling relevance, purely for operator-facing job accounting/reporting (bjobs -l/bhist-style views), as opposed to bug-1735's memory+CPU-time core, which is a prerequisite for future scheduling decisions. Worker (collect_job_rusages(), xcecd_cgroups.c) already measures and sends 'memory_swap' and 'tasks_current' per job, but the controller has NO field to receive them at all (xce_host_runusage_t only has cpu_time/mem_usage -- no swap or task-count field exists) -- these two are silently dropped entirely, not just misnamed. Separately, xce_job_runusage_t's peak/efficiency fields (peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency) have no computation logic anywhere in the codebase (grepped) -- nothing populates them, ever, regardless of bug-1735's fix.", "file": "src/worker/src/xcecd_cgroups.c (collect_job_rusages -- already sends memory_swap/tasks_current, no controller-side field to receive them), src/include/xce/types.h (xce_host_runusage_t -- would need new fields for swap/task-count; xce_job_runusage_t -- the peak/efficiency fields with no producer), src/controller/misc.c (update_host_runusage -- would need new parse branches), src/controller/job.c (update_job_rusages -- same open FIXME as bug-1735, this is the other half of what it would need to cover)", "root_cause": "See bug-1735's root_cause for the full original investigation (resname-mismatch mechanism, environment-limitation note re: this session's sandbox having no real cgroup-backed worker to live-verify against) -- this entry covers only the subset of that finding split out as lower-priority. Unlike bug-1735's memory/cpu_time (which have existing-but-mismatched fields), memory_swap/tasks_current have NO receiving field in xce_host_runusage_t at all -- the worker's measurement is a complete dead end on the controller side, not a naming mismatch. The peak/efficiency fields (xce_job_runusage_t's own top-level fields) are simply never computed by anything, anywhere -- confirmed via grep, no partial implementation exists to build on.", "fix": "PARTIALLY FIXED (2026-08-30). Of the two gaps this entry covers:\n\n(1) memory_swap/tasks_current -- FIXED. Added mem_swap (double)/tasks_current (int) fields to xce_host_runusage_t (types.h), matching capnp fields (memSwap @3/tasksCurrent @4) appended to the HostRunusage schema (controller.capnp) for WAL field-completeness, parsing in update_host_runusage() (misc.c) for the worker's 'memory_swap'/'tasks_current' resnames, and per-host copy in merge_job_runusage()'s (job.c) existing-host branch (the new fields would otherwise have been silently dropped on WAL replay/catchup for a host already seen).\n\n(2) peak_cpu_time -- FIXED. aggregate_job_runusage() (misc.c) now tracks it as the running MAX of the per-call cpu_time sum across every report a job has ever received (not just the latest), guarding against a later report reducing it (e.g. an out-of-order merge/replay).\n\nSTILL NOT FIXED, DELIBERATELY (documented scope decision, not an oversight -- see aggregate_job_runusage()'s own doc comment in misc.c): peak_cpu_duration and the three efficiency fields (mem_efficiency/avg_cpu_efficiency/peak_cpu_efficiency). These need a 'used vs. requested' ratio against a job's resource request (a resreq expression tree, not a plain core/slot count) and/or an unspecified duration semantic -- genuine product/design decisions, not a formula that can be safely guessed. Left at their existing zero value rather than populated with an invented number that could mislead an operator.\n\nVerified: full `meson test --suite controller` clean, Ok:6 Fail:0, 1220/1220 individual controller_unit_test cases pass (zero regression from touching the shared xce_host_runusage_t struct + its capnp schema), plus 2 new regression tests in test_host_runusage.c (7/7 cases pass).", @@ -24324,25 +11406,29 @@ "static-analysis-only", "unreproduced-in-sandbox", "needs-design-not-just-fix", - "partially-fixed", "peak-cpu-time-fixed", "swap-taskcount-fixed", - "efficiency-fields-remain-scope-decision", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "efficiency-fields-remain-scope-decision" ], "related_bugs": [ "bug-1735" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "5c2c6664" + ], + "record": ".wolf/bugs/bug-1736.md" }, { "id": "bug-1737", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-08-30T22:05:00Z", - "error_message": "User-reported (live 3-node HA cluster + worker, 50-element array job `sleep 100 && hostname`, xce_scheduler -i -s vs controller joblist): job 0-8-1788143749-0[38] shows EXIT in the scheduler's replay (`info jobs`) but DONE in the controller's joblist -- confirmed a REAL state divergence (JOB_STAT_EXIT=0x04 and JOB_STAT_DONE=0x08 are genuinely different bits, src/include/xce/types.h), not cosmetic.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (live 3-node HA cluster + worker, 50-element array job `sleep 100 && hostname`…\n\nUser-reported (live 3-node HA cluster + worker, 50-element array job `sleep 100 && hostname`, xce_scheduler -i -s vs controller joblist): job 0-8-1788143749-0[38] shows EXIT in the scheduler's replay (`info jobs`) but DONE in the controller's joblist -- confirmed a REAL state divergence (JOB_STAT_EXIT=0x04 and JOB_STAT_DONE=0x08 are genuinely different bits, src/include/xce/types.h), not cosmetic.", "file": "src/controller/sched.c (sched_job_exit_fun, renamed sched_job_finish_fun), src/scheduler/src/handler.c (sched_handle_notify's NotifyData_data_schedExitJob case), src/schema/controller.capnp (schedExitJob field), src/include/typemap.h (xce_notify_data_t's sched_exit_job union member), src/lib/helper.c (xce_notify_data_ptr)", "root_cause": "CONFIRMED via direct code reading (both static and, indirectly, live -- the fixed behavior was verified against a real captured replay; the pre-fix defect itself was not separately re-reproduced live in this session since fixing it was done in the same pass). HOOK_DETAIL_JOB_EXIT fires for EVERY job finish, both success and failure -- job.c's record_job_finish_fact()/apply_job_finish_transition() decide DONE (exit_code==0) vs EXIT (exit_code!=0) BEFORE firing this exact hook detail either way (mirrors persist.c's own WAL_JOB_FINISH -> HOOK_DETAIL_JOB_EXIT mapping, `case HOOK_DETAIL_JOB_EXIT: return WAL_JOB_FINISH;`). sched.c's sched_job_exit_fun() (the HOOK_DETAIL_JOB_EXIT subscriber that notifies the scheduler) receives xce_jobfinish_t* info (which HAS an exit_code field) but never read it -- it unconditionally sent a bare jobid via NotifyData_data_schedExitJob (controller.capnp's schedExitJob was a plain UInt64, no exit_code at all on the wire). scheduler/src/handler.c's sched_handle_notify() then unconditionally mapped ANY schedExitJob to JOB_STAT_EXIT, with no way to ever land on JOB_STAT_DONE via this live path. Under STABLE connectivity this was normally self-correcting: the SAME WAL apply also fires HOOK_DETAIL_JOB_UPDATE moments later (persist.c:4854-4855, immediately after the HOOK_DETAIL_JOB_EXIT fire, same lock scope), which carries the job's REAL, already-final job->job.state via schedUpdateJob -> XCE_CORE_EVENT_JOB_UPDATE's `job->state = updated->state;` (core_events.c), overwriting the wrongly-forced EXIT with the correct DONE within the same notify batch. But this is TWO SEPARATE notify() calls, not atomic -- if the scheduler's connection blipped between them (the same reconnect-flakiness bug-1731/bug-1734 already documented live under this exact 50-element-array load), the correcting schedUpdateJob could be dropped (c->sched.ep NULL at fire time) while the earlier, wrong schedExitJob(EXIT) had already landed -- leaving the scheduler permanently stuck at EXIT for a job the controller correctly recorded as DONE. This exactly explains why only ONE array element (out of 50) showed the divergence: only that job's particular pair of notifies happened to straddle a reconnect gap.", "fix": "FIXED (2026-08-30). Reused the EXISTING JobFinishInfo wire shape (jobid + code) apiserver_job_exit_fun() (apiserver.c) already sends for this identical hook, rather than inventing a new capnp struct: controller.capnp's schedExitJob field changed from `UInt64` to `JobFinishInfo`; typemap.h's xce_notify_data_t.sched_exit_job union member changed from `uint64_t` to `xce_jobfinish_t*`; sched.c's sched_job_exit_fun() (renamed sched_job_finish_fun() for accuracy -- it covers both DONE and EXIT outcomes, not just EXIT) now sends `.sched_exit_job = info` (the whole struct, mirroring apiserver_job_exit_fun's `.finishinfo = info` pattern) instead of `info->jobid`; scheduler/src/handler.c's sched_handle_notify() now reads finish->exit_code and mirrors job.c's own decision exactly: exit_code==0 -> JOB_STAT_DONE, else -> JOB_STAT_EXIT, instead of unconditionally assuming EXIT. helper.c's xce_notify_data_ptr() updated to match the new pointer type (dropped a now-redundant void* cast). TDD: added test_handle_notify_job_exit_maps_exit_code_to_done_or_exit (test_handlers.c) -- exit_code==0 -> DONE, exit_code!=0 -> EXIT, NULL payload -> -EINVAL (no crash). Full lib+controller+scheduler+worker build and test suite (both a dedicated full build via xce-build.sh and an incremental rebuild+retest after further edits): 61/61 Ok for controller+scheduler (66/67 overall including lib, the 1 failure a pre-existing unrelated ASAN-leak fixture bug already logged separately, not caused by this diff), xceadmin 288/288. Live-verified: deployed a fresh 3-node HA cluster + worker with the fixed binaries, submitted a 51-job workload (50-element array + 1 standalone job, all exiting 0), let everything run to completion -- the scheduler's replay showed DONE for every single job, matching the controller's joblist exactly, with zero EXIT/DONE divergence.", @@ -24355,7 +11441,6 @@ "job-state", "exit-code", "done-vs-exit", - "fixed", "confirmed-live" ], "related_bugs": [ @@ -24363,15 +11448,20 @@ "bug-1734", "bug-1738" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1737.md" }, { "id": "bug-1738", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-30T22:10:00Z", - "error_message": "Found investigating Symptom 4 (EXIT vs DONE divergence) alongside bug-1737: a SEPARATE, genuinely different mechanism that can ALSO clobber a correct terminal job state on the scheduler side, independent of the schedExitJob exit_code gap. Not separately live-reproduced this session (no post_exec-stage job failure was exercised against the live cluster) -- confirmed by direct code reading with an exact, unambiguous mechanism.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Found investigating Symptom 4 (EXIT vs DONE divergence) alongside bug-1737: a SEPARATE, genuinely…\n\nFound investigating Symptom 4 (EXIT vs DONE divergence) alongside bug-1737: a SEPARATE, genuinely different mechanism that can ALSO clobber a correct terminal job state on the scheduler side, independent of the schedExitJob exit_code gap. Not separately live-reproduced this session (no post_exec-stage job failure was exercised against the live cluster) -- confirmed by direct code reading with an exact, unambiguous mechanism.", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_STATE_CHANGE case, and the JOBS_PUSH duplicate-jobid merge bug-1734 added), src/controller/sched.c (sched_job_fail_fun, comment only)", "root_cause": "CONFIRMED via direct code reading. HOOK_DETAIL_JOB_FAIL is a narrower event than HOOK_DETAIL_JOB_EXIT/finish -- job.c's apply_job_fail_transition() only forces JOB_STAT_EXIT for a genuinely pre-exit failure (the `default:` case, no prior exit info exists yet); for JOB_EXEC_STAGE_POST_EXEC/Q_POST_EXEC (a post-exec script failing on an otherwise-already-decided job) it MERGES a JOB_STAT_PERR flag onto whatever DONE/EXIT the job's own finish already decided -- it never re-decides DONE vs EXIT after the fact. But sched.c's sched_job_fail_fun() (the HOOK_DETAIL_JOB_FAIL subscriber) always sends the SAME bare-jobid schedFailJob notify regardless of which case applied, and scheduler/src/handler.c's sched_handle_notify() unconditionally maps THAT to JOB_STAT_EXIT via XCE_CORE_EVENT_JOB_STATE_CHANGE, which (before this fix) had NO guard against overwriting an already-terminal state: `job->state = new_state;` unconditionally. Net effect: a job that finished successfully (DONE, correctly notified live) and then has an UNRELATED post-exec script fail afterward would have the scheduler's view of it forcibly clobbered from DONE back to EXIT -- a real, distinct correctness bug from bug-1737's exit_code gap. SEPARATELY, this session's own bug-1734 fix (landed earlier the same session, src/scheduler/src/core_events.c, XCE_CORE_EVENT_JOBS_PUSH's duplicate-jobid merge path) has the IDENTICAL class of gap: on a resync redelivery carrying a DIFFERENT state than the existing tracked job, it merged `existing->state = job->state` unconditionally with no ordering/terminal-permanence check -- a stale or out-of-order resync redelivery claiming EXIT for a job already correctly DONE would silently regress it. Both gaps share one missing invariant: once a job reaches a terminal state (DONE or EXIT) on the scheduler's own tracked copy, that state must be PERMANENT -- mirroring the controller's own job_state_is_terminal() doctrine (job.c: apply_job_finish_transition() no-ops a duplicate FINISH once terminal) -- and no single-job notify or resync redelivery should ever be able to change it to something DIFFERENT.", "fix": "FIXED (2026-08-30). Added one shared static helper, job_state_regresses_terminal(current_state, new_state) (core_events.c, just above xce_core_push_events()): true iff current_state is already DONE or EXIT AND new_state is different. Applied it at BOTH vulnerable call sites: (1) XCE_CORE_EVENT_JOB_STATE_CHANGE's case -- rejects (WARN-logged, no-op) a new_state that would regress an already-terminal job, before the existing accounts/allocs-cleanup + state-assignment logic runs; (2) the JOBS_PUSH duplicate-jobid merge path (bug-1734) -- same guard, applied before deciding how to merge. Also added a clarifying comment to sched.c's sched_job_fail_fun() documenting the POST_EXEC-vs-default distinction and why this core-side guard (not a sched.c-side fix) is the right place to close the gap, since the scheduler has no JOB_STAT_PERR-equivalent concept of its own to merge instead -- 'never regress a terminal state' is simpler and sufficient. TDD: added test_job_state_change_does_not_regress_terminal_state (drives a job to DONE via JOB_STATE_CHANGE, then a second JOB_STATE_CHANGE claiming EXIT must be ignored) and test_jobs_push_duplicate_jobid_does_not_regress_terminal_state (same invariant via the JOBS_PUSH merge path) in test_core_events.c. Full scheduler+controller suite: 61/61 Ok, zero regressions -- the pre-existing bug-1734 test (PEND -> DONE merge, a legitimate forward transition) still passes unchanged, confirming the new guard only blocks a REGRESSION (terminal -> different), not a legitimate non-terminal -> terminal merge.", @@ -24380,23 +11470,27 @@ "job-state", "terminal-state", "job-fail", - "regression-guard", - "fixed" + "regression-guard" ], "related_bugs": [ "bug-1734", "bug-1737", "bug-1739" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1738.md" }, { "id": "bug-1739", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-30T22:15:00Z", - "error_message": "User-reported (live 3-node HA cluster + worker, 50-element array job): `info summary` showed 30 elements of an array marked RUN in `info jobs`, but `info worker pc13v` only listed 22 'assigned job:' lines (matching the worker's real 22-core capacity) -- 8 elements showed job-level state RUN with NO corresponding worker/slot-assignment record at all, a 'phantom RUN' exceeding real hardware capacity.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (live 3-node HA cluster + worker, 50-element array job): `info summary` showed 30…\n\nUser-reported (live 3-node HA cluster + worker, 50-element array job): `info summary` showed 30 elements of an array marked RUN in `info jobs`, but `info worker pc13v` only listed 22 'assigned job:' lines (matching the worker's real 22-core capacity) -- 8 elements showed job-level state RUN with NO corresponding worker/slot-assignment record at all, a 'phantom RUN' exceeding real hardware capacity.", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge path)", "root_cause": "CONFIRMED via direct code reading (this exact mechanism was identified and fixed before a dedicated live reproduction of the PRE-fix phantom-RUN state was attempted; the FIX itself was verified live -- a real 50-element array job run against the fixed binary showed zero phantom RUN jobs at any inspection point). This session's own bug-1734 fix (landed earlier the same session) added a merge for XCE_CORE_EVENT_JOBS_PUSH's duplicate-jobid (EEXIST) case: on a resync redelivery carrying a DIFFERENT state than the existing tracked job, if the new state was RUN, it set `existing->state = job->state` (RUN) directly -- but did NOT transfer any allocation data. core_build_job() (this same file, called earlier in the SAME event to construct the redelivered `job` clone from the wire xce_job_t) already adopts the wire payload's `allocs` field into a fresh xce_sched_job_t exactly the same way a brand-new RUN job's JOBS_PUSH would (`job->accounts = make_job_account(job->allocs)` when `j->allocs != NULL`) -- i.e. the redelivered clone DOES carry real allocation data when the controller's resync wire payload includes it (xcesched_push_jobs(), sched.c, collects the controller's live job_t structs directly, whose ->job.allocs is the real dispatch record for a running job). But bug-1734's merge branch discarded this clone entirely (FREEUP(job) at the end of the EEXIST handler, with allocs/accounts explicitly left out of scope in that fix's own comment) without ever copying job->allocs/job->accounts onto `existing` -- leaving `existing->state == RUN` with `existing->allocs == NULL`: a job the scheduler's own resource/slot bookkeeping (`info worker`'s 'assigned job:' scan, which walks core->jobs' allocations) has no record of at all, exactly the reported phenotype.", "fix": "FIXED (2026-08-30), same edit pass as bug-1738's core_events.c guard (both touch the same JOBS_PUSH EEXIST branch). When the merge's terminal-permanence guard (bug-1738) allows the merge to proceed and the redelivered state is RUN: if the redelivered clone carries real allocation data (job->accounts != NULL && job->allocs != NULL), TRANSFER it onto `existing` (free whatever stale accounts/allocs `existing` had, then move the pointers over -- `existing->accounts = job->accounts; existing->allocs = job->allocs; job->accounts = NULL; job->allocs = NULL;` so the later FREEUP(job) doesn't double-free them) instead of leaving them untouched/discarded. If the redelivery claims RUN but carries NO allocation data (a defensive case -- an older/degraded controller payload, or a race where the dispatch record wasn't attached yet), the merge into RUN is REFUSED (WARN-logged, existing state kept unchanged) rather than manufacturing a phantom RUN from a different cause. TDD: added test_jobs_push_duplicate_jobid_run_state_transfers_allocation (RUN + real allocs -> existing->state==RUN AND existing->allocs/accounts non-NULL, transferred not discarded) and test_jobs_push_duplicate_jobid_run_state_without_allocation_is_refused (RUN + no allocs -> existing stays at its prior safe state) in test_core_events.c. Full scheduler+controller suite: 61/61 Ok, zero regressions -- bug-1734's own original test (PEND -> DONE merge, no allocation involved) still passes unchanged. LIVE RE-VERIFICATION: rebuilt, redeployed a fresh 3-node HA cluster + 22-core worker, submitted a 50-element overlapping-runtime array job (sleep 60 && hostname) plus one standalone job against the 22-slot capacity, captured a mid-run scheduler recording (19 jobs genuinely RUN, matching the worker's real capacity minus a few still-queued elements) and inspected `info worker`/`info jobs`: RUN-state job count matched 'assigned job:' count exactly at every inspection point (23 assigned = 19 RUN + 4 jobs with an in-flight allocation reservation not yet finalized to RUN, a benign timing artifact of catching mid-dispatch state, not a phantom RUN) -- no job ever showed RUN with a missing worker assignment. Could not force the SAME reconnect-storm conditions that originally exposed bug-1734's gap (this deployment's controller/scheduler connection stayed stable throughout), so the EEXIST/resync branch itself was exercised only via the unit tests above, not a live resync redelivery -- high confidence from code + unit tests, not a live end-to-end repro of the exact original trigger.", @@ -24405,26 +11499,30 @@ "job-state", "phantom-run", "allocation", - "resync", - "fixed" + "resync" ], "related_bugs": [ "bug-1734", "bug-1738", "bug-1737" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1739.md" }, { "id": "bug-1740", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-08-31T22:00:00Z", - "error_message": "The bug-1735/resource-accounting.md reservation-vs-usage reconciliation feature (sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage(), resource.c, committed this session) was wired into schedobj.c's sched_apply_job_rusage() -- but that function has exactly ONE caller anywhere in this codebase: scheduler.c's sched_update(), which itself has NO live caller at all (confirmed by exhaustive grep -- superseded by driver.c's xce_sched_driver_translate_pending_events()/xce_core_push_events() mechanism, per driver.c's own comments: \"Replaces sched_update()'s inline handling\"). The REAL, live production path for job rusage -- XCE_CORE_EVENT_JOB_RUSAGE in core_events.c's xce_core_push_events(), the SAME core-level event both the normal-mode driver (driver.c) and the sim/replay driver (sim_loop.c) translate their own rusage events into -- has its OWN separate, duplicate inline implementation of \"store rusage on job\" that never called sched_apply_job_rusage() or the new reconciliation logic at all. Result: the entire reconciliation feature was completely inert in both normal-mode (live cluster) and sim/replay mode -- exactly the context the user's original bug reports (phantom RUN jobs, EXIT/DONE divergence, mem always 0) were observed through -- despite passing all 11 of its own unit tests, which called the reconciliation functions directly, bypassing the real event-dispatch path entirely.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "The bug-1735/resource-accounting.md reservation-vs-usage reconciliation feature…\n\nThe bug-1735/resource-accounting.md reservation-vs-usage reconciliation feature (sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage(), resource.c, committed this session) was wired into schedobj.c's sched_apply_job_rusage() -- but that function has exactly ONE caller anywhere in this codebase: scheduler.c's sched_update(), which itself has NO live caller at all (confirmed by exhaustive grep -- superseded by driver.c's xce_sched_driver_translate_pending_events()/xce_core_push_events() mechanism, per driver.c's own comments: \"Replaces sched_update()'s inline handling\"). The REAL, live production path for job rusage -- XCE_CORE_EVENT_JOB_RUSAGE in core_events.c's xce_core_push_events(), the SAME core-level event both the normal-mode driver (driver.c) and the sim/replay driver (sim_loop.c) translate their own rusage events into -- has its OWN separate, duplicate inline implementation of \"store rusage on job\" that never called sched_apply_job_rusage() or the new reconciliation logic at all. Result: the entire reconciliation feature was completely inert in both normal-mode (live cluster) and sim/replay mode -- exactly the context the user's original bug reports (phantom RUN jobs, EXIT/DONE divergence, mem always 0) were observed through -- despite passing all 11 of its own unit tests, which called the reconciliation functions directly, bypassing the real event-dispatch path entirely.", + "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_RUSAGE handler -- live fix site), src/scheduler/src/schedobj.c (sched_apply_job_rusage -- dead-code path, left in place), src/scheduler/src/scheduler.c (sched_update -- unreachable caller), src/scheduler/src/resource.c (sched_reconcile_job_resource_usage/sched_reconcile_host_resource_usage), src/scheduler/src/driver.c (xce_sched_driver_translate_pending_events/xce_core_push_events -- the live path that supersedes sched_update), src/scheduler/testsuite/test_core_events.c (new regression test) [file field reconstructed post-recovery from context in error_message/root_cause/fix -- not the verbatim original string, see recovery note]", "root_cause": "Classic 'tested the function in isolation, never proved the real event path reaches it' gap. schedobj.c's sched_apply_job_rusage() and core_events.c's XCE_CORE_EVENT_JOB_RUSAGE handler are two independent, duplicate reimplementations of the same 'store rusage on job' bookkeeping (a pre-existing architectural duplication, likely left over from the scheduler-core rewrite that introduced the core/driver split -- sched_update() and its whole XCE_SCHED_OP_* dataq-dispatch mechanism appear to be the OLD, pre-rewrite path, fully superseded by driver.c's XCE_CORE_EVENT_* translation but never deleted). The two implementations had already drifted: they call DIFFERENT picker hooks (on_job_rusage vs plugin_job_rusage_update). Adding new logic to the dead one instead of the live one was an easy mistake to make without first confirming which path is actually reachable in production -- found only by directly grepping for every call site of sched_apply_job_rusage() and sched_update(), not by reading either function's own code or comments, which gave no hint of the duplication.", "fix": "FIXED (2026-08-31). Added a call to sched_reconcile_job_resource_usage() directly inside core_events.c's XCE_CORE_EVENT_JOB_RUSAGE handler (the real, live path), right after its existing job->rusages bookkeeping. schedobj.c's sched_apply_job_rusage() keeps its own copy of the reconciliation call too (harmless dead code, but still exercised by its own existing unit tests, so left in place rather than removed as an unrelated cleanup). Added a genuine end-to-end regression test, test_job_rusage_reconciles_mem_reservation_against_usage (test_core_events.c), that drives the REAL xce_core_push_events() with a dispatched job carrying a real 4MB mem allocation, then a JOB_RUSAGE event claiming 7MB actual usage, and asserts the 3MB overage lands on both the account and the resource pool -- this is the test that would have caught the original wiring gap immediately, since it exercises the actual event dispatch instead of calling the reconciliation function directly. Required white-box #include-ing resource.c into test_core_events.c (previously only core.c/core_events.c) plus two link-satisfying stubs (plugin_prepare_allocation/sched_ctx_get_uncomit_host, matching test_resource_accounting.c's own established pattern).", - "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_RUSAGE handler -- live fix site), src/scheduler/src/schedobj.c (sched_apply_job_rusage -- dead-code path, left in place), src/scheduler/src/scheduler.c (sched_update -- unreachable caller), src/scheduler/src/resource.c (sched_reconcile_job_resource_usage/sched_reconcile_host_resource_usage), src/scheduler/src/driver.c (xce_sched_driver_translate_pending_events/xce_core_push_events -- the live path that supersedes sched_update), src/scheduler/testsuite/test_core_events.c (new regression test) [file field reconstructed post-recovery from context in error_message/root_cause/fix -- not the verbatim original string, see recovery note]", "tags": [ "scheduler", "resource-accounting", @@ -24435,19 +11533,23 @@ "core-events", "mem", "duplicate-implementation", - "fixed", "found-via-self-review" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1740.md" }, { "id": "bug-1741", "status": "fixed", "status_updated": "2026-09-27", "status_note": "0cad6850: OPCode_notifyAcked -- job start/finish/fail acknowledged after commit, resent until acked; FF fallback for old controllers", + "component": "lib", + "severity": "unknown", "timestamp": "2026-08-31T23:45:00Z", - "error_message": "User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster + xcecd_virtual worker, build.dbg): after submitting a 50-element array job (jobsub -a 50 -c hostname) immediately following a fresh primary election, `xce_test`'s `joblist` shows array elements [6] through [12] (7 consecutive elements) permanently stuck in state RUN (FinishTime=0) while every other element (0-5, 13-49) correctly reached DONE with real FinishTimes, including elements submitted/dispatched both before and after the stuck range and well after the cluster had fully stabilized (up to ~46s post-submit). The user confirmed via repeated `joblist` checks over several minutes that elements 6-12 never progress -- not a transient lag.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster + xcecd_virtual worker…\n\nUser-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster + xcecd_virtual worker, build.dbg): after submitting a 50-element array job (jobsub -a 50 -c hostname) immediately following a fresh primary election, `xce_test`'s `joblist` shows array elements [6] through [12] (7 consecutive elements) permanently stuck in state RUN (FinishTime=0) while every other element (0-5, 13-49) correctly reached DONE with real FinishTimes, including elements submitted/dispatched both before and after the stuck range and well after the cluster had fully stabilized (up to ~46s post-submit). The user confirmed via repeated `joblist` checks over several minutes that elements 6-12 never progress -- not a transient lag.", "file": "src/worker/src/controller_connector.c (xcecd_job_finished), src/lib/client.c (xcecli_notify, xcecli_notify_jobfinish), src/lib/client_internal.h (XCECLI_FF_CALL), src/controller/job.c (on_job_finish, never invoked for the 6 lost reports), src/controller/cluster.c (do_notify/handle_notify)", "root_cause": "PARTIALLY ROOT-CAUSED via direct log correlation across worker, controller, and scheduler logs from the live incident (not yet independently live-reproduced in isolation this session -- found while investigating the unrelated bug-1698 HA-hang occurrence in the same incident's data, but happened ~19s BEFORE ctrl-1's hang began and is confirmed unrelated to it: ctrl-1 was fully healthy and primary at the time these 7 finish reports were lost). Timeline reconstructed from three independent logs for elements 6-12: xcecd_worker.log shows the xcecd_virtual master daemon spawned one subprocess per array element (its own established per-job-process architecture), each connecting to the master via a local unix-socket IPC (uv_ipc.c) and reporting 'Completed' for all 7 elements within a ~600ms window (16:25:00.943 to 16:25:01.535). ctrl-1.log shows dispatch_a_job_/on_job_start logged correctly for all 7 elements, but job.c's on_job_finish() (traced via 'job finish jobid ... exit code' TRACE, its own first log line) fired for element [6] ONLY -- elements [7] through [12]'s finish reports never reached on_job_finish at all, i.e. the loss is upstream of job.c entirely, not a hook-firing gap (distinct from bug-1731's fix, which addressed apply_committed_entry() firing HOOK_DETAIL_JOB_START/EXIT -- that fix only helps once on_job_finish/WAL_JOB_FINISH already happened; here it never happens for 6 of 7 jobs). scheduler.log's later repeated 'sched_add_job failed ... rc=-17, merging state into existing tracked job' WARNs for exactly elements 6-12 (recurring at 16:25:23, 16:25:41, 16:25:46 across multiple post-failover JOBS_PUSH resyncs from the new primary ctrl-2) confirm the loss is durable, not just delayed: ctrl-2 inherited ctrl-1's own WAL/replicated state, which never recorded a finish for these 6 elements either, so every resync just re-merges the same stale RUN state -- there is no surviving source of truth anywhere in the cluster that ever learned these jobs finished. Traced the worker-side send path for the likely drop mechanism: xcecd_job_finished() (controller_connector.c:1680) -> xcecli_notify_jobfinish() -> xcecli_notify() (client.c:610, explicitly commented 'Fire-and-forget chokepoint') -> XCECLI_FF_CALL (client_internal.h:290-302), whose own doc comment states plainly: 'single-shot, NOT a retry loop... evaluate EXPR exactly ONCE... A lost send rc is surfaced as-is, never retried.' All 7 near-simultaneous finish notifications for this burst go out over the SAME single xcecli_ctx/network connection (one xcecd master daemon process per worker host, matching this codebase's one-connection-per-daemon model) via this explicitly best-effort, non-retrying path -- exactly the kind of rapid same-connection burst most likely to hit whatever failure mode XCECLI_FF_CALL's own comment already acknowledges as a known risk ('A failed reconnect is non-fatal -- the send below is simply best-effort'). This is the SAME XCECLI_FF_CALL mechanism bug-1548 already flagged for a related-but-distinct reliability gap (silently self-healing the transport across ticks, masking a needed rejoin) -- this occurrence is a second, independent real-world manifestation of the same primitive's fire-and-forget design causing silent data loss, this time for job-finish reports specifically. NOT YET CONFIRMED: the exact internal reason xce_notify()'s single-shot send fails/drops 6 of 7 calls in this specific burst (whether it's a genuine transport error being silently discarded because nothing checks xcecd_job_finished()'s return value, some reentrancy/buffer-reuse issue in xce_notify()'s own encode/send path when called back-to-back without waiting for the prior call's completion, or contention on ac->failover.lock combined with a narrow race) -- would need either a live gdb/strace-instrumented repro of this exact burst pattern, or a close read of xce_notify()'s implementation and whether xcecd_job_finished()'s callers check/log its return code at all (not yet done this session). USER-PROMPTED REFINEMENT (2026-08-31, same session): the user hypothesized this might be a stale-primary artifact (worker kept reporting to the old primary and never resent to the new one). That exact framing doesn't fit the millisecond timing (these 7 finishes/sends happened at 16:25:00.9-01.5, before any failover was detected by anyone), but chasing it surfaced an important adjacent fact from the worker daemon's OWN controller-connection log (xcecd.log, root-owned -- required sudo to read, distinct from xcecd_worker.log's per-job-subprocess-only content used for the rest of this investigation): the WORKER's client-side failover logic logged its own first sign of trouble with ctrl-1 at 16:25:12.164 ('xcecli failover: attempt 1 -> localhost:9901') -- 8 SECONDS BEFORE ctrl-1 itself logged anything wrong ('standby[0] silent for 12s ... evicting', 16:25:20 per bug-1698's own timeline), and about 11-19 seconds AFTER the lost job-finish notifies for elements 6-12 were sent (16:25:00.943-01.535). The worker successfully failed over to the new primary at 16:25:22.332 ('rejoin after transparent reconnect succeeded', following attempt 2 -> localhost:9902 at 16:25:22.165). This reframes the likely relationship between this bug and bug-1698: rather than two unrelated defects that happened to occur in the same incident, the timing is consistent with a SINGLE underlying trigger -- ctrl-1's own I/O/connection handling apparently began degrading measurably before ctrl-1 itself noticed (silently failing to process/ack the worker's job-finish sends circa 16:25:00-01, then dropping the worker's connection entirely by ~16:25:12) -- with ctrl-1's own self-diagnosis (the 12s standby-liveness timeout) simply being the slowest of the three symptoms to surface. If that shared-trigger theory holds, this bug's fix should NOT rely solely on 'the worker eventually fails over correctly' (bug-1698's territory, and confirmed here to work correctly once detected, ~10s after the fact) -- XCECLI_FF_CALL's fire-and-forget send has zero resend/replay mechanism, so a notify lost during exactly this kind of covert pre-failover degradation window is gone forever regardless of how well the subsequent failover behaves. Still not proven whether ctrl-1's degradation genuinely started as early as 16:25:00 (matching the lost notifies) or only at 16:25:12 (matching the worker's own first detected symptom) with the 16:25:00-01 notify loss being a coincidentally-timed SEPARATE glitch -- distinguishing these needs the same live gdb/strace capture bug-1698's own next-steps already call for, ideally instrumented to also watch ctrl-1's handling of inbound OPCode_notify requests specifically, not just its outbound peer reconnect attempts.", "fix": "NOT YET FIXED -- found and root-caused-to-this-depth this session while investigating bug-1698 in the same incident's log data; not yet independently reproduced or fixed. Next steps: (1) confirm whether xcecd_job_finished()'s callers (worker_job_callback / process_exit_callback path in xcecd_worker.c) check and log its return code -- if not, that alone is worth fixing regardless of the deeper cause, so a real send failure is at least loud instead of silent; (2) read xce_notify()'s actual implementation (xceapi.c, shared with every other XCECLI_FF_CALL user) to determine whether it is safe to call back-to-back in a tight loop on the same api_ctx_t without waiting for a prior call's underlying I/O to fully complete, or whether it has an internal single-buffer/reentrancy hazard; (3) attempt a targeted live repro (a single worker host completing 5+ jobs within under a second, e.g. a small array of near-instant jobs against one host with concurrency >= the array size) with the worker daemon under gdb/strace to catch xce_notify()'s return code and underlying syscall result for the calls that get lost; (4) once root-caused, likely fix directions include making job-finish notifications durable/retried (not fire-and-forget) given their consequence (a permanently stuck job holding worker slots/resources forever is a correctness issue, not just a display one -- unlike heartbeats, a job-finish report has no self-healing follow-up message the way bug-1737's DONE-vs-EXIT divergence did), or serializing/queuing bursts of notify calls from the same worker daemon so they cannot race or overwrite each other on the shared connection.", @@ -24469,13 +11571,10 @@ "partially-root-caused", "needs-live-repro", "needs-owner", - "unfixed", "related-to-bug-1548", "distinct-from-bug-1731", "distinct-from-bug-1698", - "possibly-same-root-cause-as-bug-1698", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "possibly-same-root-cause-as-bug-1698" ], "related_bugs": [ "bug-1548", @@ -24485,18 +11584,23 @@ "bug-1739", "bug-1698" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "0cad6850" + ], + "record": ".wolf/bugs/bug-1741.md" }, { "id": "bug-1742", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-09-01T04:10:00Z", - "gitea_issue": 45, - "error_message": "Gitea issue #45: `xcecli jobs kill ` fails for every job-control action (kill/stop/resume/checkpoint) with `request to /xce/v1/jobs//control failed: []` -- an empty error body, not a normal ReturnError JSON payload. Reproduced by static analysis of the full CLI->apiserver call path (job ID conversion verified correct by hand: MakeJobID(1,0,1788186846,0,1) == 73932643848159233, matching the URL in the report, so the numeric-vs-string ID mismatch in the bug report is a red herring).", - "file": "src/frontend/apiserver/controllers/jobs.go (validateJobControl:2997-3072, foundJob:3077-3089, ControlJobByID:2515-2599), src/frontend/apiserver/models/jobs.go (JobInfo.JobStatus/.ArrayIdx:489,493), src/frontend/apiserver/store/convert.go (StructToOrderMap:10-35), src/frontend/apiserver/router/routers.go (gin.Default() Recovery middleware:19), src/frontend/xcecli (request.go:219 HandleErrorResponse empty-body fallback)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Gitea issue #45: `xcecli jobs kill ` fails for every job-control action…\n\nGitea issue #45: `xcecli jobs kill ` fails for every job-control action (kill/stop/resume/checkpoint) with `request to /xce/v1/jobs//control failed: []` -- an empty error body, not a normal ReturnError JSON payload. Reproduced by static analysis of the full CLI->apiserver call path (job ID conversion verified correct by hand: MakeJobID(1,0,1788186846,0,1) == 73932643848159233, matching the URL in the report, so the numeric-vs-string ID mismatch in the bug report is a red herring).", + "file": "src/frontend/apiserver/controllers/jobs.go (validateJobControl:2997-3072, foundJob:3077-3089, ControlJobByID:2515-2599), src/frontend/apiserver/models/jobs.go (JobInfo.JobStatus/.ArrayIdx:489, 493), src/frontend/apiserver/store/convert.go (StructToOrderMap:10-35), src/frontend/apiserver/router/routers.go (gin.Default() Recovery middleware:19), src/frontend/xcecli (request.go:219 HandleErrorResponse empty-body fallback)", "root_cause": "models.JobInfo declares JobStatus and ArrayIdx as plain Go `int` (jobs.go:489,493), not `int64`. store.StructToOrderMap builds jobinfo.Data via reflection (val.Field(i).Interface()), which preserves the field's concrete runtime type -- so Data[\"job_status\"] and Data[\"array_idx\"] are boxed as `int`, never `int64`. validateJobControl's guard at jobs.go:3014 (`if jobStatus, ok := rawStatus.(int64); ok`) therefore ALWAYS evaluates false for every job, unconditionally routing into the `else` error branch (3052-3060). That branch immediately does an UNCHECKED single-return assertion at jobs.go:3057, `int(jobinfo.Data[\"array_idx\"].(int64))` -- since array_idx is actually `int`, this panics with `interface conversion: interface {} is int, not int64` on every single job-control call, regardless of job status, action, or whether the job is an array member (the reporter's job happening to be an array index is incidental; the same unsafe `.(int64)` pattern is repeated at jobs.go:3024/3031/3040/3048/3066, and in foundJob at 3081-3082 for the array-index-lookup path). Gin's default Recovery() middleware (routers.go:19) catches the panic and returns an empty 500 body with no JSON, which is exactly what the xcecli client's error-fallback path (request.go:219, `HandleErrorResponse`) renders as `\"%s failed: [%s]\"` with an empty httpResp.String() -- producing the reported `failed: []` message. Confirmed the type mismatch directly by reading the struct declarations; confirmed the panic site's code by reading validateJobControl in full; not yet confirmed via a live repro/server-log stack trace (static analysis only this session).", "fix": "FIXED (2026-09-01). validateJobControl (jobs.go:2997) now converts job_status and array_idx via utils.IntOrInt64() (already used elsewhere for this exact int/int64 mismatch, e.g. processJobData) instead of bare `.(int64)` assertions -- a failed conversion is now reported as a normal failedJobs entry with a descriptive msg instead of panicking. foundJob (jobs.go:3077) gets the same fix for its array-index lookup path. TDD: added TestValidateJobControl_HandlesRealJobInfoTypes (builds a real models.JobInfo and converts it via store.StructToOrderMap(), the same reflection path production uses, then asserts validateJobControl doesn't panic and returns a correctly-typed success job -- this is the test that would have caught the original bug, since it exercises the real int/int64 boundary instead of hand-built int64 map literals), TestValidateJobControl_RejectsNonKillableStatus, and TestFoundJob_MatchesRealArrayIdxType (jobs_test.go). Confirmed red (panic at jobs.go:3057, \"interface conversion: interface {} is int, not int64\") before the fix, green after. Full apiserver suite (store/utils/controllers/backend/backend/auth) passes with zero regressions.", "tags": [ @@ -24511,34 +11615,28 @@ "empty-error-body", "new", "found-static-analysis", - "needs-live-repro", - "fixed" + "needs-live-repro" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1742.md", + "gitea_issue": 45, + "gitea_synced_status": "fixed" }, { "id": "bug-1743", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "config", + "severity": "unknown", "timestamp": "2026-09-01T04:10:00Z", - "gitea_issue": 46, - "related_bugs": [ - "bug-1673", - "bug-1556", - "bug-1784", - "bug-1785", - "bug-1786", - "bug-1787", - "bug-1788", - "bug-1789", - "bug-1790" - ], - "error_message": "Gitea issue #46 (\u8fd8\u9700\u8981\u4e00\u4e2aslots\u7684\u6982\u5ff5): host test01 shows NCPUS=16, MAX_SLOT=16 via `xcecli hosts list`, but of 9 submitted single-slot `sleep` jobs on queue `normal`, only 8 reach RUN and the 9th is stuck PEND -- dispatch stops at 8/16 host capacity. Secondary ask: `hosts list` should show slots used/free, not just NCPUS/MAX_SLOT.", - "file": "src/scheduler/src/resource.c (sched_check_host_satisfy_:328-377), src/scheduler/src/plugins/limit.c (build_vres_name/per-user-queue-project virtual resource accounting:183-227,312-376), src/configs/resources.yaml (slots resource:4, consumable:true), src/configs/limits.yaml (ug1_normal_slots example, value:8, lines 41-49), src/frontend/apiserver/models/hosts.go (HostInfo.NumRun:87)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Gitea issue #46 (还需要一个slots的概念): host test01 shows NCPUS=16, MAX_SLOT=16 via `xcecli hosts list`…\n\nGitea issue #46 (还需要一个slots的概念): host test01 shows NCPUS=16, MAX_SLOT=16 via `xcecli hosts list`, but of 9 submitted single-slot `sleep` jobs on queue `normal`, only 8 reach RUN and the 9th is stuck PEND -- dispatch stops at 8/16 host capacity. Secondary ask: `hosts list` should show slots used/free, not just NCPUS/MAX_SLOT.", + "file": "src/scheduler/src/resource.c (sched_check_host_satisfy_:328-377), src/scheduler/src/plugins/limit.c (build_vres_name/per-user-queue-project virtual resource accounting:183-227, 312-376), src/configs/resources.yaml (slots resource:4, consumable:true), src/configs/limits.yaml (ug1_normal_slots example, value:8, lines 41-49), src/frontend/apiserver/models/hosts.go (HostInfo.NumRun:87)", "root_cause": "Two independent findings. (1) Dispatch-stops-at-8 is most likely NOT a slot-counting bug in the scheduler -- MAX_SLOT genuinely gates per-host admission in sched_check_host_satisfy_() the same way `mem` does (per .kiro/steering/resource-accounting.md, reservation==usage for virtual resources like slots), but there is a SEPARATE, independent admission gate: cluster-wide usergroup+queue slot limits enforced by the scheduler's limit plugin (limit.c), configured in limits.yaml. This repo's example limits.yaml defines `ug1_normal_slots`: users in group ug1 on queue `normal` share a pool capped at exactly 8 (limits.yaml:41-49) -- an exact match to the reporter's queue (`normal`) and the observed 8/16 cutoff. This exact scenario (a queue/usergroup slot limit as an alternate explanation to a host-capacity bug) was explicitly predicted-but-unconfirmed in bug-1673 (filed against the related earlier report Gitea #40) and appears to be that prediction now reproduced. CAVEAT: limits.yaml/users.yaml in this repo look like sample/demo config (users.yaml's stock ug1 group only contains user1/user2, not `admin`) -- confirming this requires checking the reporter's actual deployed limits.yaml/users.yaml or their job's pend-reason output, neither of which was available this session (static analysis only). (2) The feature-request half (show slot usage in hosts list) is NOT starting from scratch: models.HostInfo.NumRun (hosts.go:87) already exists in the schema and is already an output column in `xcecli hosts list` (matching the reporter's own output), but `grep -rn \"\\.NumRun\\s*=\" apiserver --include=*.go` returns zero hits anywhere in the apiserver -- the field is never assigned and is permanently stuck at Go's zero value, which is exactly why the reporter's own output shows NUM_RUN=0 while 8 jobs are actually running on that host. Same for NumSSusp/NumUSusp.", - "fix": "PARTIALLY FIXED (2026-09-01). Part 2 (hosts-list slot visibility) is FIXED: added computeHostSlotUsage() (controllers/hosts.go) which scans store.Jobs' live ExecHosts JSONB (the same {\"host_slots\":[{\"host_name\",\"num_slots\"}],...} shape backend/convert_pure.go populates) and JobStatus bits (JOBSTAT[\"RUN\"]/[\"SSUSP\"]/[\"USUSP\"]) to count occupied slots per host, per sub-state -- counting SLOTS (not just job count) so the numbers are directly comparable to max_slot, matching the reporter's actual ask ('host\u4e0a\u9700\u8981\u4f53\u73b0\u7528\u4e86\u591a\u5c11slots'). processHostData() now overwrites row.Data[\"num_run\"/\"num_ssusp\"/\"num_ususp\"] with these live counts instead of leaving HostInfo's always-zero struct fields untouched. TDD: added TestProcessHostData_ComputesSlotUsageFromJobs (host_control_test.go) covering plain RUN, USUSP, a 2-slot SSUSP job (verifies slot-counting, not job-counting), a PEND job with no exec host (must not count), and a job on a different host (must not bleed across hosts). Confirmed red (num_run=0, want 6) before the fix, green after. Full apiserver suite passes with zero regressions.\n\nPart 1 (dispatch-stops-at-8 root cause): FULLY RESOLVED (2026-09-07/08, follow-up session). Confirmed live (no reporter config needed after all -- the leading hypothesis was right, just broken at a much deeper layer than 'is a limit configured'): the entire usergroup/queue limit-matching machinery in the limit plugin was silently non-functional, a 6-bug cascade -- see bug-1784 (named usergroups never registered into the scheduler), bug-1785 (unconfigured scope matched no one instead of falling back to all), bug-1786 (election loop aborted after the first matched limit), bug-1787 (extra pointer indirection broke a resmgr handle lookup), bug-1788 (instantiate_limit() conflated not-found with a real error), and bug-1789/bug-1790 (even once matching worked, the limit's own consumption computation read the wrong request vector and always saw a 0 'wanted' value, so the numeric cap itself never bound). All six fixed and live-verified. Final confirmation (2026-09-08): fresh single-node deploy at current HEAD, 9 single-slot jobs submitted as user1 (a genuine ug1 member) on queue normal -- the exact repro shape from this issue -- scheduler trace shows limit1/ug1_slots/ug1_normal_slots all correctly matching and, for the first time, a real nonzero wanted value ('unsatisfy_ resource 35 total 3.000000 inuse 3.000000 uncomit 0.000000 wanted 1.000000' / 'unsatisfy_ unmet resource 35 (limit=limit1) shortfall 1.000000'), correctly PENDing jobs past the cap. Posted as a comment on gitea#46 and closed the issue (2026-09-08).", + "fix": "PARTIALLY FIXED (2026-09-01). Part 2 (hosts-list slot visibility) is FIXED: added computeHostSlotUsage() (controllers/hosts.go) which scans store.Jobs' live ExecHosts JSONB (the same {\"host_slots\":[{\"host_name\",\"num_slots\"}],...} shape backend/convert_pure.go populates) and JobStatus bits (JOBSTAT[\"RUN\"]/[\"SSUSP\"]/[\"USUSP\"]) to count occupied slots per host, per sub-state -- counting SLOTS (not just job count) so the numbers are directly comparable to max_slot, matching the reporter's actual ask ('host上需要体现用了多少slots'). processHostData() now overwrites row.Data[\"num_run\"/\"num_ssusp\"/\"num_ususp\"] with these live counts instead of leaving HostInfo's always-zero struct fields untouched. TDD: added TestProcessHostData_ComputesSlotUsageFromJobs (host_control_test.go) covering plain RUN, USUSP, a 2-slot SSUSP job (verifies slot-counting, not job-counting), a PEND job with no exec host (must not count), and a job on a different host (must not bleed across hosts). Confirmed red (num_run=0, want 6) before the fix, green after. Full apiserver suite passes with zero regressions.\n\nPart 1 (dispatch-stops-at-8 root cause): FULLY RESOLVED (2026-09-07/08, follow-up session). Confirmed live (no reporter config needed after all -- the leading hypothesis was right, just broken at a much deeper layer than 'is a limit configured'): the entire usergroup/queue limit-matching machinery in the limit plugin was silently non-functional, a 6-bug cascade -- see bug-1784 (named usergroups never registered into the scheduler), bug-1785 (unconfigured scope matched no one instead of falling back to all), bug-1786 (election loop aborted after the first matched limit), bug-1787 (extra pointer indirection broke a resmgr handle lookup), bug-1788 (instantiate_limit() conflated not-found with a real error), and bug-1789/bug-1790 (even once matching worked, the limit's own consumption computation read the wrong request vector and always saw a 0 'wanted' value, so the numeric cap itself never bound). All six fixed and live-verified. Final confirmation (2026-09-08): fresh single-node deploy at current HEAD, 9 single-slot jobs submitted as user1 (a genuine ug1 member) on queue normal -- the exact repro shape from this issue -- scheduler trace shows limit1/ug1_slots/ug1_normal_slots all correctly matching and, for the first time, a real nonzero wanted value ('unsatisfy_ resource 35 total 3.000000 inuse 3.000000 uncomit 0.000000 wanted 1.000000' / 'unsatisfy_ unmet resource 35 (limit=limit1) shortfall 1.000000'), correctly PENDing jobs past the cap. Posted as a comment on gitea#46 and closed the issue (2026-09-08).", "tags": [ "scheduler", "slots", @@ -24556,22 +11654,35 @@ "needs-live-repro", "related-to-bug-1673", "related-to-bug-1556", - "partially-fixed", - "needs-reporter-confirmation", - "fixed" + "needs-reporter-confirmation" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "related_bugs": [ + "bug-1673", + "bug-1556", + "bug-1784", + "bug-1785", + "bug-1786", + "bug-1787", + "bug-1788", + "bug-1789", + "bug-1790" + ], + "fix_commits": [], + "record": ".wolf/bugs/bug-1743.md", + "gitea_issue": 46, + "gitea_synced_status": "fixed" }, { "id": "bug-1744", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-27", "status_note": "b970ab75: apiserver checks rusage[] syntax at submit via libxce's grammar; select/order names still scheduler-side only", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-01T05:45:00Z", - "gitea_issue": 47, - "error_message": "Gitea issue #47 (rusage\u662f\u5426\u505a\u4e86\uff1f): `xcecli jobs submit -R \"rusage[mem=100000000a0]\"` -- a malformed resreq string (trailing non-numeric \"a0\" on the mem value) -- is silently ACCEPTED and returns a normal 'Job <...> is submitted' success message. Reporter's original comment: '\u4e71\u5199\u4e5f\u53ef\u4ee5\u63d0\u4ea4' (garbage input can be submitted too). Maintainer (rshen) already commented on the issue: this is a front-end (apiserver+cli) validation gap by design intent.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Gitea issue #47 (rusage是否做了？): `xcecli jobs submit -R \"rusage[mem=100000000a0]\"` -- a malformed…\n\nGitea issue #47 (rusage是否做了？): `xcecli jobs submit -R \"rusage[mem=100000000a0]\"` -- a malformed resreq string (trailing non-numeric \"a0\" on the mem value) -- is silently ACCEPTED and returns a normal 'Job <...> is submitted' success message. Reporter's original comment: '乱写也可以提交' (garbage input can be submitted too). Maintainer (rshen) already commented on the issue: this is a front-end (apiserver+cli) validation gap by design intent.", "file": "src/frontend/xcecli/cmd/jobs/submit.go (resReq flag, no value validation, just passed through: jobReq.Resreq = js.resReq), src/frontend/apiserver/controllers/jobs.go:1602-1606 (only length-checks Resreq, no syntax/grammar validation), src/lib/resreq.c (parse_resreq/parse_rusage -- the real PEG-generated grammar parser), src/lib/rusage.peg (rusage_param grammar, mem value must match `number <- [0-9]+('.'[0-9]*)?`, so \"100000000a0\" is a genuine grammar violation), src/scheduler/src/core_events.c:236-245 (parse_resreq call site inside core_build_job()) and :751-760 (JOBS_PUSH caller: on core_build_job()==NULL, WARNs server-side and `continue`s -- job silently dropped, never added to core->jobs)", "root_cause": "The ONLY resreq validation in the entire submission path is a length check (apiserver/controllers/jobs.go:1602-1606, `len(req.Resreq) > 4094`) -- there is no syntax/grammar validation anywhere client-side or in the apiserver before a job is accepted and persisted. The REAL validator, a PEG-generated grammar parser (parse_resreq()/parse_rusage(), src/lib/resreq.c, built from src/lib/rusage.peg + xce_resreq.peg via meson custom_target), exists but is only ever invoked deep inside the C scheduler at core_build_job() time (core_events.c:236), i.e. asynchronously, well AFTER the apiserver has already accepted the job, returned a submission-success response to the client, and persisted it. When parse_resreq() fails on malformed syntax (confirmed via rusage.peg's `number` rule, `[0-9]+('.'[0-9]*)?` -- \"100000000a0\" cannot match this and leaves \"a0\" unconsumed, a genuine PEG parse failure), core_build_job() logs an ERROR and returns NULL (core_events.c:238-245); its caller in the XCE_CORE_EVENT_JOBS_PUSH handler (core_events.c:756-760) only logs a server-side WARN and `continue`s to the next job in the batch -- the malformed job is silently dropped from the scheduler's core->jobs registry entirely. Net effect is worse than 'garbage accepted': the job is accepted, shows the user a success message, is almost certainly persisted and shown as PEND via `jobs list` (this session did not confirm the exact persisted/listed state live), and then sits PEND FOREVER with zero indication to the user that anything is wrong -- the only trace is a server-side log line the user never sees. This is architecturally the same class of gap as bug-1740 (a real, working validator/reconciler exists but is wired into the wrong -- or in this case, too-late -- point in the pipeline) and matches the maintainer's own comment that front-end validation is the intended design, not yet implemented.", "fix": "NOT YET FIXED -- investigated and design-confirmed this session; DEPRIORITIZED, deferred to later (explicit user decision, not urgent). Confirmed fix approach: validate resreq synchronously in the apiserver's submit handler (controllers/jobs.go:1602, alongside the existing length check), reusing libxce's existing PEG parser rather than reimplementing the grammar in Go. Specifically parse_rusage() (src/lib/resreq.c:154, declared resreq.h:301 -- `int parse_rusage(xlist_t*, char*, char**)`) needs NO resmgr/resource-manager context, unlike the fuller parse_resreq()/parse_select() (resreq.h:303,305 -- both take a `var resmgr` to validate resource NAMES in select[]/order[] clauses, which the apiserver doesn't currently construct) -- so a resreq-name-agnostic rusage[]-only syntax check is a small, low-risk CGo wrapper, not a larger resmgr-plumbing project. apiserver/backend already has the exact split pattern to follow: a `//go:build cgo` file with the real implementation (e.g. clib.go, which already #includes libxce headers and links -lxce) plus a `//go:build !cgo` stub (stub.go, returns an errNoCGO-style error) for the CGO-disabled test path. Full parse_resreq()/resmgr-backed select[]/order[] validation is explicitly out of scope for now -- would be a separate, larger follow-up if wanted.\n\nSCOPE DECISION (2026-09-01): will be implemented as a direct bugfix when picked up, NOT a kiro spec -- small, contained blast radius (one new CGo wrapper + stub in apiserver/backend, one call site in controllers/jobs.go, matching the size/shape of the already-shipped bug-1742/bug-1743 fixes), no HA/persistence/RBAC involvement, and no open design questions left (exact function, signature, and existing cgo/stub pattern to follow are already nailed down). Only the FULLER resmgr-backed select[]/order[] validation (out of scope here) would warrant at least a /kiro-spec-quick, since it needs new apiserver-side state (a resmgr context kept in sync with resource definitions) -- not needed for this rusage-only fix.", @@ -24590,19 +11701,27 @@ "related-to-bug-1740", "not-yet-fixed", "deprioritized", - "deferred", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "deferred" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "related_bugs": [], + "fix_commits": [ + "b970ab75" + ], + "record": ".wolf/bugs/bug-1744.md", + "gitea_issue": 47, + "gitea_synced_status": "fixed" }, { "id": "bug-1745", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-09-01T00:20:00Z", - "error_message": "kiro-validate-design worker-rejoin-resync (design phase, spec never implemented yet): initial pass found 2 critical issues in design.md, self-correction during the fix pass found 2 more. None are runtime bugs (no code exists for this spec yet) -- this entry records design-document defects caught before implementation, same category as bug-1514.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "kiro-validate-design worker-rejoin-resync (design phase, spec never implemented yet): initial pass…\n\nkiro-validate-design worker-rejoin-resync (design phase, spec never implemented yet): initial pass found 2 critical issues in design.md, self-correction during the fix pass found 2 more. None are runtime bugs (no code exists for this spec yet) -- this entry records design-document defects caught before implementation, same category as bug-1514.", "file": ".kiro/specs/worker-rejoin-resync/design.md", "root_cause": "(1) design.md's Data Models section cited bug-1678 (WAL_JOB_DISPATCH not carrying job->allocs through replay) as a still-open, tracked gap justifying record_job_reconcile_assign_fact() not populating an allocation -- bug-1678 was actually fixed 2026-08-25, before this design.md was finalized; the citation was stale, and the design missed that persist_submit_job_dispatch_fact() (the fixed primitive) already has a tested NULL-alloc path it could directly reuse instead of inventing a new WAL fact shape.\n(2) design.md's dispatch-gating reuse (Requirement 5.1, XCE_WORKER_RECONCILING via dispatch_a_job_'s w->host.status != 0 check) was researched before scheduler-dispatch-commit (2026-08-24/25) added xce_classify_dispatch_fail()'s bounded-retry-then-mark_job_fail() machinery behind that exact check -- the design's own 'Revalidation Triggers' section named this exact check as a trigger but it had not actually been re-checked against current code. A RECONCILING host could be selected as a dispatch candidate by the scheduler (whose own filter only checks IS_WORKER_UNAVAIL, confirmed via grep) and then get retried-then-terminal-failed by the controller purely for being mid-reconciliation.\n(3) Fixing (1) surfaced bug-1631 (job.allocs never released on dispatch_a_job_'s retryable-failure paths, deliberately left unfixed because nothing in the system re-admitted an already-RUN job to the scheduler's PEND pool) -- worker-rejoin-resync's own Requirement 4.6 (unrecognized job -> return to PEND, eligible for re-dispatch) IS such a re-admission path, and design.md's record_job_reconcile_pending_fact spec did not release job->job.allocs, reactivating bug-1631's exact deferred trigger condition.\n(4) Fixing (3) by adding an eager job->job.allocs release exposed a self-introduced inconsistency: apply_job_update_transition() only mutates job->job.state, deferred until apply_committed_entry applies the committed fact (confirmed by reading record_job_finish_fact()'s identical deferred pattern) -- an eager allocs release paired with a deferred state change would produce a real, brief RUN-state-with-NULL-allocs window. Resolved by modeling both new job.c functions on dispatch_a_job_'s own eager-mutation-plus-explicit-persist_notify_job_updated_already_persisted pattern (bug-1679's precedent) instead, rather than record_job_finish_fact's deferred one.", "fix": "All 4 corrected directly in design.md during the same validation session (not deferred to implementation): (1) Data Models' 'Documented scope decision' + Components section rewritten -- record_job_reconcile_assign_fact() now specified to call the existing, fixed persist_submit_job_dispatch_fact(alloc=NULL) directly. (2) Added a new Scheduler-side component (scheduler.c:1211's candidate filter gets one added disjunct, IS_WORKER_RECONCILING, mirroring the existing IS_WORKER_UNAVAIL check) as the primary Requirement 5.1 enforcement, with dispatch_a_job_'s existing check kept as an unmodified defensive backstop; Requirement 5.4's disconnected-worker case given its own pre-classifier branch (bypassing xce_classify_dispatch_fail entirely, not a 3rd enum value, to avoid forcing ~5 unrelated switch sites to handle a case they can't hit). (3) record_job_reconcile_pending_fact() now specified to release job->job.allocs eagerly. (4) Both new job.c functions redesigned around dispatch_a_job_'s eager-mutation + persist_notify_job_updated_already_persisted pattern, explicitly not record_job_finish_fact's deferred one, with the rationale (worker-confirmed ground truth vs. an unconfirmed local admin command) documented inline so a future editor doesn't accidentally revert to the deferred pattern. spec.json's design.approved flipped to true with a validation_note recording this trail. No code was written -- this is a design-document-only fix; implementation-time discovery may still surface further refinement, per design.md's own remaining 'must confirm at implementation time' caveats.", @@ -24614,9 +11733,7 @@ "bug-1631", "bug-1679", "scheduler-dispatch-commit", - "not-a-runtime-bug", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "not-a-runtime-bug" ], "related_bugs": [ "bug-1678", @@ -24624,15 +11741,20 @@ "bug-1679", "bug-1514" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1745.md" }, { "id": "bug-1746", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-09-01T00:00:00Z", - "error_message": "worker-rejoin-resync task 2.2 implementation: persist_submit_job_dispatch_fact(p, jobid, state, ts, alloc=NULL) crashes with a NULL-pointer SEGV instead of producing the \"no allocation\" WAL_JOB_DISPATCH payload design.md (Data Models / Out-of-Boundary section) and bug-1745's own design-review fix claimed was already \"a tested NULL-alloc path\" reusable verbatim. Confirmed via AddressSanitizer: SEGV on unknown address 0x0 in encode_JobAlloc (lib/controller.capnp.c:9805), called from capnp_encode_joballoc (controller/persist.c:235), called from persist_submit_job_dispatch_fact (controller/persist.c:7856), called from the new record_job_reconcile_assign_fact (controller/job.c) -- reproduced twice, deterministically, both via the full meson controller-test suite and via an isolated -t test_worker_resync run.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 2, + "error_message": "worker-rejoin-resync task 2.2 implementation: persist_submit_job_dispatch_fact(p, jobid, state…\n\nworker-rejoin-resync task 2.2 implementation: persist_submit_job_dispatch_fact(p, jobid, state, ts, alloc=NULL) crashes with a NULL-pointer SEGV instead of producing the \"no allocation\" WAL_JOB_DISPATCH payload design.md (Data Models / Out-of-Boundary section) and bug-1745's own design-review fix claimed was already \"a tested NULL-alloc path\" reusable verbatim. Confirmed via AddressSanitizer: SEGV on unknown address 0x0 in encode_JobAlloc (lib/controller.capnp.c:9805), called from capnp_encode_joballoc (controller/persist.c:235), called from persist_submit_job_dispatch_fact (controller/persist.c:7856), called from the new record_job_reconcile_assign_fact (controller/job.c) -- reproduced twice, deterministically, both via the full meson controller-test suite and via an isolated -t test_worker_resync run.", "file": "src/controller/persist.c", "root_cause": "persist_submit_job_dispatch_fact() (persist.c:7844) unconditionally calls capnp_encode_joballoc(alloc, ...) regardless of whether alloc is NULL. capnp_encode_joballoc() (persist.c:225) calls encode_JobAlloc(cs, &cja, (xce_job_alloc_t*)a) directly -- NOT the NULL-safe encode_JobAlloc_ptr() wrapper that already exists in the same generated file (lib/controller.capnp.c:9388, used correctly by every OTHER encode call site) and already has an explicit NULL guard (if s==NULL, sets a capn_null pointer instead of encoding). encode_JobAlloc() itself (lib/controller.capnp.c:9322, generated capnp binding) has no NULL guard at all: its first statement dereferences its xce_job_alloc_t* parameter unconditionally (reads s->jobid). design.md (worker-rejoin-resync) and bug-1745's own design-validation fix pass both asserted this call is safe -- quoting design.md: bug-1678 already gave WAL_JOB_DISPATCH a persist_submit_job_dispatch_fact() primitive with both apply seams decoding an optional trailing JobAlloc, a NULL alloc being the documented, tested backward-compatible case -- but that claim conflates two different things: the DECODE/apply seams (apply_committed_entry, apply_wal_entry) genuinely ARE NULL-safe and tested for a payload with no trailing alloc bytes (test_wal_job_dispatch_fact.c constructs such payloads by hand, e.g. test_wjd_apply_committed_entry_transitions_pend_to_run) -- but NO existing test, and no existing production call site (dispatch_a_job_ is the only one, job.c:1731, and it always passes a freshly-calloc'd non-NULL j->job.allocs), ever calls persist_submit_job_dispatch_fact() itself with alloc=NULL. The function's own doc comment (persist.h, near line 1447) actually states the opposite precondition: alloc must be non-NULL, the real call site (dispatch_a_job_) always has one by construction. The ENCODE side was never made NULL-safe when bug-1678 fixed the DECODE side.", "fix": "FIXED (coordinator-authorized scope expansion of worker-rejoin-resync task 2.2 to include exactly this one function in persist.c, mirroring task 1.1's typemap.h/clusterconfig.h expansion precedent -- no schema or generated-code change). Added a NULL guard as the first statement of capnp_encode_joballoc() (persist.c:225): `if (a == NULL) { *out = NULL; *out_len = 0; return 0; }`. Produces a genuinely ZERO-LENGTH buffer for NULL input -- NOT a capnp-encoded null-pointer JobAlloc (what the existing encode_JobAlloc_ptr() wrapper's NULL branch would produce, which is non-zero bytes on the wire). This distinction is load-bearing: both apply seams (apply_committed_entry persist.c:4206-4227, apply_wal_entry persist.c:4759-4781) discriminate 'is there an allocation to decode' purely by `alloc_len > 0` on the trailing bytes after the {jobid,state[,start_time]} header -- a non-zero null-pointer-encoding byte count would have been misdecoded as a real (if empty/garbage) allocation to adopt, silently producing a non-NULL-but-garbage job.allocs on replay instead of the required NULL. Verified: (1) record_job_reconcile_assign_fact()'s 2 new tests (test_reconcile_assign_applies_running_no_allocs_and_notifies, test_reconcile_assign_survives_restart_replay -- the latter exercising a real disk WAL append + persist_recover() restart-replay round trip) now PASS cleanly, no SEGV. (2) record_job_reconcile_pending_fact()'s 3 existing task-2.1 tests still PASS unchanged. (3) dispatch_a_job_()'s existing non-NULL-alloc call site (job.c:1731, the only other caller) is completely unaffected -- its 5 dispatch-commit tests in test_dispatch_commit.c (test_dc_success_commits_alloc_state_and_resets_counter, test_dc_success_stamps_start_time_synchronously, test_dc_success_notifies_job_update_hook_exactly_once, test_dc_worker_not_found_leaves_job_pend_and_alloc_uncommitted, and critically test_dc_success_appends_wal_job_dispatch_durably -- a real disk WAL + persist_recover() round trip through the non-NULL-alloc path) all still PASS unchanged, confirming the new guard is a pure early-return for the NULL case with zero effect on the non-NULL path. (4) Full controller test suite (meson test -C src/xce.build 'controller test'): 6/6 OK, zero regressions anywhere. Also confirmed no secondary UBSan issue from the resulting `memcpy(dst, NULL, 0)` in persist_submit_job_dispatch_fact() when alloclen==0 (allocbuf stays NULL) -- this build runs under UBSAN_OPTIONS=halt_on_error=1 and the full suite passed clean, so glibc's memcpy tolerates a NULL/0 call here in practice.", @@ -24645,24 +11767,27 @@ "bug-1745", "null-pointer-deref", "asan", - "record_job_reconcile_assign_fact", - "fixed" + "record_job_reconcile_assign_fact" ], "related_bugs": [ "bug-1678", "bug-1745", "bug-1631" ], - "occurrences": 2, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1746.md" }, { "id": "bug-1747", "status": "fixed", "status_updated": "2026-09-26", "status_note": "FINISH half eec2596c; FAIL half 1d6c04c2: WAL_JOB_FAIL trailer carries finish_time, record_job_fail_fact_at() for resync.", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-01T00:00:00Z", - "error_message": "Pre-existing, systemic gap predating worker-rejoin-resync entirely, discovered while implementing task 3.2 (resync_apply_report, resync.c): WAL_JOB_FAIL carries no fail-time/finish-time slot in its wire payload at ANY layer, and apply_job_fail_transition() (job.c) unconditionally stamps job.finish_time = time(NULL) at APPLY time (not even receipt time) for EVERY fail-reporting caller in this codebase -- not just reconciliation. This means Requirement 4.2 ('the Controller shall record the finish time the Worker reported ... rather than the time the Controller processed the reconciliation') is currently unsatisfiable for a reconciled finished-FAILURE outcome via any existing path. UPDATE (2026-09-01, coordinator-directed re-scope): the sibling FINISH-success half of this gap is FIXED -- see 'fix' field below; only the FAIL half remains open.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "systemic gap predating worker-rejoin-resync entirely, discovered while implementing task 3.2…\n\nPre-existing, systemic gap predating worker-rejoin-resync entirely, discovered while implementing task 3.2 (resync_apply_report, resync.c): WAL_JOB_FAIL carries no fail-time/finish-time slot in its wire payload at ANY layer, and apply_job_fail_transition() (job.c) unconditionally stamps job.finish_time = time(NULL) at APPLY time (not even receipt time) for EVERY fail-reporting caller in this codebase -- not just reconciliation. This means Requirement 4.2 ('the Controller shall record the finish time the Worker reported ... rather than the time the Controller processed the reconciliation') is currently unsatisfiable for a reconciled finished-FAILURE outcome via any existing path. UPDATE (2026-09-01, coordinator-directed re-scope): the sibling FINISH-success half of this gap is FIXED -- see 'fix' field below; only the FAIL half remains open.", "file": "src/controller/job.c", "root_cause": "job_failinfo_t (xce/types.h) is {jobid, stage, code, detail} -- no finish_time field, and WAL_JOB_FAIL's durable payload (record_job_fail_fact(), job.c: persist_submit_job_fact_ex(&c->persist, WAL_JOB_FAIL, jobid, 0, WAL_JOB_FAIL_STAGE_FLAG|stage)) carries no finish-time slot at all -- confirmed by direct inspection of the payload construction. apply_job_fail_transition() (job.c, the guarded core BOTH the live job_ctrl path (mark_job_fail) and the deferred apply seam (apply_committed_entry's WAL_JOB_FAIL case) invoke) unconditionally executes `job->job.finish_time = time(NULL);` with no external value ever consulted, at APPLY time -- meaning even a caller willing to plumb a real value through record_job_fail_fact's WAL payload would still need a apply_job_fail_transition() signature change to have it actually land in job.job.finish_time. This is a pre-existing, already-documented residual: job.c's own comment near apply_job_postdone_transition (task 4.1) already states 'finish_time is not part of the wal_jobstate_t wire format ... has no apply-time equivalent' for a sibling fact type, confirming this is systemic to the wal_jobstate_t-based fact family, not something worker-rejoin-resync introduced. Fixing it for real would require extending WAL_JOB_FAIL's wire payload (wal_types.h, mirroring bug-1689's own WAL_JOB_FINISH trailing-fields precedent) PLUS an apply_job_fail_transition() signature/behavior change reaching every fail-reporting call site in the system (job_ctrl's mark_job_fail, on_job_fail's ordinary worker-fail receipt path, and this feature's own reconciliation caller) -- real, broad-blast-radius work, not a narrow patch.", "fix": "SPLIT (2026-09-01, coordinator-directed): the FINISH-success half of this gap is FIXED as part of worker-rejoin-resync task 3.2 -- added record_job_finish_fact_at(cluster_t*, job_t*, xce_jobfinish_t*, uint64_t finish_time_ms) (job.c), record_job_finish_fact()'s own worker-reported-finish-time sibling (identical guard/hook-fire/durable-append body, differing only in passing the caller-supplied finish_time -- converted ms->s -- to persist_submit_job_finish_fact() instead of time(NULL)); record_job_finish_fact() itself is UNMODIFIED, its own caller (on_job_finish, an ordinary near-instantaneous worker report) correctly keeps using receipt-time. resync_apply_report()'s finishedOk branch now calls record_job_finish_fact_at(c, job, &finish, entry->finish_time_ms). Verified via a real restart-replay test (test_apply_report_finished_success_uses_worker_reported_finish_time, test_worker_resync.c): a finished-success entry with finish_time_ms set to 'now minus 1 day' recovers job.finish_time equal to that worker-reported value after persist_recover(), not reconciliation-processing wall-clock time. This was possible narrowly for FINISH because persist_submit_job_finish_fact() already accepted an explicit finish_time parameter at the primitive level -- record_job_finish_fact()'s specific call site just never exposed it. The FAIL half (see root_cause) is NOT YET FIXED and is NOT task 3.2's (or any single narrow task's) job to fix -- it needs the broader WAL_JOB_FAIL payload extension + apply_job_fail_transition() signature change described above, deliberately deferred pending a dedicated task, matching the same 'deliberately deferred, documented, not blocking' treatment bug-1631's own sibling gap received earlier in this same feature. The reconciled job's STATE (EXIT) is always correct for a finished-failure outcome either way -- only its recorded finish_time reflects apply-processing time rather than the worker's true report.", @@ -24679,11 +11804,8 @@ "pre-existing", "predates-feature", "systemic", - "partially-fixed", "resync_apply_report", - "record_job_finish_fact_at", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "record_job_finish_fact_at" ], "related_bugs": [ "bug-1689", @@ -24691,15 +11813,23 @@ "bug-1746", "bug-1631" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "eec2596c", + "1d6c04c2" + ], + "record": ".wolf/bugs/bug-1747.md" }, { "id": "bug-1748", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-01T02:00:00Z", - "error_message": "worker-rejoin-resync task 3.2 (resync_apply_running_entry, resync.c), round-2 independent review: the round-1 fix for the PEND/unassigned-job regression (bug-1747-adjacent, not itself in bug-1747) introduced a NEW Requirement 4.4 violation. The round-1 guard `if (!resync_job_is_terminal(job->job.state) && resync_job_assigned_to_host(job, w->host.name)) { no-op } else { record_job_reconcile_assign_fact(...) }` short-circuits the `&&` to false whenever job->job.state IS terminal, falling into the `else` branch and calling record_job_reconcile_assign_fact() on an ALREADY-TERMINAL (DONE/EXIT) job -- which has no backward-transition guard of its own and unconditionally sets job->job.state = JOB_STAT_RUN, clobbering a terminal job back to RUN. Confirmed live and, worse, CERTIFIED AS CORRECT by the round-1 implementation's own test: test_apply_report_all_five_outcomes_updates_job_table seeded j_extra at JOB_STAT_DONE with a comment claiming this 'satisfies Req 4.8's trigger,' then asserted j_extra->job.state == JOB_STAT_RUN afterward and the suite reported PASS -- the test baked the bug in as an expected outcome instead of catching it.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "worker-rejoin-resync task 3.2 (resync_apply_running_entry, resync.c), round-2 independent review…\n\nworker-rejoin-resync task 3.2 (resync_apply_running_entry, resync.c), round-2 independent review: the round-1 fix for the PEND/unassigned-job regression (bug-1747-adjacent, not itself in bug-1747) introduced a NEW Requirement 4.4 violation. The round-1 guard `if (!resync_job_is_terminal(job->job.state) && resync_job_assigned_to_host(job, w->host.name)) { no-op } else { record_job_reconcile_assign_fact(...) }` short-circuits the `&&` to false whenever job->job.state IS terminal, falling into the `else` branch and calling record_job_reconcile_assign_fact() on an ALREADY-TERMINAL (DONE/EXIT) job -- which has no backward-transition guard of its own and unconditionally sets job->job.state = JOB_STAT_RUN, clobbering a terminal job back to RUN. Confirmed live and, worse, CERTIFIED AS CORRECT by the round-1 implementation's own test: test_apply_report_all_five_outcomes_updates_job_table seeded j_extra at JOB_STAT_DONE with a comment claiming this 'satisfies Req 4.8's trigger,' then asserted j_extra->job.state == JOB_STAT_RUN afterward and the suite reported PASS -- the test baked the bug in as an expected outcome instead of catching it.", "file": "src/controller/resync.c", "root_cause": "Requirement 4.8's actual trigger condition is 'no existing non-terminal record of that job assigned elsewhere' (requirements.md AC 4.8; record_job_reconcile_assign_fact's own doc comment and design.md's Service Interface Preconditions: 'job is non-terminal ... or has no existing non-terminal assignment'). Terminal-vs-non-terminal is NOT the axis that decides assign-eligibility on its own -- Requirement 4.4 ('shall not overwrite that state with a less final or older report') independently and unconditionally forbids ever assign-routing a terminal job, regardless of host. The round-1 guard conflated two orthogonal conditions (terminal-ness, and host-assignment) into a single boolean expression via `&&`, which is only correct when BOTH must jointly hold for the no-op case -- but Req 4.4's terminal exclusion needs to apply UNCONDITIONALLY, independent of the host check, not only when the host check also happens to be true. A two-way boolean cannot express three independent outcomes (terminal-always-noop; non-terminal-but-already-assigned-noop; non-terminal-and-unassigned-assign) without this exact class of short-circuit bug.", "fix": "FIXED (2026-09-01, round-2 review, task 3.2). resync_apply_running_entry() rewritten as a genuine three-way `if`/`else if`/`else` split, not a boolean guard: (a) resync_job_is_terminal(job->job.state) -> ALWAYS a no-op, checked FIRST and independently, regardless of host (closes the Req 4.4 violation). (b) non-terminal AND job->job.allocs != NULL (already has SOME live non-terminal host assignment -- to the reporting worker's own host, or to a different one, it does not matter which) -> a no-op; also tightened from round 1's own host-specific check (resync_job_assigned_to_host(job, w->host.name)) to a host-agnostic allocs!=NULL check, since round-1's version would have ALSO wrongly routed a job assigned to a DIFFERENT host into the assign branch, resetting its start_time and re-appending a spurious WAL_JOB_DISPATCH fact for a job that was never actually lost. (c) non-terminal AND allocs == NULL (genuinely unassigned) -> record_job_reconcile_assign_fact(), Req 4.8's real trigger. Verified: (1) test_apply_report_all_five_outcomes_updates_job_table's j_extra is now seeded TERMINAL and asserted COMPLETELY UNTOUCHED (state stays DONE, no WAL append) -- the exact inversion of the round-1 assertion that had certified the bug. (2) test_apply_report_unqueried_extra_already_tracked_is_noop (branch (b), assigned to a DIFFERENT host) strengthened with a start_time-unchanged assertion (record_job_reconcile_assign_fact unconditionally stamps time(NULL), bug-1697 -- invisible to a bare state/allocs check since the job was already RUN either way) and an LSN-unchanged assertion (no spurious WAL_JOB_DISPATCH append) -- round 1's version of this test could not have caught either defect. (3) test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned (branch (c), genuinely unassigned/PEND) already covers the real assign-success case with full restart-replay rigor, added in the same round-1 fix pass. Full controller test suite (meson test -C src/xce.build 'controller test'): all green.", @@ -24714,23 +11844,27 @@ "guard-logic", "short-circuit", "review-caught", - "test-baked-in-bug", - "fixed" + "test-baked-in-bug" ], "related_bugs": [ "bug-1747", "bug-1697", "bug-1631" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1748.md" }, { "id": "bug-1750", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "worker", + "severity": "unknown", "timestamp": "2026-09-01T09:00:00Z", - "error_message": "worker-rejoin-resync task 4.3, round-1 review: the resync_pending gate was checked as checking_connection()'s very first statement (an entry guard), returning early before xcecli_hello() or the needs_rejoin-triggered rejoin_call() could run. xcecd_resync_handle_query_page() sets resync_pending = true on receiving a job-state query's first page and only clears it when called again with a later page. If the worker's connection to the controller drops after a non-final page of a multi-page OPCode_jobStateQuery arrives (a query spanning more non-terminal jobs than fit in one page -- explicitly in scope per Requirement 8.1/8.3), no further page ever arrives, so xcecd_resync_handle_query_page() is never invoked again and resync_pending is stuck true forever -- permanently disabling checking_connection()'s entire body, including the worker's ONLY connection-recovery mechanism (xcecli_hello() / needs_rejoin-triggered rejoin). Directly violated Requirement 6.1 (the worker must be able to disconnect-and-rejoin during an incomplete reconciliation round) and regressed the intent of the pre-existing invariant test test_checking_connection_never_permanently_disables_worker, though none of round 1's own new tests drove the multi-page-then-disconnect sequence so it passed the full suite undetected.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "worker-rejoin-resync task 4.3, round-1 review: the resync_pending gate was checked as…\n\nworker-rejoin-resync task 4.3, round-1 review: the resync_pending gate was checked as checking_connection()'s very first statement (an entry guard), returning early before xcecli_hello() or the needs_rejoin-triggered rejoin_call() could run. xcecd_resync_handle_query_page() sets resync_pending = true on receiving a job-state query's first page and only clears it when called again with a later page. If the worker's connection to the controller drops after a non-final page of a multi-page OPCode_jobStateQuery arrives (a query spanning more non-terminal jobs than fit in one page -- explicitly in scope per Requirement 8.1/8.3), no further page ever arrives, so xcecd_resync_handle_query_page() is never invoked again and resync_pending is stuck true forever -- permanently disabling checking_connection()'s entire body, including the worker's ONLY connection-recovery mechanism (xcecli_hello() / needs_rejoin-triggered rejoin). Directly violated Requirement 6.1 (the worker must be able to disconnect-and-rejoin during an incomplete reconciliation round) and regressed the intent of the pre-existing invariant test test_checking_connection_never_permanently_disables_worker, though none of round 1's own new tests drove the multi-page-then-disconnect sequence so it passed the full suite undetected.", "file": "src/worker/src/controller_connector.c", "root_cause": "design.md directed mirroring the existing needs_rejoin boolean pattern for the new resync_pending gate, but needs_rejoin is self-healing by construction -- it gates one step INSIDE checking_connection() (the rejoin action itself), so the function's body including xcecli_hello() still runs every tick regardless of its value, giving the worker a fresh chance to detect and recover every cycle. Round 1 placed resync_pending's check as the function's entry guard instead, which is not actually the same pattern in the load-bearing respect that matters: it makes the gate's only exit condition (a later query page arriving) depend on the very connection health the gate is supposed to be resilient to, with no independent path back to a working state.", "fix": "FIXED (2026-09-01, round-2 review, task 4.3). Two changes, defense-in-depth: (a) narrowed the gate so xcecd_resync_is_pending() is checked only inside the needs_rejoin branch, guarding solely rejoin_call() -- xcecli_hello() and the needs_rejoin check itself now run unconditionally on every checking_connection() tick, restoring the self-healing property. (b) added xcecd_resync_reset(), called from xcecd_on_reconnect() (the same signal that sets needs_rejoin, confirmed to fire only after xce_failover_rotate()/xcecli_invoke_rejoin() has already repaired a real, previously-detected transport failure -- never spuriously on a healthy connection), which clears resync_pending and discards any stalled partial multi-page accumulation (frees and zeroes the accumulator, not just the boolean), bounding the worst case to at most one reconnect cycle instead of forever. Verified: round-2 reviewer independently reintroduced the round-1 entry-guard bug, rebuilt, and confirmed a new structural regression test (test_checking_connection_hello_and_needs_rejoin_precede_resync_gate_in_source, asserting strict source-order) fails while the companion behavioral test (test_checking_connection_survives_permanently_stuck_resync_pending) does NOT catch the reintroduced bug alone -- establishing that placement-sensitive gates need a structural/ordering assertion, not just a behavioral one. Full worker suite green after the fix (xcecd_resync_test 28/28, controller_connector_test 22/22 including the pre-existing test_checking_connection_never_permanently_disables_worker invariant). One advisory, non-blocking follow-up noted but not fixed: resync_pending/query_accum are unsynchronized statics touched from two different OS threads (connector thread vs. a vtaskmgr pool thread) -- a narrow theoretical race on xcecd_resync_reset()'s free() vs. a concurrent read, mirroring needs_rejoin's own accepted unsynchronized-static risk profile but riskier since it frees a heap pointer.", @@ -24745,22 +11879,26 @@ "gate-placement", "checking_connection", "needs_rejoin", - "review-caught", - "fixed" + "review-caught" ], "related_bugs": [ "bug-1749", "bug-1748" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1750.md" }, { "id": "bug-1751", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-01T09:00:00Z", - "error_message": "worker-rejoin-resync task 5 (scheduler candidate-filter exclusion for XCE_WORKER_RECONCILING): design.md's Components/Architecture/Testing Strategy sections all cite scheduler.c:1211 (sched_reset_worker_(), the IS_WORKER_UNAVAIL skip inside the legacy sched_reset_workers()/sched_begin_session() path) as 'the existing candidate-host filter' to extend with the new reconciling-bit disjunct. Investigating before implementing (grep for every IS_WORKER_UNAVAIL skip-style check across src/scheduler/src/*.c) found sched_begin_session() has NO live callers anywhere in the current driver.c/sim_loop.c production paths -- both call xce_core_begin_session() (core_session.c, added by the scheduler-core refactor) directly instead, which runs core_reset_workers() (core_session.c:102), an independently-maintained mirror of sched_reset_worker_() with the IDENTICAL IS_WORKER_UNAVAIL-only check, per that function's own header comment ('mirrors scheduler.c's sched_reset_workers/sched_reset_worker_'). Extending only scheduler.c:1211 as design.md literally specifies would leave the task's own Observable Completion Criterion (a RECONCILING host is never selected as a dispatch candidate while the flag is set) factually false on both real drivers, since neither ever executes the code path being changed. bug-1688's own investigation (2026-08-26) independently confirms core_reset_workers()/xce_core_begin_session() as the actively-traced, live production path. No test in src/scheduler/testsuite exercises sched_reset_worker_()'s or sched_begin_session()'s IS_WORKER_UNAVAIL behavior either (design.md's Testing Strategy claim of an 'existing IS_WORKER_UNAVAIL candidate-filter test' to extend does not correspond to any test that actually exists).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "worker-rejoin-resync task 5 (scheduler candidate-filter exclusion for XCE_WORKER_RECONCILING)…\n\nworker-rejoin-resync task 5 (scheduler candidate-filter exclusion for XCE_WORKER_RECONCILING): design.md's Components/Architecture/Testing Strategy sections all cite scheduler.c:1211 (sched_reset_worker_(), the IS_WORKER_UNAVAIL skip inside the legacy sched_reset_workers()/sched_begin_session() path) as 'the existing candidate-host filter' to extend with the new reconciling-bit disjunct. Investigating before implementing (grep for every IS_WORKER_UNAVAIL skip-style check across src/scheduler/src/*.c) found sched_begin_session() has NO live callers anywhere in the current driver.c/sim_loop.c production paths -- both call xce_core_begin_session() (core_session.c, added by the scheduler-core refactor) directly instead, which runs core_reset_workers() (core_session.c:102), an independently-maintained mirror of sched_reset_worker_() with the IDENTICAL IS_WORKER_UNAVAIL-only check, per that function's own header comment ('mirrors scheduler.c's sched_reset_workers/sched_reset_worker_'). Extending only scheduler.c:1211 as design.md literally specifies would leave the task's own Observable Completion Criterion (a RECONCILING host is never selected as a dispatch candidate while the flag is set) factually false on both real drivers, since neither ever executes the code path being changed. bug-1688's own investigation (2026-08-26) independently confirms core_reset_workers()/xce_core_begin_session() as the actively-traced, live production path. No test in src/scheduler/testsuite exercises sched_reset_worker_()'s or sched_begin_session()'s IS_WORKER_UNAVAIL behavior either (design.md's Testing Strategy claim of an 'existing IS_WORKER_UNAVAIL candidate-filter test' to extend does not correspond to any test that actually exists).", "file": "src/scheduler/src/scheduler.c, src/scheduler/src/core_session.c", "root_cause": "design.md's research (Architecture > Existing Architecture Analysis) predates, or did not re-trace, the scheduler-core refactor's replacement of sched_begin_session() with xce_core_begin_session() as the driver-invoked session-start entry point on both the normal-mode driver (driver.c:751) and the simulation driver (sim_loop.c:919). sched_reset_worker_()/sched_reset_workers()/sched_begin_session() (scheduler.c) are consequently dead code on both live paths today, referenced only in historical doc comments ('the still-live sched_begin_session()...main.c loop', core_session.c:27-30, itself now stale) -- the design cited the pre-refactor site without noticing its post-refactor mirror in core_session.c had become the one that actually runs.", "fix": "FIXED (2026-09-01, task 5, found live during implementation, same in-boundary-companion-fix pattern as bug-1746/bug-1747 in this same spec). Applied the identical one-line disjunct (IS_WORKER_UNAVAIL(w->status) || IS_WORKER_RECONCILING(w->status)) to BOTH scheduler.c:1211 (sched_reset_worker_(), matching design.md's literal instruction and File Structure Plan entry verbatim) AND core_session.c:102 (core_reset_workers(), the live mirror, necessary for the task's Observable Completion Criterion to hold on either real driver). Comment wording in scheduler.c/driver.c deliberately avoids the literal substring 'resync' anywhere (caught once live: an earlier comment draft containing 'worker-rejoin-resync' tripped test_no_worker_resync_symbols_in_scheduler_publish_path, since strstr() matches inside compound words too) -- fixed by rewording to 'job-state reconciliation' with no spec-name reference. New regression coverage added to test_core_session.c (white-box #include of core.c/core_session.c/core_clock.c/decision.c, the file's existing convention) since core_session.c's core_reset_workers() is reachable from a lightweight fixture (key_add() a worker directly into core->workers, populate resInUse with a known non-zero value, call the real xce_core_begin_session(), assert resInUse is/isn't zeroed) whereas scheduler.c's sched_reset_worker_() is static and scheduler.c's own dependency closure (client.h network I/O, plugin registration) is already established codebase precedent (test_snapshot_load_allocator_regression.c's header comment) as too large to white-box #include for a single new test. Five new tests: baseline available-host reset, baseline unavailable-host skip (unchanged), new reconciling-host skip, both-bits-set truth-table completeness (guards the exact orthogonal-condition mistake class flagged for this spec's task 3.2), and reconciling-bit-clears re-includes-on-next-session. Full scheduler suite (xce-build.sh scheduler, ASAN, --incremental): 54/55 OK, sched_core_session_test 13/13, sched_scheduler_decision_redelivery_test's test_no_worker_resync_symbols_in_scheduler_publish_path green and unmodified; the one remaining failure (test_job_rusage_reconciles_mem_reservation_against_usage, sched_core_events_test) is pre-existing and unrelated -- core_events.c was already modified in this worktree before this task started and this task's diff never touches it.", @@ -24778,8 +11916,7 @@ "core_reset_workers", "design-reality-mismatch", "found-live-during-implementation", - "symbol-guard-test", - "fixed" + "symbol-guard-test" ], "related_bugs": [ "bug-1745", @@ -24788,15 +11925,20 @@ "bug-1748", "bug-1750" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1751.md" }, { "id": "bug-1752", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "worker", + "severity": "unknown", "timestamp": "2026-09-01T10:00:00Z", - "error_message": "worker-rejoin-resync task 6.1 (worker-side config-delivery gap): tasks.md's own task text says to have 'the worker's existing config-receive handling call xcecd_finished_job_retention_set_window_ms() with the received value' -- premised on some existing per-field config-receive dispatch already existing at the worker that just needed one more field wired in. Investigating before implementing (reading src/worker/src/controller_connector.c's myreq_handler dispatch switch and src/lib/xcectx.c's vtask_entry, which decodes every inbound request before myreq_handler ever sees it) found this premise false: myreq_handler's switch had NO case for OPCode_push at all (only OPCode_startjob/runtasks/jobcontrol/fetchfile/jobStateQuery existed) -- a config push (Request_body_configpushmsg, the body_which xce_pushconfig_req()/worker_push_config() send) fell straight through to `default: break;` and was silently discarded in its entirety, for EVERY field including the pre-existing clustername, not just the not-yet-added workerResyncWindowMs. xcecd_finished_job_retention_set_window_ms()'s own doc comment (xcecd_worker_utils.h, task 4.1) had already independently found and documented the same gap ('the worker daemon has no runtime delivery path for this ClusterConfig field today... worker_push_config() only ever pushes hosttype/hostmodel/resource confvars via collect_confvars()... this setter is the seam a later task wires the real operator-configured value into once that delivery path exists') -- task 6.1 is that later task, but its own task text still assumed the delivery path itself already existed when it did not.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "worker-rejoin-resync task 6.1 (worker-side config-delivery gap): tasks.md's own task text says to…\n\nworker-rejoin-resync task 6.1 (worker-side config-delivery gap): tasks.md's own task text says to have 'the worker's existing config-receive handling call xcecd_finished_job_retention_set_window_ms() with the received value' -- premised on some existing per-field config-receive dispatch already existing at the worker that just needed one more field wired in. Investigating before implementing (reading src/worker/src/controller_connector.c's myreq_handler dispatch switch and src/lib/xcectx.c's vtask_entry, which decodes every inbound request before myreq_handler ever sees it) found this premise false: myreq_handler's switch had NO case for OPCode_push at all (only OPCode_startjob/runtasks/jobcontrol/fetchfile/jobStateQuery existed) -- a config push (Request_body_configpushmsg, the body_which xce_pushconfig_req()/worker_push_config() send) fell straight through to `default: break;` and was silently discarded in its entirety, for EVERY field including the pre-existing clustername, not just the not-yet-added workerResyncWindowMs. xcecd_finished_job_retention_set_window_ms()'s own doc comment (xcecd_worker_utils.h, task 4.1) had already independently found and documented the same gap ('the worker daemon has no runtime delivery path for this ClusterConfig field today... worker_push_config() only ever pushes hosttype/hostmodel/resource confvars via collect_confvars()... this setter is the seam a later task wires the real operator-configured value into once that delivery path exists') -- task 6.1 is that later task, but its own task text still assumed the delivery path itself already existed when it did not.", "file": "src/worker/src/controller_connector.c", "root_cause": "No prior worker-rejoin-resync task (nor any task before this spec) ever needed the worker to consume ANY field off an OPCode_push config push, so myreq_handler's dispatch switch was never extended to handle that opcode at all -- xcectx.c's vtask_entry only special-cases the OTHER OPCode_push shape (Request_body_pushmsg, the generic job/worker/resource list-table push used for scheduler/apiserver connections) before handing off to the registered handler; a Request_body_configpushmsg-shaped OPCode_push passes through unmodified into myreq_handler's default: break; case. This predates worker-rejoin-resync entirely and was simply never exercised or noticed because nothing at the worker previously needed any pushed ClusterConfig field.", "fix": "FIXED (2026-09-01, task 6.1, found live during implementation, same design-vs-reality-correction pattern as bug-1746/1747/1748/1751 earlier in this spec). Added a new case OPCode_push: to myreq_handler's dispatch switch, routing to a new handle_pushconfig() function that checks body_which == Request_body_configpushmsg and configdata->data_which == ConfigData_data_workerconfig before reading conf4worker->worker_resync_window_ms and calling xcecd_finished_job_retention_set_window_ms() with it -- scoped narrowly to closing the one seam task 6.1 needs (workerResyncWindowMs), not a general config-receive rework: clustername/confvars remain unconsumed by the worker, unchanged and explicitly out of this task's boundary. New tests in src/worker/testsuite/test_controller_connector.c (white-box, real wire encode via xce_pushconfig_req + real xce_decode + myreq_handler dispatch) cover: the real value reaching the retention setter exactly once, a second/different value also applied (proves live read, not memoized), a non-workerconfig ConfigData arm being ignored, and a NULL configpushmsg being a safe no-op.", @@ -24808,8 +11950,7 @@ "requirement-3.7", "config-push", "design-reality-mismatch", - "found-live-during-implementation", - "fixed" + "found-live-during-implementation" ], "related_bugs": [ "bug-1746", @@ -24817,17 +11958,22 @@ "bug-1748", "bug-1751" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1752.md" }, { "id": "bug-1753", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-01T20:30:00Z", - "error_message": "worker-rejoin-resync task 6.2 (controller-side OPCode_jobStateReport dispatch): design.md's Error Handling section only anticipated the STANDBY-role case for this opcode pair (\"verify... whether OPCode_jobStateQuery/OPCode_jobStateReport need an explicit redirect-reply path... Expected answer: no... but this must be confirmed, not assumed, given bug-1700's documented history of this exact class of gap recurring across two other opcodes\"). Auditing ha_dispatch_gate() (ha.c) in full per that instruction found the STANDBY case was indeed already correctly handled automatically (OPCode_jobStateReport, left absent from the CTRL_STANDBY allow-list, falls to the generic default: return -XCE_ERR_STANDBY branch, and cluster.c's existing bug-1700 post-gate block already fills the redirect hint for any such rejection with zero opcode-specific code needed) -- but the same audit surfaced a DIFFERENT, unanticipated gap in the SAME function: a CTRL_PRIMARY controller that has temporarily lost write quorum rejects OPCode_jobStateReport outright with -XCE_ERR_NOT_READY, because the opcode was absent from ha_dispatch_gate's quorum-loss \"fact-bearing ops\" allowlist (OPCode_notify/OPCode_update/OPCode_setpreason/OPCode_clrpreason, ha.c ~line 4686-4696) that already exists specifically to prevent this exact class of loss (bug-396) for other worker/scheduler fact reports.", - "root_cause": "OPCode_jobStateReport is sent fire-and-forget (xce_job_state_report_req(), request.c/xce_request()) with no redelivery on rejection -- structurally identical to OPCode_notify, the opcode bug-396's fix (ha-commit-barrier-rollout) originally added this allowlist to protect. Since worker-rejoin-resync's two new opcodes were added to controller.capnp's OPCode enum without also being added to ha_dispatch_gate's quorum-loss allowlist, a job-state reconciliation report arriving at a primary during a transient write-quorum-loss window (e.g. mid-failover, the exact scenario this whole feature exists to recover from) would be rejected outright and never retried by the worker -- reproducing bug-396's own loss mode for precisely the facts worker-rejoin-resync (bug-1741) exists to recover.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "worker-rejoin-resync task 6.2 (controller-side OPCode_jobStateReport dispatch): design.md's Error…\n\nworker-rejoin-resync task 6.2 (controller-side OPCode_jobStateReport dispatch): design.md's Error Handling section only anticipated the STANDBY-role case for this opcode pair (\"verify... whether OPCode_jobStateQuery/OPCode_jobStateReport need an explicit redirect-reply path... Expected answer: no... but this must be confirmed, not assumed, given bug-1700's documented history of this exact class of gap recurring across two other opcodes\"). Auditing ha_dispatch_gate() (ha.c) in full per that instruction found the STANDBY case was indeed already correctly handled automatically (OPCode_jobStateReport, left absent from the CTRL_STANDBY allow-list, falls to the generic default: return -XCE_ERR_STANDBY branch, and cluster.c's existing bug-1700 post-gate block already fills the redirect hint for any such rejection with zero opcode-specific code needed) -- but the same audit surfaced a DIFFERENT, unanticipated gap in the SAME function: a CTRL_PRIMARY controller that has temporarily lost write quorum rejects OPCode_jobStateReport outright with -XCE_ERR_NOT_READY, because the opcode was absent from ha_dispatch_gate's quorum-loss \"fact-bearing ops\" allowlist (OPCode_notify/OPCode_update/OPCode_setpreason/OPCode_clrpreason, ha.c ~line 4686-4696) that already exists specifically to prevent this exact class of loss (bug-396) for other worker/scheduler fact reports.", "file": "src/controller/ha.c (ha_dispatch_gate, CTRL_PRIMARY quorum-loss branch, ~line 4686-4699)", + "root_cause": "OPCode_jobStateReport is sent fire-and-forget (xce_job_state_report_req(), request.c/xce_request()) with no redelivery on rejection -- structurally identical to OPCode_notify, the opcode bug-396's fix (ha-commit-barrier-rollout) originally added this allowlist to protect. Since worker-rejoin-resync's two new opcodes were added to controller.capnp's OPCode enum without also being added to ha_dispatch_gate's quorum-loss allowlist, a job-state reconciliation report arriving at a primary during a transient write-quorum-loss window (e.g. mid-failover, the exact scenario this whole feature exists to recover from) would be rejected outright and never retried by the worker -- reproducing bug-396's own loss mode for precisely the facts worker-rejoin-resync (bug-1741) exists to recover.", "fix": "FIXED (2026-09-01, task 6.2, found live during the task's own mandated standby-redirect investigation -- not assumed safe, confirmed by reading the full gate function as instructed). Added `case OPCode_jobStateReport:` alongside the existing OPCode_notify/OPCode_update/OPCode_setpreason/OPCode_clrpreason cases in ha_dispatch_gate's quorum-loss fact-bearing-ops allowlist, so a report arriving while the primary temporarily lacks write quorum is admitted (not rejected) and can flow into the same RETAIN/pending-facts WAL-append mechanism those sibling opcodes already rely on. OPCode_jobStateQuery needed no corresponding change: the controller only ever SENDS that opcode (controller -> worker), never receives it, so it never reaches ha_dispatch_gate at all. New regression test test_gate_no_quorum_admits_jobstatereport_fact (test_commit_barrier_rollout.c), mirroring test_gate_no_quorum_admits_notify_fact's exact fixture/assertion shape; also added to test_gate_with_quorum_admits_all_opcodes's opcode list for parity. The CTRL_STANDBY case needed and received no code change at all -- proven correct as-is by three new tests in the new src/controller/testsuite/test_jobstatereport_dispatch.c (rejects-when-not-primary, standby-with-known-primary-redirects, standby-no-known-primary-no-redirect), mirroring test_sched_admin_dispatch.c's identical bug-1700 regression tests for OPCode_schedAdmin, the 'real existing primary-only opcode' ground truth this task's brief called for.", "tags": [ "worker-rejoin-resync", @@ -24839,8 +11985,7 @@ "bug-396-class", "quorum-loss", "design-reality-mismatch", - "found-live-during-implementation", - "fixed" + "found-live-during-implementation" ], "related_bugs": [ "bug-396", @@ -24851,18 +11996,20 @@ "bug-1751", "bug-1752" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1753.md" }, { "id": "bug-1754", - "gitea_synced_status": "fixed", - "gitea_issue": 266, "status": "fixed", "status_updated": "2026-09-26", "status_note": "6876e4f6: WAL_JOB_FAIL carries an int32 code trailer; both apply seams adopt it into exit_info (non-POST_EXEC stages). Also fixed apply_wal_entry assigning wire state 0 for STAGE_FLAG FAIL entries.", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-01T13:00:00Z", - "error_message": "Discovered while building worker-rejoin-resync task 7.1's full round-trip integration test (test_full_rejoin_reconciliation_round_trip, test_worker_resync.c): a worker-reported exit code for a finished-FAILURE outcome is never durably recorded by ANY existing path in this codebase, reconciliation or ordinary. record_job_fail_fact()'s WAL_JOB_FAIL payload (job.c, persist_submit_job_fact_ex(..., WAL_JOB_FAIL, jobid, 0, WAL_JOB_FAIL_STAGE_FLAG|stage)) carries no exit-code slot at all, and apply_job_fail_transition() (job.c:2831-2853) never touches job.job.exit_info for the FAIL path -- it stays NULL unconditionally for every job failure. The new round-trip test asserts this honestly (TEST_ASSERT_NULL on job.job.exit_info with an inline comment) rather than encoding a wrong 'correct' expectation.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Discovered while building worker-rejoin-resync task 7.1's full round-trip integration test…\n\nDiscovered while building worker-rejoin-resync task 7.1's full round-trip integration test (test_full_rejoin_reconciliation_round_trip, test_worker_resync.c): a worker-reported exit code for a finished-FAILURE outcome is never durably recorded by ANY existing path in this codebase, reconciliation or ordinary. record_job_fail_fact()'s WAL_JOB_FAIL payload (job.c, persist_submit_job_fact_ex(..., WAL_JOB_FAIL, jobid, 0, WAL_JOB_FAIL_STAGE_FLAG|stage)) carries no exit-code slot at all, and apply_job_fail_transition() (job.c:2831-2853) never touches job.job.exit_info for the FAIL path -- it stays NULL unconditionally for every job failure. The new round-trip test asserts this honestly (TEST_ASSERT_NULL on job.job.exit_info with an inline comment) rather than encoding a wrong 'correct' expectation.", "file": "src/controller/job.c", "root_cause": "Distinct from and broader than bug-1747 (which documents only the FAIL finish-time mistiming, via time(NULL) instead of a worker-reported value). This gap is about the exit CODE, not the finish TIME: WAL_JOB_FAIL's wire payload was never given a code slot in the first place, so no caller of record_job_fail_fact()/on_job_fail() -- worker-reported or otherwise -- has ever been able to persist or apply a job's actual failure exit code. Systemic and pre-existing; unrelated to worker-rejoin-resync's own changes, which only exercise (and correctly do not attempt to fix) this existing gap.", "fix": "NOT FIXED -- out of scope for worker-rejoin-resync (a test-only validation task, task 7.1, correctly declined to patch a production defect it incidentally surfaced). Recommend a dedicated future task: extend WAL_JOB_FAIL's wire payload with an exit-code field (mirroring bug-1747's eventual FAIL-side fix for finish-time, if/when that lands) and have apply_job_fail_transition() populate job.job.exit_info from it. Until then, job.job.exit_info is NULL after any job failure, worker-rejoin-resync reconciliation included -- callers/UIs must not assume this field is populated for a failed job.", @@ -24877,22 +12024,29 @@ "wal_job_fail", "requirement-4", "discovered-not-fixed", - "systemic", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "systemic" ], "related_bugs": [ "bug-1747" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "6876e4f6" + ], + "record": ".wolf/bugs/bug-1754.md", + "gitea_issue": 266, + "gitea_synced_status": "fixed" }, { "id": "bug-1755", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-02T00:30:00Z", - "error_message": "Discovered while building worker-rejoin-resync task 7.4's large job-count pagination test (test_large_job_count_pagination_correctness, test_worker_resync.c): the controller's real production OPCode_jobStateQuery send site, worker_send_resync_query() (src/controller/worker.c:730), calls xce_job_state_query_req(ep, c->myhost, &query, NULL, NULL, 0) with a hardcoded pktlimit of 0. xce_job_state_query_req()'s own pktlimit<=0 convention (request.c:1822-1824, \"if (pktlimit <= 0) { pktlimit = nrows; }\") means this ALWAYS sends the entire query as exactly ONE wire page, however many non-terminal jobs the controller queries a worker about -- there is no bound at all. This is asymmetric with the worker's own real report-send site (controller_connector.c's handle_jobstatequery -> XCECD_RESYNC_REPORT_PKTLIMIT == 100), which DOES genuinely cap and paginate OPCode_jobStateReport in production. The query-direction pagination mechanism (xce_job_state_query_req()'s own do/while page loop, task 1.4) is fully implemented and correct -- task 7.4's own test calls it directly with an explicit page size and proves it genuinely spans many wire pages with no drops/duplicates -- but it is effectively dead code in production today because its only call site never supplies a bound.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Discovered while building worker-rejoin-resync task 7.4's large job-count pagination test…\n\nDiscovered while building worker-rejoin-resync task 7.4's large job-count pagination test (test_large_job_count_pagination_correctness, test_worker_resync.c): the controller's real production OPCode_jobStateQuery send site, worker_send_resync_query() (src/controller/worker.c:730), calls xce_job_state_query_req(ep, c->myhost, &query, NULL, NULL, 0) with a hardcoded pktlimit of 0. xce_job_state_query_req()'s own pktlimit<=0 convention (request.c:1822-1824, \"if (pktlimit <= 0) { pktlimit = nrows; }\") means this ALWAYS sends the entire query as exactly ONE wire page, however many non-terminal jobs the controller queries a worker about -- there is no bound at all. This is asymmetric with the worker's own real report-send site (controller_connector.c's handle_jobstatequery -> XCECD_RESYNC_REPORT_PKTLIMIT == 100), which DOES genuinely cap and paginate OPCode_jobStateReport in production. The query-direction pagination mechanism (xce_job_state_query_req()'s own do/while page loop, task 1.4) is fully implemented and correct -- task 7.4's own test calls it directly with an explicit page size and proves it genuinely spans many wire pages with no drops/duplicates -- but it is effectively dead code in production today because its only call site never supplies a bound.", "file": "src/controller/worker.c", "root_cause": "worker_send_resync_query() was written to pass pktlimit=0 to xce_job_state_query_req(), which the callee treats as \"unbounded / exactly one page\" rather than \"use a real per-page cap.\" No ClusterConfig field or fixed constant analogous to XCECD_RESYNC_REPORT_PKTLIMIT was ever wired in for the query direction, unlike the report direction. Requirement 8.1 ('regardless of how many non-terminal jobs... without... failing solely because of the number of jobs involved') is still technically satisfied -- an unbounded single message doesn't fail merely due to job count, and no hard message-size cap was found elsewhere in the transport/capnp layer that would make a very large single-page query actually fail or truncate. But design.md's own stated intent (both opcodes described as 'paginated'; the Testing Strategy's Performance/Load line about the exchange genuinely spanning multiple pages) is not actually exercised by any real caller for the query direction -- only the report direction is genuinely bounded/paginated in production as shipped.", "fix": "FIXED (2026-09-01, worker-rejoin-resync task 7.5). worker_send_resync_query() (src/controller/worker.c:730, now ~745) no longer hardcodes pktlimit=0 in its xce_job_state_query_req() call -- it now passes a new, named page-size cap, XCE_RESYNC_QUERY_PKTLIMIT (worker.c, value 100), mirroring the worker's own already-wired XCECD_RESYNC_REPORT_PKTLIMIT (controller_connector.c, also 100) for the report direction. Same value used for both directions: a JobStateQuery entry (bare jobid) is smaller on the wire than a JobStateReport entry (jobid + status + exitCode + finishTimeMs + queried), so the query direction safely supports at least the same page size, with no reason found to invent a smaller one. This closes the query-direction gap task 7.4 found and correctly declined to fix (a test-only task) -- the query direction now genuinely, observably paginates in production, not just at the xce_job_state_query_req() mechanism level task 7.4's own test proved. Verified with a new RED->GREEN test exercising the REAL production call site directly (not a xce_job_state_query_req() bypass): test_worker_send_resync_query_paginates_across_multiple_wire_pages (test_worker_join.c) arms a reconciliation round over 250 non-terminal jobs (> the 100-job page cap, not a multiple of it) and asserts worker_send_resync_query() itself splits the query into exactly 3 wire pages (100+100+50), each with correct pos/count/n_total, every jobid appearing exactly once, no gaps or overlaps. Confirmed genuinely RED beforehand by reverting the fix and rerunning: failed with 'no single page may exceed the configured page cap' (a single collapsed page, matching the pre-fix hardcoded-0 behavior). Confirmed the fix does not regress small-job-count behavior: both existing task 6.1 tests (test_worker_send_resync_query_sends_armed_round, 2 jobs; test_worker_send_resync_query_sends_nothing_when_no_round, 0 jobs) and task 7.4's own test_large_job_count_pagination_correctness (which exercises xce_job_state_query_req() directly with an explicit page-size override, still bypassing the wrapper on purpose -- kept unmodified, it targets the mechanism, not the wrapper) all still pass unchanged. Full controller test suite (meson test -C src/xce.build 'controller test'): 6/6 OK, zero regressions.", @@ -24907,22 +12061,26 @@ "pagination", "design-reality-mismatch", "asymmetric-with-report-direction", - "task-7.5", - "fixed" + "task-7.5" ], "related_bugs": [ "bug-1746", "bug-1747" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1755.md" }, { "id": "bug-1756", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "worker", + "severity": "unknown", "timestamp": "2026-09-02T02:40:00Z", - "error_message": "worker-rejoin-resync task 4.4 (Requirement 3.8, resource_sync.c): AddressSanitizer abort \"Joining already joined thread, aborting.\" while building the task's own new resync_pending-gate tests for resource_sync_thread(). Discovered live: the new tests are the first callers ever to exercise resource_sync_create()/resource_sync_start()/resource_sync_stop()/resource_sync_destroy() as a full lifecycle from a test binary (the two pre-existing tests only ever called collect_load_indices() directly). Each test calls resource_sync_stop(sync) then resource_sync_destroy(sync) back-to-back -- the exact same sequence xcecd.c's own real shutdown path (xcecd_cleanup(), src/worker/src/xcecd.c:580-582) already uses in production.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "worker-rejoin-resync task 4.4 (Requirement 3.8, resource_sync.c): AddressSanitizer abort \"Joining…\n\nworker-rejoin-resync task 4.4 (Requirement 3.8, resource_sync.c): AddressSanitizer abort \"Joining already joined thread, aborting.\" while building the task's own new resync_pending-gate tests for resource_sync_thread(). Discovered live: the new tests are the first callers ever to exercise resource_sync_create()/resource_sync_start()/resource_sync_stop()/resource_sync_destroy() as a full lifecycle from a test binary (the two pre-existing tests only ever called collect_load_indices() directly). Each test calls resource_sync_stop(sync) then resource_sync_destroy(sync) back-to-back -- the exact same sequence xcecd.c's own real shutdown path (xcecd_cleanup(), src/worker/src/xcecd.c:580-582) already uses in production.", "file": "src/worker/src/resource_sync.c", "root_cause": "resource_sync_stop() calls pthread_join(sync->thread, NULL) but never resets sync->thread to 0 (or any invalid sentinel) afterward. resource_sync_destroy() unconditionally calls resource_sync_stop() again as its first action, with no guard on whether the module was already stopped -- so a caller pattern of stop() then destroy() (used verbatim by both xcecd_cleanup() in production and the new task 4.4 tests) pthread_join()s the same already-joined thread handle a second time. This is undefined behavior; under ASAN it aborts hard instead of silently misbehaving. connection_handler.c's analogous network_handler_destroy() already guards its own equivalent second stop call with `if (handler->running)`, showing this exact hazard is a known pattern elsewhere in the same daemon that resource_sync.c's destroy simply never adopted.", "fix": "FIXED (2026-09-02, worker-rejoin-resync task 4.4, in-boundary companion fix -- resource_sync.c was already in this task's file boundary and the double-join blocked the task's own new tests from ever completing). resource_sync_destroy() now guards its internal resource_sync_stop() call with `if (atomic_load(&sync->running))`, mirroring network_handler_destroy()'s established idiom exactly -- if the caller already stopped the module explicitly, destroy() no longer re-joins the same thread handle. Verified via 5 consecutive full runs of the rebuilt resource_sync_test binary under ASAN: 5 Tests, 0 Failures each run, no abort (previously deterministic abort on the very first run reaching a stop()+destroy() pair). Full worker suite (meson test --suite worker) also green: 7/7 OK, no regressions, including controller_connector_test and xcecd_resync_test (task 4.3's own suites).", @@ -24937,22 +12095,26 @@ "pthread", "thread-lifecycle", "companion-fix", - "requirement-3.8", - "fixed" + "requirement-3.8" ], "related_bugs": [ "bug-1519", "bug-1757" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1756.md" }, { "id": "bug-1757", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "worker", + "severity": "unknown", "timestamp": "2026-09-02T02:45:00Z", - "error_message": "worker-rejoin-resync task 4.4 (resource_sync.c): after fixing bug-1756's double-join abort, the same new resync_pending-gate tests still failed deterministically -- \"No load indices to report\" logged, and both test_resource_sync_thread_resumes_load_indices_send_once_resync_clears and test_resource_sync_thread_sends_load_indices_normally_when_not_pending failed asserting the load-indices send call count stayed 0. Both tests run AFTER an earlier test in the same binary has already completed a full resource_sync_create()/start()/stop()/destroy() cycle.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "worker-rejoin-resync task 4.4 (resource_sync.c): after fixing bug-1756's double-join abort, the…\n\nworker-rejoin-resync task 4.4 (resource_sync.c): after fixing bug-1756's double-join abort, the same new resync_pending-gate tests still failed deterministically -- \"No load indices to report\" logged, and both test_resource_sync_thread_resumes_load_indices_send_once_resync_clears and test_resource_sync_thread_sends_load_indices_normally_when_not_pending failed asserting the load-indices send call count stayed 0. Both tests run AFTER an earlier test in the same binary has already completed a full resource_sync_create()/start()/stop()/destroy() cycle.", "file": "src/worker/src/resource_sync.c", "root_cause": "init_builtin_load_indices() guarded its own body with a function-local `static bool initialized` that, once set true, permanently no-ops the function for the rest of the process -- but resource_sync_destroy() -> resource_sync_cleanup_load_indices() frees g_load_indices and resets it to NULL on every destroy. collect_load_indices() (the function's only caller) already re-checks `if (g_load_indices == NULL) { init_builtin_load_indices(); }` before every collection, correctly wanting to re-populate the registry after a destroy -- but the stale `initialized` static silently blocked that re-population forever after the first create/destroy cycle in a process, leaving g_load_indices permanently NULL/empty (a pure test-lifecycle gap: the real daemon calls resource_sync_create() exactly once per process lifetime, so it never previously exercised a second populate-after-teardown cycle).", "fix": "FIXED (2026-09-02, worker-rejoin-resync task 4.4, in-boundary companion fix, same file as bug-1756 and blocking the same new tests). Removed the redundant function-local `static bool initialized` guard from init_builtin_load_indices() entirely -- its only caller, collect_load_indices(), already provides the exact same one-time-per-populated-window guarantee via its own `g_load_indices == NULL` check, and no other call site exists (grepped) and no concurrent caller (collect_load_indices() runs only on resource_sync_thread()'s single dedicated pthread in production, or single-threaded test code). Verified via 5 consecutive full runs of the rebuilt resource_sync_test binary: 5 Tests, 0 Failures each run (previously deterministic 1-2 failures on tests running after the first create/destroy cycle). Full worker suite green: 7/7 OK.", @@ -24964,21 +12126,25 @@ "collect_load_indices", "stale-guard", "test-lifecycle", - "companion-fix", - "fixed" + "companion-fix" ], "related_bugs": [ "bug-1756" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1757.md" }, { "id": "bug-1758", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-02T03:23:00Z", - "error_message": "scheduler suite (meson test --suite scheduler): sched_core_events_test FAILs deterministically -- test_core_events.c:1364:test_job_rusage_reconciles_mem_reservation_against_usage:FAIL: Expected 4 Was 0. Discovered as one of 2 failures (the other is a pre-existing, unrelated libxce/meson.build ASAN-flag gap) during worker-rejoin-resync feature-level /kiro-validate-impl full-suite run; confirmed NOT caused by worker-rejoin-resync.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "scheduler suite (meson test --suite scheduler): sched_core_events_test FAILs deterministically…\n\nscheduler suite (meson test --suite scheduler): sched_core_events_test FAILs deterministically -- test_core_events.c:1364:test_job_rusage_reconciles_mem_reservation_against_usage:FAIL: Expected 4 Was 0. Discovered as one of 2 failures (the other is a pre-existing, unrelated libxce/meson.build ASAN-flag gap) during worker-rejoin-resync feature-level /kiro-validate-impl full-suite run; confirmed NOT caused by worker-rejoin-resync.", "file": "src/scheduler/testsuite/test_core_events.c", "root_cause": "NOT YET ROOT-CAUSED (logged, not investigated -- out of scope for the session that found it). Confirmed via git blame that the ENTIRE failing test function, including the failing sanity assertion at line 1364 itself, was newly added by commit eb06f16d (\"fix(scheduler): wire mem reservation reconciliation into live rusage event path\", .kiro/specs/scheduler-dispatch-commit/), not by worker-rejoin-resync or any earlier commit -- this is that commit's own new test failing against its own new code, not a regression it caused in pre-existing coverage. The failure is at the test's OWN pre-condition sanity check (line 1364, before the rusage/reconciliation event the test is actually about is even pushed): a JOBS_PUSH-dispatched job with a real 4.0 \"mem\" xce_alloc_t is asserted to have already charged 4.0 into w->resInUse (\"matching sched_charge_account()'s existing behavior\", per the test's own comment) -- but val[0] reads back 0. Two live hypotheses, neither yet confirmed: (a) core_events.c's XCE_CORE_EVENT_JOBS_PUSH handler (used here to simulate a dispatch by directly attaching job1->allocs) never actually calls sched_charge_account()/an equivalent resInUse-charging step for a pushed RUN job carrying allocs -- JOBS_PUSH may be resync-shaped (report already-existing state) rather than dispatch-shaped (perform a fresh charge) by design, making the test's own sanity-check premise wrong; or (b) charging happens but keyed off a different resource-index/worker-pointer path than this test's fixture reads back from. Needs an actual debugging pass (e.g. kiro-debug) to distinguish; not attempted here.\n\nRESOLVED (2026-09-08, follow-up session): hypothesis (a) confirmed, but with the opposite implication than it first suggested -- this is NOT a production gap. core_session.c's own core_begin_session_internal() doc comment (lines 156-176) already documents the real architecture: resInUse/sharedInUse is never charged incrementally at JOBS_PUSH/dispatch time. It is fully RE-DERIVED once per round by sched_account_jobs() (scheduler.c:1434) -- iterates every currently-RUN job in core->jobs and calls sched_charge_account() on each one's accounts -- called by BOTH the real normal-mode driver (driver.c:761) and the sim driver (sim_loop.c:1118), always BEFORE anything else in that round reads resInUse. JOBS_PUSH intentionally does none of this itself (confirmed by reading its full handler in core_events.c and builtin_fcfs_on_job_push() -- neither calls sched_charge_account/sched_uncharge_account anywhere), and neither does the terminal-state cleanup path (JOB_STATE_CHANGE/JOB_UPDATE just free the accounts structure) -- both are consistent with a 'recompute from source of truth each round' design that needs no incremental charge/uncharge pairing at all. The test's own precondition assertion (checking resInUse immediately after JOBS_PUSH, with no round boundary simulated) was checking an invariant that was never the real contract -- a test bug, not a production one.", "fix": "FIXED (2026-09-08, follow-up session). test_job_rusage_reconciles_mem_reservation_against_usage (test_core_events.c) now calls sched_charge_account(&core, tracked->accounts) right after JOBS_PUSH and before the resInUse sanity assertion -- simulating the exact reconciliation step sched_account_jobs() would perform for this test's one RUN job, the same primitive sched_account_jobs() itself calls internally (test_core_events.c already white-box #includes resource.c, where sched_charge_account() lives; sched_account_jobs() itself is scheduler.c, not included here, so its single-job effect is reproduced directly rather than pulling in that whole file). No production code changed -- the bug was entirely in the test's precondition assumption, not the scheduler. Verified: full scheduler suite xce-build.sh scheduler asan --incremental: 77/77 Ok (was 76/77, this test's own prior deterministic failure now gone); full xce-build.sh all asan --incremental: 94/94 Ok, zero regressions.", @@ -24990,20 +12156,23 @@ "mem-reservation", "resInUse", "unrelated-branch-breakage", - "discovered-by-worker-rejoin-resync-validation", - "not-fixed", - "fixed" + "discovered-by-worker-rejoin-resync-validation" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1758.md" }, { "id": "bug-1759", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-02T03:47:00Z", - "error_message": "Live simulation run (xce_scheduler --sim, user-observed, not reproduced by me in this session) shows a JOBS_PUSH batch of 50 jobs where the first 22 (indices [0]-[21]) receive [sim] DISPATCH decisions, immediately followed by [sim] PEND decisions for ALL 50 jobs (indices [0]-[49]) -- including the 22 that had JUST been dispatched. Design expectation (confirmed correct, see root_cause): a job that has received a DISPATCH decision must be treated as RUN immediately, without waiting for any downstream job-start acknowledgment, and must never be reconsidered as a fresh scheduling candidate afterward -- in BOTH the live scheduler and simulation/replay mode.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Live simulation run (xce_scheduler --sim, user-observed, not reproduced by me in this session)…\n\nLive simulation run (xce_scheduler --sim, user-observed, not reproduced by me in this session) shows a JOBS_PUSH batch of 50 jobs where the first 22 (indices [0]-[21]) receive [sim] DISPATCH decisions, immediately followed by [sim] PEND decisions for ALL 50 jobs (indices [0]-[49]) -- including the 22 that had JUST been dispatched. Design expectation (confirmed correct, see root_cause): a job that has received a DISPATCH decision must be treated as RUN immediately, without waiting for any downstream job-start acknowledgment, and must never be reconsidered as a fresh scheduling candidate afterward -- in BOTH the live scheduler and simulation/replay mode.", "file": "src/scheduler/src/sim_loop.c, src/scheduler/src/plugins/builtin_fcfs.c, src/scheduler/src/scheduler.c", "root_cause": "ROOT-CAUSED FOR REAL (2026-09-07), and turns out to be a SPEC CONFLICT, not a logic bug -- a patch was attempted and REVERTED after it broke a pre-existing, passing regression test. Confirmed mechanism: sim mode's REAL DISPATCH decisions come entirely from sim_reconcile_unaccounted_jobs() (sim_loop.c), which reconstructs a job's dispatch PURELY from replay data (populating job->allocs/job->accounts, reporting a synthesized DISPATCH decision) but never calls picker->picker_iface->on_dispatch() -- unlike the real/live allocator path (sched_allocator(), scheduler.c line 507-510), which ALWAYS calls on_dispatch() to retire the job from the picker's own candidate list (fcfs_queue_t.pendJobs) before building the decision. Because reconciliation skips this, the job stays a live candidate in the picker's bookkeeping. sim_picker_thread_fn() (sim_loop.c) -- a SEPARATE, always-running thread that walks next_job() every round and unconditionally PENDs whatever it is still offered, since it has no real resource-matching allocation of its own -- keeps re-offering and re-PENDing that job every subsequent round until an explicit JOB_STATE_CHANGE-to-RUN event (also replayed from the original recording, can legitimately arrive several rounds later) removes it from pendJobs via builtin_fcfs_on_job_state_change(). CRITICAL DISCOVERY: this exact behavior is NOT accidental -- it is DOCUMENTED, INTENTIONAL, and covered by an existing passing test, test_sim_verify_no_false_mismatch_across_round_separated_dispatch (test_sim_verify.c, scheduler-sim-eager-reconciliation spec task 3.2), whose own header comment states verbatim: 'sim_reconcile_unaccounted_jobs() captures job->accounts but never calls picker_iface->on_dispatch() to retire the job from the picker's own internal pendJobs (design.md's Non-Goals/Revalidation Triggers explicitly leave live-picker DISPATCH emission to a future scheduler-allocation spec)'. That spec's own design.md (.kiro/specs/scheduler-sim-eager-reconciliation/design.md, Non-Goals section) explicitly lists 'Any change to simulation mode's live picker (still PEND-only for freshly-arriving jobs, per scheduler-allocation)' as OUT OF SCOPE. A fix attempt (calling on_dispatch() from the reconciliation FOUND branch, mirroring sched_allocator()'s exact ordering) was implemented with a new TDD test, confirmed the bug reproduces deterministically (RED), and confirmed the fix resolves it (GREEN) -- but broke test_sim_verify_no_false_mismatch_across_round_separated_dispatch, whose fixture explicitly bakes in 'PEND(jobid) recorded in the SAME round as its own DISPATCH, and again for every round until the RUN confirmation arrives' as the CORRECT, expected --sim-verify-clean shape. The fix was REVERTED (both the sim_loop.c change and its new test) rather than kept, since it silently violates an already-agreed, tested design contract from a sibling spec.", "fix": "FIXED FOR REAL (2026-09-07), after a first fix attempt was correctly reverted (see prior root_cause note) upon discovering it broke test_sim_verify_no_false_mismatch_across_round_separated_dispatch. That test's OWN fixture turned out to be baking in the bug's exact symptom as 'expected' -- but its parent spec's own design.md ('Integration with --sim-verify') already documented the CORRECT assumption ('the picker never re-decides an already-resolved job'), which the implementation had silently drifted from. This was a genuine implementation/test drift from an already-agreed design intent, not a live spec conflict requiring a product decision -- confirmed by re-reading both scheduler-sim-decision-parity's and scheduler-sim-eager-reconciliation's own design.md files (both updated in place with 2026-09-07 correction notes explaining this). Fix: sim_reconcile_unaccounted_jobs() (sim_loop.c) now resolves the job picker and calls picker_iface->on_dispatch() immediately after reconstructing job->allocs/job->accounts in its FOUND branch, mirroring sched_allocator()'s own live-dispatch ordering (scheduler.c line 507-515) exactly. This retires the reconciled job from the picker's own pendJobs bookkeeping the moment its dispatch is reconstructed, so sim_picker_thread_fn()'s unconditional-PEND thread never re-offers/re-PENDs it on a later round. TDD: new test test_reconcile_removes_reconciled_job_from_picker_candidates (test_sim_reconcile_unaccounted.c) directly proves a job is no longer offered by next_job() after reconciliation, using sim_bootstrap_picker() to register a real, resolvable builtin_fcfs instance -- confirmed RED before the fix, GREEN after. test_sim_verify_no_false_mismatch_across_round_separated_dispatch (test_sim_verify.c) was updated to match the corrected behavior: round 1 now records DISPATCH-only (new helper append_replay_session_with_dispatch(), no accompanying PEND), and rounds 2-3 are now empty ReplaySessions (matching round 4's own existing pattern) instead of baking in a stale re-PEND echo as expected. Full scheduler suite: 76/77 pass (same pre-existing unrelated test_core_events.c failure, bug-1762, untouched by this fix).", @@ -25012,7 +12181,6 @@ "discovered-live", "dispatch", "dispatch-implies-run", - "fixed", "job-state", "needs-debug", "pend", @@ -25028,16 +12196,20 @@ "bug-1567", "bug-1623" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1759.md" }, { "id": "bug-1760", "status": "fixed", "status_updated": "2026-09-24", "status_note": "88b0ac99", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-02T04:10:00Z", - "error_message": "Live-tested against a real 3-node HA cluster + xcecd_virtual worker + xce_scheduler (worker-rejoin-resync soak-harness development): after the CURRENT primary is killed and a NEW primary is elected+reconverged, a job submitted through the new primary stays PEND for 60+ seconds with ZERO scheduler.log activity of any kind (no WARN/INFO/ERROR, no reconnect attempt) -- not a slow recovery, a genuinely stuck one. Reproduced twice in a row with the same shape.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "Live-tested against a real 3-node HA cluster + xcecd_virtual worker + xce_scheduler…\n\nLive-tested against a real 3-node HA cluster + xcecd_virtual worker + xce_scheduler (worker-rejoin-resync soak-harness development): after the CURRENT primary is killed and a NEW primary is elected+reconverged, a job submitted through the new primary stays PEND for 60+ seconds with ZERO scheduler.log activity of any kind (no WARN/INFO/ERROR, no reconnect attempt) -- not a slow recovery, a genuinely stuck one. Reproduced twice in a row with the same shape.", "file": "src/scheduler/src/scheduler.c (sched_needs_rejoin:487, sched_on_reconnect:507, sched_publish_check_rejoin:1474), src/scheduler/src/main.c", "root_cause": "CONFIRMED via source read (not yet live-instrumented/gdb-confirmed): the scheduler has NO periodic connection-liveness/reconnect check analogous to the worker daemon's checking_connection() (src/worker/src/controller_connector.c:505, called every xcecd_ctx.update_interval seconds from xcecd.c's main loop via xcecli_hello()). The scheduler's ONLY rejoin mechanism is sched_publish_check_rejoin() (scheduler.c:1474), which re-issues xcecli_join() ONLY as a side effect of its own NEXT SUCCESSFUL publish call (sched_publish_dispatch()/sched_publish_pend_reason() via sched_publish_retry_loop(), which calls sched_publish_check_rejoin() only after attempt_fn() already returned >=0). sched_needs_rejoin itself is set correctly by sched_on_reconnect() (the client library's own transparent-reconnect callback), so the SCHEDULER's own underlying connection to the cluster likely IS aware it reconnected somewhere -- but nothing forces a FRESH xcecli_join() against the actual NEW primary until the scheduler has something to publish. After a real controller failover, the scheduler has nothing queued to publish (no job ever reaches its picker/allocator, since the CONTROLLER-side registration of \"where the scheduler is\" is presumed to be live-connection-scoped state on whichever controller instance the scheduler last explicitly joined -- i.e. the dead primary -- and does not appear to carry over to the newly-elected primary), so the scheduler can never publish anything to trigger its own rejoin: a chicken-and-egg deadlock. NOT YET independently confirmed via gdb/live instrumentation which exact half of this loop is stuck (client-library-level reconnect vs. controller-side scheduler-registration loss) -- this is a plausible, evidence-consistent hypothesis from static reading + the observed symptom, not a fully root-caused finding.", "fix": "NOT FIXED -- discovered as a side effect of unrelated worker-rejoin-resync soak-harness development; deliberately not investigated further or fixed in that session (explicit instruction: log and continue). This is a potentially serious standing HA-availability gap (every job submitted after ANY controller failover could stay PEND forever until the scheduler process is manually restarted) that deserves its own dedicated debugging investigation (e.g. /kiro-debug) scoped to component-failover-adoption's scheduler-rejoin design, ideally with live gdb/log instrumentation to confirm which half of the chicken-and-egg loop is actually broken. Pragmatic workaround adopted ONLY in the worker-rejoin-resync soak harness (.claude/skills/xce-test-ha/run-ha-soak.sh): restart the scheduler process after each round's controller failover, the same way the harness already restarts the downed CONTROLLER node -- a fresh scheduler process calls sched_init(), which tries all 3 candidates in order and joins whichever is actually reachable, sidestepping the stuck reactive-rejoin path entirely. This workaround is scoped to test infrastructure only and does NOT fix the underlying production behavior.\n\nUPDATE (2026-09-04): FIXED. sched_publish_check_rejoin() (scheduler.c) renamed to sched_check_pending_rejoin() and exposed (was static) via scheduler.h. main.c's round loop (while(1) { xce_sched_driver_run_normal_mode_round(); ...; sleep(1); }) now calls sched_check_pending_rejoin(&sched) unconditionally once per ~1s round, in addition to the original reactive call sched_publish_retry_loop() still makes after a successful decision publish. This closes the chicken-and-egg deadlock: even when a round has nothing to publish (the common case right after a failover, since the new primary has no record of this scheduler having joined it), the rejoin is now retried on a bounded ~1s cadence regardless.\n\nDeliberately placed in main.c, not inside driver.c's xce_sched_driver_run_normal_mode_round() (the first attempt): driver.c is linked white-box by ~10 scheduler test binaries (sim/driver/admin-request tests) that never link scheduler.c (where sched_check_pending_rejoin/sched_needs_rejoin live) -- a driver.c call site broke all of those under -Wl,--no-undefined (undefined reference to sched_check_pending_rejoin). main.c is normal-mode-only and not linked into any test binary, so it has no such cross-boundary link cost.\n\nTDD: two new tests added to test_scheduler_decision_publish_retry.c. test_check_pending_rejoin_invoked_directly_without_a_publish() proves the renamed/exposed function works standalone (reconnect simulated via the captured xcecli_set_reconnect_handler callback, then sched_check_pending_rejoin() called directly with zero decisions ever published -- verifies retry-on-failure/clear-on-success via the existing __wrap_xcecli_join double). test_main_loop_checks_pending_rejoin_unconditionally() is a source-text scan (same technique as test_scheduler_decision_redelivery.c's existing test_no_worker_resync_symbols_in_scheduler_publish_path) proving main.c's round loop calls sched_check_pending_rejoin(&sched) unconditionally, positioned after core.stop's break check but before sleep(1) -- i.e. every live round, not gated on publish success. Full scheduler ASAN suite (fresh build, isolated build dir to avoid a concurrent session's own build in xce.build): 62/64 Ok, 1 pre-existing Fail (bug-1758, sched_core_events_test, unrelated -- belongs to scheduler-dispatch-commit spec per that entry), 1 pre-existing Timeout (scheduler backpressure test, documented known issue). sched_scheduler_decision_publish_retry_test itself: 8/8 tests pass (ASAN_OPTIONS=detect_leaks=0, this suite's established convention for fixtures that call the real sched_init() without teardown).\n\nNot independently live-verified against a real 3-node HA failover in this session (the fix is a straightforward, well-reasoned closing of a plainly-diagnosed gap -- a periodic check where only a reactive one existed -- verified by unit test rather than a live repro this time).\n\nUPDATE (2026-09-04, live-repro + partial fix session): Root cause CONFIRMED live, exactly as hypothesized. Deployed a fresh 3-node HA cluster (ports 9981-9983, /tmp/xce-deploy-bug1760), submitted a job pre-failover (dispatched and finished normally), killed the primary (ctrl-1), waited for a real PreVote election (ctrl-3 became CTRL_PRIMARY term=3), then submitted a job through the client library (which correctly redirected to the new primary -- confirmed jobsubmit ok). That job stayed PEND indefinitely with ZERO scheduler.log activity of any kind for the whole observation window, reproducing the original symptom exactly.\n\nPARTIAL FIX APPLIED: sched_publish_check_rejoin() (scheduler.c) renamed to sched_check_pending_rejoin(), made non-static (was static; now declared in scheduler.h), and called from a SECOND site: main()'s round loop (scheduler/src/main.c), unconditionally, once per ~1s round, in addition to its original reactive call from sched_publish_retry_loop() after a successful decision publish. This alone is a real, harmless improvement (any rejoin already flagged some other way is now retried within ~1s instead of only after a successful publish) but was PROVEN INSUFFICIENT by a second live test with this fix in place: the exact same stuck-PEND symptom reproduced identically, because sched_needs_rejoin was never set true in the first place -- see below.\n\nFULL ROOT CAUSE, now fully confirmed (both halves of the original open question resolved): sched_needs_rejoin is set ONLY by sched_on_reconnect(), the client library's transparent-reconnect callback, which itself only fires as a side effect of an ACTUAL xcecli_* call (XCECLI_FAILOVER_CALL/XCECLI_FF_CALL, client_internal.h) detecting a dead connection and rotating to a live candidate. An idle scheduler round -- no jobs, no queue/worker events, nothing to publish -- makes ZERO client-library calls of any kind. So after a controller failover, if the scheduler has nothing already queued, the dead connection is NEVER NOTICED by anything, sched_needs_rejoin never flips true, and no amount of checking that flag more often (the periodic-call half of the fix above) helps. This is a pure detection gap, not a retry-cadence gap.\n\nARCHITECTURAL CONFLICT DISCOVERED (this is the actual remaining blocker, not further coding): the obvious fix -- add an active liveness probe (xcecli_hello()) call to the same periodic round-loop site, mirroring the worker's own checking_connection()/xcecli_hello() pattern (controller_connector.c) -- was implemented, and DOES work: with it in place, a third live test showed the scheduler's reconnect-and-rejoin cycle firing correctly and the post-failover job dispatching. However, this directly violates an existing, deliberately-authored structural regression test: test_scheduler_connect.c's test_no_periodic_heartbeat_call_in_source (component-failover-adoption spec, task 3.3, Requirement 3 AC3), which source-text-scans scheduler.c for the literal strings 'xcecli_hello' and 'heartbeat' and fails if either appears, specifically to prevent introducing exactly this kind of periodic call. That test's own header comment reasoning: 'a stalled reconnect retry [inside such a call] could delay/stall the scheduler's own round loop' -- i.e. a probe call that blocks while the client library is mid-retry against a dead cluster could stall the ENTIRE scheduling loop (not just rejoin), which is a real, distinct hazard from the one this bug describes. Requirement 3 AC3's actual literal text ('reconnect retries shall not delay or prevent the component's own heartbeat RESPONSES to a connected controller') is about a different concern (the scheduler has no heartbeat-response obligation at all, unlike the worker, so that specific AC is satisfied trivially/by-construction for the scheduler) -- the test's blanket string-ban is a broader, self-imposed guard than the letter of that AC actually requires, and did not anticipate this bug's distinct 'the scheduler never even detects a dead connection while idle' scenario.\n\nDECISION MADE THIS SESSION: did NOT unilaterally override this existing, deliberately-tested design decision. Reverted the xcecli_hello probe (and its test-file/meson.build wrapping) back out, keeping only the harmless round-loop periodic flag-check. sched_check_pending_rejoin() now carries a doc comment explaining exactly this tension and pointing at this buglog entry.\n\nNEXT STEP (needs a design decision, not just code): either (a) amend component-failover-adoption's Requirement 3 AC3 and its guarding test to explicitly allow a BOUNDED/non-blocking reconnect-detection probe (distinct from an unbounded heartbeat-response obligation) -- e.g. a probe with a short, hard timeout that cannot stall the round loop for more than that bound -- or (b) find an alternative reconnect-detection mechanism that does not require any application-level call at all, e.g. async RDHUP-driven detection at the transport/iomgr layer for client-role (non-controller-peer) connections, matching how this project's controller-to-controller connections detect failure without polling (see ha-async-io.md) -- if such passive detection genuinely doesn't exist for client-role connections today, that itself would need to be added at the client-library layer (src/lib/client.c/xcectx.c), not scheduler.c. Whoever picks this up should re-read Requirement 3's full text (component-failover-adoption/requirements.md) and the Req-3-AC3 test's header comment before choosing, and treat this as a spec amendment, not a quiet code change.\n\nCleanup: cluster (ports 9981-9983, /tmp/xce-deploy-bug1760) fully torn down, all started processes killed, confirmed via ps aux. Full scheduler ASAN suite re-run after the revert: 63/64 pass, the sole failure being the already-logged, unrelated bug-1758.\n\nUPDATE (2026-09-04, live verification of the passive-detection fix): CONFIRMED FIXED end to end. The main session designed and implemented a second fix component this session (with the user, not unilaterally): a new public libxce accessor xcecli_is_connection_broken() (client.h/client.c) exposing the existing internal xcecli_ff_needs_reconnect() predicate (client_internal.h) -- a pure, socket-free, side-effect-free read of ctx->ctx.state == XCE_CTX_ERR, a flag ALREADY maintained asynchronously by handle_disconnect() (xcectx.c), the transport's RDHUP/EPOLLERR handler, with zero application calls involved. sched_check_pending_rejoin() (scheduler.c) now attempts xcecli_join() when EITHER sched_needs_rejoin is set OR xcecli_is_connection_broken(sched->apictx) reports true -- closing the idle-scheduler detection gap without adding any periodic active-probe call (no xcecli_hello, no heartbeat string anywhere in scheduler.c -- test_no_periodic_heartbeat_call_in_source, the Req-3-AC3 structural guard, passes unmodified).\n\nLive re-verification (fresh 3-node cluster, ports 9971-9973, ASAN build in src/xce.build.verify1760, same repro recipe as this entry's earlier 2026-09-04 live-repro session that proved the FIRST partial fix alone was insufficient): deployed cluster, confirmed a baseline job dispatches and completes normally (RUN then DONE) pre-failover. Killed ctrl-1 (the primary). New primary (ctrl-2, term=3) elected 15s later. Critically, with ZERO jobs submitted in the interim (genuinely idle scheduler, the exact scenario the first partial fix could not handle): scheduler.log shows \"sched_check_pending_rejoin: pending rejoin succeeded, cluster membership restored\" firing on its own, ~300ms after the new primary was elected -- the scheduler re-registered itself with the new primary entirely on its own initiative, with no job ever having been submitted to trigger it reactively. Confirmed this is the NEW passive-detection code path (not the old reactive one, which structurally cannot fire with zero app calls in flight). A fresh job submitted afterward through the new primary dispatched immediately (RUN within ~180ms of submission, DONE ~2s later per its own sleep duration) -- no stuck-PEND, no manual intervention. Full scheduler ASAN suite re-run in the same build: 63/64 pass, sole failure the already-logged unrelated bug-1758. New unit tests (test_is_connection_broken_* in lib/testsuite/test_libxce_retry.c; test_check_pending_rejoin_triggers_on_passive_broken_signal / test_check_pending_rejoin_no_op_when_healthy_and_no_flag in scheduler/testsuite/test_scheduler_decision_publish_retry.c) all pass. Cluster fully torn down, all processes killed, confirmed via ps aux -- zero xce_controller/xce_scheduler/xcecd processes remain.\n\nThis closes bug-1760 -- both halves (reactive + passive detection) are now live-verified, not just unit-tested.", @@ -25052,7 +12224,6 @@ "chicken-and-egg", "discovered-live", "severity-high-suspected", - "fixed", "verified-2026-09-04", "partially-diagnosed", "live-repro-confirmed-2026-09-04", @@ -25062,15 +12233,22 @@ "live-verified-2026-09-04" ], "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "88b0ac99" + ], + "record": ".wolf/bugs/bug-1760.md" }, { "id": "bug-1761", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-02T04:30:00Z", - "error_message": "cli-ux audit of xce_scheduler's --interactive REPL (scheduler-sim-repl-verbosity spec discussion): repl_cmd_delete() (repl_break.c) parses its argument with atoi(), which silently returns 0 on non-numeric input instead of signalling an error -- `delete abc` reports \"[sim] delete: no breakpoint with id 0\" (a well-formed-but-missing-id message) rather than rejecting \"abc\" as not a number, masking an operator typo as a legitimate lookup miss.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cli-ux audit of xce_scheduler's --interactive REPL (scheduler-sim-repl-verbosity spec discussion)…\n\ncli-ux audit of xce_scheduler's --interactive REPL (scheduler-sim-repl-verbosity spec discussion): repl_cmd_delete() (repl_break.c) parses its argument with atoi(), which silently returns 0 on non-numeric input instead of signalling an error -- `delete abc` reports \"[sim] delete: no breakpoint with id 0\" (a well-formed-but-missing-id message) rather than rejecting \"abc\" as not a number, masking an operator typo as a legitimate lookup miss.", "file": "src/scheduler/src/repl_break.c (repl_cmd_delete)", "root_cause": "CONFIRMED via source read: repl_cmd_delete() used `int id = atoi(tokens[1]);` with no validation of atoi()'s return, unlike this file family's own documented \"fail loud\" convention -- repl.c's repl_cmd_step() doc comment explicitly rejects the equivalent atoi()-style silent-coercion failure mode and uses strtol()+endptr instead specifically to catch non-numeric/trailing-garbage input. delete's atoi() call was never updated to match when that convention was established, so `delete abc`/`delete 1x` both silently coerced to some numeric prefix (or 0) instead of erroring.", "fix": "FIXED. repl_cmd_delete() (repl_break.c) now uses strtol()+endptr, mirroring repl_cmd_step()'s exact validation shape: rejects a token with no valid leading integer, trailing non-numeric garbage, or a non-positive value (breakpoint ids are assigned via pre-incremented next_bp_id in repl_cmd_break(), so 1 is the smallest id that can ever legitimately exist), printing \"[sim] delete: '' is not a valid breakpoint id -- usage: delete |all\" instead of silently proceeding. Added test_delete_non_numeric_id_rejected (test_repl_breakpoints_inspection.c, registered via RUN_TEST) covering \"abc\", \"0\", \"-1\", and \"1x\", all asserting the existing breakpoint is left untouched. Verified via /xce-build scheduler: clean compile, test_delete_non_numeric_id_rejected PASSed (testlog.txt confirms all 4 cases -- \"abc\", \"0\", \"-1\", \"1x\" -- hit the new rejection path), sched_repl_breakpoints_inspection_test suite OK. One unrelated pre-existing failure in the same run (test_job_rusage_reconciles_mem_reservation_against_usage, test_core_events.c) -- different file, different subsystem (mem-reservation reconciliation, not REPL/breakpoint parsing), not touched by this fix.", @@ -25079,20 +12257,23 @@ "scheduler-sim", "cli-ux", "input-validation", - "fail-loud", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fail-loud" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1761.md" }, { "id": "bug-1762", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-02T07:50:00Z", - "error_message": "scheduler-sim-repl-verbosity task 1.2 (Jobid Range Compressor, repl_fmt_jobid_range()): task 1.1's approved RED-phase meson.build target sched_repl_output_compression_test (src/scheduler/testsuite/meson.build) lists ONLY test_repl_output_compression.c as its source, with no source providing the function's implementation -- so even after adding a correct repl_fmt_jobid_range() definition to src/scheduler/src/repl.c (as design.md's File Structure Plan literally directs: 'implements the new shared jobid-range-compression helper ... in repl.c'), the target still fails to link: undefined reference to repl_fmt_jobid_range.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "scheduler-sim-repl-verbosity task 1.2 (Jobid Range Compressor, repl_fmt_jobid_range()): task 1.1's…\n\nscheduler-sim-repl-verbosity task 1.2 (Jobid Range Compressor, repl_fmt_jobid_range()): task 1.1's approved RED-phase meson.build target sched_repl_output_compression_test (src/scheduler/testsuite/meson.build) lists ONLY test_repl_output_compression.c as its source, with no source providing the function's implementation -- so even after adding a correct repl_fmt_jobid_range() definition to src/scheduler/src/repl.c (as design.md's File Structure Plan literally directs: 'implements the new shared jobid-range-compression helper ... in repl.c'), the target still fails to link: undefined reference to repl_fmt_jobid_range.", "file": "src/scheduler/testsuite/meson.build (sched_repl_output_compression_test), src/scheduler/src/repl.c", "root_cause": "CONFIRMED empirically, but the FIRST fix attempt below drew the wrong conclusion from it. Adding repl.c itself as a bare extra SOURCE FILE to the test target's sources does not fix the link -- it trades one undefined reference for five: repl_dispatch() (a function in repl.c, never called by this test) references repl_cmd_break/repl_cmd_delete/repl_cmd_info/repl_cmd_snapshot/repl_cmd_set, implemented in sibling files repl_break.c/repl_info.c/repl_params.c that this deliberately-lightweight test target does not build. A directly-linked .o file (unlike an archive/static-library member) requires ALL of its undefined symbols resolved regardless of whether the calling code path is ever exercised at runtime -- there is no dead-code elimination at the object-file granularity without -ffunction-sections + -Wl,--gc-sections, which this build does not use. HOWEVER: this only rules out the narrow 'bare .c file in the source list' fix -- it does NOT mean repl_fmt_jobid_range() must live in its own translation unit. sched_scheduler_connect_test (src/scheduler/testsuite/meson.build) already establishes the correct alternative for exactly this shape of problem: link the target against the REAL scheduler_srcs object list (a `foreach f : scheduler_srcs` loop over the production source list, minus src/main.c) instead of hand-picking source files, which resolves repl_dispatch()'s cross-file externs for real (repl_break.c/repl_info.c/repl_params.c are simply also linked in) rather than needing them excluded. This alternative was available and untried at the time of the original fix below, which additionally violated design.md's explicit Architecture Integration commitment ('no new production .c file needed') by introducing repl_jobid_range.c.", "fix": "REMEDIATED (superseding the original fix below, which was reviewed and rejected for violating design.md's 'no new production .c file needed' commitment). repl_fmt_jobid_range() and its private helpers now live in src/scheduler/src/repl.c itself, exactly as design.md's File Structure Plan directs, with only one new #include (, for the internal repl_jr_appendf() vararg helper) -- no new production .c file. sched_repl_output_compression_test (src/scheduler/testsuite/meson.build) now links the real scheduler_srcs object list via a `foreach f : scheduler_srcs` loop (mirroring sched_scheduler_connect_test's own established technique) instead of hand-picking source files, so repl_dispatch()'s cross-file externs resolve for real; no `-Wl,--wrap=` or `-DXCE_SCHEDULER_C_PATH` needed since this test never calls sched_init() or reaches networking code. repl_jobid_range.c deleted; 'src/repl_jobid_range.c' removed from scheduler_srcs (src/scheduler/meson.build); repl_internal.h's doc comment on repl_fmt_jobid_range() updated to drop the now-false 'lives in its own translation unit' rationale. Verified: xce-build.sh scheduler --incremental succeeds with zero linker errors; sched_repl_output_compression_test's own binary output shows all 5 Unity tests PASS (5 Tests 0 Failures 0 Ignored); full scheduler suite still 55/56, the one failure being the pre-existing unrelated test_job_rusage_reconciles_mem_reservation_against_usage (test_core_events.c, mem-reservation subsystem, untouched by this change); repl.c stays at 794/800 lines after the addition (up from 531 before this task).", @@ -25102,21 +12283,23 @@ "build-wiring", "meson", "jobid-range-compression", - "test-infrastructure", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "test-infrastructure" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1762.md" }, { "id": "bug-1763", "status": "open", "status_updated": "2026-09-24", "status_note": "test_sim_record_output.c stack-local sim read by unjoined picker thread (ASAN)", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-02T13:00:00Z", - "error_message": "sched_sim_record_output_test (scheduler-sim-repl-verbosity task 3.2 GREEN-phase build): AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151), thread T1 (sim_picker_thread_fn, sim_loop.c:456), reading the stack of test_output_file_is_decodable_with_full_stream_symmetry's stack-local `sim` (xce_sched_sim_driver_t, test_sim_record_output.c:341) after that frame returned. Fired once during a `/xce-build scheduler --incremental` run after this task's repl_break.c edits; 4/4 direct re-runs of the isolated binary afterward were clean (OK).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "sched_sim_record_output_test (scheduler-sim-repl-verbosity task 3.2 GREEN-phase build)…\n\nsched_sim_record_output_test (scheduler-sim-repl-verbosity task 3.2 GREEN-phase build): AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151), thread T1 (sim_picker_thread_fn, sim_loop.c:456), reading the stack of test_output_file_is_decodable_with_full_stream_symmetry's stack-local `sim` (xce_sched_sim_driver_t, test_sim_record_output.c:341) after that frame returned. Fired once during a `/xce-build scheduler --incremental` run after this task's repl_break.c edits; 4/4 direct re-runs of the isolated binary afterward were clean (OK).", "file": "src/scheduler/testsuite/test_sim_record_output.c", "root_cause": "Same class already documented by bug-1598/bug-1632, not a new defect: sched_sim_driver_run() spawns sim_picker_thread_fn on an unjoined pthread that reads its xce_sched_sim_driver_t* for the rest of the process's life; any stack-local `sim` fixture is therefore a latent stack-use-after-return the moment its owning test function returns, and whether ASAN's shadow memory actually catches a given run is purely a function of incidental stack-layout/timing -- bug-1598's own root cause entry documents an unrelated new test shifting stack layout enough to make this fire deterministically for a different file. test_sim_record_output.c #includes repl_break.c white-box (test_sim_record_output.c:78), so this task's repl_break.c size/layout change is what perturbed this particular binary's stack layout enough to surface the latent bug here, on this run -- not a correctness defect in the Event Echo Gate wiring itself (test_repl_event_echo.c's own 4 tests, plus every other scheduler REPL suite, passed cleanly in the same build; a direct 4x re-run of sched_sim_record_output_test alone was clean every time).", "fix": "NOT FIXED -- out of scope for scheduler-sim-repl-verbosity task 3.2 (repl_break.c wiring only, no sim_loop.c/test_sim_record_output.c changes in this task's boundary). Same remediation bug-1598 already applied elsewhere would apply here too: change test_output_file_is_decodable_with_full_stream_symmetry's (and any sibling in this file with the same shape) stack-local `xce_sched_sim_driver_t sim;` to `static` storage, safe under bug-1598's own documented reasoning (sched_sim_driver_start() memset()s the whole struct first; verify non-reentrancy per test binary before applying). Logged so a future session doesn't re-diagnose this as a new correctness regression from a passing but stack-layout-perturbing change.", @@ -25128,23 +12311,26 @@ "stack-use-after-return", "flaky", "thread-lifecycle", - "scheduler-sim-repl-verbosity", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "scheduler-sim-repl-verbosity" ], "related_bugs": [ "bug-1598", "bug-1632" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1763.md" }, { "id": "bug-1764", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-03T03:01:26Z", - "error_message": "info worker's \"assigned jobs\" listing (repl_info_worker_one(), repl_info.c) can show a job that has already reached a terminal state (DONE/EXIT) as still assigned to a worker, contradicting info summary's running count. Live-observed in a real interactive session (session=59): info summary reported \"100 jobs (0 pending, 7 running)\" and worker pc13v's slots showed used=7.00, but the same info worker output listed assigned jobs: 0-11-1788164699-0[0-49] -- the full 50-index range of one array job, all still shown assigned.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "info worker's \"assigned jobs\" listing (repl_info_worker_one(), repl_info.c) can show a job that…\n\ninfo worker's \"assigned jobs\" listing (repl_info_worker_one(), repl_info.c) can show a job that has already reached a terminal state (DONE/EXIT) as still assigned to a worker, contradicting info summary's running count. Live-observed in a real interactive session (session=59): info summary reported \"100 jobs (0 pending, 7 running)\" and worker pc13v's slots showed used=7.00, but the same info worker output listed assigned jobs: 0-11-1788164699-0[0-49] -- the full 50-index range of one array job, all still shown assigned.", "file": "src/scheduler/src/sim_loop.c", "root_cause": "sim_reconcile_unaccounted_jobs() (sim_loop.c:664) triggers reconciliation on job->accounts == NULL alone (sim_loop.c:690), with no check on job->state. This trigger was deliberately widened to be state-independent by the scheduler-sim-eager-reconciliation spec (task 1.1, test_sim_reconcile_unaccounted.c's own Test 5 header comment: \"capture eligibility no longer depends on job->state\") so a PEND-state job's early-recorded DISPATCH decision could be captured before its RUN confirmation replays. That widening only considered the forward case (PEND -> captured early -> later RUN) and missed the terminal case: a job that already ran to completion has job->accounts/job->allocs correctly freed by xce_core_push_events()'s normal end-of-life cleanup (core_events.c:1127-1135 JOB_STATE_CHANGE, :1179-1186 JOB_UPDATE) -- which also leaves job->accounts == NULL. If ANY later round's decoded_session->decisions still carries an unconsumed DISPATCH entry for that same jobid (plausible in a real recording with duplicate/retried dispatch attempts, or per-round decision-emission timing not lining up cleanly with per-job completion for an array job), sim_reconcile_unaccounted_jobs() matches it and resurrects job->allocs/job->accounts on the already-terminal job, with no state check to prevent it. Dynamically confirmed via a temporary probe test (added to test_sim_reconcile_unaccounted.c, run once, then reverted -- never committed): a job driven to JOB_STAT_DONE via xce_core_push_events() (which correctly nulled its allocs/accounts) was then fed a second decoded_session containing a fresh DISPATCH entry for the same jobid; sim_reconcile_unaccounted_jobs() unconditionally re-populated both job->allocs and job->accounts, contradicting job->state == JOB_STAT_DONE. repl_info_worker_one()'s assigned-jobs filter (repl_info.c:686, if (job->allocs == NULL) continue;) has no independent state check either, so it faithfully (and misleadingly) displays the resurrected allocation. The already-established defensive pattern for exactly this class of problem, job_state_regresses_terminal() (core_events.c:696-701, \"terminal states are permanent\"), is not reused here.", "fix": "FIXED (2026-09-03, scheduler-sim-reconcile-terminal-guard spec, task 1.2). sim_reconcile_unaccounted_jobs()'s per-job trigger check (sim_loop.c) now also excludes JOB_IS_DONE(job->state) || JOB_IS_EXIT(job->state), alongside the existing job->accounts != NULL skip -- a terminal job is never searched for a matching recorded DISPATCH entry, mirroring job_state_regresses_terminal()'s existing 'terminal is permanent' invariant (core_events.c). TDD: task 1.1's RED-phase tests (test_reconcile_never_reallocates_a_done_job_despite_stray_dispatch_entry, test_reconcile_never_reallocates_an_exit_job_despite_stray_dispatch_entry, test_sim_reconcile_unaccounted.c) pinned the exact reproduction before the fix landed; both pass after. Confirmed no regression to the pre-existing PEND eager-capture case or to core_events.c's JOBS_PUSH resync-merge terminal-state tests. The analogous resync-merge branch was investigated and confirmed already correct (job_state_regresses_terminal() call), no change needed there.", @@ -25155,9 +12341,7 @@ "reconciliation", "job-lifecycle", "stale-state", - "array-job", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "array-job" ], "related_bugs": [ "bug-1611", @@ -25165,16 +12349,21 @@ "bug-1738", "bug-1739" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1764.md" }, { "id": "bug-1765", "status": "open", "status_updated": "2026-09-24", "status_note": "frontend tests/regression/run.sh jobid regex rejects dashed jobid format", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-09-03T14:51:54Z", - "error_message": "tests/regression/run.sh's 'jobs submit' test FAILs deterministically (28 passed / 1 failed / 1 skipped instead of 29/0/0), with the cascading 'jobs kill' test SKIPped as a result -- reproduced identically regardless of which build system (Makefile vs Meson) produced the xcecli binary under test, confirming it predates and is unrelated to the eliminate-frontend-makefile migration.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "tests/regression/run.sh's 'jobs submit' test FAILs deterministically (28 passed / 1 failed / 1…\n\ntests/regression/run.sh's 'jobs submit' test FAILs deterministically (28 passed / 1 failed / 1 skipped instead of 29/0/0), with the cascading 'jobs kill' test SKIPped as a result -- reproduced identically regardless of which build system (Makefile vs Meson) produced the xcecli binary under test, confirming it predates and is unrelated to the eliminate-frontend-makefile migration.", + "file": "", "root_cause": "CONFIRMED via source + git history. src/frontend/apiserver/controllers/jobs.go:1518 formats the submit-response message with a plain-digit '%d' placeholder ('Job <%d> is submitted...'), but src/frontend/xcecli/cmd/jobs/submit.go:500 post-processes that message client-side, replacing the '<%d>' substring via format.FormatJobID() (src/frontend/xcecli/common/format/jobid.go:40-57), which renders the packed 64-bit job ID as the dashed '{cluster}-{apiserver}-{timestamp}-{sequence}' form (e.g. '1-0-1788446837-0'). tests/regression/run.sh:143's assertion regex ('Job <[0-9]+> is submitted') was never updated for this dashed format. The CLI-side reformatting was introduced by commit d6008ead ('improve frontend jobid display', 2026-05-07) -- nearly four months before this discovery -- so the test has been silently broken (as a false FAIL, not a false PASS) since then, not a new regression.", "fix": "NOT FIXED -- out of scope for the eliminate-frontend-makefile spec's task 3.5 (pure verification task, no file changes permitted). Fix would be updating run.sh:143's regex to accept the dashed job-ID format (e.g. 'Job <[0-9-]+> is submitted' or delegating to the same format.FormatJobID() pattern). A second, independent, lower-severity issue was found in the same file during this investigation: run.sh:155's assert_not_contains 'jobs list submit_time not dash' calls grep -qF \"- \" without a '--' separator, so GNU grep treats the leading-dash argument as an invalid option and errors out -- this coincidentally still yields the intended PASS (grep's error-exit is indistinguishable here from 'no match'), so it doesn't affect reported counts, but isn't actually testing what it appears to test.", "tags": [ @@ -25182,21 +12371,24 @@ "test-infra", "jobid-format", "false-fail", - "pre-existing", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "pre-existing" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1765.md" }, { "id": "bug-1766", "status": "fixed", "status_updated": "2026-09-25", "status_note": "601b245e", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-04T08:03:28Z", - "error_message": "OPEN, NOT A CRASH -- discovered while reviewing the scheduler-allocation spec requirements against live scheduler.c source: sched_make_compound_allocation() does not roll back already-succeeded primaries when a LATER primary in the same COMPOUND resreq fails allocation, and its own post-loop code can misattribute a partial allocation to a failed compound.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "c source: sched_make_compound_allocation() does not roll back already-succeeded primaries when a…\n\nOPEN, NOT A CRASH -- discovered while reviewing the scheduler-allocation spec requirements against live scheduler.c source: sched_make_compound_allocation() does not roll back already-succeeded primaries when a LATER primary in the same COMPOUND resreq fails allocation, and its own post-loop code can misattribute a partial allocation to a failed compound.", + "file": "", "root_cause": "sched_make_compound_allocation() (src/scheduler/src/scheduler.c:206-238) loops over a COMPOUND resreq's primaries, calling sched_make_primary_alloc_(sched, &ctx) for each with ONE shared ctx across the whole loop. On a mid-loop failure (rc <= 0) it just `break`s -- it never undoes the primaries that already succeeded (no zeroing of ctx.alloc, no reversal of any resource state those primaries touched). Worse: seqjob/parjob's job_alloc_fn sets ctx->alloc directly on success (see seqjob_allocator.c my_alloc()), so if primary 1 of a 2-primary compound succeeds (leaving ctx.alloc non-NULL) and primary 2 then fails, the function's unconditional post-loop `if (ctx.alloc != NULL) { MOVE_PTR(obj->alloc, ctx.alloc); }` still fires regardless of the overall `rc <= 0` -- so a COMPOUND job that is reported as allocation-FAILED can still end up with obj->alloc set to primary 1's partial, orphaned allocation. requirements.md for scheduler-allocation (Req 9 AC2) had assumed, based on the sibling scheduler-framework brief's own text, that this path already 'tracks allocations in ctx and cleans up on failure' -- that claim does not hold against the actual code; requirements.md was corrected during this review. Not yet confirmed whether the orphaned ctx.alloc is ever read for a job later found to be PEND/failed (i.e. whether this is dormant/leak-only or produces an observably wrong result) -- not investigated further since it is out of the scheduler-allocation spec's own boundary (Req 9 AC2 explicitly freezes COMPOUND/ALTER dispatch as unchanged).", "fix": "NOT FIXED -- out of scope for the scheduler-allocation spec (which explicitly leaves COMPOUND/ALTER dispatch unchanged, Req 9 AC2) and no other spec currently owns sched_make_compound_allocation(). Discovered only because scheduler-allocation's own NEW per-plan rollback (Req 4's rollback_plan_allocation(), for the parjob multi-host-bucket case) needed a reusable precedent to model itself on; this path was found not to provide one. Needs its own follow-up: likely either (a) track which primaries in the compound loop actually succeeded and explicitly free/clear their contribution to ctx.alloc before returning on a later failure, or (b) defer the MOVE_PTR(obj->alloc, ctx.alloc) until the loop confirms overall success, freeing ctx.alloc on the failure path instead. Whether this ever produces an observably wrong result (vs. a dormant orphaned pointer) is unconfirmed and should be checked first.", "tags": [ @@ -25206,22 +12398,26 @@ "partial-allocation", "open-gap", "scheduler-allocation-spec", - "requirements-review", - "not-fixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "requirements-review" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "601b245e" + ], + "record": ".wolf/bugs/bug-1766.md" }, { "id": "bug-1767", "status": "fixed", "status_updated": "2026-09-27", "status_note": "1dfdf42c: snapshot_load.c now uses the shared xce_core_build_schedobj()", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-04T08:03:28Z", - "error_message": "OPEN, NOT A CRASH -- recurring pattern finding, not a single defect: snapshot_load.c's local, hand-duplicated reconstruction of xce_sched_obj_t/schedobj (kept independent of core_events.c's core_build_schedobj() because that helper is `static` and snapshot.c/snapshot_load.c are compiled as separate objects) has now caused THREE independent field-completeness gaps in the same narrow reconstruction path, each discovered separately.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "not a single defect: snapshot_load.c's local, hand-duplicated reconstruction of…\n\nOPEN, NOT A CRASH -- recurring pattern finding, not a single defect: snapshot_load.c's local, hand-duplicated reconstruction of xce_sched_obj_t/schedobj (kept independent of core_events.c's core_build_schedobj() because that helper is `static` and snapshot.c/snapshot_load.c are compiled as separate objects) has now caused THREE independent field-completeness gaps in the same narrow reconstruction path, each discovered separately.", + "file": "", "root_cause": "snapshot.c's own header comment (Why not reuse core_events.c's core_build_job/core_build_queue/core_build_worker, snapshot.c:296-314) documents the deliberate choice to keep snapshot_load.c's job/queue/worker/schedobj reconstruction as a second, independent, field-for-field-identical implementation rather than exporting core_events.c's static builders -- justified there for core_build_job/queue/worker specifically (they also need an independent SAVE-direction implementation regardless, and funneling through xce_core_push_events() reintroduces a config/queue double-application bug). That reasoning does not equally apply to core_build_schedobj() alone: schedobj is never wire-captured (no build_wire_schedobj(), no SAVE-direction counterpart needed) and has no xce_core_push_events() involvement, so the only real objection to exporting it is 'larger, unrequested change to core_events.c's surface' -- a scope call, not an architectural wall. This duplication has now independently produced: (1) scheduler-snapshot-job-allocation spec's own motivating bug -- job->allocs/job->accounts never restored from snapshot (documented, deliberate scope decision at the time, later fixed); (2) the same spec's own further finding -- every snapshot-restored job (PENDING or RUNNING) got schedobj==NULL, an undocumented latent crash risk, fixed by adding a local field-for-field mirror of core_build_schedobj() to snapshot_load.c; (3) scheduler-allocation spec (this one) -- the new xce_resreq_fingerprint_t resreq_fp field added to xce_primary_schedobj_t (Req 7) is computed at core_build_schedobj()'s call site but has no reason to be computed by snapshot_load.c's independent copy, silently degrading Req 7's unsatisfiable-resreq cache to a permanent miss for any session started via snapshot+replay (the documented common real-investigation path) -- addressed in scheduler-allocation's own requirements.md (Req 7 AC 6) as a one-line addition, not by eliminating the duplication.", "fix": "NOT FIXED -- each individual instance above was/will be fixed at its own site (additively, preserving the duplication), not by addressing the duplication itself. Recommended follow-up, out of scope for any single already-scoped spec: export core_build_schedobj() (with a signature change to take user/queue/project strings directly instead of a full xce_job_t*, since snapshot_load.c has the restored strings but not a real xce_job_t/submit struct at reconstruction time) and have snapshot_load.c call it instead of maintaining its own copy -- eliminating the ONE remaining independent-reconstruction risk in this path that isn't justified by a genuine SAVE-direction or xce_core_push_events() concern the way core_build_job/queue/worker's duplication is. Tag for persistence.md's 'Snapshot Save/Load Field-Completeness (Scheduler)' section and xce-audit-invariants scheduler dimension 4 -- this is exactly the class of gap that invariant was formalized to catch, and has now recurred a third time in the same function.", "tags": [ @@ -25233,24 +12429,28 @@ "open-gap", "persistence", "xce-audit-invariants", - "not-fixed", - "recurring-pattern", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "recurring-pattern" ], "related_bugs": [ "bug-1766" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "1dfdf42c" + ], + "record": ".wolf/bugs/bug-1767.md" }, { "id": "bug-1768", "status": "fixed", "status_updated": "2026-09-25", "status_note": "b4e9a099", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-04T00:00:00Z", - "error_message": "OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). seqjob_prepare_allocation() (src/scheduler/src/plugins/seqjob_allocator.c) and parjob_prepare_allocation() (src/scheduler/src/plugins/parjob_allocator.c) both silently `return 0` whenever `ctx->schedobj->tag != XCE_RESREQ_PRIMARY`. For a primary resource request nested inside a COMPOUND (or, transitively, an ALTER) job, `ctx->schedobj` remains the enclosing COMPOUND object for the entire per-primary allocation loop -- it is never swapped to a per-primary object -- so this guard fires for every nested primary and both prepare-allocation hooks no-op unconditionally.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "out-of-scope gap discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring)\n\nOPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). seqjob_prepare_allocation() (src/scheduler/src/plugins/seqjob_allocator.c) and parjob_prepare_allocation() (src/scheduler/src/plugins/parjob_allocator.c) both silently `return 0` whenever `ctx->schedobj->tag != XCE_RESREQ_PRIMARY`. For a primary resource request nested inside a COMPOUND (or, transitively, an ALTER) job, `ctx->schedobj` remains the enclosing COMPOUND object for the entire per-primary allocation loop -- it is never swapped to a per-primary object -- so this guard fires for every nested primary and both prepare-allocation hooks no-op unconditionally.", + "file": "", "root_cause": "CONFIRMED via direct code read and `git log --follow -p`: the `ctx->schedobj->tag != XCE_RESREQ_PRIMARY` guard predates the entire scheduler-allocation feature (introduced in commit 12dcc2a6, months before this branch's base commit 5a670676); task 3.1's seqjob rewrap (b18c5f78) did not touch or introduce it. Effect: `ctx->alloc_req.host`/`ctx->alloc_req.shared` are never populated for a nested primary's resource want, so `sched_check_host_satisfy()` trivially passes against an empty want vector regardless of a candidate host's real free capacity. Only the SELECT expression (if the resreq specifies one) still gates candidate hosts for a nested primary -- slot count and rusage (memory, etc.) are unconditionally treated as satisfied. This means every COMPOUND job (and transitively every ALTER job, since ALTER delegates to COMPOUND dispatch) with a nested primary resource request has never had its real slots/rusage availability checked by the scheduler, since this guard was introduced.", "fix": "NOT FIXED -- discovered and characterized during scheduler-allocation task 5.1's independent review (2026-09-04); deliberately left out of that task's scheduler.c-dispatch boundary. Fixing requires either (a) constructing/staging a transient per-primary schedobj view when dispatching a nested primary through prepare_allocation (mirroring task 5.1's own sched_primary_cache_obj_() stack-local-schedobj pattern used for cache-key synthesis), or (b) relaxing seqjob_prepare_allocation()/parjob_prepare_allocation()'s guard to operate directly on ctx->primary (already available on ctx regardless of the enclosing schedobj's tag) instead of gating on ctx->schedobj->tag at all. Needs its own dedicated bugfix task/spec given the severity and blast radius (all COMPOUND/ALTER jobs, not scoped to scheduler-allocation).\n\nUPDATE (2026-09-25): 2026-09-25 (bug-1769 fix 1ecd348f): with ALTER now allocating for real, this gap is what prevents a test where an ALTER alternative genuinely fails and the next one is used -- compound/alter-nested primaries never check slot/rusage availability, so every alternative 'fits'.", "tags": [ @@ -25260,23 +12460,28 @@ "alter-resreq", "pre-existing", "severity-high", - "scheduler-allocation-discovery", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "scheduler-allocation-discovery" ], "related_bugs": [ "bug-1767" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "b4e9a099" + ], + "record": ".wolf/bugs/bug-1768.md" }, { "id": "bug-1769", "status": "fixed", "status_updated": "2026-09-25", "status_note": "1ecd348f", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-04T00:00:00Z", - "error_message": "OPEN, NOT A CRASH (but reachable) -- pre-existing, out-of-scope type confusion discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). core_build_alter_schedobj() (src/scheduler/src/core_events.c) stores raw `xce_compound_schedobj_t*` (sizeof == 24 bytes) into `obj->u.alters` (an xlist_t), but sched_make_alter_allocation() (src/scheduler/src/scheduler.c) reads each entry back cast as `xce_sched_obj_t*` (sizeof == 176 bytes, with `offsetof(xce_sched_obj_t, u) == 136`) and dereferences `obj->u.compound->resreq` through that mis-typed pointer.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "out-of-scope type confusion discovered during scheduler-allocation task 5.1's review…\n\nOPEN, NOT A CRASH (but reachable) -- pre-existing, out-of-scope type confusion discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). core_build_alter_schedobj() (src/scheduler/src/core_events.c) stores raw `xce_compound_schedobj_t*` (sizeof == 24 bytes) into `obj->u.alters` (an xlist_t), but sched_make_alter_allocation() (src/scheduler/src/scheduler.c) reads each entry back cast as `xce_sched_obj_t*` (sizeof == 176 bytes, with `offsetof(xce_sched_obj_t, u) == 136`) and dereferences `obj->u.compound->resreq` through that mis-typed pointer.", + "file": "", "root_cause": "CONFIRMED via direct code read at both the construction site (core_events.c) and the read site (scheduler.c), plus a compiled offsetof/sizeof probe against the real headers during task 5.1's review: this is a genuine struct-layout mismatch, not a false alarm -- the two types do not share a compatible prefix layout. Any real ALTER-tagged job (the resreq grammar's `{A} || {B}` alternate syntax legitimately produces XCE_RESREQ_ALTER per xce_resreq.peg) dereferencing through this cast reads ~112 bytes past the actual 24-byte allocation, an out-of-bounds heap read. Separately confirmed during the same investigation that test_snapshot_load_allocator_regression.c's header comment claiming to already exercise this ALTER path is itself mistaken: its `\"type == any\"` fixture has no `{`/`}` braces, so per xce_resreq.peg's grammar it parses as XCE_RESREQ_PRIMARY, not ALTER -- it never actually drives sched_make_alter_allocation()'s u.alters read path. No test anywhere in the tree currently constructs a real multi-entry u.alters list and drives it through sched_make_alter_allocation(), so this defect is both real and currently completely untested.", "fix": "NOT FIXED -- discovered and characterized during scheduler-allocation task 5.1's independent review (2026-09-04); deliberately left out of that task's scheduler.c-dispatch boundary. Needs a dedicated bugfix task: either change core_build_alter_schedobj() to store properly-typed xce_sched_obj_t* entries into u.alters (matching what sched_make_alter_allocation() expects), or change sched_make_alter_allocation() to read xce_compound_schedobj_t* entries directly without the intermediate xce_sched_obj_t* cast. Should be paired with a new regression test that actually constructs an ALTER-tagged job (real `{A}||{B}` resreq syntax) and drives it end to end, since the existing test that was believed to cover this does not.", "tags": [ @@ -25287,22 +12492,28 @@ "pre-existing", "severity-high", "untested", - "scheduler-allocation-discovery", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "scheduler-allocation-discovery" ], "related_bugs": [ "bug-1768" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "1ecd348f" + ], + "record": ".wolf/bugs/bug-1769.md" }, { "id": "bug-1770", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-04T00:00:00Z", - "error_message": "FIXED (2026-09-06, resreq-maxtasks-wiring feature, commits 799a71ec..5c87f1cc). Originally: pre-existing, out-of-scope gap discovered during scheduler-allocation task 5.3's end-to-end integration test. The REAL job-ingest path could NEVER produce a primary resource request with maxTasks > 1, regardless of what task count a real job submission specified -- parjob_allocator.c's my_alloc() (maxTasks > 1) was therefore UNREACHABLE from any real production job submission.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "FIXED (2026-09-06, resreq-maxtasks-wiring feature, commits 799a71ec..5c87f1cc)\n\nFIXED (2026-09-06, resreq-maxtasks-wiring feature, commits 799a71ec..5c87f1cc). Originally: pre-existing, out-of-scope gap discovered during scheduler-allocation task 5.3's end-to-end integration test. The REAL job-ingest path could NEVER produce a primary resource request with maxTasks > 1, regardless of what task count a real job submission specified -- parjob_allocator.c's my_alloc() (maxTasks > 1) was therefore UNREACHABLE from any real production job submission.", + "file": "", "root_cause": "CONFIRMED via direct code read and a genuine RED-phase run of test_parjob_e2e_multislot.c (src/scheduler/testsuite/): parse_resreq_primary() (src/lib/resreq.c:431-432) unconditionally hardcodes `primary->minTasks = 1; primary->maxTasks = 1;` after parsing a resreq string's select/rusage/span/same/affinity clauses, regardless of their content -- there is no resreq-string syntax for task count at all (LSF-style task count is a job-submission-level parameter, `job_submit_data_t.mintasks/maxtasks`, sibling to the resreq string, not nested inside it). Neither of this codebase's two job-ingest constructors -- core_build_job()/core_build_primary_schedobj() (src/scheduler/src/core_events.c, the real production JOBS_PUSH path) nor make_sched_job() (src/scheduler/src/schedobj.c, the sibling non-event construction path) -- ever copies the job's own submitted job_submit_data_t.mintasks/maxtasks (correctly propagated onto the TOP-LEVEL xce_sched_job_t.mintasks/maxtasks by both functions, confirmed working) onto the NESTED xce_primary_resreq_t.minTasks/maxTasks that sched_make_primary_alloc_()'s maxTasks>1 dispatch check (scheduler.c:290) and parjob_allocator.c's my_alloc() actually read. Net effect: every job's primary resreq that parses successfully has maxTasks pinned to 1, so sched_make_primary_alloc_() always routes it to seqjob_allocator instead of parjob_allocator (scheduler.c:290-294) -- parjob_allocator's real combinatorial multi-host allocation (scheduler-allocation spec, tasks 4.1-4.3) is fully implemented and unit-tested but cannot be exercised by any real job submission as the system stands today. Live-confirmed: test_parjob_e2e_multislot.c's own RED-phase run (maxTasks patch temporarily disabled) showed the job routing to seqjob and dispatching onto exactly 1 host instead of the intended 2 (`Expected 2 Was 1`).", "fix": "FIXED (2026-09-06, resreq-maxtasks-wiring feature, commits 799a71ec..5c87f1cc, 12 tasks). Added apply_job_task_count(xce_resreq_t*, uint32_t mintasks, uint32_t maxtasks) (src/lib/resreq.c/resreq.h) -- overwrites a standalone top-level primary's minTasks/maxTasks with the job's real requested count (no-op for COMPOUND/ALTER nested primaries, deliberately left out of scope per bug-1768; defensive min>max clamp). Wired at both live ingest call sites: core_build_job() (core_events.c, fresh submission) using job->submit->mintasks/maxtasks, and the snapshot-restore path (snapshot_load.c, --sim/replay driver) using the job's already-restored mintasks/maxtasks -- both called strictly before their respective schedobj/resreq_fp builders read the task count. make_sched_job() (schedobj.c) intentionally NOT wired -- confirmed dead code (unreachable via sched_update(), which has zero live callers). Verified end-to-end via a new xce-test-multihost skill: a real multi-task job (and a multi-task array-job element) submitted via plain jobsub is genuinely split across two real xcecd_virtual hosts by the scheduler's own decision and reaches DONE on both. Full regression clean (scheduler/controller/worker suites, only pre-existing unrelated bug-1758/bug-1723 remain) and HA soak clean (sequential + chaotic, zero job loss/divergence). test_parjob_e2e_multislot.c's PATCH_MAXTASKS_FOR_BUG_1770 workaround is removed; that test now drives its scenario entirely through unmodified real ingest. Sibling gaps deliberately NOT fixed by this feature, tracked separately: bug-1772 (snapshot-restore resreq_fp gap), bug-1773 (resource-request update never taking effect for scheduling), bug-1774 (make_sched_job()'s dead-code resreq_fp gap).", "tags": [ @@ -25313,25 +12524,31 @@ "pre-existing", "severity-high", "scheduler-allocation-discovery", - "task-5.3", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-5.3" ], "related_bugs": [ "bug-1766", "bug-1768", "bug-1769" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "799a71ec", + "5c87f1cc" + ], + "record": ".wolf/bugs/bug-1770.md" }, { "id": "bug-1771", "status": "fixed", "status_updated": "2026-09-25", "status_note": "1ecd348f", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-04T00:00:00Z", - "error_message": "ASAN SEGV (READ of address 0x0) in fcfs_del_job() (src/scheduler/src/plugins/fcfs.c:318, `job->queue->name`), reached via fcfs_commit() <- plugin_commit_decision() <- sched_commit_decision() <- commit_decision() <- sched_allocator(), the very first time a real DISPATCH decision was committed through the real, live sched_allocator() thread in this test directory. Discovered live during scheduler-allocation task 5.3's own end-to-end integration test construction (test_parjob_e2e_multislot.c) -- full ASAN backtrace captured.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "ASAN SEGV (READ of address 0x0) in fcfs_del_job() (src/scheduler/src/plugins/fcfs.c:318…\n\nASAN SEGV (READ of address 0x0) in fcfs_del_job() (src/scheduler/src/plugins/fcfs.c:318, `job->queue->name`), reached via fcfs_commit() <- plugin_commit_decision() <- sched_commit_decision() <- commit_decision() <- sched_allocator(), the very first time a real DISPATCH decision was committed through the real, live sched_allocator() thread in this test directory. Discovered live during scheduler-allocation task 5.3's own end-to-end integration test construction (test_parjob_e2e_multislot.c) -- full ASAN backtrace captured.", + "file": "", "root_cause": "CONFIRMED via direct code read and the live ASAN backtrace. plugin_commit_decision() (src/scheduler/src/plugin.c:442-475) unconditionally fans a committed decision out to EVERY registered plugin's own `.hooks.commit_hook`, regardless of which plugin is the currently-ACTIVE job picker -- this includes the legacy 'xce_sched_fcfs' plugin (src/scheduler/src/plugins/fcfs.c), still registered alongside the modern default 'builtin_fcfs' by sched_register_plugins() (scheduler.c) even when it is never the resolved picker. fcfs_commit() (fcfs.c:326-337) calls fcfs_del_job(plugin, decision->u.alloc->jobid) for every XCE_DECISION_DISPATCH, and fcfs_del_job() (fcfs.c:303-324) unconditionally dereferences `job->queue->name` with no NULL check on `job->queue` -- sched_get_queue() (core.c:257-271) is get-ONLY (never auto-creates), so any job whose xce_sched_job_t.queue is NULL (i.e. its submitted queue name was never registered via a real QUEUE_UPDATE/CONFIG_PUSH event before the job reached DISPATCH) crashes this unrelated legacy plugin's own housekeeping, even though that plugin never elected or tracked the job at all. Every prior real-scheduler_srcs test in this directory that reaches a real sched_commit_decision() (test_scheduler_cache_dispatch.c, test_scheduler_cache_dispatch_compound.c, test_snapshot_load_allocator_regression.c) only ever produces PEND decisions (fcfs_commit()'s `default: break;` branch, never touching job->queue), so this NULL-deref had never been triggered before. Not reachable in real production as the system stands today (a real job's queue is always registered via controller-pushed config before any job can target it), but IS a genuine, real, unguarded NULL dereference in a code path that runs unconditionally for every real DISPATCH decision regardless of test/production context.", "fix": "NOT FIXED -- discovered during scheduler-allocation task 5.3's end-to-end test construction (2026-09-04). test_parjob_e2e_multislot.c's own fixture was corrected instead (added a real XCE_CORE_EVENT_QUEUE_UPDATE push registering the job's queue before the JOBS_PUSH event, matching real production sequencing where queues are always configured before any job can target them) -- this is the correct fix for that specific test, not a production defect this task's own boundary covers. The underlying hardening gap (fcfs_del_job() missing a `job->queue == NULL` guard before dereferencing `job->queue->name`) remains open; recommended minimal fix for a dedicated follow-up: `if ((job == NULL) || (job->queue == NULL)) { return 0; }` in fcfs_del_job(), mirroring the function's own existing `if (job == NULL) { return 0; }` early-return style.", "tags": [ @@ -25343,22 +12560,26 @@ "pre-existing", "severity-low", "scheduler-allocation-discovery", - "task-5.3", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-5.3" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "1ecd348f" + ], + "record": ".wolf/bugs/bug-1771.md" }, { "id": "bug-1772", - "gitea_synced_status": "fixed", - "gitea_issue": 52, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-05T00:00:00Z", - "error_message": "OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during requirements scoping for the resreq-maxtasks-wiring spec (bug-1770's fix). snapshot_build_primary_schedobj() (src/scheduler/src/snapshot_load.c:81-96) never computes primary->resreq_fp at all, unlike core_build_primary_schedobj() (src/scheduler/src/core_events.c:130-135), which sets resreq_fp.min_tasks/max_tasks/select_node_id/rusage_hash from the parsed xce_primary_resreq_t immediately after parsing. Every job reconstructed from a snapshot (post-restart or post-failover restore) therefore gets a zeroed resreq_fp -- the same class of gap as bug-1767's broader 'recurring schedobj-reconstruction-duplication' pattern (snapshot_load.c independently re-implements core_events.c's schedobj construction rather than sharing it, and each independent copy has repeatedly drifted field-by-field). TRIGGER CORRECTION (2026-09-05, resreq-maxtasks-wiring design review): this entry's original phrasing ('post-restart or post-failover restore') overstates the live trigger. xce_sched_core_load_snapshot()'s ONLY real (non-test) caller anywhere in src/ is sim.c's sim_locate_and_load_snapshot(), reachable only via main.c's --sim FILE offline replay flag. Normal-mode main.c has no snapshot-load call or flag at all; a live scheduler restart (including post-failover, e.g. run-ha-soak.sh's restart_scheduler_for_new_primary()) starts with empty in-memory state and rejoins via on_scheduler_join() -> the same core_build_job() path fresh ingest uses -- never touching snapshot_load.c. The accurate trigger is: a job in a .replay-recorded snapshot loaded by the --sim simulation/replay driver, not any live-cluster crash-recovery or failover scenario. This does not change the bug's validity or fix (the gap is real for --sim/replay fidelity, which matters per this project's driver-equivalence invariant since --sim drives the identical allocator dispatch) -- it only corrects when it's actually reachable, which matters for scoping test coverage (an HA soak does NOT exercise this path at all).", + "last_seen": "2026-09-23T10:05:00Z", + "occurrences": 1, + "error_message": "out-of-scope gap discovered during requirements scoping for the resreq-maxtasks-wiring spec…\n\nOPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during requirements scoping for the resreq-maxtasks-wiring spec (bug-1770's fix). snapshot_build_primary_schedobj() (src/scheduler/src/snapshot_load.c:81-96) never computes primary->resreq_fp at all, unlike core_build_primary_schedobj() (src/scheduler/src/core_events.c:130-135), which sets resreq_fp.min_tasks/max_tasks/select_node_id/rusage_hash from the parsed xce_primary_resreq_t immediately after parsing. Every job reconstructed from a snapshot (post-restart or post-failover restore) therefore gets a zeroed resreq_fp -- the same class of gap as bug-1767's broader 'recurring schedobj-reconstruction-duplication' pattern (snapshot_load.c independently re-implements core_events.c's schedobj construction rather than sharing it, and each independent copy has repeatedly drifted field-by-field). TRIGGER CORRECTION (2026-09-05, resreq-maxtasks-wiring design review): this entry's original phrasing ('post-restart or post-failover restore') overstates the live trigger. xce_sched_core_load_snapshot()'s ONLY real (non-test) caller anywhere in src/ is sim.c's sim_locate_and_load_snapshot(), reachable only via main.c's --sim FILE offline replay flag. Normal-mode main.c has no snapshot-load call or flag at all; a live scheduler restart (including post-failover, e.g. run-ha-soak.sh's restart_scheduler_for_new_primary()) starts with empty in-memory state and rejoins via on_scheduler_join() -> the same core_build_job() path fresh ingest uses -- never touching snapshot_load.c. The accurate trigger is: a job in a .replay-recorded snapshot loaded by the --sim simulation/replay driver, not any live-cluster crash-recovery or failover scenario. This does not change the bug's validity or fix (the gap is real for --sim/replay fidelity, which matters per this project's driver-equivalence invariant since --sim drives the identical allocator dispatch) -- it only corrects when it's actually reachable, which matters for scoping test coverage (an HA soak does NOT exercise this path at all).", + "file": "", "root_cause": "snapshot_load.c's snapshot_build_primary_schedobj() is a hand-duplicated, field-for-field port of core_events.c's core_build_primary_schedobj() (kept independent because core_build_primary_schedobj() is static and the two files are compiled as separate objects -- see bug-1767's own root-cause text for the deliberate-duplication rationale). The port was written before task 1.2 (scheduler-allocation feature) added resreq_fp/hash_rusages() wiring to the core_events.c original, and the snapshot_load.c copy was never updated to match. Effect: a snapshot-restored job's unsatisfiable-resreq cache fingerprint (used by scheduler-allocation's alloc_cache.c) is always the zero value, which is a valid-looking but WRONG fingerprint -- it could produce a false cache hit against an unrelated job that also happens to have a zeroed fingerprint (extremely unlikely today since real jobs always populate resreq_fp correctly at first ingest, but a snapshot-restored job's own fingerprint stays wrong for its entire post-restore lifetime, and any code that later relies on comparing/keying by resreq_fp for a restored job silently gets 0/0/0/0). Discovered while scoping resreq-maxtasks-wiring (bug-1770's fix spec) because that fix will make resreq_fp carry real, meaningful, non-degenerate maxTasks values for the first time -- increasing the practical consequence of this gap versus its current all-zero-everywhere baseline.", "fix": "FIXED (2026-09-23, TDD, /xce-audit-invariants scheduler dimension-4 audit -- independently re-found by both a dedicated subagent and direct investigation into bug-1845; this entry's own prior 'fixed'/'closed-by-scheduler-sim-live-allocation' tags were STALE/WRONG -- direct code inspection confirmed snapshot_build_primary_schedobj() still had no resreq_fp computation immediately before this fix). Added the same four-field resreq_fp computation core_build_primary_schedobj() (core_events.c) performs -- min_tasks/max_tasks/select_node_id/rusage_hash -- to snapshot_build_primary_schedobj() (snapshot_load.c), matching that function's own documented 'field-for-field port' contract, which had simply never included this field. Added test_snapshot_load_computes_resreq_fingerprint_for_restored_primary() (test_snapshot_save_load.c) -- a job restored from a snapshot with mintasks=4/maxtasks=6 must have schedobj->u.primary->resreq_fp.min_tasks/max_tasks reflect those real values, not {0,0,0,0}. Verified genuine red->green: temporarily reverted just the new resreq_fp assignment and reran -- scheduler snapshot save load test FAILED (exit 1); restored the fix and reran -- full 85/85 scheduler suite green.", "tags": [ @@ -25369,7 +12590,6 @@ "pre-existing", "schedobj-duplication", "severity-medium", - "fixed", "tdd", "audit-reopened" ], @@ -25377,16 +12597,23 @@ "bug-1767", "bug-1770" ], - "occurrences": 1, - "last_seen": "2026-09-23T10:05:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1772.md", + "gitea_issue": 52, + "gitea_synced_status": "fixed" }, { "id": "bug-1773", "status": "fixed", "status_updated": "2026-09-27", "status_note": "41c3694d", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-05T00:00:00Z", - "error_message": "OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during design review for the resreq-maxtasks-wiring spec (bug-1770's fix). A job's resource-request UPDATE (JOB_UPDATE core event / sched_update_existing_job()) never takes effect for scheduling purposes, for ANY field -- not just task count. sched_update_existing_job() (src/scheduler/src/schedobj.c:597-656) and core_events.c's JOB_UPDATE handler (src/scheduler/src/core_events.c:1200-1234) both do `free(job->resreq); job->resreq = newreq;` (a shallow free via FREEUP, utils.h:114) but never touch job->schedobj. ADDENDUM (2026-09-05, independent follow-up investigation): sched_update_existing_job() itself is dead code (unreachable -- its sole caller is scheduler.c's sched_update(), which has zero live callers, same root architecture as bug-1740). core_events.c's JOB_UPDATE handler IS live (traced main.c -> driver.c's xce_sched_driver_translate_pending_events() -> xce_core_push_events() -> the JOB_UPDATE case), reached routinely by ordinary job-lifecycle notifications (state/rusage/postdone), not only by a genuine resource-request change -- but grepping the whole controller finds no code path that ever mutates an existing job's submit->resreq after initial submission, so this live handler's parse_resreq() branch always re-parses the job's original, unchanged resreq string today. No real 'update a job's resource request' feature exists yet in this codebase; this bug describes what WOULD happen if one were added without also fixing job->schedobj reconstruction.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "out-of-scope gap discovered during design review for the resreq-maxtasks-wiring spec (bug-1770's…\n\nOPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during design review for the resreq-maxtasks-wiring spec (bug-1770's fix). A job's resource-request UPDATE (JOB_UPDATE core event / sched_update_existing_job()) never takes effect for scheduling purposes, for ANY field -- not just task count. sched_update_existing_job() (src/scheduler/src/schedobj.c:597-656) and core_events.c's JOB_UPDATE handler (src/scheduler/src/core_events.c:1200-1234) both do `free(job->resreq); job->resreq = newreq;` (a shallow free via FREEUP, utils.h:114) but never touch job->schedobj. ADDENDUM (2026-09-05, independent follow-up investigation): sched_update_existing_job() itself is dead code (unreachable -- its sole caller is scheduler.c's sched_update(), which has zero live callers, same root architecture as bug-1740). core_events.c's JOB_UPDATE handler IS live (traced main.c -> driver.c's xce_sched_driver_translate_pending_events() -> xce_core_push_events() -> the JOB_UPDATE case), reached routinely by ordinary job-lifecycle notifications (state/rusage/postdone), not only by a genuine resource-request change -- but grepping the whole controller finds no code path that ever mutates an existing job's submit->resreq after initial submission, so this live handler's parse_resreq() branch always re-parses the job's original, unchanged resreq string today. No real 'update a job's resource request' feature exists yet in this codebase; this bug describes what WOULD happen if one were added without also fixing job->schedobj reconstruction.", + "file": "", "root_cause": "CONFIRMED via direct code read: the scheduler's actual dispatch reads job->schedobj (scheduler.c:483, sched_make_allocation(sched, job->schedobj)), not job->resreq. sched_make_primary_allocation() (scheduler.c:315) reads xce_primary_resreq_t* primary = obj->u.primary->resreq -- i.e. job->schedobj->u.primary->resreq, a pointer set exactly once, at ingest, by core_build_primary_schedobj()/make_primary_schedobj() (primary->resreq = resreq, aliasing job->resreq->u.primary at that moment). job->schedobj is built ONLY in core_build_job() (core_events.c:259-265) and make_sched_job() (schedobj.c:194-202) -- confirmed via grepping every `job->schedobj =` assignment in the scheduler; neither update path is among them. So after any resreq update, job->schedobj->u.primary->resreq keeps pointing at the ORIGINAL, ingest-time xce_primary_resreq_t forever, completely disconnected from the freshly-parsed newreq the update path installed into job->resreq. Any change an operator makes to a pending job's select/rusage/task-count via a resource-request update is silently invisible to the scheduler's own dispatch decision -- only a job's original, at-submission resreq is ever actually evaluated.", "fix": "NOT FIXED -- discovered during resreq-maxtasks-wiring's design review (2026-09-05); deliberately kept out of that spec's scope (it is not about task-count wiring, and fixing it changes long-standing update semantics far beyond this feature's boundary). Needs its own dedicated bugfix task/spec: the update paths must rebuild (or re-point) job->schedobj's primary->resreq to the newly-parsed newreq, mirroring how core_build_job()/make_sched_job() construct it at ingest, probably by extracting a shared 'build/refresh primary schedobj from a resreq' helper both ingest and update can call.", "tags": [ @@ -25396,23 +12623,28 @@ "schedobj", "pre-existing", "severity-high", - "resreq-maxtasks-wiring-discovery", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "resreq-maxtasks-wiring-discovery" ], "related_bugs": [ "bug-1770" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "41c3694d" + ], + "record": ".wolf/bugs/bug-1773.md" }, { "id": "bug-1774", "status": "wontfix", "status_updated": "2026-09-24", "status_note": "accepted: dead code path (make_primary_schedobj resreq_fp), zero live impact", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-05T00:00:00Z", - "error_message": "OPEN, NOT A CRASH, DEAD-CODE-ONLY -- pre-existing gap discovered during design review for the resreq-maxtasks-wiring spec, sibling to bug-1772, CORRECTED 2026-09-05: this entry originally claimed make_sched_job()'s ingest path was \"confirmed live, not legacy\" -- that claim was WRONG and has been retracted (see root_cause). make_primary_schedobj() (src/scheduler/src/schedobj.c:45-63, backing make_sched_job()) never computes primary->resreq_fp, unlike core_build_primary_schedobj() (src/scheduler/src/core_events.c:130-135) which sets resreq_fp.min_tasks/max_tasks/select_node_id/rusage_hash from the parsed xce_primary_resreq_t immediately after parsing -- but make_sched_job() itself is unreachable in production today (see root_cause), so this gap currently has ZERO live impact.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "gap discovered during design review for the resreq-maxtasks-wiring spec, sibling to bug-1772…\n\nOPEN, NOT A CRASH, DEAD-CODE-ONLY -- pre-existing gap discovered during design review for the resreq-maxtasks-wiring spec, sibling to bug-1772, CORRECTED 2026-09-05: this entry originally claimed make_sched_job()'s ingest path was \"confirmed live, not legacy\" -- that claim was WRONG and has been retracted (see root_cause). make_primary_schedobj() (src/scheduler/src/schedobj.c:45-63, backing make_sched_job()) never computes primary->resreq_fp, unlike core_build_primary_schedobj() (src/scheduler/src/core_events.c:130-135) which sets resreq_fp.min_tasks/max_tasks/select_node_id/rusage_hash from the parsed xce_primary_resreq_t immediately after parsing -- but make_sched_job() itself is unreachable in production today (see root_cause), so this gap currently has ZERO live impact.", + "file": "", "root_cause": "CONFIRMED via direct code read that make_primary_schedobj() calloc's the primary schedobj and leaves resreq_fp all-zero, with no equivalent computation anywhere in the function -- identical failure shape to bug-1772 (snapshot_load.c's snapshot_build_primary_schedobj() has the same gap), a third instance of the recurring schedobj-reconstruction-duplication pattern already tracked by bug-1767. HOWEVER, independently re-verified 2026-09-05 (two separate investigations, exhaustive grep of every call site of sched_update(), sched_update_jobs(), and make_sched_job() across src/ outside testsuite/): sched_update(xce_scheduler_t*) (scheduler.c:1200) has ZERO live callers anywhere in this codebase -- only its own declaration/definition and doc-comments describing it as dead/superseded exist (matching bug-1740's already-established finding for the sibling sched_apply_job_rusage() case, same root architecture). sched_update_jobs() (scheduler.c:1109) is called only from inside sched_update()'s switch, and make_sched_job() (schedobj.c:153) is called only from sched_update_jobs() -- so the ENTIRE chain is unreachable in production. The real, live production job-ingest paths are exactly two: core_build_job() (core_events.c, fresh submission via the driver-translated XCE_CORE_EVENT_JOBS_PUSH path) and snapshot_load.c's restore path -- both confirmed live by the same grep rigor. This entry's original \"confirmed live, not legacy\" parenthetical was an unverified, incorrect aside and is hereby retracted; make_sched_job()'s resreq_fp gap is real but currently dead-code-only, with no production consequence unless/until something makes this path reachable again.", "fix": "NOT FIXED, and NOT CURRENTLY WORTH FIXING given zero live impact -- discovered during resreq-maxtasks-wiring's design review (2026-09-05), corrected same day once the dead-code status was established. resreq-maxtasks-wiring's own Requirement 5.4 does NOT need an exception for this path (it can be excluded from consideration entirely, not merely excluded-and-documented, since it never ingests any real job) -- that spec's actual Req 5.4 exception is for bug-1772's snapshot-restore gap, which IS live. If make_sched_job()'s dead code path is ever revived or repurposed in a future change, revisit this fix then: add the same resreq_fp computation core_build_primary_schedobj() does to make_primary_schedobj(), or better, per bug-1767's broader recommended remediation, share one real schedobj-construction implementation across all three call sites instead of maintaining three independent, drift-prone copies (two live, one dead).", "tags": [ @@ -25423,24 +12655,28 @@ "pre-existing", "severity-low", "resreq-maxtasks-wiring-discovery", - "corrected", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "corrected" ], "related_bugs": [ "bug-1767", "bug-1770", "bug-1772" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1774.md" }, { "id": "bug-1775", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-09-06T00:00:00Z", - "error_message": "xce-test-multihost's run-multihost-verify.sh (resreq-maxtasks-wiring task 4.3, array-job scenario): verify_task_started_on_host()'s grep -E pattern embedded a raw jobid string directly in an extended-regex alternation. A bracketed array-element jobid (e.g. \"[0]\") has its \"[0]\" misparsed by grep -E as a one-character bracket-expression class (matching a literal '0') instead of the literal 3-character substring \"[0]\" -- causing a false negative: `grep -qE 'Job base[0] ntasks is:'` does not match the literal log line `\"Job base[0] ntasks is: 5\"`.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-test-multihost's run-multihost-verify.sh (resreq-maxtasks-wiring task 4.3, array-job…\n\nxce-test-multihost's run-multihost-verify.sh (resreq-maxtasks-wiring task 4.3, array-job scenario): verify_task_started_on_host()'s grep -E pattern embedded a raw jobid string directly in an extended-regex alternation. A bracketed array-element jobid (e.g. \"[0]\") has its \"[0]\" misparsed by grep -E as a one-character bracket-expression class (matching a literal '0') instead of the literal 3-character substring \"[0]\" -- causing a false negative: `grep -qE 'Job base[0] ntasks is:'` does not match the literal log line `\"Job base[0] ntasks is: 5\"`.", + "file": "", "root_cause": "Extended regular expressions treat unescaped `[...]` as a bracket-expression (character class), not a literal substring. Array-element jobids introduced by task 4.3 are the first jobid format in this harness containing ERE metacharacters (plain jobids are digits/hyphens only); the pre-existing plain-job code path never hit this because grep -E on a digits-and-hyphens-only string has no metacharacters to misparse.", "fix": "FIXED (2026-09-06, resreq-maxtasks-wiring task 4.3): added ere_escape() to run-multihost-verify.sh, applied to any jobid string before use in a grep -E pattern (escapes '[' and ']', the only metacharacters a jobid can contain per fmt_jobid_ex/REGEX_JOBARRAY_ID's digit/hyphen/bracket-only format). Confirmed a byte-identical no-op for plain (non-array) jobids. Reproduced the false negative standalone before the fix and confirmed the escaped pattern matches correctly after, independently re-verified by review.", "tags": [ @@ -25449,22 +12685,25 @@ "regex", "array-job", "xce-test-multihost", - "severity-low", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "severity-low" ], "related_bugs": [ "bug-1770" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1775.md" }, { "id": "bug-1776", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-06T21:05:00Z", - "error_message": "User improvement request (simulator REPL feedback): 'info worker's assigned jobs' listing gives no indication of how many 'slots' each assigned job actually consumes on that worker -- every assigned job looks identical regardless of whether it occupies 1 slot or many (e.g. a fully-packed 22-slot worker running one 2-slot job shows the exact same line shape as a 1-slot job).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User improvement request (simulator REPL feedback): 'info worker's assigned jobs' listing gives no…\n\nUser improvement request (simulator REPL feedback): 'info worker's assigned jobs' listing gives no indication of how many 'slots' each assigned job actually consumes on that worker -- every assigned job looks identical regardless of whether it occupies 1 slot or many (e.g. a fully-packed 22-slot worker running one 2-slot job shows the exact same line shape as a 1-slot job).", "file": "src/scheduler/src/repl_info.c", "root_cause": "NOT A BUG -- feature gap. repl_info_worker_one()'s assigned-jobs loop already walks each job's xce_host_alloc_t (per-host allocation) to match it against the current worker's name, but only extracted job->jobid and discarded the xce_host_alloc_t itself (`break`) without ever inspecting its `allocs[]` array, which already carries a per-resource allocation list (same struct/pattern the controller's own get_nslots_in_alloc(), job.c, already scans for the identical 'slots' resname) -- so the slot count was always available with no new wire/plumbing needed, just never read at this call site.", "fix": "FIXED (2026-09-06). Added repl_worker_alloc_nslots() (repl_info.c) mirroring get_nslots_in_alloc()'s resname==\"slots\" scan (duplicated rather than shared, since that helper is file-static to job.c and this is a read-only REPL display concern). The assigned-jobs loop now also collects a parallel assigned_slots[] array (repl_append_int(), new int-array counterpart to the existing repl_append_u64()). When every assigned job on a worker consumed the SAME slot count and that count is >1, the existing compressed jobid-range line gets a trailing '*N' (e.g. 'assigned jobs: 0-14-1788751435-0*2'); a single-slot-per-job worker (the common case) renders identically to before. When assigned jobs on the SAME worker have DIFFERING slot counts, a single '*N' suffix on a compressed range would be ambiguous, so the listing falls back to the pre-existing uncompressed per-job lines (same fallback tier a too-small compression buffer already used), each annotated with its own count. New tests in test_repl_resource_table.c: test_info_worker_assigned_job_listing_appends_slot_count_when_uniform, test_info_worker_assigned_job_listing_falls_back_per_job_when_slots_differ. Full scheduler suite: 76/77 pass (the one failure, test_job_rusage_reconciles_mem_reservation_against_usage in test_core_events.c, is a pre-existing unrelated mem-reservation-subsystem failure already documented in bug-1762, untouched by this change).", @@ -25473,23 +12712,26 @@ "repl", "sim", "assigned-jobs", - "ux-improvement", - "fixed" + "ux-improvement" ], "related_bugs": [ "bug-1601", "bug-1591" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1776.md" }, { "id": "bug-1777", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test workerlist MaxSwap ignores hosts.yaml override", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-06T21:10:00Z", - "error_message": "xce_test workerlist's MaxSwap column can permanently disagree with the scheduler's own resmgr/replay view for the same worker: workerlist always shows the join-time hardware_monitor stub/detected value, never a hosts.yaml per-host maxswap override, while the scheduler always uses the override. User noticed this while comparing a scheduler simulator replay (maxswap=10000) against xce_test workerlist (MaxSwap=1.00 GB) for the same host and asked whether the two should be consistent.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_test workerlist's MaxSwap column can permanently disagree with the scheduler's own…\n\nxce_test workerlist's MaxSwap column can permanently disagree with the scheduler's own resmgr/replay view for the same worker: workerlist always shows the join-time hardware_monitor stub/detected value, never a hosts.yaml per-host maxswap override, while the scheduler always uses the override. User noticed this while comparing a scheduler simulator replay (maxswap=10000) against xce_test workerlist (MaxSwap=1.00 GB) for the same host and asked whether the two should be consistent.", "file": "src/controller/testsuite/test.c", "root_cause": "CONFIRMED via code trace (not yet reproduced against a live pc13v/replay artifact -- none existed on disk at investigation time). xce_worker_t carries two separate copies of a host-attrs-style resource: attrs->maxswap (hostattrs_t, written ONLY by xcecd.c's init_host_attrs() at join time -- rejoin_worker(), worker.c:440, just re-copies the same join-time value) and resvals[idx(\"maxswap\")] (resmgr-indexed zarray, seeded from attrs->maxswap by add_hostattrs_resvals() at apply_join_state() start, worker.c, then OVERWRITTEN by mark_host_res()/update_val_res() if a hosts.yaml resource override exists for that host, worker.c:79 `array = &(w->host.resvals)`). cmd_workerlist() (test.c:1478) reads attrs->maxswap directly -- structurally incapable of ever reflecting a config override, since nothing writes an override back into attrs. The scheduler-facing path (collect_resvals_workers() -> resvals_to_sched -> xcesched_push_workers(), dataex.c/sched.c) reads resvals, which DOES pick up the override. So whenever a maxswap override is configured, the scheduler correctly uses it (and a replay/snapshot reflects it) while xce_test workerlist permanently shows the stale join-time value regardless of timing/staleness -- not a caching/race artifact, a structural gap. The Go apiserver/xcecli path (convert.go) also just casts attrs.maxswap with no override-awareness, and xcecli's hosts.yaml model doesn't even expose a MaxSwap column today.", "fix": "NOT FIXED -- user explicitly deferred (2026-09-06) to prioritize bug-1554 (real swap/tmp hardware detection, now fixed) and the assigned-jobs slot-count display improvement first. Two candidate fix directions discussed: (a) make cmd_workerlist()/the Go apiserver path read the effective resmgr/resvals value instead of attrs directly, or (b) make mark_host_res() write the override back into attrs too, so every attrs consumer (maxcpus/maxmem/maxswap/maxtmp alike) stays in sync at the source -- (b) was the recommended direction since it fixes the whole class rather than patching each display path individually, but neither is implemented yet.", @@ -25500,23 +12742,25 @@ "maxswap", "workerlist", "config-override", - "not-fixed", - "deferred", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "deferred" ], "related_bugs": [ "bug-1554" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1777.md" }, { "id": "bug-1778", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-07T06:50:00Z", - "error_message": "Live-observed (interactive sim session=29, replaying a real captured trace): three plain (non-array) jobs already at JOB_STAT_DONE (0-14-1788751409-0, 0-14-1788751435-0, 0-14-1788751504-0) still appear in `info worker pc13v`'s 'assigned jobs' listing (each annotated '*2' via bug-1776's slot-count fix), alongside the genuinely-running array job elements. User: 'The finished job(exit/done) should be de-assign from worker.'", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Live-observed (interactive sim session=29, replaying a real captured trace): three plain…\n\nLive-observed (interactive sim session=29, replaying a real captured trace): three plain (non-array) jobs already at JOB_STAT_DONE (0-14-1788751409-0, 0-14-1788751435-0, 0-14-1788751504-0) still appear in `info worker pc13v`'s 'assigned jobs' listing (each annotated '*2' via bug-1776's slot-count fix), alongside the genuinely-running array job elements. User: 'The finished job(exit/done) should be de-assign from worker.'", "file": "src/scheduler/src/repl_info.c", "root_cause": "PARTIALLY CONFIRMED, NOT FULLY ROOT-CAUSED this session (user asked to record, not fix). Confirmed facts: (1) repl_info_worker_one()'s assigned-jobs loop (repl_info.c, ~line 686) gates solely on `job->allocs != NULL`, with no independent job->state check -- if allocs is non-NULL for a terminal job, it will always display, by design (mirrors the exact structural gap bug-1764 already documented for this same function). (2) The worker's ACTUAL resource accounting is NOT confused by this: the same session's 'resource slots: value=22.00 used=20.00' correctly reflects only the 5 genuinely-running 4-slot array elements (5*4=20), excluding the 3 DONE jobs entirely -- proving core.resmgr/resInUse (the real admission-control state) is fine; only the assigned-jobs DISPLAY loop (which walks job->allocs directly, a separate field) is stale for these 3 jobs. (3) This is NOT a regression of bug-1764's fix: verified sim_loop.c:713-714 still correctly excludes JOB_IS_DONE(job->state)||JOB_IS_EXIT(job->state) from sim_reconcile_unaccounted_jobs()'s resurrection path, so that specific documented mechanism is not what's reintroducing these allocs. (4) core_events.c's XCE_CORE_EVENT_JOB_STATE_CHANGE (line ~1155-1163) and XCE_CORE_EVENT_JOB_UPDATE (line ~1207-1215) handlers both correctly free_JobAlloc_ptr()+null job->allocs atomically alongside the state write whenever `!JOB_IS_RUN(new_state)` -- so if either of THESE handlers was what drove these 3 jobs to DONE, allocs should already be NULL; something else must be true. Leading unconfirmed hypothesis: snapshot_load.c:490-493 deep-copies wj->allocs from the .replay snapshot's wire payload UNCONDITIONALLY whenever wj->allocs != NULL, with no check against the snapshot's own wj->state -- if the live production recording's snapshot-capture moment (snapshot_wire.c's save side) captured these 3 jobs' allocs and state in a mutually inconsistent way (e.g. a save-time field-read ordering gap, or the snapshot genuinely predates their JOB_STATE_CHANGE/JOB_UPDATE terminal cleanup while a separately-recorded later event updated state to DONE without a matching allocs-clearing event reaching this replay stream), the replay would faithfully reproduce that inconsistency verbatim. NOT verified with a live trace/gdb this session -- flagged per this project's own persistence.md field-completeness concern (a state that 'recovers correctly' while an unrelated field goes stale is the exact class of bug that steering calls out), not asserted as confirmed.", "fix": "INTERIM DISPLAY-LAYER MITIGATION APPLIED (2026-09-07, per user request). repl_info_worker_one()'s assigned-jobs loop (repl_info.c) now also excludes JOB_IS_DONE(job->state) || JOB_IS_EXIT(job->state), in addition to the existing job->allocs != NULL check -- mirrors the same 'terminal is permanent' guard job_state_regresses_terminal()/sim_reconcile_unaccounted_jobs() already apply elsewhere. TDD: new test test_info_worker_assigned_job_listing_excludes_terminal_job_despite_stale_allocs (test_repl_resource_table.c) directly assigns the SAME allocs struct to a DONE job and a RUN job on the same worker, forcing the DONE job's state after assigning allocs (reproducing the exact state/allocs inconsistency without needing to root-cause how it arises live) -- confirms the DONE job is excluded while the RUN job still shows. Full scheduler suite: 76/77 pass (same pre-existing unrelated test_core_events.c failure as before, bug-1762). NOTE: this masks the symptom at the display layer only -- the underlying question of WHY job->allocs stays non-NULL for a terminal job (suspected snapshot save/load inconsistency) remains unresolved; a real fix should also prevent/clear the stale allocs at its source.", @@ -25532,22 +12776,26 @@ "needs-live-repro", "related-to-bug-1764", "interim-fix", - "sim", - "fixed" + "sim" ], "related_bugs": [ "bug-1764", "bug-1776" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1778.md" }, { "id": "bug-1779", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-07T06:58:00Z", - "error_message": "Live-observed (interactive sim session=19, replaying a real captured trace, 'step 9'): within the SAME batch of scheduler decisions, jobs 0-12-1788762745-0[0..4] receive a DISPATCH decision, then immediately ALSO receive a PEND decision in the same step's output -- and `info jobs` afterward shows all 10 array elements (0-12-1788762745-0[0..9]) as PEND, including the 5 that were just dispatched. User: 'when a job has been dispatched (in scheduler), we should treat it as RUN, so it should be excluded from PEND ... we need to remove 0-12-1788762745-0[0-4] from PEND but show in RUN.'", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Live-observed (interactive sim session=19, replaying a real captured trace, 'step 9'): within the…\n\nLive-observed (interactive sim session=19, replaying a real captured trace, 'step 9'): within the SAME batch of scheduler decisions, jobs 0-12-1788762745-0[0..4] receive a DISPATCH decision, then immediately ALSO receive a PEND decision in the same step's output -- and `info jobs` afterward shows all 10 array elements (0-12-1788762745-0[0..9]) as PEND, including the 5 that were just dispatched. User: 'when a job has been dispatched (in scheduler), we should treat it as RUN, so it should be excluded from PEND ... we need to remove 0-12-1788762745-0[0-4] from PEND but show in RUN.'", "file": "src/scheduler/src/sim_loop.c", "root_cause": "ROOT-CAUSED FOR REAL (2026-09-07), and turns out to be a SPEC CONFLICT, not a logic bug -- a patch was attempted and REVERTED after it broke a pre-existing, passing regression test. Confirmed mechanism: sim mode's REAL DISPATCH decisions come entirely from sim_reconcile_unaccounted_jobs() (sim_loop.c), which reconstructs a job's dispatch PURELY from replay data (populating job->allocs/job->accounts, reporting a synthesized DISPATCH decision) but never calls picker->picker_iface->on_dispatch() -- unlike the real/live allocator path (sched_allocator(), scheduler.c line 507-510), which ALWAYS calls on_dispatch() to retire the job from the picker's own candidate list (fcfs_queue_t.pendJobs) before building the decision. Because reconciliation skips this, the job stays a live candidate in the picker's bookkeeping. sim_picker_thread_fn() (sim_loop.c) -- a SEPARATE, always-running thread that walks next_job() every round and unconditionally PENDs whatever it is still offered, since it has no real resource-matching allocation of its own -- keeps re-offering and re-PENDing that job every subsequent round until an explicit JOB_STATE_CHANGE-to-RUN event (also replayed from the original recording, can legitimately arrive several rounds later) removes it from pendJobs via builtin_fcfs_on_job_state_change(). CRITICAL DISCOVERY: this exact behavior is NOT accidental -- it is DOCUMENTED, INTENTIONAL, and covered by an existing passing test, test_sim_verify_no_false_mismatch_across_round_separated_dispatch (test_sim_verify.c, scheduler-sim-eager-reconciliation spec task 3.2), whose own header comment states verbatim: 'sim_reconcile_unaccounted_jobs() captures job->accounts but never calls picker_iface->on_dispatch() to retire the job from the picker's own internal pendJobs (design.md's Non-Goals/Revalidation Triggers explicitly leave live-picker DISPATCH emission to a future scheduler-allocation spec)'. That spec's own design.md (.kiro/specs/scheduler-sim-eager-reconciliation/design.md, Non-Goals section) explicitly lists 'Any change to simulation mode's live picker (still PEND-only for freshly-arriving jobs, per scheduler-allocation)' as OUT OF SCOPE. A fix attempt (calling on_dispatch() from the reconciliation FOUND branch, mirroring sched_allocator()'s exact ordering) was implemented with a new TDD test, confirmed the bug reproduces deterministically (RED), and confirmed the fix resolves it (GREEN) -- but broke test_sim_verify_no_false_mismatch_across_round_separated_dispatch, whose fixture explicitly bakes in 'PEND(jobid) recorded in the SAME round as its own DISPATCH, and again for every round until the RUN confirmation arrives' as the CORRECT, expected --sim-verify-clean shape. The fix was REVERTED (both the sim_loop.c change and its new test) rather than kept, since it silently violates an already-agreed, tested design contract from a sibling spec.", "fix": "FIXED FOR REAL (2026-09-07), after a first fix attempt was correctly reverted (see prior root_cause note) upon discovering it broke test_sim_verify_no_false_mismatch_across_round_separated_dispatch. That test's OWN fixture turned out to be baking in the bug's exact symptom as 'expected' -- but its parent spec's own design.md ('Integration with --sim-verify') already documented the CORRECT assumption ('the picker never re-decides an already-resolved job'), which the implementation had silently drifted from. This was a genuine implementation/test drift from an already-agreed design intent, not a live spec conflict requiring a product decision -- confirmed by re-reading both scheduler-sim-decision-parity's and scheduler-sim-eager-reconciliation's own design.md files (both updated in place with 2026-09-07 correction notes explaining this). Fix: sim_reconcile_unaccounted_jobs() (sim_loop.c) now resolves the job picker and calls picker_iface->on_dispatch() immediately after reconstructing job->allocs/job->accounts in its FOUND branch, mirroring sched_allocator()'s own live-dispatch ordering (scheduler.c line 507-515) exactly. This retires the reconciled job from the picker's own pendJobs bookkeeping the moment its dispatch is reconstructed, so sim_picker_thread_fn()'s unconditional-PEND thread never re-offers/re-PENDs it on a later round. TDD: new test test_reconcile_removes_reconciled_job_from_picker_candidates (test_sim_reconcile_unaccounted.c) directly proves a job is no longer offered by next_job() after reconciliation, using sim_bootstrap_picker() to register a real, resolvable builtin_fcfs instance -- confirmed RED before the fix, GREEN after. test_sim_verify_no_false_mismatch_across_round_separated_dispatch (test_sim_verify.c) was updated to match the corrected behavior: round 1 now records DISPATCH-only (new helper append_replay_session_with_dispatch(), no accompanying PEND), and rounds 2-3 are now empty ReplaySessions (matching round 4's own existing pattern) instead of baking in a stale re-PEND echo as expected. Full scheduler suite: 76/77 pass (same pre-existing unrelated test_core_events.c failure, bug-1762, untouched by this fix).", @@ -25556,7 +12804,6 @@ "builtin_fcfs", "decision", "dispatch", - "fixed", "job-lifecycle", "needs-live-repro", "pend", @@ -25569,15 +12816,20 @@ "related_bugs": [ "bug-1778" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1779.md" }, { "id": "bug-1780", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-09-07T07:55:00Z", - "error_message": "User-reported (xcecli): 'xcecli jobs submit -n 2 --env /run/current-system/sw/bin/sleep 100' submits successfully but the job immediately fails with Status . xcecd.log shows: '[WARN][controller_connector.c:822] No execuser provided in startjob request, defaulting to nobody', then 'Starting worker as user nobody to process command...', then 'Worker 301734 exited with code 126'.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (xcecli): 'xcecli jobs submit -n 2 --env /run/current-system/sw/bin/sleep 100'…\n\nUser-reported (xcecli): 'xcecli jobs submit -n 2 --env /run/current-system/sw/bin/sleep 100' submits successfully but the job immediately fails with Status . xcecd.log shows: '[WARN][controller_connector.c:822] No execuser provided in startjob request, defaulting to nobody', then 'Starting worker as user nobody to process command...', then 'Worker 301734 exited with code 126'.", "file": "src/frontend/apiserver/backend/convert.go", "root_cause": "modelJobToSubmit() (src/frontend/apiserver/backend/convert.go, converts a Go models.JobInfo into the C job_startinfo_t sent to the controller) populated si.cmdline/launch_type/job_pre_exec/job_post_exec but never set si.execuser, even though job_startinfo_t (src/include/xce/types.h) has an execuser field and models.JobInfo already has an ExecUser field (used only on the read/display path, in convert_pure.go's rawJobToModel). CreateJob() (controllers/jobs.go) also never populated JobInfo.ExecUser when building the job to submit -- there is no --exec-user flag on xcecli either, so the field was simply always empty at submission. The controller (src/controller/job.c) forwards job.submit->start_info verbatim to the worker without ever touching execuser. xcecd's controller_connector.c (lines 817-822) defaults an empty/NULL execuser to the literal Linux user 'nobody', then (lines 1037-1040) copies that into job_ctx->limits->user_name, which xcecd_dbus_manager.c uses as systemd-run's User= for the job's transient unit. 'nobody' has no home directory/login shell and typically cannot execute or access the real submitting user's binaries/cwd, so the transient unit fails with 'Permission denied' -- surfacing to xcecd_worker_utils.c as exit_code=126, which xcecli then reports as job Status . Every job submitted via xcecli/apiserver was silently running as 'nobody' regardless of who submitted it.", "fix": "FIXED (2026-09-07), TDD RED->GREEN. Added resolveExecUser(execUser, submitUser string) string to convert_pure.go -- a pure-Go helper (no CGO types) that returns the caller-specified exec user if present, else defaults to the submitting user; this mirrors the existing pure/CGO split in this file (rawJobToModel etc. already keep pure field-mapping logic separate from the CGO struct-building in convert.go specifically so it's testable without CGO -- and because a _test.go file can never itself import \"C\" (golang/go#13938), this split is the only way to unit-test any part of modelJobToSubmit's logic). modelJobToSubmit() now sets si.execuser = C.CString(resolveExecUser(job.ExecUser, job.SubmitUser)); freeJobSubmit() now also frees si.execuser (previously would have leaked once set). New tests in convert_test.go: TestResolveExecUser_DefaultsToSubmitUserWhenEmpty, TestResolveExecUser_PrefersExplicitExecUser, TestResolveExecUser_EmptyWhenBothEmpty -- pass under CGO_ENABLED=0 go test ./backend/... (RED confirmed by the function not existing before this change). Verified with a full /xce-build apiserver asan --incremental: exit 0, all frontend Go suites pass including apiserver/backend (7.595s, real CGO build), fresh xce_apiserver/xcecli binaries produced. Live re-run of the exact reported repro (xcecli jobs submit ... /run/current-system/sw/bin/sleep 100) not yet performed in this session -- recommend a live smoke test to confirm the job now runs as the submitting user and reaches DONE/RUN instead of EXIT before closing.", @@ -25590,21 +12842,23 @@ "nobody", "permission-denied", "worker", - "xcecli", - "fixed" + "xcecli" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1780.md" }, { "id": "bug-1781", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-07T08:35:00Z", - "gitea_issue": 45, - "error_message": "Gitea issue #45 REOPENED after bug-1742's fix. User-reproduced: after bug-1742 fixed the earlier panic/empty-error-body symptom, `xcecli jobs kill ` (and stop/resume/checkpoint) now fails with a real, non-empty error instead: `request to /xce/v1/jobs//control failed: [failed to write data: xcecli_jobctrl failed: -22]` (-22 = -EINVAL). Reproduced live for kill/stop/resume/checkpoint, all failing.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Gitea issue #45 REOPENED after bug-1742's fix\n\nGitea issue #45 REOPENED after bug-1742's fix. User-reproduced: after bug-1742 fixed the earlier panic/empty-error-body symptom, `xcecli jobs kill ` (and stop/resume/checkpoint) now fails with a real, non-empty error instead: `request to /xce/v1/jobs//control failed: [failed to write data: xcecli_jobctrl failed: -22]` (-22 = -EINVAL). Reproduced live for kill/stop/resume/checkpoint, all failing.", "file": "src/frontend/apiserver/backend/clib.go (ControlJob), src/lib/request.c (xce_jobctrl_req), src/lib/controller.capnp.h (JobControl_u_which), src/include/xce/types.h (xce_jobctrl_t), src/controller/job.c (job_ctrl)", "root_cause": "ControlJob() (clib.go) set xce_jobctrl_t.action to the job_ctrl_type_t action code (models.JOB_STOP=1, JOB_RESUME=2, JOB_KILL=3, JOB_CHKPNT=8, src/include/xce/types.h job_ctrl_type_t) -- but that field is actually the capnp JobControl_u_which UNION DISCRIMINANT (JobControl_u_siginfo=0, JobControl_u_ckptoption=1, src/lib/controller.capnp.h), a completely different enum that says which branch of xce_jobctrl_t.u (siginfo vs ckptoption) is populated. xce_jobctrl_req() (src/lib/request.c:2200) and the controller's job_ctrl() (src/controller/job.c:1972, for a job with no allocs yet) both switch on ctrinfo->action expecting the discriminant values 0/1, not the job_ctrl_type_t codes. Consequences per action: kill(3) and resume(2) never match 0 or 1, so xce_jobctrl_req's `default: return -EINVAL` fires client-side before the request is even sent to the controller -- this is the exact -22 in the bug report. checkpoint(8) hits the same default-EINVAL path, so checkpoint was ALSO silently broken (never reached the controller's real checkpoint-eligibility check). stop(1) is the worst case: it numerically collides with JobControl_u_ckptoption(1), so xce_jobctrl_req took the ckptoption branch and read `ctrinfo->u.ckptoption->method` through a pointer that was actually populated as `xce_signal_t*` (via jobctrl_set_siginfo, clib.go) -- a wrong-typed union-member read, undefined behavior in the apiserver's own address space, not merely a clean rejection. The `signo`-based dispatch already correctly threaded through job_ctrl()'s inner switch (SIGCONT->resume, SIGSTOP/SIGTSTP->stop, SIGKILL/SIGTERM/default->kill on the not-yet-started-job branch) proves the intended design: `action` is the union tag, and `u.siginfo->signo` is what actually distinguishes stop/resume/kill -- ControlJob() conflated the two enums when translating the caller's action into the wire struct.", "fix": "FIXED and LIVE-VERIFIED (2026-09-07). Added jobCtrlUnionDiscriminant(action int) int to convert_pure.go (pure Go, no CGO types, alongside resolveExecUser from bug-1780) -- returns JobControl_u_ckptoption(1) for JOB_CHKPNT, JobControl_u_siginfo(0) for JOB_STOP/JOB_RESUME/JOB_KILL. ControlJob() (clib.go) now sets `ctrl.action = C.int(jobCtrlUnionDiscriminant(action))` instead of the raw action code; the existing jobctrl_set_siginfo/jobctrl_set_ckptoption C helpers (already correct, only ever touched ctrl.u) are unchanged. TDD: TestJobCtrlUnionDiscriminant_StopResumeKillUseSiginfo and TestJobCtrlUnionDiscriminant_ChkptUsesCkptOption (convert_test.go), passing under CGO_ENABLED=0 go test (RED before the function existed). Verified with a full /xce-build apiserver asan --incremental (exit 0, apiserver/backend real-CGO suite green). LIVE-VERIFIED beyond static analysis (learning from bug-1742's own gap, which was static-analysis-only and left this exact defect in place): deployed a real single-node cluster (controller+scheduler+worker+apiserver) via deploy-ha.sh, submitted a job that stays PEND (unsatisfiable select[] so it never dispatches -- exercises job_ctrl()'s not-yet-started branch, the exact code path this bug hits, without needing job execution), and confirmed via the real built xcecli binary: `jobs kill` succeeds (job removed from listing, previously -22), `jobs stop`/`jobs resume` succeed with no apiserver crash (previously read through the wrong union member -- confirmed apiserver process survived and stayed responsive), and `jobs checkpoint` on an unstarted job now correctly gets a clean business-rule rejection (\"cannot be \") instead of a spurious client-side -EINVAL.", @@ -25620,23 +12874,27 @@ "union-discriminant", "undefined-behavior", "gitea-45", - "live-verified", - "fixed" + "live-verified" ], "related_bugs": [ "bug-1742" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1781.md", + "gitea_issue": 45, + "gitea_synced_status": "fixed" }, { "id": "bug-1782", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "apiserver", + "severity": "unknown", "timestamp": "2026-09-07T08:40:00Z", - "gitea_issue": 45, - "error_message": "Not a defect -- API ergonomics follow-up discussed while fixing bug-1781/gitea#45. The job control REST endpoint (`/xce/v1/jobs/{jobids}/control`) only accepted the packed numeric job ID; xcecli already translates the human-readable form (e.g. `1-0-1788768547-0`, as printed by `jobs list`) to that numeric form client-side (format.ToAPIJobID) before calling the API, so a caller hitting the endpoint directly (not through xcecli) had to reimplement that bit-packing itself.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Not a defect -- API ergonomics follow-up discussed while fixing bug-1781/gitea#45. The job control…\n\nNot a defect -- API ergonomics follow-up discussed while fixing bug-1781/gitea#45. The job control REST endpoint (`/xce/v1/jobs/{jobids}/control`) only accepted the packed numeric job ID; xcecli already translates the human-readable form (e.g. `1-0-1788768547-0`, as printed by `jobs list`) to that numeric form client-side (format.ToAPIJobID) before calling the API, so a caller hitting the endpoint directly (not through xcecli) had to reimplement that bit-packing itself.", "file": "src/frontend/apiserver/backend/convert_pure.go (ParseHumanJobID, new), src/frontend/apiserver/controllers/jobs.go (ControlJobByID)", "root_cause": "N/A -- enhancement, not a bug. Existing numeric-only behavior was correct and intentional.", "fix": "ADDED (2026-09-07), additive/backward-compatible -- the numeric form is untouched and still works exactly as before. Added ParseHumanJobID(s string) (uint64, bool) to convert_pure.go: parses a plain human-readable job ID (\"{cluster}-{apiserver}-{timestamp}-{sequence}\", no array brackets -- controllers/jobs.go's parseArrayName already strips any \"[idx]\" suffix before this runs) back into the packed numeric ID, reusing the bit-layout constants this file already duplicates from xcecli/common/format/jobid.go for the read path (FormatJobID/ExtractArrayBaseID). Works correctly for both a plain job's own ID and an array child's dash-quad prefix (verified equal to ExtractArrayBaseID(original) in the array case -- the two enums are NOT ambiguous here, unlike bug-1781). ControlJobByID (jobs.go) now falls back to ParseHumanJobID when jobid.KeyID isn't numeric, before erroring; numeric IDs skip this path entirely. TDD: TestParseHumanJobID_NonArrayRoundTripsWithFormatJobID, TestParseHumanJobID_ArrayChildPrefixMatchesBaseID, TestParseHumanJobID_RejectsNumeric, TestParseHumanJobID_RejectsGarbage (convert_test.go), passing under CGO_ENABLED=0 go test. Verified with a full /xce-build apiserver asan --incremental (exit 0). LIVE-VERIFIED: against the same deployed cluster used for bug-1781, called POST /xce/v1/jobs/{jobids}/control directly via curl with the URL-encoded human-readable ID (bypassing xcecli's own translation entirely) and confirmed the job was correctly found and killed (HTTP 202, job removed from listing).", @@ -25647,23 +12905,27 @@ "human-readable-id", "gitea-45", "enhancement", - "live-verified", - "fixed" + "live-verified" ], "related_bugs": [ "bug-1781" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1782.md", + "gitea_issue": 45, + "gitea_synced_status": "fixed" }, { "id": "bug-1784", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-07T09:00:00Z", - "gitea_issue": 46, - "error_message": "Discovered while live-verifying gitea#46's Part 1 (dispatch-stops-at-8, previously hypothesized in bug-1743 to be limits.yaml's ug1_normal_slots cap=8 working as designed, never live-confirmed). Live test: submitted 10 single-slot jobs as user1 (a genuine ug1 member per users.yaml) on queue normal against a 22-slot worker -- all 10 ran, the supposed cap=8 usergroup limit never blocked anything.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Discovered while live-verifying gitea#46's Part 1 (dispatch-stops-at-8, previously hypothesized in…\n\nDiscovered while live-verifying gitea#46's Part 1 (dispatch-stops-at-8, previously hypothesized in bug-1743 to be limits.yaml's ug1_normal_slots cap=8 working as designed, never live-confirmed). Live test: submitted 10 single-slot jobs as user1 (a genuine ug1 member per users.yaml) on queue normal against a 22-slot worker -- all 10 ran, the supposed cap=8 usergroup limit never blocked anything.", "file": "src/lib/group.c (groupmgr_load_configs, new), src/include/group.h, src/scheduler/src/scheduler.c (sched_init_config)", "root_cause": "Named usergroups/hostgroups (users.yaml's top-level \"usergroups:\", hosts.yaml's \"hostgroups:\") are carried down to the scheduler via xce_conf4sched_t.ugroups/hgroups (src/controller/sched.c populates these from c->myconf->cluster_config->ugroups/hgroups), but nothing on the scheduler side ever registered them into a group manager by name: groupmgr_add() (src/lib/group.c) -- the obvious candidate -- had ZERO callers anywhere in production code (confirmed via full-tree grep). When a limit's own per-limit consumer group (plugins/limit.c's build_consumer()) references a named group like \"ug1\" as a member, make_g_member() (group.c) calls get_group(mgr, \"ug1\"), finds nothing (since it was never registered), and falls back to treating \"ug1\" as a literal, ungrouped member name that will never strcmp-match a real username via is_in_group(). Confirmed via source inspection (build_group(), a plausible alternative entry point declared in group.h, has NO implementation anywhere either -- a second dead prototype) and via live behavior (user1, a real ug1 member, ran unthrottled against a configured cap).", "fix": "FIXED (2026-09-07). Added groupmgr_load_configs(var mgr, int count, xce_group_config_t** configs) to group.c/group.h: iterates the config list and registers each named entry into mgr via the existing, already-correct makeGroup(mgr, config, config->name). Wired into scheduler.c's sched_init_config(): groupmgr_load_configs() for both core.ugmgr/config->ugroups and core.hgmgr/config->hgroups, called BEFORE plugin_config_update() so any limit's own build_consumer() call (which runs inside plugin_config_update -> limit_update_config -> build_limit) can resolve a named-group reference at build time. TDD: 3 new tests in src/lib/testsuite/test_group.c -- test_load_configs_registers_each_group_by_name, test_load_configs_enables_group_name_as_member_reference (the exact regression: a limit-style unnamed consumer group referencing a named group by member, mirroring build_consumer()'s own call shape), test_load_configs_zero_count_is_a_noop. All pass (313 Tests 0 Failures under CGO... err, under ASAN_OPTIONS=detect_leaks=0 xcelib_test). Full scheduler suite: 76/77 (pre-existing unrelated bug-1762 test_core_events.c failure). LIVE-VERIFIED: fresh single-node deploy, user1 (ug1 member) submitting jobs on queue normal now correctly has match_limit() report a match (previously always rc=0/no-match) for ug1_slots and ug1_normal_slots -- confirmed via scheduler trace logs. (Full cap enforcement required two further fixes in the same session -- see bug-1786/1788 -- and one remaining gap not yet fixed, see bug-1789.)", @@ -25674,8 +12936,7 @@ "hostgroup", "group-manager", "gitea-46", - "live-verified", - "fixed" + "live-verified" ], "related_bugs": [ "bug-1743", @@ -25685,17 +12946,22 @@ "bug-1788", "bug-1789" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1784.md", + "gitea_issue": 46, + "gitea_synced_status": "fixed" }, { "id": "bug-1785", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-07T09:15:00Z", - "gitea_issue": 46, - "error_message": "Found immediately after fixing bug-1784: with named-usergroup resolution working, a limit scoped to only SOME of the four dimensions (users.yaml's demo limits.yaml sets users+queues on ug1_normal_slots but never projects/hosts) still matched no one at all.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Found immediately after fixing bug-1784: with named-usergroup resolution working, a limit scoped…\n\nFound immediately after fixing bug-1784: with named-usergroup resolution working, a limit scoped to only SOME of the four dimensions (users.yaml's demo limits.yaml sets users+queues on ug1_normal_slots but never projects/hosts) still matched no one at all.", "file": "src/lib/xcelimit.c (build_consumer)", "root_cause": "build_consumer() (xcelimit.c) is documented to default an unconfigured scope to the builtin \"all\" group (matching everyone) when g_config is NULL. But capnp encodes LimitConfig's users/queues/projects/hosts as embedded GROUPS, not nullable pointers (xceadmin's mapping.py: _build_membership_group() no-ops on an absent YAML key, but the embedded capnp struct is still present on the wire as an empty-but-non-NULL group) -- so an unconfigured scope arrives here as a non-NULL xce_group_config_t with zero members/ex_members, never as NULL. build_consumer()'s g_config!=NULL branch then builds a real, empty, UNNAMED group via makeGroup() -- a group that (per is_in_group()'s own logic) matches no one, the exact OPPOSITE of the intended \"unrestricted\" default. Every limit in this repo's demo limits.yaml configures only a subset of the four scopes, so every one of them was silently rejecting every job on its unconfigured dimensions.", "fix": "FIXED (2026-09-07). build_consumer() now normalizes an empty-but-present g_config (members.count==0 AND ex_members.count==0) to NULL before the existing g_config!=NULL/NULL branch runs, restoring the documented \"fall back to all\" behavior for a genuinely-unconfigured scope. An operator who wants \"no one\" for a scope still has the builtin \"empty\" group to reference by name explicitly, so this doesn't remove any real capability. TDD: new test_build_consumer_empty_config_defaults_to_all (src/lib/testsuite/test_limit.c) -- builds a zero-valued xce_group_config_t{0}, asserts build_consumer() falls back to a group where is_in_group() matches an arbitrary name. Passes (313 lib tests, 0 failures). Full scheduler suite: 76/77 (pre-existing bug-1762). LIVE-VERIFIED as part of the same trace as bug-1784/1786/1787/1788 -- combined, these let match_limit() and instantiate_limit() run to completion for all 7 limits in limits.yaml on every job submission (previously it silently rejected on the FIRST unconfigured scope it hit).", @@ -25705,8 +12971,7 @@ "capnp", "group-config", "default-semantics", - "gitea-46", - "fixed" + "gitea-46" ], "related_bugs": [ "bug-1784", @@ -25715,17 +12980,22 @@ "bug-1788", "bug-1789" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1785.md", + "gitea_issue": 46, + "gitea_synced_status": "fixed" }, { "id": "bug-1786", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-07T09:30:00Z", - "gitea_issue": 46, - "error_message": "Found immediately after bug-1784/1785: with group resolution and default-scope semantics both fixed, match_limit() finally reported a genuine match (rc=1) for the FIRST time in this codebase's live history -- and the very next step (usage/threshold accounting) immediately broke: limit_job_election()'s loop aborted after the first matching limit, so no limit past it in iteration order was ever evaluated for that job.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Found immediately after bug-1784/1785: with group resolution and default-scope semantics both…\n\nFound immediately after bug-1784/1785: with group resolution and default-scope semantics both fixed, match_limit() finally reported a genuine match (rc=1) for the FIRST time in this codebase's live history -- and the very next step (usage/threshold accounting) immediately broke: limit_job_election()'s loop aborted after the first matching limit, so no limit past it in iteration order was ever evaluated for that job.", "file": "src/scheduler/src/plugins/limit.c (match_job_global_limit_)", "root_cause": "match_job_global_limit_(), right after a successful match_limit(), does `rc = key_find(privdata->instances, vresname, &vres); if (rc <= 0) { return rc; }` before the `if (vres == NULL) { instantiate_limit(...) }` block below it. On the FIRST-EVER lookup of a given vresname (always true immediately after bug-1784/1785 made matching work for the first time), key_find() signals \"not found\" as a NEGATIVE rc (confirmed live: rc=-2/-ENOENT, not 0) -- this codebase's kvmgr convention elsewhere (get_group(), is_host_based_limit()) treats rc<=0 uniformly as \"not found\", so this specific early-return, written assuming rc==0 meant not-found and only negative meant a real error, unconditionally caught EVERY not-found case and returned before instantiate_limit() could ever run. limit_job_election()'s own loop breaks on any rc<0 from match_job_global_limit_, so this also silently aborted evaluation of every remaining limit for that job. Same defect class already fixed once before in this codebase, in a different pair of functions: bug-1495 (sched_get_user()/sched_get_project(), core.c) hit and fixed the identical rc<0-vs-not-found conflation.", "fix": "FIXED (2026-09-07). Removed the erroneous `if (rc <= 0) { return rc; }` guard entirely; `vres` (calloc'd NULL, only ever reassigned by a successful key_find) is already the correct, sufficient signal for \"needs instantiation\" via the existing `if (vres == NULL)` block immediately below -- mirrors this file's own build_vres_name/key_add error-handling style, just without conflating a legitimate not-found miss with failure. No new unit test added for this specific control-flow fix (see bug-1788's fix note for why: exercising limit_job_election() end-to-end requires a fully-wired xce_sched_core_t fixture -- resmgr, sharedInst, a real xce_sched_obj_t with initialized xlist_t/zarray_t fields -- judged not worth the fragility for a single control-flow correction already covered by full live E2E verification). Full scheduler suite: 76/77 (pre-existing bug-1762). LIVE-VERIFIED: after this fix (combined with bug-1787/1788, needed to get all the way through), all 7 limits in limits.yaml now appear in match_job_global_limit_'s own trace log for every job submission, where previously only the first one in iteration order ever did.", @@ -25736,8 +13006,7 @@ "error-code-convention", "not-found-vs-error", "gitea-46", - "live-verified", - "fixed" + "live-verified" ], "related_bugs": [ "bug-1495", @@ -25747,17 +13016,22 @@ "bug-1788", "bug-1789" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1786.md", + "gitea_issue": 46, + "gitea_synced_status": "fixed" }, { "id": "bug-1787", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-07T09:45:00Z", - "gitea_issue": 46, - "error_message": "Found immediately after bug-1786: match_job_global_limit_'s own key_find rc=-2 masking fixed, but the FOLLOWING step -- instantiate_limit() -- now returned rc=-22/-EINVAL outright, still aborting the loop after the first limit.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Found immediately after bug-1786: match_job_global_limit_'s own key_find rc=-2 masking fixed, but…\n\nFound immediately after bug-1786: match_job_global_limit_'s own key_find rc=-2 masking fixed, but the FOLLOWING step -- instantiate_limit() -- now returned rc=-22/-EINVAL outright, still aborting the loop after the first limit.", "file": "src/scheduler/src/plugins/limit.c (match_job_global_limit_, match_job_host_limit_)", "root_cause": "is_host_based_limit() and instantiate_limit() (src/lib/xcelimit.c) both take a plain `var resmgr` parameter -- core.h declares core.resmgr itself as a plain `var` (a handle, not var*), matching how every other manager consumer in this codebase (get_group(), is_in_group(), key_find() itself) takes its manager by value. All four call sites in limit.c (2 in match_job_global_limit_, 2 in the host-scoped sibling match_job_host_limit_) instead passed &(sched->core.resmgr) -- the ADDRESS of the field, one level of indirection too many. RTTI's key_find() dispatch received what amounts to a var* misinterpreted as a var (a meta_obj_t* pointing at raw, non-RTTI memory) and rejected it: silently inside is_host_based_limit() (whose own rc<=0 guard turns ANY key_find failure, including this type-confusion, into a uniform -ENOENT -- see bug-1786's note on why that specific defensive pattern isn't itself wrong), but fatally inside instantiate_limit(), whose own key_find() call on the SAME mistyped resmgr returned -EINVAL and (before this session's other fixes) propagated straight out.", "fix": "FIXED (2026-09-07). Removed the erroneous & at all 4 call sites (2 in match_job_global_limit_, 2 in match_job_host_limit_): `is_host_based_limit(sched->core.resmgr, limit)` / `instantiate_limit(&vres, sched->core.resmgr, limit, ...)`. Confirmed via live diagnostic instrumentation (temporary, removed before landing) that instantiate_limit()'s own top-of-function NULL-argument dump showed every argument non-NULL yet still failing -- proving the -EINVAL came from RTTI's own internal type/handle validation inside key_find(), not the function's explicit guard clause, which directly confirmed the type-confusion theory before touching production code. Full scheduler suite: 76/77 (pre-existing bug-1762). LIVE-VERIFIED as part of the combined bug-1784/1785/1786/1788 trace.", @@ -25768,8 +13042,7 @@ "pointer-indirection", "type-confusion", "gitea-46", - "live-verified", - "fixed" + "live-verified" ], "related_bugs": [ "bug-1784", @@ -25778,17 +13051,22 @@ "bug-1788", "bug-1789" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1787.md", + "gitea_issue": 46, + "gitea_synced_status": "fixed" }, { "id": "bug-1788", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-07T10:00:00Z", - "gitea_issue": 46, - "error_message": "Found immediately after bug-1787: with the resmgr type-confusion fixed, instantiate_limit() still returned rc=-2/-ENOENT, still aborting limit_job_election()'s loop after the first limit -- reproduced identically on a scheduler's FIRST-EVER join to a freshly-started controller, ruling out any rejoin/resync-specific explanation.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Found immediately after bug-1787: with the resmgr type-confusion fixed, instantiate_limit() still…\n\nFound immediately after bug-1787: with the resmgr type-confusion fixed, instantiate_limit() still returned rc=-2/-ENOENT, still aborting limit_job_election()'s loop after the first limit -- reproduced identically on a scheduler's FIRST-EVER join to a freshly-started controller, ruling out any rejoin/resync-specific explanation.", "file": "src/lib/xcelimit.c (instantiate_limit)", "root_cause": "The EXACT same defect class as bug-1786, recurring twice more, plus a related truthiness bug, all inside instantiate_limit() itself: (1) `rc = key_find(resmgr, vresname, &(vres->vres)); if (rc < 0) { return rc; } if (rc == 0) { add_vres(...); }` -- live-confirmed via targeted diagnostic instrumentation that this key_find's own \"not found\" signal is rc=-2/-ENOENT (not 0) for THIS resmgr-backed lookup too, so the `if (rc<0) return rc;` guard caught every first-ever vresname lookup before the add_vres()-creates-it branch could run. (2) The immediately following lookup of the limit's underlying resource (limit->val.resname, e.g. \"slots\") had the identical `if (rc==0) return -ENOENT;` pattern, vulnerable to the same rc convention mismatch (not hit in this specific trace only because \"slots\" was already registered by the time it was reached, but latent). (3) `if (is_host_based_limit(resmgr, limit)) { shared = 0; }` checked bare truthiness of a function that can return a negative errno -- a non-zero, therefore truthy, value -- so a transient is_host_based_limit() failure would have incorrectly flagged a limit as host-based.", "fix": "FIXED (2026-09-07). All three corrected in instantiate_limit(): (1) `if (vres->vres == NULL) { add_vres(...); }` replaces the rc<0/rc==0 checks -- vres->vres is calloc'd NULL and only ever reassigned by a successful key_find, the same reliable-signal pattern as bug-1786's fix. (2) `if (vres->res == NULL) { return -ENOENT; }` replaces the equivalent rc<0/rc==0 pair for the resource-existence check. (3) `is_host_based_limit(resmgr, limit) > 0` replaces the bare truthiness check. No new unit test added (same reasoning as bug-1786: this function's correctness is now covered by full live E2E verification, and hand-building a resmgr+sched_core fixture for a unit test was judged not worth the fragility versus the live proof already obtained -- see bug-1786/1789 discussion). Full scheduler suite: 76/77 (pre-existing bug-1762). LIVE-VERIFIED: combined with bug-1784/1785/1786/1787, all 7 limits in limits.yaml (ug1_slots, ug2_slots_per_user, night_mem_pct, hg1_slots_per_host, ug1_normal_slots, limit1, limits2) now correctly appear in the trace and complete matching+instantiation for every job submission on a completely fresh, never-before-touched cluster deploy -- confirmed reproducible across 3 independent from-scratch redeploys, ruling out any deploy-specific or rejoin-specific artifact.", @@ -25800,8 +13078,7 @@ "not-found-vs-error", "truthiness-bug", "gitea-46", - "live-verified", - "fixed" + "live-verified" ], "related_bugs": [ "bug-1495", @@ -25811,17 +13088,22 @@ "bug-1787", "bug-1789" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1788.md", + "gitea_issue": 46, + "gitea_synced_status": "fixed" }, { "id": "bug-1789", - "gitea_synced_status": "fixed", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-07T10:30:00Z", - "gitea_issue": 46, - "error_message": "After bug-1784 through bug-1788 all fixed and live-verified (all 7 limits in limits.yaml now correctly match and instantiate usage tracking for every job), the actual numeric cap still does not reject anyone: 10 single-slot jobs submitted as user1 on queue normal against ug1_normal_slots's configured value:8 cap all still reach RUN, none PEND.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "After bug-1784 through bug-1788 all fixed and live-verified (all 7 limits in limits.yaml now…\n\nAfter bug-1784 through bug-1788 all fixed and live-verified (all 7 limits in limits.yaml now correctly match and instantiate usage tracking for every job), the actual numeric cap still does not reject anyone: 10 single-slot jobs submitted as user1 on queue normal against ug1_normal_slots's configured value:8 cap all still reach RUN, none PEND.", "file": "src/scheduler/src/resource.c (sched_fill_limit_requests)", "root_cause": "NOT YET ROOT-CAUSED TO A FIX -- traced to a specific, likely-defective computation, but the correct intended semantics were not established this session (deliberately stopped here rather than guess at production logic). Chain traced and CONFIRMED CORRECT up to one point: match_job_global_limit_ populates obj->glimits (fixed, bug-1784-1788) -> plugin.c's plugin_prepare_allocation calls sched_fill_limit_requests(sched, &(ctx->alloc_req.shared), &(obj->glimits)) when ctx->worker==NULL (the global/non-host-scoped case, correct for ug1_normal_slots) -> seqjob_allocator.c/parjob_allocator.c both correctly call sched_check_shared_satisfy(sched, uncomit, &(ctx->alloc_req.shared), &(ctx->unmeet)) and correctly reject (PREASON_NO_RESOURCE, rc=0) when a shared resource's wanted amount would exceed its tracked total (itself correctly set to limit->val.val=8 by update_limit_res_value(), which DOES run correctly now). The break is inside sched_fill_limit_requests() itself (resource.c:912-931): for each matched limit, it computes the job's \"wanted\" quantity of the limit's virtual resource via `val = get_resval_by_id(vec, vres->res->idx)` where `vec` is `ctx->alloc_req.shared` ITSELF -- i.e. it looks up how many units of the limit's UNDERLYING resource (e.g. \"slots\") the job already requests, inside the SAME shared-resource-request vector it is about to write the limit's own synthetic entry into. But \"slots\" is inherently a per-host resource, requested via ctx->alloc_req.HOST (task/slot count per host), not ctx->alloc_req.shared -- so this lookup always misses, get_resval_by_id's own not-found fallback returns 0.0, and every job is recorded as requesting ZERO units of ug1_normal_slots regardless of how many slots it actually asks for. sched_check_shared_satisfy then trivially \"satisfies\" a wanted=0 request against any total, no matter how many jobs are already running -- the cap can never bind. NOT CONFIRMED whether this is a straightforward bug (should read from ctx->alloc_req.host or from resreq directly) or reflects a genuine, unaddressed design gap in how a GLOBAL (non-host-scoped) limit is meant to interact with a per-host-scoped underlying resource like slots -- the fix requires understanding what \"consumption count\" SHOULD mean for this combination, not just correcting an obvious return-code mistake like bug-1784-1788 all were.", "fix": "FIXED (2026-09-07, follow-up session). Root-caused via codegraph trace of plugin_prepare_allocation()'s full hook loop (not just the two call sites the prior session had traced): seqjob_prepare_allocation()/parjob_prepare_allocation() (the allocator plugins' own prepare_allocation_hook) ALWAYS stage the job's wanted slot count into ctx->alloc_req.host -- even during the ctx->worker==NULL/global pass -- because \"slots\" is deliberately excluded from calc_wanted_global_resvals()/calc_wanted_host_resvals() (it's per-host, not XCE_RESF_GLOBAL) and is staged by a separate, unbranched follow-up store. This confirmed the prior session's NEXT-STEPS item (1): the job's real wanted slot count (ctx->nslots, set earlier in that SAME prepare_allocation call) is already available and correct for both phases -- 1/seqjob or primary->maxTasks/parjob-global or the current bucket's count/parjob-per-host -- it just was never read from the right place. FIRST ATTEMPT (reverted): tried mirroring the ctx->worker branch into the seqjob/parjob \"add slots\" store itself (writing into ctx->alloc_req.shared during the global phase) -- this broke 6 previously-passing tests (seqjob/parjob dispatch tests going from DISPATCH to PEND) because ctx->alloc_req.shared is also read by sched_check_shared_satisfy() against sched->core.sharedInst, the GLOBAL capacity pool, which never tracks \"slots\" (a per-host-only resource) -- injecting a spurious \"slots\" want there made EVERY job's general satisfy check fail, not just limit-scoped ones. REAL FIX: read-side only, confined to limit evaluation. sched_fill_limit_requests() (resource.c) now takes an added xce_sched_ctx_t* ctx parameter; when a matched limit's underlying resource is \"slots\" (by name, matching this codebase's existing convention of hardcoding \"slots\" by name in seqjob/parjob's own key_find() calls), it uses (float)ctx->nslots directly as the wanted quantity instead of the always-missing get_resval_by_id(vec, ...) lookup; every other resource keeps the original vec-lookup behavior unchanged. plugin.c's two call sites updated to pass ctx; scheduler.h's extern declaration updated; all 23 testsuite files providing a link-satisfying no-op stub for the old 3-arg signature mechanically updated to the new 4-arg one (added (void)ctx; each, zero behavior change). TDD: 2 new focused unit tests in test_resource_accounting.c (white-box #include of resource.c) -- test_fill_limit_requests_global_slots_limit_uses_ctx_nslots (builds a bare xce_vres_t{res=\"slots\", vres=\"limit=X\"}, ctx.nslots=6, empty vec, asserts the limit's own vres entry gets stored with val=6.0) and test_fill_limit_requests_non_slots_resource_reads_from_vec (same shape for a \"mem\"-resname'd limit, vec pre-staged with 3.0, ctx.nslots=99 deliberately mismatched, asserts the mem path is unaffected and still reads 3.0 from vec, not ctx->nslots). Full scheduler suite: 76/77 (only pre-existing, unrelated bug-1762 failure remains -- test_job_rusage_reconciles_mem_reservation_against_usage, mem-reservation subsystem, untouched by this change). LIVE-VERIFIED: fresh single-node deploy (localhost:19997), 10 single-slot jobs submitted as user1 (ug1 member) on queue normal. hosts.yaml's default host template has max_slot: -1 (unlimited), ruling out raw host capacity as a factor. Scheduler trace confirms match_job_global_limit_ matching ug1_slots/ug1_normal_slots/limit1 (all three global, slots-resname'd) for every job, and resource.c's unsatisfy_() now logging a REAL non-zero wanted value for the limit's virtual resource on every charge (previously always 0.0, per this bug's original root_cause) -- e.g. 'unsatisfy_ resource 35 total 3.000000 inuse 3.000000 uncomit 0.000000 wanted 1.000000' for limit1 (cap 3.0, matches every non-rshen user including user1) once its 3rd job committed, correctly PENDing the 4th and all subsequent jobs with a genuine shortfall. limit1 (cap 3, broader scope: all users except rshen) bound before ug1_normal_slots (cap 8, narrower: ug1+normal only) could, since both match user1 identically and limit1's cap is lower -- expected overlapping-limit interaction (flagged in an earlier session's own observation), not a gap in this fix; both run through the identical, now-fixed code path. REVISED (2026-09-07, same session, user feedback): the name-based `strcmp(vres->res->name, \"slots\")` special-case above was flagged as unreliable (correctly -- it silently fails to generalize to any future per-host resource that isn't literally named \"slots\"). Replaced with a flag-based dispatch: sched_fill_limit_requests() now takes xce_sched_alloc_req_t* alloc_req instead of xce_sched_ctx_t* ctx (nothing else from ctx was ever used), and for each matched limit reads its underlying resource's want from whichever of alloc_req->shared/alloc_req->host that resource's own XCE_RESF_GLOBAL flag indicates -- the SAME flag calc_wanted_global_resvals()/calc_wanted_host_resvals() (resreq.c) themselves already dispatch on to decide where THEY write a resource's want, so this is not a new rule, just the existing one applied consistently. plugin.c's two call sites, scheduler.h, and all 23 testsuite stubs updated accordingly (mechanical param rename). Replaced the 2 name-based unit tests with 3 flag-based ones in test_resource_accounting.c: test_fill_limit_requests_global_limit_on_host_resource_reads_host (bug-1789's actual case: a GLOBAL limit's own write target is alloc_req->shared, but its per-host underlying resource's want must be read from alloc_req->host), test_fill_limit_requests_host_limit_on_host_resource_reads_host (already-working sibling case, unaffected), and test_fill_limit_requests_host_limit_on_global_resource_reads_shared (the new symmetric case this refactor adds real coverage for: a HOST-scoped limit's write target is alloc_req->host, but a genuinely-XCE_RESF_GLOBAL-flagged underlying resource's want must be read from alloc_req->shared instead -- proves the dispatch is a real bidirectional rule, not a one-off special case). Full scheduler suite re-run: 76/77 (same pre-existing bug-1762 failure, no new failures). Live re-verified end to end with a fully fresh redeploy (not just a scheduler restart against old job state, to keep the signal clean): identical result to the first live verification, byte-identical scheduler trace line ('unsatisfy_ resource 35 total 3.000000 inuse 3.000000 uncomit 0.000000 wanted 1.000000'), confirming the refactor is behavior-preserving for the current config. KNOWN RESIDUAL GAP (explicitly out of scope, not silently solved): a hypothetical NEW global limit on some per-host resource OTHER than \"slots\" (e.g. an absolute-value global mem cap, as opposed to the existing percent-based host-scoped mem limits) would still hit an analogous bug -- calc_wanted_global_resvals() itself skips any non-XCE_RESF_GLOBAL-flagged resource entirely, so such a resource's want is never computed anywhere during the global phase; \"slots\" only works because seqjob/parjob_prepare_allocation() stage it via a separate, always-run step outside the normal rusages loop, which no other resource has. Fixing that would mean teaching calc_wanted_global_resvals() (or an equivalent) to also compute a non-global resource's want when a currently-matched limit needs it -- a materially larger change, not needed by any limit in the current limits.yaml (the two mem-based limits, night_mem_pct/limits2, are both host-scoped, unaffected either way) and deliberately not attempted here. FURTHER SIMPLIFIED (2026-09-07, same session, user request): sched_fill_limit_requests()'s 1st parameter (xce_scheduler_t* sched) was entirely unused in the function body (only alloc_req and limits are referenced) -- a leftover from an earlier signature shape, not flagged by the prior same-day refactor that removed the old vec parameter. Dropped it: signature is now sched_fill_limit_requests(xce_sched_alloc_req_t* alloc_req, xlist_t* limits). Updated scheduler.h's extern declaration, both call sites in plugin.c's plugin_prepare_allocation() (the sched local var stays -- still used elsewhere in that function for plugin iteration), all 23 link-satisfying testsuite stubs (mechanical Python replace, byte-identical body across all 23), and the 3 real call sites in test_resource_accounting.c (the local `sched` var in each test stays too -- still needed for xce_sched_core_init()/makeResource() resource setup, just no longer passed to this call). Full scheduler suite re-verified after this change: 76/77 (same pre-existing, unrelated bug-1762 failure, no new failures).", @@ -25830,8 +13112,7 @@ "limit-plugin", "resource-allocation", "gitea-46", - "live-verified", - "fixed" + "live-verified" ], "related_bugs": [ "bug-1784", @@ -25841,15 +13122,22 @@ "bug-1788", "bug-1790" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1789.md", + "gitea_issue": 46, + "gitea_synced_status": "fixed" }, { "id": "bug-1790", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-09-07T00:00:00Z", - "error_message": "OPEN, NOT A CRASH -- known residual gap explicitly carved out of bug-1789's fix, filed for its own tracking per user request. A hypothetical NEW global limit defined on a per-host resource OTHER than \"slots\" (e.g. a future absolute-value global mem cap, as opposed to the existing percent-based host-scoped mem limits in limits.yaml) would silently under-enforce the same way bug-1789's slots limits did, for a different reason than bug-1789's root cause.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "filed for its own tracking per user request\n\nOPEN, NOT A CRASH -- known residual gap explicitly carved out of bug-1789's fix, filed for its own tracking per user request. A hypothetical NEW global limit defined on a per-host resource OTHER than \"slots\" (e.g. a future absolute-value global mem cap, as opposed to the existing percent-based host-scoped mem limits in limits.yaml) would silently under-enforce the same way bug-1789's slots limits did, for a different reason than bug-1789's root cause.", "file": "src/lib/resreq.c", "root_cause": "bug-1789 fixed sched_fill_limit_requests()'s READ side: it now picks whichever of alloc_req->shared/alloc_req->host to read a limit's underlying resource's want from by checking that resource's own XCE_RESF_GLOBAL flag on vres->res (and separately picks the WRITE destination for the limit's own vres entry by checking vres->vres's flag) -- see resource.c:928-955. That fix only works because alloc_req->host already has SOME value for \"slots\" to read, staged by seqjob_prepare_allocation()/parjob_prepare_allocation() via a separate, unconditional step outside the normal rusages loop, in EVERY phase including the global one. No other per-host resource gets this special treatment. The general population path -- calc_wanted_global_resvals() (src/lib/resreq.c:734) and calc_wanted_host_resvals() (src/lib/resreq.c:793), both called from seqjob_allocator.c/parjob_allocator.c's prepare_allocation hooks -- each unconditionally skip any resource whose XCE_RESF_GLOBAL flag doesn't match their own phase (calc_wanted_global_resvals() line 763: `if ((r->flags & XCE_RESF_GLOBAL) == 0) continue;`; calc_wanted_host_resvals() line 822: `if ((r->flags & XCE_RESF_GLOBAL) != 0) continue;`), and only calc_wanted_global_resvals() runs during the ctx->worker==NULL global phase. So for any per-host resource other than \"slots\", alloc_req->host is simply never populated with a want during the global phase -- sched_fill_limit_requests()'s now-correct read-side dispatch would read a genuine zero from an empty vector, not a population bug on its own side, but the net effect (a global limit on a per-host resource never seeing a nonzero want) is the same silent under-enforcement bug-1789 fixed for slots.", "fix": "FIXED (2026-09-07, follow-up session, user-driven design). Corrected an earlier, wrong candidate fix (calling calc_wanted_host_resvals() during the global phase) identified during design discussion as invalid: that function's PER_HOST formula returns one host's own share (v->value, no multiplier) and, for parjob specifically, its nslots argument is reset PER BUCKET (parjob_allocator.c's parjob_try_plan_(), ctx->nslots = plan->bucket_slots[b], set right before each candidate host is tried) -- neither matches what a GLOBAL limit needs, which is the job-WIDE total, not any one host's/bucket's share.\n\nREAL FIX: added calc_wanted_host_resval_job_totals() (src/lib/resreq.c), a new function for resources NOT flagged XCE_RESF_GLOBAL that computes the JOB-WIDE TOTAL want using the SAME formulas calc_wanted_global_resvals() already uses for GLOBAL resources (PER_TASK: value*nslots, PER_JOB: value, PER_HOST: value*nhosts -- nslots/nhosts being the whole job's totals, set during the global phase, not any bucket's share). Both functions now share a common static helper (calc_wanted_job_total_resvals_()) parameterized only by which flag direction to match, avoiding duplicating the ~30-line loop body. seqjob_prepare_allocation()/parjob_prepare_allocation()'s global-phase branch (ctx->worker == NULL) now calls this new function into ctx->alloc_req.host immediately alongside the existing calc_wanted_global_resvals() call into .shared.\n\nSAFETY, traced precisely rather than assumed: plugin_prepare_allocation() (plugin.c:607-613) resets ONLY .shared at the top of a global-phase call and ONLY .host at the top of a per-host call -- never the other -- so .host is NOT reset when this new write happens. sched_fill_limit_requests() for obj->glimits runs later in that SAME plugin_prepare_allocation() invocation (plugin.c:644-647), reading and transferring the value into the limit's own permanent .shared record BEFORE any later per-host call ever resets .host. The value is therefore fully consumed and irrelevant by the time .host gets correctly wiped and repopulated with real per-host/per-bucket data for the actual capacity check -- two different consumers at two different times, not a conflict, confirmed by reading parjob_allocator.c's own per-bucket reset code directly (not inferred).\n\nTDD: two new regression tests calling seqjob_prepare_allocation()/parjob_prepare_allocation() directly with ctx.worker == NULL and a \"mem\"-named (non-\"slots\", non-global) rusage entry -- test_seqjob_prepare_allocation_global_phase_stores_host_resource_job_total (test_seqjob_allocator.c) and test_parjob_prepare_allocation_global_phase_stores_host_resource_job_total (test_parjob_combinatorial.c, maxTasks=3 specifically to prove the formula uses the job's WHOLE nslots=3, giving 2.0*3=6.0, not any bucket's share). Both assert .shared stays empty (mem correctly excluded, not globally flagged) and .host carries both the new mem entry and the pre-existing unconditional \"slots\" entry. Both PASS standalone with full Unity PASS lines confirmed (not just suite-level OK).\n\nFull `xce-build.sh all asan --incremental`: 92/94 Ok. The 2 failures are both pre-existing and unrelated, confirmed by cross-referencing this exact buglog: test_job_rusage_reconciles_mem_reservation_against_usage (bug-1762) and libxce test's 456-byte/29-allocation leak in test_xcealloc.c's make_alloc()/make_host_alloc() calls (bug-1733's fix note already confirms this exact leak signature -- 456 bytes, 29 allocations -- present byte-for-byte identically via an independent git-stash A/B rebuild in an earlier, unrelated session). LIVE-VERIFIED (2026-09-08, follow-up session). Added ug1_global_mem to limits.yaml -- a permanent, real GLOBAL limit on \"mem\" (a per-host resource, not \"slots\"), capping ug1's total cluster-wide mem usage at 5,000,000,000 -- exactly the previously-hypothetical scenario this bug was about, now real example config instead of untested. Fresh single-node deploy, 3 jobs submitted as user1 (ug1 member) each requesting rusage[mem=2000000000] (2GB) on queue normal: scheduler trace confirms ug1_global_mem matching (rc=1) and, for the first time, correctly computing a nonzero job-wide-total wanted value via calc_wanted_host_resval_job_totals() -- 'unsatisfy_ resource 36 total 5000000000.000000 inuse 4000000000.000000 uncomit 0.000000 wanted 2000000000.000000' / 'unsatisfy_ unmet resource 36 (limit=ug1_global_mem) shortfall 1000000000.000000' -- correctly detecting the 3rd job (4GB inuse + 2GB wanted = 6GB > 5GB cap) and PENDing it until an earlier job's mem freed up. Deploy config validated cleanly with zero new warnings. This closes the residual live-verification gap entirely -- bug-1790 is now proven correct for a resource other than \"slots\", not just unit-tested. REFACTORED (2026-09-08, same follow-up session, user-driven design review): the fix above worked and was live-verified, but put limit-plugin-specific logic (computing a job-total for a per-host resource, needed only when some GLOBAL limit references it) inside the ALLOCATOR plugins (seqjob_prepare_allocation()/parjob_prepare_allocation()) -- the wrong owner: those plugins should only ever compute what the job itself wants for capacity-satisfy purposes, and the duplication across two allocator plugins (a third would need it too) plus unconditionally computing totals for every per-host resource regardless of whether any limit needs one were both symptoms of that. Moved ownership to sched_fill_limit_requests() (resource.c) itself, called once centrally from plugin_prepare_allocation() regardless of which allocator ran: removed calc_wanted_host_resval_job_totals() and its calls from both allocator plugins entirely; added calc_wanted_job_total_resval() (resreq.c) -- a single-resource on-demand lookup (by name, no resmgr/key_find needed since the caller already has the resolved xce_resource_t), computing the SAME job-wide-total formula only for the one (scope, underlying-resource) combination that needs it (a GLOBAL limit on a per-host resource) -- the other three combinations are unchanged, still reading straight out of alloc_req. sched_fill_limit_requests() gained back rusages/nhosts/nslots parameters (undoing part of this session's earlier signature-trimming, but for a genuine need this time, not a re-added unused one); plugin.c's two call sites pass obj->u.primary->resreq->rusages (guarded on obj->tag == XCE_RESREQ_PRIMARY -- NULL for COMPOUND/ALTER, matching this codebase's existing scope convention for those tags) and ctx->nhosts/ctx->nslots (already correctly set by whichever allocator just ran, same invocation). All 23 testsuite link-satisfying stubs mechanically updated to the new 5-arg signature; the two allocator-level regression tests added by the first pass (test_seqjob_allocator.c/test_parjob_combinatorial.c) removed -- the behavior they proved no longer exists by design -- and test_resource_accounting.c's test_fill_limit_requests_global_limit_on_host_resource_reads_host rewritten to exercise the new on-demand path directly (job rusages instead of pre-staged .host, nslots=3 to prove the job-WIDE total is used, matching what the removed parjob-specific test proved). Full xce-build.sh all asan --incremental: 94/94 Ok, zero regressions. Live re-verified against the SAME ug1_global_mem deploy/scenario: byte-identical trace ('unsatisfy_ resource 36 total 5000000000.000000 inuse 4000000000.000000 uncomit 0.000000 wanted 2000000000.000000' / 'shortfall 1000000000.000000') to the pre-refactor result, confirming behavior-preserving. REGRESSION CAUGHT AND FIXED (2026-09-08, same follow-up session, user prompted \"should we regression the test of gitea#46\" after the ownership refactor above). calc_wanted_job_total_resval() scans `rusages` by name -- but \"slots\" is NEVER a declared rusage entry in this codebase: seqjob_prepare_allocation()/parjob_prepare_allocation() derive its want directly from the job's own task count (1.0 for seqjob, always maxTasks==1; (float)ctx->nslots for parjob), via their own unconditional key_find(\"slots\", ...) + store, entirely outside the normal rusages loop -- confirmed by the user (\"all resources except slots should be included in the rusage[] for request\"). The refactor's on-demand lookup therefore always missed for \"slots\" and silently left `val` at its 0.0 default. Live-confirmed against a fresh deploy of the ORIGINAL gitea#46 scenario (9 plain sleep jobs, user1, queue normal, no -R flag -- exactly the issue's own repro): all 3 slots-based limits (limit1/ug1_slots/ug1_normal_slots) matched correctly (rc=1), but ZERO unsatisfy_ lines appeared anywhere and all 9 jobs dispatched with no cap enforcement at all -- gitea#46's exact original symptom, reintroduced by the refactor. The refactor's own new unit test had masked this: it artificially added a \"slots\" entry to its test rusages, which doesn't reflect production reality. FIX: sched_fill_limit_requests() (resource.c) now falls back to `nslots` directly (the SAME value both allocators already store as \"slots\"'s own per-host want) whenever the rusages lookup misses AND the resource is named \"slots\" -- the one documented exception to the rusages-lookup rule, matching this codebase's own existing \"slots\" special-casing convention (both allocators already key_find it by literal name, outside their normal rusages-driven computation). New test test_fill_limit_requests_global_limit_on_slots_uses_nslots (test_resource_accounting.c) proves the fallback with NO \"slots\" entry in rusages at all, matching the real shape; the original test renamed/reworked to use \"mem\" instead, so it now genuinely exercises the general on-demand path rather than accidentally exercising (and masking a gap in) the special case. Full xce-build.sh all asan --incremental: 94/94 Ok. Live re-verified against BOTH the original gitea#46 scenario (byte-identical to the pre-refactor trace: 'unsatisfy_ resource 35 total 3.000000 inuse 3.000000 uncomit 0.000000 wanted 1.000000' / 'shortfall 1.000000' for limit1) and the ug1_global_mem scenario from the refactor's own prior verification -- both correct. SIMPLIFIED (2026-09-08, same follow-up session, user insight: \"alloc_req should have enough information? slots should be added into it?\"). The nslots-fallback fix above worked but was an unnecessary name-based special case: alloc_req->host ALREADY holds exactly \"slots\"'s job-wide total during the global phase -- both allocators' own unconditional \"add slots\" step stages it there regardless of phase, and plugin_prepare_allocation() (plugin.c) never resets .host during the global phase (confirmed: sched_ctx_init(), schedobj.c, gives every job a fresh, empty alloc_req, so by the time sched_fill_limit_requests() runs -- right after the allocator's own hook, same invocation -- .host contains exactly \"slots\"'s total and nothing else). Replaced the strcmp(vres->res->name, \"slots\") check with reading alloc_req->host FIRST (get_resval_by_id()) and falling back to the rusages-based on-demand computation only when nothing is staged there (val stays the function's 0.0 default, get_resval_by_id()'s own \"not found\" signal -- safe here since \"slots\" is never legitimately 0: seqjob only runs for maxTasks==1, parjob only for maxTasks>1, so nslots is always >=1). This is data-driven, not name-based -- matching the same design philosophy bug-1789's own earlier flag-based-dispatch revision established -- and generalizes for free to any future resource an allocator plugin similarly pre-stages outside its normal rusages loop, with no further sched_fill_limit_requests() changes needed. Test renamed test_fill_limit_requests_global_limit_on_slots_reads_host, now pre-staging alloc_req->host exactly as the allocators do (rather than relying on the removed nslots fallback) with rusages left empty, matching production exactly. Full xce-build.sh all asan --incremental: 94/94 Ok. Live re-verified against BOTH the original gitea#46 scenario and the ug1_global_mem scenario: byte-identical scheduler traces to every prior verification round, confirming this simplification is behavior-preserving.", @@ -25860,21 +13148,25 @@ "global-limit", "pre-existing", "severity-low", - "fixed", "live-verified" ], "related_bugs": [ "bug-1789" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1790.md" }, { "id": "bug-1791", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-09T08:50:58.000Z", - "error_message": "controller_unit_test: interruptcall.c:142: xcetest_worker_main: Assertion \\`unlock_rc == 0' failed (pthread_mutex_unlock returned EPERM)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller_unit_test: interruptcall.c:142: xcetest_worker_main: Assertion \\`unlock_rc == 0' failed…\n\ncontroller_unit_test: interruptcall.c:142: xcetest_worker_main: Assertion \\`unlock_rc == 0' failed (pthread_mutex_unlock returned EPERM)", "file": "src/controller/testsuite/interruptcall.c", "root_cause": "xce-test-cli-ux task 1.5's per-connection lock registry initialized each connection's pthread_mutex_t with PTHREAD_MUTEX_ERRORCHECK, intended to catch double-unlock/never-unlocked regressions. But this design's own ownership-handoff protocol (design.md/research.md's 'per-connection lock' decision) REQUIRES a legitimate cross-thread release on the abandonment path: the calling (REPL) thread acquires the lock via pthread_mutex_trylock(), but on abandonment the still-running detached WORKER thread is the one that must release it later, once fn() finishes -- ownership of 'release the lock' passes to the worker at the same moment ownership of the heap call block does. PTHREAD_MUTEX_ERRORCHECK (and _RECURSIVE) mutexes track a single owning thread and reject pthread_mutex_unlock() from any other thread with EPERM, which fired immediately on the very first same-key-serialization test.", "fix": "Switched the per-connection lock registry (xcetest_conn_lock_for() in interruptcall.c) to initialize each mutex with the DEFAULT type (pthread_mutex_init(&node->mutex, NULL)) instead of PTHREAD_MUTEX_ERRORCHECK. glibc default (PTHREAD_MUTEX_NORMAL-equivalent) mutexes do not track a per-thread owner, so a worker thread may validly unlock a mutex the REPL thread locked. Kept the assert(pthread_mutex_unlock(...) == 0) safety-net checks at every release site (computed outside the assert() expression itself, so the unlock call is never skipped under NDEBUG) as a lower-cost guard against genuine corruption, without relying on error-checking-mutex semantics that conflict with the intentional cross-thread handoff. SUPERSEDED 2026-09-09 (task 1.5 remediation round 1, independent review): an external reviewer correctly pointed out that POSIX classifies non-owning-thread pthread_mutex_unlock() as undefined behavior for PTHREAD_MUTEX_NORMAL/DEFAULT too, not just ERRORCHECK/_RECURSIVE -- NORMAL is merely *permitted* not to detect the violation, not declared well-defined for it. The DEFAULT-mutex fix above was therefore empirically safe on XCE's glibc/NPTL-exclusive Nix build but relied on non-portable, POSIX-UB behavior, under-documented as such. Replaced the pthread_mutex_t per-connection lock with a POSIX semaphore (sem_t, binary, sem_init(&sem, 0, 1)) throughout: sem_trywait() in place of pthread_mutex_trylock() (EAGAIN in place of EBUSY) in the interruptible poll loop, and sem_post() in place of pthread_mutex_unlock() at every release site (both the caller-normal-completion path and the worker-abandoned-cleanup cross-thread path). sem_t has no ownership concept, so this cross-thread release is now fully POSIX-defined with no UB reliance at all, at the same code complexity. All 1285 controller_unit_test tests (including the 3 task-1.5-specific tests) continued to pass unchanged after the migration.", @@ -25890,20 +13182,23 @@ "design-review-caught-by-test", "posix-ub", "sem_t", - "semaphore-migration", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "semaphore-migration" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1791.md" }, { "id": "bug-1792", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-09T12:40:00.000Z", - "error_message": "test_test_cli.c tab-completion test (xce-test-cli-ux task 2.3): Tab keystroke produced zero effect (no bell, no redraw, no completion) when driven through the existing clitest_spawn() pty harness -- 'wo' + Tab + Enter submitted literally \"wo\" (\"wo: No such command for xce test.\"), never \"worker\".", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "test_test_cli.c tab-completion test (xce-test-cli-ux task 2.3): Tab keystroke produced zero effect…\n\ntest_test_cli.c tab-completion test (xce-test-cli-ux task 2.3): Tab keystroke produced zero effect (no bell, no redraw, no completion) when driven through the existing clitest_spawn() pty harness -- 'wo' + Tab + Enter submitted literally \"wo\" (\"wo: No such command for xce test.\"), never \"worker\".", "file": "src/controller/testsuite/test_test_cli.c", "root_cause": "clitest_spawn() (task 1.7) deliberately clears kernel ECHO on the pty slave BEFORE forking, documented at the time as affecting ONLY the line discipline's own re-echo of injected bytes, 'not readline()/libedit's own prompt/redraw output, which stays intact'. An A/B probe (standalone C program linked against this project's exact pinned libedit, driven both ways via a raw openpty()+fork() harness, controlling for controlling-terminal presence too) showed that assumption does not hold for THIS libedit build: when the slave's ECHO bit already reads off at the moment readline() is invoked, libedit treats the session as a silent/non-interactive (password-style) input mode and suppresses ALL of its own live editing feedback, not just kernel echo -- Tab is accepted as a no-op, the line is only ever consumed whole on Enter, and no bell/redraw/candidate-list ever appears. With kernel ECHO left at its default-on state (libedit reconfigures the tty itself once its own readline() call takes over, taking on echo/redraw duties without any raw kernel double-echo reaching the captured stream), the identical Tab sequence produced the expected bell + in-place completion + ambiguous-match candidate listing. Controlling-terminal setup (setsid()/TIOCSCTTY) was also probed and made no observable difference -- ECHO state was the sole deciding factor.", "fix": "Refactored clitest_spawn() into clitest_spawn_ex(sess, path, argv, disable_echo) with the ECHO-clearing block made conditional on the new parameter; clitest_spawn() itself is preserved as a one-line wrapper calling clitest_spawn_ex(..., true), so every existing (output-only) caller is unaffected. The new xce-test-cli-ux task 2.3 tab-completion test calls clitest_spawn_ex(..., false) directly to keep kernel ECHO on, which restored full interactive completion behavior end-to-end (single-match auto-complete, ambiguous-match double-Tab listing, all verified via the real xce_test binary). Documented the finding in both clitest_spawn_ex()'s own comment and the file's top-of-file API summary so future tasks in this spec (3.5, 4.1-4.4) that add more interactive-feedback-dependent tests to this same file know to use clitest_spawn_ex(..., false) rather than rediscovering this the hard way.", @@ -25917,21 +13212,23 @@ "xce-test-cli-ux", "task-2.3", "test_test_cli.c", - "silent-input-mode", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "silent-input-mode" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1792.md" }, { "id": "bug-1793", "status": "open", "status_updated": "2026-09-24", "status_note": "xce_test join maxswap/maxtmp defaults are stale byte values (join_worker_args.c)", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-09T16:24:19Z", - "error_message": "xce_test's worker->join command's maxswap (1024) and maxtmp (2048) defaults have the same stale-MB-scale bug as bug-1557's maxmem -- discovered while fixing bug-1557 (xce-test-cli-ux task 2.6)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_test's worker->join command's maxswap (1024) and maxtmp (2048) defaults have the same…\n\nxce_test's worker->join command's maxswap (1024) and maxtmp (2048) defaults have the same stale-MB-scale bug as bug-1557's maxmem -- discovered while fixing bug-1557 (xce-test-cli-ux task 2.6)", "file": "src/controller/testsuite/join_worker_args.c (join_worker_parse_attrs, hostattrs_t default initialization)", "root_cause": "Same root cause as bug-1557: maxswap/maxtmp were reasonable defaults (1KB/2KB) back when these fields were a smaller unit, but an earlier session converted maxswap/maxtmp to bytes-canonical (confirmed via resources.yaml 'max swap/tmp disk space in bytes' and hardware_monitor.c's raw-byte sysinfo()/statvfs() computations) without updating this test-tool default. 1024 bytes of swap / 2048 bytes of tmp are just as nonsensical as the old maxmem=64 bytes.", "fix": "NOT FIXED -- deliberately deferred (xce-test-cli-ux task 2.6). USAGE_JOIN_WORKER's help text currently documents these exact stale values accurately, so fixing only the code without also fixing the docs would create a new code/docs mismatch outside task 2.6's Requirement-12.1-scoped mandate (which covers maxmem only). Recommended: a coordinated fix changing both defaults (e.g. maxswap to a byte-scale value matching typical swap sizing, maxtmp similarly) AND USAGE_JOIN_WORKER's documented defaults, together. A regression test (test_join_worker_sibling_defaults_unchanged in test_join_worker_args.c) currently locks in today's stale values specifically so a future fix here doesn't collide with bug-1557's maxmem-specific test assertions -- update/remove that test's maxswap/maxtmp assertions when this bug is actually fixed.", @@ -25942,23 +13239,25 @@ "cmd_join_worker", "stale-default", "units", - "unfixed", - "deferred", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "deferred" ], "related_bugs": [ "bug-1557" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1793.md" }, { "id": "bug-1794", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-09-09T19:03:51Z", - "error_message": "xce_test cmd_jobrun crashes (ASAN allocation-size-too-big abort in alloc_packet, requested size ~0xffffffffc3c3c4b0) when run live against a real controller, after xce-test-cli-ux task 3.2 wrapped its xcecli_jobdispatch() call through the interruptible-call adapter (heap-allocated ctx block instead of a stack-local xce_job_alloc_t).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce_test cmd_jobrun crashes (ASAN allocation-size-too-big abort in alloc_packet, requested size…\n\nxce_test cmd_jobrun crashes (ASAN allocation-size-too-big abort in alloc_packet, requested size ~0xffffffffc3c3c4b0) when run live against a real controller, after xce-test-cli-ux task 3.2 wrapped its xcecli_jobdispatch() call through the interruptible-call adapter (heap-allocated ctx block instead of a stack-local xce_job_alloc_t).", "file": "src/controller/testsuite/test.c (cmd_jobrun, fill_job_alloc, jobrun_call_t/jobrun_thunk)", "root_cause": "PRE-EXISTING, latent: fill_job_alloc() (test.c) only ever sets xce_job_alloc_t.nhosts/.allocs -- it never initializes the struct's other two fields, nres_shared and allocs_shared (xce/types.h). Before task 3.2, xce_job_alloc_t jalloc was a plain stack-local in cmd_jobrun, so those two never-set fields held whatever incidental value that stack frame happened to have -- harmless in practice, apparently usually small/zero. Task 3.2 moved this struct into a heap-allocated jobrun_call_t block (required by the interruptible-call adapter's contract: fn must only touch heap-owned memory, never the caller's stack, since an abandoned call's worker thread can outlive the caller's own stack frame) allocated via plain malloc(), which leaves ALL bytes as genuine heap garbage -- confirmed live (real single-node deploy, jobrun -j -m worker-1:1): the garbage nres_shared/allocs_shared corrupted xce_jobdispatch_req's wire-size computation in lib/request.c, producing a multi-exabyte allocation request that ASAN correctly aborted on.", "fix": "FIXED (2026-09-09, xce-test-cli-ux task 3.2). Every per-call-site heap block (all ~26 _call_t typedefs test.c's interrupt-wrapping introduces) now allocates via calloc(1, sizeof(*call)) instead of malloc(sizeof(*call)), zero-initializing every field uniformly rather than auditing each wrapped request struct for a similarly-never-fully-set field one at a time -- the same defensive default an accidentally-zero stack frame was already silently relied on for. Verified by re-running the exact live repro (jobrun against a real single-node deploy) after the fix: completes normally with no ASAN error.", @@ -25973,22 +13272,25 @@ "asan", "interruptcall", "xce-test-cli-ux", - "task-3.2", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "task-3.2" ], "related_bugs": [ "bug-1791" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1794.md" }, { "id": "bug-1795", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-09-09T20:10:00Z", - "error_message": "leave_ctx() (test.c, called by cmd_up/cmd_quit) could free a connection's api_ctx_t (xcecli_stop()+xcecli_ctx_fini(), src/lib/client.c) while an abandoned worker thread from a prior Ctrl-C-interrupted RPC on that SAME connection was still running and still reading/writing it -- a genuine use-after-free, found by kiro-review during xce-test-cli-ux task 3.2's own round-1 review (not reproduced live/under ASAN on that run; identified structurally by tracing the code paths).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "leave_ctx() (test.c, called by cmd_up/cmd_quit) could free a connection's api_ctx_t…\n\nleave_ctx() (test.c, called by cmd_up/cmd_quit) could free a connection's api_ctx_t (xcecli_stop()+xcecli_ctx_fini(), src/lib/client.c) while an abandoned worker thread from a prior Ctrl-C-interrupted RPC on that SAME connection was still running and still reading/writing it -- a genuine use-after-free, found by kiro-review during xce-test-cli-ux task 3.2's own round-1 review (not reproduced live/under ASAN on that run; identified structurally by tracing the code paths).", "file": "src/controller/testsuite/test.c (leave_ctx), src/controller/testsuite/interruptcall.c/.h (xcetest_wait_connection_idle, new), src/lib/client.c (xcecli_wakeup, xcecli_stop, xcecli_ctx_fini, read-only for this fix)", "root_cause": "Task 3.2's own interruptible-call adapter (tasks 1.4/1.5, interruptcall.c) runs every blocking xce_* RPC on a detached worker thread; when the operator interrupts (Ctrl-C) a blocked call, xcetest_run_interruptible() returns -EINTR to the REPL immediately, but the worker thread it spawned keeps running -- possibly for up to its own ~10s cbuf_timedget deadline -- reading/writing whatever connection state its ctx closes over. Before this task, the REPL was fully synchronous, so leave_ctx() (test.c, shared by cmd_up's '..' and cmd_quit) could never run while an RPC was outstanding on that connection; task 3.2's adapter broke that invariant by letting control return to the prompt WHILE a worker is still abandoned-but-running. leave_ctx() already called xcecli_wakeup(active_ctx->ctx, -EINTR) before tearing the connection down, but that is only a best-effort nudge (it wakes the underlying condvar wait sooner than the worker's own deadline) -- nothing forced leave_ctx() to actually WAIT for that woken worker to finish before proceeding to xcecli_stop() (tears down the live socket + failover lock) and xcecli_ctx_fini() (unconditionally free()s the whole api_ctx_t). If the operator's very next action after abandoning a hung command is 'up' or 'quit' -- an entirely ordinary thing to do -- the still-running worker can be mid-read/write on memory that just got freed out from under it. This directly contradicts the Interruptible Call Adapter's own documented invariant ('at most one worker thread -- running or abandoned-but-still-running -- is ever active per connection at a time', interruptcall.h/design.md) -- that invariant was enforced only BETWEEN xcetest_run_interruptible() calls (via the per-connection semaphore registry, tasks 1.4/1.5), never against connection TEARDOWN, since leave_ctx() never itself goes through xcetest_run_interruptible().", "fix": "FIXED (2026-09-09, xce-test-cli-ux task 3.2 remediation round 1). Added xcetest_wait_connection_idle(void* conn_key) to interruptcall.c/.h, reusing the EXACT SAME per-connection semaphore registry lookup (xcetest_conn_lock_for()) that xcetest_run_interruptible() already builds and maintains -- not a new/parallel mechanism. It does a plain BLOCKING sem_wait() (deliberately not the interruptible sem_trywait()-poll variant used elsewhere -- this is a narrow, safety-critical wait where correctness outranks Ctrl-C responsiveness, and it resolves quickly in practice since xcecli_wakeup() already nudges the worker) then immediately sem_post()s the semaphore back rather than holding it -- the semantic is 'prove no worker is active on this connection right now', not 'hold the connection locked'. leave_ctx() now calls xcetest_wait_connection_idle(ctx->ctx) right after the existing xcecli_wakeup() call and right before xcecli_stop()/xcecli_ctx_fini() -- since both cmd_up and cmd_quit call this SAME shared leave_ctx(), both paths are covered by a single call site. New black-box regression test test_interrupt_then_immediate_teardown_no_uaf (test_test_cli.c, reuses the existing clitest_start_blackhole_listener() harness helper) interrupts a blocked hello RPC against a black-hole TCP peer and sends 'up' IMMEDIATELY afterward -- with no settling wait for the interrupted prompt to redisplay first, unlike the sibling interrupt test -- to race leave_ctx()'s teardown against the abandoned worker on purpose; asserts the session exits only via a normal (non-signal) status and that stderr never contains an 'AddressSanitizer' diagnostic anywhere across the whole session. Verified under the asan build profile.", @@ -26006,23 +13308,25 @@ "semaphore", "xce-test-cli-ux", "task-3.2", - "kiro-review-finding", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "kiro-review-finding" ], "related_bugs": [ "bug-1794" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1795.md" }, { "id": "bug-1796", "status": "open", "status_updated": "2026-09-24", "status_note": "qctrl/hostctrl perror() prints stale errno instead of RPC rc (src/harness/test.c)", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-09T22:00:00Z", - "error_message": "cmd_qctrl/cmd_hostctrl/cmd_jobkill's RPC-failure branches perror() a fixed label (e.g. \"xcecli_queuectrl\") without ever setting errno to reflect the RPC's actual returned status, so the printed reason (e.g. \"Interrupted system call\", \"Resource temporarily unavailable\", \"Inappropriate ioctl for device\") is whatever errno happened to be left over from an unrelated earlier libc call in the same process, not the real failure reason.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "cmd_qctrl/cmd_hostctrl/cmd_jobkill's RPC-failure branches perror() a fixed label (e.g\n\ncmd_qctrl/cmd_hostctrl/cmd_jobkill's RPC-failure branches perror() a fixed label (e.g. \"xcecli_queuectrl\") without ever setting errno to reflect the RPC's actual returned status, so the printed reason (e.g. \"Interrupted system call\", \"Resource temporarily unavailable\", \"Inappropriate ioctl for device\") is whatever errno happened to be left over from an unrelated earlier libc call in the same process, not the real failure reason.", "file": "src/controller/testsuite/test.c (cmd_qctrl, cmd_hostctrl, cmd_jobkill RPC-error return paths)", "root_cause": "Each of these commands does `if (rc < 0) { perror(\"xcecli_...\"); return rc; }` where `rc` is the RPC's own negative-errno-style return value (e.g. a clean -ENOENT for \"no such queue/host\"), but perror() always reads the CURRENT process errno, not `rc` -- and nothing in this branch ever does `errno = -rc` first. Found during xce-test-cli-ux task 4.2's live confirmation-gate verification: a bypassed qctrl/hostctrl RPC against a deliberately bogus (nonexistent) queue/host name printed different, unrelated-looking error text across different runs of the SAME scenario, root-caused (and independently reproduced 3 times by the task 4.2 reviewer) to stale errno left over from prior, unrelated syscalls/signal-decline paths in the same process -- e.g. a destructive-command Ctrl-C decline explicitly sets errno = EINTR at its own call site and never resets it, so a LATER unrelated RPC failure in the same session then misreports itself as \"Interrupted system call\". Confirmed pre-existing (commit 41f56fa6e, 2025-11-05), predating the whole xce-test-cli-ux spec; NOT a defect in the interruptible-call adapter (tasks 1.4/3.2) or the confirmation gate (task 2.1/3.3) -- both were independently confirmed to behave correctly in every case tested, this is purely a misleading-diagnostic-text bug.", "fix": "NOT YET FIXED (found and deliberately deferred -- out of boundary for task 4.2, a pure live-verification task with no source changes permitted). Recommended: set `errno = -rc;` immediately before each `perror(...)` call on these RPC-error paths (or replace with `fprintf(stderr, \"...: %s\\n\", strerror(-rc));` to avoid the errno indirection entirely), and audit other cmd_* handlers for the same perror()-without-errno-set pattern.", @@ -26039,21 +13343,23 @@ "xce-test-cli-ux", "task-4.2", "pre-existing", - "deferred", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "deferred" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1796.md" }, { "id": "bug-1797", "status": "open", "status_updated": "2026-09-24", "status_note": "cmd_qctrl leaks strdup'd queue name on RPC error (src/harness/test.c)", + "component": "controller", + "severity": "leak", "timestamp": "2026-09-09T22:00:00Z", - "error_message": "cmd_qctrl leaks its strdup()'d queue name on the xcecli_queuectrl() RPC-error return path (ASAN LeakSanitizer-confirmed, 14 bytes).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 3, + "error_message": "cmd_qctrl leaks its strdup()'d queue name on the xcecli_queuectrl() RPC-error return path (ASAN…\n\ncmd_qctrl leaks its strdup()'d queue name on the xcecli_queuectrl() RPC-error return path (ASAN LeakSanitizer-confirmed, 14 bytes).", "file": "src/controller/testsuite/test.c (cmd_qctrl)", "root_cause": "cmd_qctrl does `queue = strdup(optarg)` while parsing -q, then on `if (rc < 0) { perror(\"xcecli_queuectrl\"); return rc; }` (the RPC-error branch) returns without `free(queue)` -- only the success path frees it. Distinct from the adjacent, intentional non-free on the EINTR/abandoned-worker path (ownership legitimately transferred there per the interruptible-call adapter's contract). Found during xce-test-cli-ux task 4.2's live confirmation-gate verification (a bypassed qctrl against a bogus queue name triggered this exact RPC-error branch under the asan build). Confirmed pre-existing (commit 41f56fa6e, 2025-11-05), predating the whole xce-test-cli-ux spec and untouched by task 3.1's flagspec migration; not caused by any change in this spec.", "fix": "NOT YET FIXED (found and deliberately deferred -- out of boundary for task 4.2, a pure live-verification task with no source changes permitted). Recommended: add `free(queue);` on the RPC-error return path (or restructure with a single cleanup label), and check `cmd_hostctrl`/`cmd_jobkill`'s sibling strdup'd-argument fields for the same pattern.", @@ -26067,24 +13373,25 @@ "xce-test-cli-ux", "task-4.2", "pre-existing", - "deferred", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "deferred" ], "related_bugs": [ "bug-1796" ], - "occurrences": 3, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1797.md" }, { "id": "bug-1798", - "gitea_synced_status": "fixed", - "gitea_issue": 267, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-10T06:57:00Z", - "error_message": "Fresh (from-scratch) build fails intermittently with `fatal error: controller.capnp.h: No such file or directory`, reported live by the user running `./tools/xce-build.sh all debug` -- first for a libxce-internal consumer (utils.c.o), then AGAIN after the first fix landed, this time for scheduler/src/sim_record.c.o and scheduler/src/core_events.c.o (files entirely outside libxce). This is the root-cause fix for a class of failures already recorded across many prior recurrences in this buglog (see related_bugs) as an unexplained/unfixed 'ninja-race' -- this entry identifies and fixes the actual missing-dependency-edge cause at the meson.build level, in three parts (see fix), and additionally audited the whole project for other instances of this same bug class.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Fresh (from-scratch) build fails intermittently with `fatal error: controller.capnp.h: No such…\n\nFresh (from-scratch) build fails intermittently with `fatal error: controller.capnp.h: No such file or directory`, reported live by the user running `./tools/xce-build.sh all debug` -- first for a libxce-internal consumer (utils.c.o), then AGAIN after the first fix landed, this time for scheduler/src/sim_record.c.o and scheduler/src/core_events.c.o (files entirely outside libxce). This is the root-cause fix for a class of failures already recorded across many prior recurrences in this buglog (see related_bugs) as an unexplained/unfixed 'ninja-race' -- this entry identifies and fixes the actual missing-dependency-edge cause at the meson.build level, in three parts (see fix), and additionally audited the whole project for other instances of this same bug class.", "file": "src/lib/meson.build", "root_cause": "THREE separate but related missing-dependency-edge gaps, all in src/lib/meson.build, all following the identical shape -- a multi-output custom_target() (a .c + a paired .h) whose .h output is reached by some OTHER file only via #include, with libxce_src or libxce_dep only ever referencing the .c output's index [0]: (1) the capnp code-generation custom_target (`gen_srcs`, output controller.capnp.c/.h) -- `libxce_src` only listed `gen_srcs[0]`, so libxce's OWN other .c files (utils.c, hook.c, threshold.c, xlist.c, window.c, xceapi.c, wrap.c, zarray.c, htab.c, xstring.c, etc.) that #include controller.capnp.h had no build edge forcing codegen to finish first. (2) The SAME gen_srcs custom_target was also unreachable, via any dependency edge, from EXTERNAL consumers: `libxce_dep = declare_dependency(link_with: libxce)` had no `sources:` kwarg, so `link_with` only ordered a consumer's LINK step after libxce was built, never a consumer's own COMPILE steps -- and controller/worker/scheduler all reach controller.capnp.h directly via a raw `-Ilib`/`-I../lib` include path, formally depending on nothing. Fix (1) alone only closed the race for files inside libxce itself, which is why the user's SECOND from-scratch `all debug` run still failed, now in scheduler instead. (3) Found via a follow-up audit (user asked 'check source code generated by peg'): the SAME shape recurs for all four packcc PEG-parser custom_targets (section_parser, rusage_parser, xce_resreq_parser, xce_select_parser -- each producing a .c + a paired .h: section.h, rusage.h, xce_resreq.h, xce_select_peg.h). `libxce_src` only listed each parser's `[0]` (.c) index; `lib/resreq.c` (confirmed via a project-wide grep to be the ONLY consumer of any of these four headers, entirely inside src/lib/, so no libxce_dep-level fix was needed for this part) #includes all four headers but has no declared build edge to any of the four codegen steps. On an INCREMENTAL build all three gaps are invisible (the generated files from a prior build already exist on disk), which is why this session's own extensive controller-only `--incremental` rebuilds throughout the xce-test-cli-ux spec never hit any of them -- they only manifest on a genuinely from-scratch build (`rm -rf` + fresh `meson setup`, which `xce-build.sh ` does by default without `--incremental`), and even then only when ninja's scheduler happens to pick an unlucky order under parallelism -- explaining the long history of the capnp variant being reported as an intermittent, unfixed 'ninja-race' across at least 5 prior recorded recurrences (2026-07-16 through 2026-07-26) with no meson.build-level fix ever applied, only the workaround 'retry the build once'. A project-wide search confirmed these are the ONLY multi-output C/H code-generation custom_target()s anywhere in the codebase (every other custom_target() -- xceadmin, frontend/apiserver, frontend/xcecli, autoscaler/providers -- produces a single Go-binary/stamp output with no equivalent header-only-reached-via-#include hazard).", "fix": "FIXED (2026-09-10), in three parts, all in src/lib/meson.build. Part 1: changed `libxce_src`'s `gen_srcs[0]` element to the unindexed `gen_srcs`, closing gap (1) for capnp. Part 2 (the fix that actually closes the general, project-wide capnp race): added `sources: [gen_srcs]` to `libxce_dep`'s `declare_dependency()` call, closing gap (2) -- `declare_dependency()`'s `sources:` kwarg exists specifically for this case (generated files that must be built before ANY consumer of the dependency compiles), and since controller/worker/scheduler all already list `libxce_dep` in their own `dependencies:`, this single change propagates the missing build-order edge to every current and future consumer with no per-component changes needed. Part 3: changed `libxce_src`'s `section_parser[0]`/`rusage_parser[0]`/`xce_resreq_parser[0]`/`xce_select_parser[0]` elements to their unindexed forms, closing gap (3) for all four PEG parsers (no libxce_dep change needed here since nothing outside lib/ consumes these headers). Every part uses the same pattern: passing a multi-output custom_target unindexed into a `sources:`-accepting list still compiles only its recognized-source (.c) output, while ordering the WHOLE enclosing target's (or dependency's every consumer's) build after ALL of that custom_target's outputs are generated. (Tried first, and reverted: a `depends: [gen_srcs]` kwarg directly on the `static_library()` call -- rejected immediately by this project's pinned Meson 1.9.1 with 'Got unknown keyword arguments \"depends\"', not supported on `static_library()`/`build_target()` in this version.) Verified incrementally as each gap was found: two consecutive from-scratch `lib debug` builds passed with Part 1 alone (313/313 lib tests) -- but this was INSUFFICIENT, confirmed live by the user's own second `./tools/xce-build.sh all debug` run still failing in scheduler; after Part 2, two consecutive full from-scratch `./tools/xce-build.sh all debug` builds (1173/1173 targets each, the user's own exact repro command) both succeeded with zero failures and no retry; after adding Part 3 (found via the follow-up PEG audit), a third consecutive full from-scratch `all debug` build also succeeded clean (1173/1173), plus a fresh `lib debug` build + full `xcelib_test` run (313/313) and a fresh incremental `controller asan --incremental` rebuild. That incremental rebuild's FIRST run showed `1331 Tests 1 Failures` in `test_smoke_launch_send_echo_capture_stdout_and_exit_code` ('xce_test did not exit within the harness's safety timeout after quit') -- investigated and confirmed to be an unrelated, system-load-induced timing flake (the controller build ran concurrently with/immediately after the just-completed full `all debug` build): re-running the SAME binary standalone on an otherwise-idle system passed this exact test and the full suite cleanly (1331/1331) twice in a row; this is a pre-existing PTY-harness timing sensitivity, not a defect introduced by any of the three meson.build parts, all of which only affect build-graph ordering, never runtime process-exit behavior. Three consecutive full-project clean-build successes do not, by the race's own probabilistic nature, provide absolute certainty against some other unlucky ninja ordering recurring -- but Parts 1-3 address the actual general-case structural root cause for every multi-output codegen step in the project (every consumer of libxce_dep, plus every internal libxce_src consumer, now has a real build-graph edge on the relevant codegen step), so this is expected to eliminate this entire race class project-wide rather than merely reduce its likelihood. Does not address the separate, distinct apiserver<-libxce.a CGO-link race noted in one prior recurrence (2026-07-26) -- that is a different meson target (frontend/apiserver's go_build custom_target referencing @BUILD_ROOT@/lib/libxce.a via an env-var string, not a `dependencies:`/`libxce_dep` relationship) and remains unfixed; also worth a follow-up ticket for the test_test_cli.c PTY-harness safety-timeout flakiness under system load, observed here but out of this bug's own scope.", @@ -26097,25 +13404,27 @@ "missing-dependency-edge", "libxce", "src/lib/meson.build", - "root-cause-fix", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "root-cause-fix" ], "related_bugs": [ "bug-1439" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1798.md", + "gitea_issue": 267, + "gitea_synced_status": "fixed" }, { "id": "bug-1799", - "gitea_synced_status": "fixed", - "gitea_issue": 268, "status": "fixed", "status_updated": "2026-09-24", "status_note": "25fb5c11", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-11T07:32:00Z", - "error_message": "User-reported (xce_scheduler -i -s , live cluster-captured replay /tmp/scheduler-replay-20260911-064117.replay, worker pc13v, 22 real slots): `info worker pc13v` at session=98 shows `resource slots: value=22.00 used=22.00` while the printed `assigned job:` list for that same worker sums to 26 slots (jobs [5]-[9] each *4 = 20, plus six single-slot jobs from a second batch = 6); the very next session (99, no new DISPATCH/PEND events, same assigned-jobs list) shows `used=26.00` -- exceeding the worker's own `value=22.00` capacity, an admission-invariant violation (max slots exceeded). Session 98's own event stream shows two back-to-back `WORKERS_PUSH`+`JOB_PUSH` (njobs=20) events for the same worker/jobids, consistent with a live scheduler-connection resync/reconnect being faithfully replayed (matching bug-1734/bug-1739's documented resync scenario), plus `sched_add_job failed ... rc=-17, merging state into existing tracked job instead of discarding` warnings for nearly every job in the resync batch, and one `resync claims RUN for jobid ...[7]/[8] with no allocation data -- refusing to merge a phantom RUN` (bug-1739's guard firing as designed).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (xce_scheduler -i -s , live cluster-captured replay…\n\nUser-reported (xce_scheduler -i -s , live cluster-captured replay /tmp/scheduler-replay-20260911-064117.replay, worker pc13v, 22 real slots): `info worker pc13v` at session=98 shows `resource slots: value=22.00 used=22.00` while the printed `assigned job:` list for that same worker sums to 26 slots (jobs [5]-[9] each *4 = 20, plus six single-slot jobs from a second batch = 6); the very next session (99, no new DISPATCH/PEND events, same assigned-jobs list) shows `used=26.00` -- exceeding the worker's own `value=22.00` capacity, an admission-invariant violation (max slots exceeded). Session 98's own event stream shows two back-to-back `WORKERS_PUSH`+`JOB_PUSH` (njobs=20) events for the same worker/jobids, consistent with a live scheduler-connection resync/reconnect being faithfully replayed (matching bug-1734/bug-1739's documented resync scenario), plus `sched_add_job failed ... rc=-17, merging state into existing tracked job instead of discarding` warnings for nearly every job in the resync batch, and one `resync claims RUN for jobid ...[7]/[8] with no allocation data -- refusing to merge a phantom RUN` (bug-1739's guard firing as designed).", "file": "src/scheduler/src/scheduler.c, src/scheduler/src/core_session.c", "root_cause": "Interaction between two independently-reasonable pieces of code that were never reconciled with each other. (1) bug-1751 (2026-09-01, worker-rejoin-resync task 5) deliberately added an `IS_WORKER_RECONCILING(w->status)` skip to `core_reset_workers()` (core_session.c:136-142, the sim/normal-mode-shared per-round resInUse-reset helper called from `xce_core_begin_session()`): a worker mid-resync has its `w->resInUse` zeroing SKIPPED that round, on purpose, so its in-flight usage isn't wiped while reconciliation is still in progress. (2) `sched_account_jobs()` (scheduler.c:1434-1461) is the ONLY thing that re-derives `resInUse`/`sharedInUse` each round -- it walks every job in `core->jobs` and, for each one with `JOB_IS_RUN(job->state) && job->accounts != NULL`, calls `sched_charge_account()` (an unconditional ADD via `sched_update_res_accounts(..., charge=1)`, resource.c:608-633) -- with NO equivalent RECONCILING check and NO reset of its own. Its own documented contract (core.h:311-322) is explicit that callers 'must call this strictly after the session's resval reset' -- a precondition bug-1751's reset-skip silently violates for exactly the RECONCILING window it targets. Consequence: for every round a worker stays RECONCILING (entered via a WORKERS_PUSH re-announce during a scheduler reconnect/resync, exactly what session 98's two WORKERS_PUSH events reflect), every currently-RUN job's slot charge on that worker gets RE-ADDED on top of the NOT-reset resInUse from the prior round, instead of the intended zero-then-resum. This produces exactly the observed shape: `used=` (accumulating additively) trails one round behind the true `assigned jobs` sum at first (22 vs 26 at session 98, since only some of the newly-RUN jobs' charges had been double-counted-in by that round's pass), then overtakes real capacity outright (26 at session 99). This is NOT the same mechanism as bug-1739 (JOBS_PUSH resync-merge phantom-RUN guard, which governs whether `job->allocs`/`state` get attached at all) nor bug-1624 (already-fixed dispatch-to-RUN synchronization gap) nor bug-1727 (already-fixed `info worker` display double-count of `value=`) -- all four live in the same resource-accounting neighborhood but are structurally distinct bugs. Root cause identified via static trace only (codegraph_explore + direct read of core_session.c/scheduler.c/core.h); not yet reproduced with an isolated minimal repro or instrumented trace confirming the exact round-by-round resInUse progression.", "fix": "NOT YET FIXED -- investigation only, per user request (this session's live replay-driven Q&A, not yet escalated to an implementation task). Likely fix direction: give `sched_account_jobs()` the SAME `IS_WORKER_RECONCILING` awareness `core_reset_workers()`/`sched_reset_worker_()` already have (bug-1751) -- either (a) skip charging jobs whose host is currently RECONCILING (mirroring the reset-skip so the pair stays consistent: neither reset nor resum touches a RECONCILING worker's resInUse until it clears), or (b) have `core_reset_workers()` NOT skip the reset for a RECONCILING worker but instead have `sched_account_jobs()` re-derive from job state alone regardless of RECONCILING (i.e. resInUse is always a clean per-round sum, and RECONCILING-safety is enforced elsewhere, e.g. admission control refusing new dispatches to a RECONCILING host, which per bug-1751/bug-1745 already exists as a separate candidate-filter check). Needs the worker-rejoin-resync spec's own design intent re-read (.kiro/specs/worker-rejoin-resync/) before picking (a) vs (b), plus a live repro (re-run this exact replay file with an instrumented sched_account_jobs() logging resInUse before/after each round) to confirm the round-by-round mechanism before implementing, and full TDD (scheduler-driver-equivalence steering requires normal-mode/sim-mode parity for any fix here).", @@ -26133,11 +13442,8 @@ "max-capacity-exceeded", "core_reset_workers", "bug-1751-interaction", - "not-fixed", "user-reported", - "found-live", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "found-live" ], "related_bugs": [ "bug-1751", @@ -26147,17 +13453,24 @@ "bug-1727", "bug-1613" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [ + "25fb5c11" + ], + "record": ".wolf/bugs/bug-1799.md", + "gitea_issue": 268, + "gitea_synced_status": "fixed" }, { "id": "bug-1800", - "gitea_synced_status": "fixed", - "gitea_issue": 269, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-11T07:55:00Z", - "error_message": "User request (live replay session, worker pc13v): `info worker`'s CONSUMABLE value=/used= line for a bytes-valued resource (mem/swp/tmp) printed raw byte counts with no unit scaling -- e.g. `resource mem: value=20684476416.00 used=87960930222080.00` -- unlike every OTHER bytes-valued field in the same listing (maxmem/maxswap/maxtmp already render `value=33182072832.000000 (30.90 GB)`, and mem/tmp/swp themselves render human-scaled via the Load Metrics path when NOT flagged consumable). bug-1727's own header comment had explicitly scoped this out: 'no bytes-valued CONSUMABLE resource is exercised anywhere in this codebase's tests today.'", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User request (live replay session, worker pc13v): `info worker`'s CONSUMABLE value=/used= line for…\n\nUser request (live replay session, worker pc13v): `info worker`'s CONSUMABLE value=/used= line for a bytes-valued resource (mem/swp/tmp) printed raw byte counts with no unit scaling -- e.g. `resource mem: value=20684476416.00 used=87960930222080.00` -- unlike every OTHER bytes-valued field in the same listing (maxmem/maxswap/maxtmp already render `value=33182072832.000000 (30.90 GB)`, and mem/tmp/swp themselves render human-scaled via the Load Metrics path when NOT flagged consumable). bug-1727's own header comment had explicitly scoped this out: 'no bytes-valued CONSUMABLE resource is exercised anywhere in this codebase's tests today.'", "file": "src/scheduler/src/repl_info.c", "root_cause": "repl_info_worker_one()'s and repl_info_resource_one()'s CONSUMABLE Value_u_fvalue branches both hardcoded `%.2f`/`%.2f` for value=/used=, deliberately bypassing repl_format_numeric_resource_value()'s existing units_mode-aware/repl_is_bytes_valued_resource()-gated formatting that every other numeric field in this listing already goes through -- correct for a non-bytes-valued consumable (slots, licenses) but wrong for mem/tmp/swp, which are bytes-valued CONSUMABLE resources in live practice even though no test exercised that combination before this session.", "fix": "FIXED (2026-09-11). Added a new shared static helper repl_format_consumable_value_and_used() (repl_info.c, right after repl_format_numeric_resource_value()) that routes value=/used= through the existing repl_format_numeric_resource_value() formatter (units_mode-aware, plus the swp MB->bytes special case) when repl_is_bytes_valued_resource(rname) or rname==\"swp\" is true, and keeps the exact pre-existing \"%.2f\"/\"%.2f\" formatting otherwise (slots/licenses unaffected). Wired into both repl_info_worker_one()'s per-host CONSUMABLE case and repl_info_resource_one()'s GLOBAL/SHARED CONSUMABLE case, which share the fix via this one helper so they can't disagree. Updated the stale doc comments at repl_info.c's Resource Value Formatter section header and both call sites (previously asserted this pair was 'NOT units_mode-aware' and that no bytes-valued CONSUMABLE resource existed in tests -- no longer true). TDD: added 4 new tests to test_repl_resource_table.c (test_info_worker_units_mode_both_renders_raw_and_human_for_consumable_bytes_resource, test_info_worker_units_mode_raw_renders_bare_values_for_consumable_bytes_resource, test_info_worker_units_mode_converts_swp_megabytes_for_consumable_resource, test_info_resource_units_mode_both_renders_raw_and_human_for_consumable_bytes_resource) covering both/raw modes, the swp MB special case, and the info-resource twin. RED confirmed first (4 new failures, 76 pre-existing tests still green, `xce-build.sh scheduler asan xce.build --incremental`); GREEN confirmed after the fix (77/77 Ok, zero regressions). All existing pinned CONSUMABLE value=/used= tests (bug-1601/1613/1727, using \"slots\"/\"licenses\"/\"gpu_licenses\") were verified to use non-bytes-valued resource names before the fix, so they were unaffected by construction, not just by accident.", @@ -26175,7 +13488,6 @@ "consumable", "display-feature", "user-requested", - "fixed", "tdd" ], "related_bugs": [ @@ -26183,17 +13495,22 @@ "bug-1601", "bug-1613" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1800.md", + "gitea_issue": 269, + "gitea_synced_status": "fixed" }, { "id": "bug-1801", - "gitea_synced_status": "fixed", - "gitea_issue": 270, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-11T08:35:00Z", - "error_message": "Follow-up fix for bug-1799 (user asked to 'fix bug-1799', this session): the resInUse over-commit caused by core_reset_workers()'s RECONCILING/UNAVAIL reset-skip (bug-1751) having no matching skip in sched_account_jobs()'s per-round re-derive pass.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Follow-up fix for bug-1799 (user asked to 'fix bug-1799', this session): the resInUse over-commit…\n\nFollow-up fix for bug-1799 (user asked to 'fix bug-1799', this session): the resInUse over-commit caused by core_reset_workers()'s RECONCILING/UNAVAIL reset-skip (bug-1751) having no matching skip in sched_account_jobs()'s per-round re-derive pass.", "file": "src/scheduler/src/resource.c", "root_cause": "See bug-1799's own entry for the full mechanism. Confirmed exactly as diagnosed: sched_update_account() (resource.c, the function both sched_charge_account() and sched_account_jobs()'s per-round resum ultimately call) had no IS_WORKER_UNAVAIL/IS_WORKER_RECONCILING check in its per-host charge loop, so a job on a worker mid-resync got its charge re-added every round on top of the not-reset total.", "fix": "FIXED (2026-09-11). Added `if (IS_WORKER_UNAVAIL(w->status) || IS_WORKER_RECONCILING(w->status)) { continue; }` to sched_update_account()'s per-host loop (resource.c), right after resolving the worker `w`, before the sched_update_res_accounts() charge call -- the exact same combined status check core_reset_workers()/sched_reset_worker_() already use for the reset half (bug-1751), restoring the 'resum only what reset also touched' symmetry. Scoped to the per-host branch only: core->sharedInUse's own reset (core_reset_shared_resval()) is unconditional with no per-worker gating, so the shared-resource charge is untouched. This is a single-point fix (sched_update_account() backs both sched_charge_account()/sched_uncharge_account()), so it also applies (harmlessly/defensively -- a RECONCILING/UNAVAIL worker should never be a fresh DISPATCH target per the existing bug-1745/1751 candidate-filter exclusion) to the dispatch-time charge path via sched_commit_job_alloc(). TDD: added test_charge_account_skips_reconciling_or_unavail_worker_host_charge (test_resource_accounting.c) -- charges a job account against a worker flagged RECONCILING then UNAVAIL (per-host resInUse must stay 0 both times, shared resInUse still charges normally), then clears the status bit and re-charges (must resume normally on the very next call, proving this is a per-call skip, not a permanent block). RED confirmed by temporarily reverting the skip and rebuilding (`xce-build.sh scheduler asan xce.build --incremental`): exactly this one new test failed (`Expected 0 Was 2.5`), 76 other tests green. GREEN confirmed after restoring the fix: 77/77 Ok, zero regressions.", @@ -26209,7 +13526,6 @@ "over-commit", "slots", "bug-1751-interaction", - "fixed", "tdd", "user-requested" ], @@ -26221,17 +13537,22 @@ "bug-1624", "bug-1727" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1801.md", + "gitea_issue": 270, + "gitea_synced_status": "fixed" }, { "id": "bug-1802", - "gitea_synced_status": "fixed", - "gitea_issue": 271, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-11T08:35:00Z", - "error_message": "User request (same live replay session as bug-1799/1800): 'how can we step to the next session which is not dummy(something occurs)?' -- the REPL's `break ` vocabulary required setting all ten per-event breakpoint kinds individually to catch 'the next session with ANY event at all,' with no single convenience kind for that common workflow.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User request (same live replay session as bug-1799/1800): 'how can we step to the next session…\n\nUser request (same live replay session as bug-1799/1800): 'how can we step to the next session which is not dummy(something occurs)?' -- the REPL's `break ` vocabulary required setting all ten per-event breakpoint kinds individually to catch 'the next session with ANY event at all,' with no single convenience kind for that common workflow.", "file": "src/scheduler/src/repl_break.c, src/scheduler/src/repl_internal.h", "root_cause": "Not a bug -- a missing convenience feature. BP_KIND_TABLE (repl_break.c) only offered the ten specific per-event kinds (CONFIG_PUSH/JOB_PUSH/.../JOB_RUSAGE) plus BEGIN_ROUND/DECISION; repl_breakpoint_matches_event() compared bp->kind for exact equality against repl_bp_kind_for_core_event(ev->type), with no wildcard option.", "fix": "FIXED (2026-09-11). Added a new XCE_REPL_BP_ANY enum member (repl_internal.h) and a `{\"ANY\", XCE_REPL_BP_ANY}` BP_KIND_TABLE entry (listed first for discoverability). repl_breakpoint_matches_event() (repl_break.c) now special-cases bp->kind == XCE_REPL_BP_ANY to match any event whose type maps to a real per-event kind (repl_bp_kind_for_core_event() returning nonzero) instead of requiring exact kind equality -- still subject to the breakpoint's own filters (though a bare `break ANY`, the intended usage, has none). Deliberately does NOT match BEGIN_ROUND (fires on every session including dummy ones -- matching it would defeat the whole point) or DECISION (a separate drained-decision hook). `break ANY` + `continue` now stops the REPL at the very next session that decodes at least one real event, transparently skipping any number of dummy (virtual-clock-only) sessions in between, in one command instead of ten. TDD: added test_any_breakpoint_skips_dummy_sessions_to_first_real_event (test_repl_breakpoints_inspection.c, full replay-file + REPL-script Group D fixture) -- two dummy sessions (rounds 1000/2000, zero events each) followed by one JOB_STATE_CHANGE event, `break ANY` + `continue` must land exactly at event #1 having processed both dummy rounds as 'idle' (round-rollup evidence), and must never print BEGIN_ROUND. Also extended the two existing BP_KIND_TABLE token-enumeration tests (test_repl_help.c's help-text test, test_repl_breakpoints_inspection.c's unrecognized-event-type-error test) to include \"ANY\" so both stay genuinely exhaustive against the real table. RED confirmed (the new test failed on the FIRST attempt for an unrelated reason -- the hit message names the breakpoint's own user-typed token, \"ANY\", not the underlying event's type name \"JOB_STATE_CHANGE\", which the test's first draft wrongly assumed; corrected the expected string, not the implementation, which was already correct). GREEN confirmed: 77/77 Ok, zero regressions, alongside bug-1801's fix in the same build.", @@ -26247,21 +13568,25 @@ "cli-ux", "feature", "user-requested", - "fixed", "tdd" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1802.md", + "gitea_issue": 271, + "gitea_synced_status": "fixed" }, { "id": "bug-1803", - "gitea_synced_status": "fixed", - "gitea_issue": 272, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-11T09:35:00Z", - "error_message": "User feedback (live replay session, immediately after bug-1802's `break ANY` shipped): 'It seems that break any command will introduce too many break in single session' -- live transcript showed `break ANY` + repeated `step` firing 'hit breakpoint 1 @ event #N: any' on EVERY event (JOB_UPDATE, JOB_STATE_CHANGE, ...) inside an already-busy session (session 40), forcing many single-event `step`s just to get back OUT of that session -- the opposite of the intended 'skip past dummy sessions in one hop' workflow. User explicitly requested removing `break ANY` and instead having `info summary` display the next non-dummy session id so `step N` can jump there directly.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User feedback (live replay session, immediately after bug-1802's `break ANY` shipped): 'It seems…\n\nUser feedback (live replay session, immediately after bug-1802's `break ANY` shipped): 'It seems that break any command will introduce too many break in single session' -- live transcript showed `break ANY` + repeated `step` firing 'hit breakpoint 1 @ event #N: any' on EVERY event (JOB_UPDATE, JOB_STATE_CHANGE, ...) inside an already-busy session (session 40), forcing many single-event `step`s just to get back OUT of that session -- the opposite of the intended 'skip past dummy sessions in one hop' workflow. User explicitly requested removing `break ANY` and instead having `info summary` display the next non-dummy session id so `step N` can jump there directly.", "file": "src/scheduler/src/repl_break.c, src/scheduler/src/repl_internal.h, src/scheduler/src/sim.c, src/scheduler/src/repl_info.c", "root_cause": "Design mismatch, not a code defect: a breakpoint is inherently a persistent, repeatedly-firing mechanism (correct behavior for its stated purpose -- it fires on every matching event until deleted), but the user's actual need ('skip past a run of dummy sessions to the next one worth looking at') is a ONE-TIME navigation query, not a recurring pause condition. Modeling it as a breakpoint kind (bug-1802) meant it kept firing on every subsequent matching event for the rest of the session, well past the point it had already served its purpose, with no natural way to say 'just this once.'", "fix": "REVERTED `break ANY` entirely (bug-1802's XCE_REPL_BP_ANY enum member, BP_KIND_TABLE entry, repl_breakpoint_matches_event() special case, and its 3 tests) back to the exact pre-bug-1802 code, confirmed via `grep -rn XCE_REPL_BP_ANY` returning nothing but the historical doc-comment note left in repl_internal.h. REPLACED with a one-time informational display: `info summary` now also prints 'next non-dummy session: N -- run `step M` to reach it' (or 'none remaining'), computed by a new sim_prescan_next_nonempty_session() (sim.c, exported via sim.h) that peeks ahead in sim->replay_file from the current read position (same ftell()/fseek()-around-a-shape-only-scan technique sim_prescan_session_count() already establishes for the startup session-count banner) for the next ReplaySession boundary preceded by at least one real event, then rewinds -- a pure read-only query, never a recurring pause. Returns a RELATIVE 'sessions ahead' count (not an absolute session number) specifically to avoid needing xce_sched_repl_state_t's full definition inside sim.c (an incomplete type there by the project's own Simulation Isolation boundary -- caught by a real compile error on the first attempt, `invalid use of incomplete typedef`, when the function tried to read sim->repl_state->session_count directly); repl_info.c (which has the full repl_internal.h definition) adds the relative count to rs->session_count for the absolute display number. TDD: added test_info_summary_reports_next_nondummy_session_when_present and test_info_summary_reports_none_remaining_when_all_dummy (test_repl_breakpoints_inspection.c). RED confirmed by temporarily hardcoding the summary's prescan result to a sentinel non-0/non-1 value and rebuilding: exactly these 2 new tests failed, 75 others green. GREEN confirmed after restoring: 77/77 Ok, zero regressions. The existing test_info_summary_reports_counts (a bare in-memory fixture with no `.replay` file, sim->replay_file==NULL) is correctly unaffected -- the new line is silently omitted whenever the prescan can't run, mirroring sched_sim_driver_start()'s own 'purely informational, silently skipped on failure' precedent for its sibling prescan function.", @@ -26277,25 +13602,29 @@ "sim_prescan_next_nonempty_session", "cli-ux", "user-requested", - "fixed", "tdd", "design-mismatch" ], "related_bugs": [ "bug-1802" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1803.md", + "gitea_issue": 272, + "gitea_synced_status": "fixed" }, { "id": "bug-1804", - "gitea_synced_status": "fixed", - "gitea_issue": 273, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-11T09:55:00Z", - "error_message": "User design assertion (same live replay session as bug-1799/1800/1803): 'according to the design, when a job has been scheduled by scheduler, we can say job is running in scheduler angle' -- directly challenging this session's own earlier explanation that a job showing DISPATCH-then-still-PEND for multiple sessions was 'by design, not a bug' (deferred-until-confirmation semantics from the scheduler-sim-eager-reconciliation spec). User asked for confirmation and, once confirmed as a genuine design fork (not a bug in the then-current implementation), explicitly directed adopting the OTHER interpretation: revise the spec to match normal-mode's own synchronous dispatch-commit semantics (bug-1624).", - "file": ".kiro/specs/scheduler-sim-eager-reconciliation/{requirements,design,research,tasks,spec.json}, src/scheduler/src/sim_loop.c", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User design assertion (same live replay session as bug-1799/1800/1803): 'according to the design…\n\nUser design assertion (same live replay session as bug-1799/1800/1803): 'according to the design, when a job has been scheduled by scheduler, we can say job is running in scheduler angle' -- directly challenging this session's own earlier explanation that a job showing DISPATCH-then-still-PEND for multiple sessions was 'by design, not a bug' (deferred-until-confirmation semantics from the scheduler-sim-eager-reconciliation spec). User asked for confirmation and, once confirmed as a genuine design fork (not a bug in the then-current implementation), explicitly directed adopting the OTHER interpretation: revise the spec to match normal-mode's own synchronous dispatch-commit semantics (bug-1624).", + "file": ".kiro/specs/scheduler-sim-eager-reconciliation/{requirements, design, research, tasks, spec.json}, src/scheduler/src/sim_loop.c", "root_cause": "Not a bug in the sense of an implementation defect -- a genuine, previously-researched design fork between two defensible readings of 'job->state during replay': (a) reconstruct what the ORIGINAL live scheduler process's own job->state field actually held (RUN, synchronously, the instant sched_commit_decision() decided to dispatch -- bug-1624), vs (b) track the SEPARATELY-recorded JOB_STATE_CHANGE/worker-confirmation event's own timing as the operative signal (the scheduler-sim-eager-reconciliation spec's originally-shipped, explicitly-researched choice -- its research.md's own 'Does bug-1624's normal-mode fix already close this gap?' entry correctly noted bug-1624 is a live in-memory write, not a recordable event, so a recording's own JOB_STATE_CHANGE event genuinely CAN land rounds after its DISPATCH decision even post-bug-1624 -- a true fact, but the spec then chose interpretation (b) on top of that fact without it being forced). Live operator use surfaced interpretation (b)'s practical cost directly: a job showing PEND in `info jobs`/`info worker` for multiple sessions after its own `[sim] DISPATCH` line had already printed, reading as a scheduler bug even though it was working exactly as the shipped spec specified.", "fix": "FIXED (2026-09-11, user-directed). Adopted interpretation (a): `sim_reconcile_unaccounted_jobs()` (sim_loop.c)'s FOUND branch now sets `job->state = JOB_STAT_RUN;` synchronously, in the same operation as the allocation capture (`job->allocs`/`job->accounts`), mirroring `scheduler.c`'s `sched_commit_decision()` (bug-1624) exactly. `sched_account_jobs()`'s own unchanged `JOB_IS_RUN` gate therefore charges the job's resInUse starting the SAME round as capture, not a later confirmation round. A later, separately-recorded `JOB_STATE_CHANGE`/`JOB_UPDATE`-to-RUN event, when it replays, is now a safe idempotent re-affirmation (RUN->RUN, `core_events.c`'s handler reassigns `job->state` unconditionally with no charging side effect) rather than the sole state-transition trigger. Revised the full `scheduler-sim-eager-reconciliation` spec in place (requirements.md Requirement 1's ACs 1.2-1.5 rewritten, Boundary Context's out-of-scope bullet corrected; design.md's Overview/Non-Goals/Boundary Commitments/Architecture flowchart/System Flows sequence diagram/Requirements Traceability/Service Interface doc/Testing Strategy all updated with dated 'Correction (2026-09-11)' notes, not silently rewritten; research.md's own 'Does bug-1624 already close this gap' entry got a superseding revision note explaining the (a) vs (b) trade-off explicitly, since the original factual finding was correct and remains true -- only the DECISION built on top of it changed; tasks.md got a new task 4/4.1/4.2 documenting the revision, both marked done; spec.json's updated_at bumped). TDD: renamed/inverted test_captured_pend_job_excluded_from_resinuse_until_run to test_captured_job_included_in_resinuse_same_round_as_capture (test_sim_reconcile_unaccounted.c) -- asserts job->state==RUN and resInUse charged immediately after capture (no separate job->state write needed), plus a new Requirement-1.4 companion check that a redundant RUN->RUN confirmation event (driven through the REAL XCE_CORE_EVENT_JOB_STATE_CHANGE handler) doesn't touch the allocation and doesn't change what gets charged after a proper core_reset_workers()-then-sched_account_jobs() round boundary (the test's own first draft skipped that reset before its second sched_account_jobs() call, an artificial double-charge that can never happen in real per-round production flow -- caught by RED/GREEN, fixed in the test, not the production code). Also rewrote test_sim_contention_parity.c's entire second test (test_contention_capacity_check_correct_across_multi_round_dispatch_confirm_gap) -- a 300+ line fixture/assertion block built around the OLD 'deferred until confirmation, partial-then-full usage across rounds 1/4/5' narrative -- to the NEW 'both dispatched jobs saturate capacity immediately at round 1; the later confirmation events are safe no-ops' narrative, since the old assertions (capacity free at round 1, only-A-charged at round 4) are now factually backwards. Audited every other file calling/depending on sim_reconcile_unaccounted_jobs() (test_snapshot_save_load.c, test_sim_snapshot_reconcile_gap.c, test_repl_resource_table.c, test_sim_round_boundary_reconciliation.c, test_sim_e2e_original_motivation.c, test_sim_verify.c) and confirmed each is unaffected: their fixtures either put the DISPATCH and JOB_STATE_CHANGE in the SAME round (a no-op change under the new synchronous-capture semantics, since the job was already RUN via the explicit event before reconciliation ever ran) or never rely on job-state/resInUse timing at all (pure --sim-verify decision-content comparisons, or jobs that are never captured by this mechanism in the first place). RED confirmed (temporarily reverted the `job->state = JOB_STAT_RUN;` line, rebuilt: exactly the 2 updated tests failed for the expected reasons, 75 others green). GREEN confirmed after restoring: full scheduler suite 77/77 Ok, zero regressions.", "tags": [ @@ -26314,7 +13643,6 @@ "user-directed", "bug-1624", "driver-equivalence", - "fixed", "tdd" ], "related_bugs": [ @@ -26324,15 +13652,22 @@ "bug-1799", "bug-1801" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1804.md", + "gitea_issue": 273, + "gitea_synced_status": "fixed" }, { "id": "bug-1805", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-14T05:07:17.752283Z", - "error_message": "xce-harness-component task 1.1: controller_unit_test had 29 test failures (all in test_test_cli.c/test_xcetest_process_entry.c) after xce_test moved from src/controller/testsuite/ to src/harness/ -- Expected 0 Was 127 and missing xce> prompt", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-harness-component task 1.1: controller_unit_test had 29 test failures (all in…\n\nxce-harness-component task 1.1: controller_unit_test had 29 test failures (all in test_test_cli.c/test_xcetest_process_entry.c) after xce_test moved from src/controller/testsuite/ to src/harness/ -- Expected 0 Was 127 and missing xce> prompt", "file": "src/controller/meson.build", "root_cause": "A bridging custom_target (xce_test_controller_link) created a symlink via command: ['ln', '-sf', '@INPUT0@', '@OUTPUT0@']. Meson expands @INPUT0@ to a build-root-relative path (harness/xce_test), but a relative symlink TARGET is resolved relative to the SYMLINK OWN DIRECTORY at dereference time, not the ln invocation cwd or the string reference frame it was written in. The resulting symlink at /controller/xce_test therefore looked for /controller/harness/xce_test, which does not exist, causing every subprocess-spawning test (resolve_xce_test_path()) to exec a missing binary (exit 127).", "fix": "FIXED (2026-09-14, xce-harness-component task 1.1). Changed the custom_target command to use the literal relative path ['ln', '-sf', '../harness/xce_test', '@OUTPUT0@'] instead of @INPUT0@, so the symlink text correctly resolves relative to controller/ up to harness/xce_test. Verified: controller_unit_test dropped from 29 failures to 0 (1228 Tests 0 Failures 0 Ignored), confirmed via the Unity binary's own printed summary line, not just meson's aggregate Ok/Fail count.", @@ -26342,21 +13677,23 @@ "custom_target", "relative-path", "build-system", - "xce-harness-component", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "xce-harness-component" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1805.md" }, { "id": "bug-1806", "status": "wontfix", "status_updated": "2026-09-24", "status_note": "accepted Meson limitation (user decision 2026-09-14)", + "component": "harness", + "severity": "unknown", "timestamp": "2026-09-14T09:20:27.103747+00:00", - "error_message": "xce-harness-component task 4.3: /xce-build controller alone still rebuilds harness/xce_test on disk under the default asan profile, apparently violating requirement 2.2 (controller-only build shall not produce xce_test as a side effect)", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "xce-harness-component task 4.3: /xce-build controller alone still rebuilds harness/xce_test on…\n\nxce-harness-component task 4.3: /xce-build controller alone still rebuilds harness/xce_test on disk under the default asan profile, apparently violating requirement 2.2 (controller-only build shall not produce xce_test as a side effect)", "file": "src/harness/meson.build, src/tools/xce-build.sh", "root_cause": "Deterministic, not a flake -- confirmed via build-log line analysis. meson test (run by /xce-build controller's default RUN_TESTS=1 asan profile via meson test --suite controller) rebuilds every registered test()'s build-order dependencies PROJECT-WIDE before filtering which tests actually RUN by --suite. harness_unit_test's test() registration (src/harness/meson.build) has depends: [xce_test] (added task 2.1, verified load-bearing in task 4.2), so any meson test invocation -- even one scoped to --suite controller -- transitively relinks xce_test. The explicit step is clean (never references xce_test); only the SUBSEQUENT meson test step touches it. Task 3.1's own validation missed this because it checked for the OLD path (controller/xce_test, correctly absent) rather than the NEW path (harness/xce_test, which does get rebuilt).", "fix": "NOT FIXED -- accepted as a documented Meson platform limitation (user decision 2026-09-14). Requirement 2.2 narrowed to scope the build system's guarantee to what it directly controls: the explicit compile-target list never names xce_test (still true and verified). Rejected alternatives: (a) add --no-rebuild to xce-build.sh's meson test calls -- non-trivial, since controller's OWN test suite already has 4 small standalone test binaries (sched_push_unit_test, sched_worker_change_unit_test, apiserver_push_unit_test, host_runusage_unit_test) not in its explicit compile target list, relying on this same auto-rebuild; fixing this properly means auditing and completing every component's explicit target list, with real regression risk; (b) restructure harness_unit_test's pty-spawning tests to drop the depends: edge and locate xce_test at runtime instead -- directly reverses the build-order guarantee task 4.2 just verified is real and load-bearing, reintroducing the stale/missing-binary risk it was added to prevent. requirements.md/design.md/tasks.md updated to reflect the narrowed, accurate guarantee.", @@ -26365,28 +13702,30 @@ "test-runner", "build-dependency", "xce-harness-component", - "accepted-limitation", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "accepted-limitation" ], "related_bugs": [ "bug-1805" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1806.md" }, { "id": "bug-1807", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "harness", + "severity": "unknown", "timestamp": "2026-09-14T15:50:00Z", - "error_message": "harness_unit_test's test_test_cli.c: 4 of 132 tests deterministically FAIL every run -- test_migrated_commands_sample_reject_unrecognized_flag (jobkill unrecognized-flag rejection), test_qctrl_hostctrl_confirm_gate_destructiveness_determination (qctrl -c or hostctrl -c prompt missing -- WHICH one shifts between builds), test_destructive_commands_prompt_by_default_and_yes_bypasses (upgradeconfig prompt missing), test_confirm_gate_session_toggle_skips_and_restores ('set +y' restore prompt missing). Each failing assertion burns its full 5000ms clitest_wait_for_text() timeout (total suite runtime ~28-29s vs ~9s once fixed) -- not a race, a genuine miss every time.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "harness_unit_test's test_test_cli.c: 4 of 132 tests deterministically FAIL every run…\n\nharness_unit_test's test_test_cli.c: 4 of 132 tests deterministically FAIL every run -- test_migrated_commands_sample_reject_unrecognized_flag (jobkill unrecognized-flag rejection), test_qctrl_hostctrl_confirm_gate_destructiveness_determination (qctrl -c or hostctrl -c prompt missing -- WHICH one shifts between builds), test_destructive_commands_prompt_by_default_and_yes_bypasses (upgradeconfig prompt missing), test_confirm_gate_session_toggle_skips_and_restores ('set +y' restore prompt missing). Each failing assertion burns its full 5000ms clitest_wait_for_text() timeout (total suite runtime ~28-29s vs ~9s once fixed) -- not a race, a genuine miss every time.", "file": "src/harness/test.c (all ~20 cmd_* handlers), src/harness/join_worker_args.c", "root_cause": "CONFIRMED via isolated pty repro (bypassing Unity/gdb entirely): every cmd_*() REPL command handler resets getopt_long() state with `optind = 1; opterr = 0;` before its own parse loop. glibc's getopt_long only re-initializes its internal static parser state (nextchar, first_nonopt/last_nonopt) when optind==0 OR on the process's very first-ever getopt_long() call (guarded by the internal __getopt_initialized flag) -- optind=1 alone is NOT sufficient once any earlier call in the same process has already run once. Since xce_test's REPL dispatches many independent cmd_*() calls (each with its OWN, differently-sized argv array) across the lifetime of one process, a command whose parse ends by consuming a short no-argument flag as its LAST token (e.g. `qctrl -q normal -a`, `hostctrl -h node01 -o`) leaves stale internal state that corrupts the NEXT unrelated command's getopt_long() call -- observed corrupting the NEXT command so badly that argv[0] (the command's own name, e.g. 'upgradeconfig' or 'jobkill') is misread as an unrecognized flag via flagspec_reject(). 100% reproducible in isolation: `qctrl -q normal -a` immediately followed by ANY other command (upgradeconfig, jobkill) corrupts that next command every time; the same session's commands run individually/first are unaffected. Explains why the specific victim test shifted between builds (bug-1807 was first seen failing on hostctrl -c, then qctrl -c after an unrelated same-file edit) -- runtime sequence-dependent, not random.", "fix": "FIXED. Changed all 20 occurrences of `optind = 1;` in src/harness/test.c and the 1 occurrence in src/harness/join_worker_args.c to `optind = 0;` (the documented glibc idiom that forces getopt_long to fully reinitialize, including nextchar/first_nonopt/last_nonopt, on every independent parse). Verified: (1) the 3 isolated minimal pty repros that previously reproduced the corruption 100% of the time now pass cleanly; (2) harness_unit_test run 10x consecutively under the debug profile: 132 Tests 0 Failures every run (was 4 Failures every run pre-fix), runtime dropped from ~28-29s to ~8-9s per run, confirming the failures were full timeout burns, not near-miss races; (3) full `/xce-build harness asan` (meson test, ASAN-instrumented) also reports 5/5 OK including harness_unit_test, no sanitizer findings.", "tags": [ "confirmed", - "fixed", "getopt", "getopt_long", "cli-parsing", @@ -26396,20 +13735,24 @@ "flagspec" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1807.md" }, { "id": "bug-1808", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "harness", + "severity": "leak", "timestamp": "2026-09-14T15:50:00Z", - "error_message": "src/harness/testsuite/test_test_cli.c: a spawned xce_test child process (clitest_spawn/clitest_spawn_ex) plus its pty master fd and stderr pipe fd leak whenever a TEST_ASSERT_* assertion fails partway through the owning test function, because every test relies on its OWN clitest_kill_and_reap()+clitest_close() calls placed inline at the bottom of the function body -- code that Unity's longjmp-based TEST_PROTECT()/TEST_ABORT() abort mechanism skips entirely on assertion failure. Directly observed: 4 live, idle `xce_test --norc` child processes still running (blocked forever on their own prompt) under one in-progress harness_unit_test invocation, discovered while investigating bug-1807's failures.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "src/harness/testsuite/test_test_cli.c: a spawned xce_test child process…\n\nsrc/harness/testsuite/test_test_cli.c: a spawned xce_test child process (clitest_spawn/clitest_spawn_ex) plus its pty master fd and stderr pipe fd leak whenever a TEST_ASSERT_* assertion fails partway through the owning test function, because every test relies on its OWN clitest_kill_and_reap()+clitest_close() calls placed inline at the bottom of the function body -- code that Unity's longjmp-based TEST_PROTECT()/TEST_ABORT() abort mechanism skips entirely on assertion failure. Directly observed: 4 live, idle `xce_test --norc` child processes still running (blocked forever on their own prompt) under one in-progress harness_unit_test invocation, discovered while investigating bug-1807's failures.", "file": "src/harness/testsuite/test_test_cli.c, src/harness/testsuite/harness_test.c", "root_cause": "Design gap, not a logic error in any specific test: clitest_kill_and_reap()/clitest_close() were documented as a mandatory safety net ('every caller is required to always end a session with clitest_kill_and_reap() regardless of which path was taken') but placed only at each test function's own normal-exit code path. Unity's RUN_TEST() macro DOES guarantee tearDown() runs after every test (pass, fail, or aborted-via-longjmp), but harness_test.c's shared tearDown() (used by all 13 Unity test groups linked into this binary, most unrelated to clitest) was a no-op beyond a cosmetic separator print -- it never reaped a still-open clitest session.", "fix": "FIXED. Added a static clitest_session_t mirror (g_last_session) in test_test_cli.c updated by clitest_spawn_ex() (set on successful spawn), clitest_kill_and_reap() (reaped flag synced), and clitest_close() (cleared) -- a STATIC-storage mirror was required rather than tracking a pointer into the aborted test's own local `sess`, since that stack frame is unwound by the longjmp and would leave a dangling pointer. Exposed a new clitest_reap_leaked_session() function, called from harness_test.c's tearDown() (now declared extern there), which force-kills+reaps+closes any session left open when a test aborted before reaching its own inline cleanup. Only one clitest session is ever open at a time in this file (verified), so a single mirror slot is sufficient. Verified via ASAN build (xce.build): no LeakSanitizer or other sanitizer findings from the new code across a clean 5/5 harness suite run.", "tags": [ - "fixed", "process-leak", "fd-leak", "unity", @@ -26421,15 +13764,20 @@ "related_bugs": [ "bug-1807" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1808.md" }, { "id": "bug-1809", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-14T16:20:00Z", - "error_message": "User-reported: live scheduler logs are flooded with WARN lines for every ordinary XCE_CORE_EVENT_JOBS_PUSH resync redelivery: \"xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid (rc=-17), merging state into existing tracked job instead of discarding\" (src/scheduler/src/core_events.c). rc=-17 is -EEXIST -- the expected, routine outcome of on_scheduler_join()'s full resync (bug-1734) or the array-push dedup race (bug-1714), already correctly handled by the bug-1734/bug-1738/bug-1739 merge logic in the same branch, not a functional defect.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported: live scheduler logs are flooded with WARN lines for every ordinary…\n\nUser-reported: live scheduler logs are flooded with WARN lines for every ordinary XCE_CORE_EVENT_JOBS_PUSH resync redelivery: \"xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid (rc=-17), merging state into existing tracked job instead of discarding\" (src/scheduler/src/core_events.c). rc=-17 is -EEXIST -- the expected, routine outcome of on_scheduler_join()'s full resync (bug-1734) or the array-push dedup race (bug-1714), already correctly handled by the bug-1734/bug-1738/bug-1739 merge logic in the same branch, not a functional defect.", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge branch, ~line 845)", "root_cause": "Not a defect -- a UX/observability request. sched_add_job() (core.c) wraps key_add()->kvtab_add() (lib/kv.c), whose only failure modes for the core->jobs table (KVMGR_KEY_INT, no keyops->assign) are -EEXIST (duplicate jobid) or -ENOMEM (allocation failure). The EEXIST case is therefore the near-universal, routine shape of this branch under a reconnect/resync storm (confirmed live in bug-1714's own investigation: 700+ of these WARNs fired from a single 8-reconnect burst) -- WARN was the wrong level for an outcome the code already handles correctly by design; it was originally added at WARN by bug-1734's own fix pass without considering steady-state log volume under the reconnect-storm conditions that same session's investigation had just captured.", "fix": "FIXED (2026-09-14). Split the single WARN call in the JOBS_PUSH EEXIST branch (core_events.c) on add_rc: when add_rc == -EEXIST, log via a new DBG(...) macro (utils.h, wraps tlog_debug() -- DEBUG level, silent by default, visible with -vvv) instead of WARN -- preserves the message (still valuable forensic evidence per bug-1734/bug-1714) without flooding default-verbosity operational logs. Any OTHER negative add_rc (e.g. -ENOMEM) still logs at WARN, since that would be a genuine anomaly, not a routine resync artifact. The new macro is named DBG, not DEBUG, matching the existing TRACE/INFO/WARN/ERROR one-line wrapper convention in utils.h: a first attempt to name it DEBUG(...) collided with the build's own bare -DDEBUG command-line define (src/meson.build, set for BOTH the debug and asan buildtypes -- confirmed via build warning \"DEBUG redefined ... this is the location of the previous definition \" across every file that includes utils.h under either profile), so DBG was used instead per user decision after the tradeoff was surfaced. Verified: xce-build.sh scheduler debug --incremental and xce-build.sh scheduler asan --incremental both compile clean (zero DBG/DEBUG-redefinition warnings in either profile); full scheduler suite 77/77 Ok, zero regressions.", @@ -26446,7 +13794,6 @@ "not-a-defect", "session-sibling-of-bug-1734", "session-sibling-of-bug-1714", - "fixed", "DBG-macro", "utils-h" ], @@ -26456,15 +13803,20 @@ "bug-1739", "bug-1738" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1809.md" }, { "id": "bug-1810", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "hang", "timestamp": "2026-09-15T03:55:00Z", - "error_message": "User-reported live hang: xce_test (harness REPL) never exits on Ctrl-D (EOF) after entering a live connection context (client/worker/scheduler). Live gdb attach on the user's machine showed the main thread parked forever in leave_ctx() -> xcecli_ctx_fini() -> xce_ctx_fini() -> vtaskmgr_fini() -> vcpu_fini()'s pthread_join(vcpu->vcpu, ...); a sibling vcpu thread was simultaneously deadlocked in cbuf_put() at mylock.c:101's pthread_mutex_lock(&(b->mutex)) on vtaskmgr->wbuf.mutex, whose __owner TID had already exited.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported live hang: xce_test (harness REPL) never exits on Ctrl-D (EOF) after entering a live…\n\nUser-reported live hang: xce_test (harness REPL) never exits on Ctrl-D (EOF) after entering a live connection context (client/worker/scheduler). Live gdb attach on the user's machine showed the main thread parked forever in leave_ctx() -> xcecli_ctx_fini() -> xce_ctx_fini() -> vtaskmgr_fini() -> vcpu_fini()'s pthread_join(vcpu->vcpu, ...); a sibling vcpu thread was simultaneously deadlocked in cbuf_put() at mylock.c:101's pthread_mutex_lock(&(b->mutex)) on vtaskmgr->wbuf.mutex, whose __owner TID had already exited.", "file": "src/lib/mylock.c (cbuf_put/cbuf_get/cbuf_timedget), src/lib/vtaskmgr.c (vcpu_idle, vcpu_entry)", "root_cause": "Two compounding defects in src/lib's shared task-pool primitives (used by every daemon + xce_test's own client/worker/scheduler REPL contexts, each of which owns a vtaskmgr_t for async I/O dispatch). (1) mylock.c's cbuf_put()/cbuf_get()/cbuf_timedget() lock b->mutex, then block in pthread_cond_wait()/pthread_cond_timedwait() with NO pthread_cleanup_push() handler installed. POSIX mandates pthread_cond_wait() reacquire its mutex before honoring a pending cancellation and running cleanup handlers -- with no handler registered, a thread cancelled while parked there terminates HOLDING the mutex, orphaning it locked forever; every other thread that later calls pthread_mutex_lock() on that same cbuf deadlocks uninterruptibly (plain mutex lock is not a cancellation point in glibc's NPTL). (2) vtaskmgr.c's teardown protocol races directly into that hazard from up to three independent angles, all torn down via pthread_cancel()+pthread_join(): (a) vtaskmgr_fini() cancels the dispatcher thread FIRST -- if it lands while the dispatcher is blocked in pick_idle_vcpu()'s cbuf_get() on the shared vtaskmgr->wbuf (empty, e.g. because real I/O work had every vcpu momentarily busy), that orphans wbuf's mutex immediately, before any vcpu teardown even starts; (b) vcpu_fini() then wakes each idle vcpu via bind_vtask(NULL, vcpu) (a 'poison pill' through the vcpu's own 1-capacity wbuf) and immediately pthread_cancel()s it, but vcpu_entry()'s loop could not distinguish that pill from 'no task yet' -- it looped straight back into vcpu_idle(), which unconditionally re-announced the vcpu into the shared (by then permanently undrained, since the dispatcher is already dead) vtaskmgr->wbuf, racing the pending cancellation against that second cbuf_put(); (c) the same poison pill arriving while a vcpu was still mid-execution of a real task queued it silently for the NEXT vcpu_idle() call, which still re-announced into the shared queue first before ever consuming it. Any of the three could orphan vtaskmgr->wbuf's mutex; every vcpu processed afterward that still touched the shared queue then deadlocked forever on pthread_mutex_lock(), and vtaskmgr_fini()'s pthread_join() for it never returned -- hanging the whole process (xce_test on Ctrl-D quit; identically reachable from any daemon that tears down a vtaskmgr_t under real concurrent I/O, e.g. controller/scheduler/worker shutdown).", "fix": "FIXED (2026-09-15), TDD RED->GREEN. Two-part fix, both required (confirmed empirically -- the vtaskmgr.c-only fix alone left the dispatcher-orphan path (2a above) unaddressed and still reproduced the hang under stress). (1) mylock.c: wrapped the pthread_cond_wait()/pthread_cond_timedwait() loops in cbuf_put()/cbuf_get()/cbuf_timedget() with pthread_cleanup_push(pthread_mutex_unlock, &b->mutex) / pthread_cleanup_pop(0) -- a cancelled waiter now always releases the mutex on unwind, so no cbuf mutex can ever be left orphaned-locked by a cancellation, regardless of which caller (dispatcher or vcpu) hits it. cbuf_timedget()'s early ETIMEDOUT return was restructured to break+flag instead of returning from inside the pushed region, since push/pop must be a matched pair at the same lexical nesting level. (2) vtaskmgr.c, defense-in-depth / closes the underlying logic bug independently of (1): vcpu_idle() now does a non-blocking cbuf_tryget() on its own wbuf BEFORE announcing itself into the shared vtaskmgr->wbuf, so an already-queued poison pill (queued while the vcpu was mid-task) is consumed directly instead of triggering a needless re-announce; vcpu_entry()'s loop now checks vcpu->task after vcpu_idle() returns and breaks (exits the thread) on NULL instead of looping back into idle -- a NULL task is unambiguously vcpu_fini()'s shutdown signal, since the dispatcher (vdispatcher_entry) never calls bind_vtask() with a NULL task. New regression test src/lib/testsuite/test_vtaskmgr_teardown.c (wired into src/lib/meson.build + testsuite/test.c): floods a burst of real VTASK_LIVENESS tasks across a 16-vcpu vtaskmgr then calls vtaskmgr_fini() on a helper thread with a pthread_timedjoin_np() 2s bounded-timeout join, self-pinning the test to a single CPU (sched_setaffinity, inherited by every spawned vtaskmgr thread at clone()) since the race needs preemptive time-slicing to reproduce reliably -- confirmed 0/hundreds of hangs on stock multi-core scheduling (true parallelism lets a vcpu almost always win the race to its own pthread_testcancel() before cancellation is even issued) vs. reliable reproduction (hung on iteration 0) once pinned to one core. Verified via: (a) this new test, RED before the fix (Expected 0 Was 110/ETIMEDOUT) and GREEN after; (b) a standalone single-core-pinned stress harness outside the Unity suite, 2000/2000 iterations clean post-fix (vs. hanging on iteration 0 with no fix and again on iteration 3 with only the vtaskmgr.c half of the fix, confirming both parts were needed); (c) full libxce test suite (314/314) and full harness suite (132/132, including the pty-based clitest tests that exercise real client-context connect+quit) both green; (d) a live pty-driven repro of the ORIGINAL user report -- launching the real xce_test binary, entering the client context, sending a real Ctrl-D (EOT) byte -- now exits in <0.1s (previously would hang indefinitely); (e) full project meson test matrix (98/98 suites) green on a clean run -- two earlier full-matrix runs showed unrelated, non-reproducing timeouts (one in harness's own quit-wait, one in controller_unit_test) that did not reproduce on isolated reruns, consistent with this sandbox's pre-existing parallel-test-suite CPU-contention flakiness (see bug-1429/bug-1580), not a regression from this fix.", @@ -26487,20 +13839,24 @@ "teardown", "race-condition", "tdd-red-green", - "live-gdb-repro", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "live-gdb-repro" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1810.md" }, { "id": "bug-1811", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-15T07:40:00Z", - "error_message": "User-reported (interactive sim session=35, `info jobs`/`info worker pc13v`): job 0-14-1789369200-0[21] shows RUN in `info jobs` but is absent from `info worker pc13v`'s assigned-jobs listing (which lists [0-20,22], skipping 21) -- user's working theory was slot overcommitment (23 running jobs vs. 22-slot worker capacity).", - "file": "src/scheduler/src/core_events.c, src/scheduler/src/sim_loop.c, .kiro/specs/scheduler-sim-decision-parity/{requirements,design}.md", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (interactive sim session=35, `info jobs`/`info worker pc13v`): job…\n\nUser-reported (interactive sim session=35, `info jobs`/`info worker pc13v`): job 0-14-1789369200-0[21] shows RUN in `info jobs` but is absent from `info worker pc13v`'s assigned-jobs listing (which lists [0-20,22], skipping 21) -- user's working theory was slot overcommitment (23 running jobs vs. 22-slot worker capacity).", + "file": "src/scheduler/src/core_events.c, src/scheduler/src/sim_loop.c, .kiro/specs/scheduler-sim-decision-parity/{requirements, design}.md", "root_cause": "NOT A BUG -- initial diagnosis was WRONG and the attempted fix was reverted. Root-caused (correctly, mechanism-wise) to XCE_CORE_EVENT_JOB_STATE_CHANGE/XCE_CORE_EVENT_JOB_UPDATE (core_events.c) setting job->state=RUN unconditionally even when job->allocs/job->accounts are still NULL -- repl_info_worker_one() (repl_info.c) walks job->allocs to build the assigned-jobs listing, so an unaccounted RUN job is correctly displayed as RUN in `info jobs` but silently absent from `info worker`'s listing. Initially misdiagnosed this AS a defect (a 'phantom RUN' analogous to bug-1739's JOBS_PUSH resync gap) and implemented a guard refusing the RUN transition without an existing allocation. That guard was WRONG: this exact 'RUN with allocs==NULL' transient window is the INTENTIONAL, designed, tested trigger condition for Recorded-Decision Reconciliation (scheduler-sim-decision-parity spec, sim_loop.c's sim_reconcile_unaccounted_jobs(), extended by bug-1804's 2026-09-11 revision to set job->state=RUN synchronously via JOB_STATE_CHANGE/JOB_UPDATE ahead of allocation capture, matching the original live scheduler's own sched_commit_decision() semantics). Confirmed via test_sim_e2e_original_motivation.c's own header comment and ANCHOR_ID fixture: the SAME core_events.c code path is used, by design, to construct a job that is 'already running, with an unaccounted allocation, when a scheduling round's own live decision-making reaches it' -- 'Recorded-Decision Reconciliation's entire reason for existing'. My guard broke this fixture (test_e2e_original_motivation_long_replay) plus test_push_events_fires_each_hook_once_per_event_with_picker_idle and test_drain_translates_and_pushes_all_ten_op_types, both of which legitimately exercise a bare JOB_STATE_CHANGE/JOB_UPDATE(RUN) with no allocation yet. Further confirmed against scheduler-sim-decision-parity/requirements.md Requirement 4 ('Graceful degradation when recorded decision data is unavailable'): AC 4.1/4.2 EXPLICITLY anticipate and accept that a job's originating DISPATCH decision may not be found in the recorded replay data, in which case the scheduler reports that job's usage as unavailable rather than fabricating a value, and continues processing -- this is precisely job 21's observed symptom, not an accident. The user's overcommitment theory is also independently wrong on the accounting side: sched_account_jobs() only charges resInUse for JOB_IS_RUN(state) && accounts != NULL, so an unaccounted RUN job is correctly EXCLUDED from the real slot-usage total (used=22.00 already correctly reflects only the 22 genuinely-accounted jobs) -- there is no real overcommitment, only a display gap for a job whose allocation reconciliation hasn't (yet, or ever, depending on whether its DISPATCH decision exists later in this replay stream) caught up.", "fix": "REVERTED. Initial fix (guard in core_events.c's XCE_CORE_EVENT_JOB_STATE_CHANGE/JOB_UPDATE refusing a RUN claim without an existing allocation, plus 6 new tests in test_core_events.c and a fixture change in test_driver_translate.c) was implemented, RED/GREEN-tested, then found to break test_sim_e2e_original_motivation.c's test_e2e_original_motivation_long_replay (the live-verify for the entire Recorded-Decision Reconciliation mechanism) plus two pre-existing test_core_events.c/test_driver_translate.c tests. All changes reverted via `git checkout --` before any commit. No production code changed. Left as an open question whether job 21 (and the user's follow-up observation that this 'occurs more frequently') ever actually reconciles later in the same replay session -- if a job's RUN-but-unaccounted display state NEVER resolves even after the replay completes, that would indicate its DISPATCH decision is genuinely missing from the recorded stream (the same category of gap scheduler-snapshot-job-allocation's PREEXIST_ID addresses for pre-recording dispatches, via snapshot-based restoration instead of replay-based reconciliation) -- worth a follow-up investigation with concrete evidence (does the job ever show up in `info worker`'s listing on a later round?) before attempting any further fix. A safe, smaller follow-up actually worth considering: improve `info worker`'s assigned-jobs display (repl_info.c) to explicitly distinguish a RUN-but-unaccounted job (e.g. an annotation) from silent omission, so operators aren't misled into an overcommitment theory the way this session's user was -- proposed to the user, not yet implemented. SUPERSEDED (2026-09-15, same session): the 'not a bug' conclusion above was itself only half right. The transient RUN-before-allocation window IS legitimate (Recorded-Decision Reconciliation), but the specific PERMANENT loss the user kept observing turned out to be a real, separate defect in the JOBS_PUSH duplicate-jobid resync merge -- see bug-1812 for the actual root cause and fix.", "tags": [ @@ -26509,24 +13865,27 @@ "scheduler", "sim-replay", "recorded-decision-reconciliation", - "reverted-fix", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "reverted-fix" ], "related_bugs": [ "bug-1739", "bug-1804", "bug-1778" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1811.md" }, { "id": "bug-1812", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-15T08:05:00Z", - "error_message": "User-reported (interactive sim session=34->35 stepped live, AND independently confirmed via a real production scheduler.log from rshen@pc13v /t/xce-ha-deploy): job 0-14-1789369200-0[21] holds allocs=1 (state=RUN) at the end of session 34; stepping into session 35 replays a JOBS_PUSH resync (njobs=60, including all of array job 0-14-1789369200-0[0-39]) followed by a JOB_UPDATE and a redundant JOB_STATE_CHANGE(RUN) for [21]; immediately after, `info job [21]` shows state=RUN but allocs=0. The real scheduler.log shows the identical mechanism live: `sched_job_state_change ... state 2` (schedStartJob confirmation) at 15:00:05.196, followed 16ms later by `JOBS_PUSH: sched_add_job failed for jobid ...[21] (rc=-17), merging state into existing tracked job instead of discarding` -- repeating in bursts on later reconnects (six WARNs in ~2s at 15:01:51-53).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (interactive sim session=34->35 stepped live, AND independently confirmed via a real…\n\nUser-reported (interactive sim session=34->35 stepped live, AND independently confirmed via a real production scheduler.log from rshen@pc13v /t/xce-ha-deploy): job 0-14-1789369200-0[21] holds allocs=1 (state=RUN) at the end of session 34; stepping into session 35 replays a JOBS_PUSH resync (njobs=60, including all of array job 0-14-1789369200-0[0-39]) followed by a JOB_UPDATE and a redundant JOB_STATE_CHANGE(RUN) for [21]; immediately after, `info job [21]` shows state=RUN but allocs=0. The real scheduler.log shows the identical mechanism live: `sched_job_state_change ... state 2` (schedStartJob confirmation) at 15:00:05.196, followed 16ms later by `JOBS_PUSH: sched_add_job failed for jobid ...[21] (rc=-17), merging state into existing tracked job instead of discarding` -- repeating in bursts on later reconnects (six WARNs in ~2s at 15:01:51-53).", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge, bug-1734/bug-1739's own code)", "root_cause": "CONFIRMED, via direct code reading plus independent live evidence from both a sim replay and a real production scheduler.log. xce_core_push_events()'s JOBS_PUSH handler, on a duplicate jobid (sched_add_job() -EEXIST, a resync redelivery of an already-tracked job), branches three ways once job_state_regresses_terminal() is ruled out: (1) resync claims RUN with real allocation data -> transfer it in (bug-1739's fix); (2) resync claims RUN with NO allocation data -> refuse, keep existing (bug-1739's own companion guard); (3) resync claims anything else (the bare `else` branch, bug-1734's original fix) -> UNCONDITIONALLY frees existing->accounts/existing->allocs and overwrites existing->state with the resync's own claim, with NO check for whether `existing` already holds a REAL, already-captured allocation the resync claim doesn't support. on_scheduler_join()'s full resync (xcesched_push_jobs(), src/controller/sched.c:214, fired on every scheduler (re)connect via join.c) is built from collect_jobs_committed() -- a committed-data snapshot gated only on 'job submission is committed', NOT on whether this specific job's DISPATCH transition has been committed yet. When a scheduler reconnects (or the connection blips) around the same moment it makes/receives confirmation of its own fresh dispatch decision for a job, the resync's committed-data snapshot can legitimately still show that job as PEND (its own dispatch commit hasn't propagated into the snapshot query yet) even though the scheduler's own live state (already set via sched_commit_decision() or the schedStartJob/JOB_STATE_CHANGE confirmation path) correctly has it RUN with a real allocation. Branch (3) blindly trusts the resync's stale PEND claim over the scheduler's own more-current, already-allocated state, freeing the real allocation and (in the observed cases) leaving state briefly PEND until a later, in-flight JOB_UPDATE/JOB_STATE_CHANGE for the SAME jobid (already queued in the same event batch, confirming the SAME dispatch) re-sets state=RUN -- but with no mechanism to restore the allocation branch (3) just destroyed. This exactly matches both the sim replay symptom (info jobs shows RUN, info worker's allocs-driven assigned-jobs listing omits it) and the production log's precise 16ms collision. Not the same mechanism as bug-1811's original (correct, but incomplete) 'Recorded-Decision Reconciliation transient window' explanation -- that window is real and self-healing, but this is a genuinely separate, permanent-until-this-fix data-loss bug triggered by resync/dispatch timing, not the reconciliation mechanism itself.", "fix": "FIXED (2026-09-15), TDD RED->GREEN. Added a guard in JOBS_PUSH's duplicate-jobid merge (core_events.c), inserted between bug-1739's `else if (JOB_IS_RUN(job->state))` branch and the original bare `else`: `else if ((existing->allocs != NULL) && !JOB_IS_DONE(job->state) && !JOB_IS_EXIT(job->state))` -- refuses the resync claim (WARNs, keeps existing->state/allocs/accounts untouched) whenever `existing` already holds a real allocation AND the resync's claimed state is not genuinely terminal. A terminal claim (DONE/EXIT) still falls through to the original branch, correctly releasing the allocation -- this is bug-1734's own legitimate 'job finished while disconnected' case and must not be blocked. Symmetric in spirit to bug-1739's own phantom-RUN guard (refuse to fabricate an unsupported RUN; now also refuse to destroy a supported RUN). TDD: added test_jobs_push_duplicate_jobid_run_with_allocation_survives_stale_nonrun_resync (test_core_events.c) -- job tracked RUN with a real allocation, redelivered via JOBS_PUSH as PEND with no allocation, must keep its RUN state and real allocation intact; and its companion test_jobs_push_duplicate_jobid_terminal_resync_releases_real_allocation -- the SAME RUN-with-real-allocation job, redelivered as DONE, must still release the allocation and transition (proves the new guard doesn't block legitimate terminal resyncs). RED confirmed (`xce-build.sh scheduler asan xce.build --incremental`): exactly the new stale-resync test failed (Expected 2 [RUN] Was 1 [PEND]), the terminal-resync companion already passed (unaffected by the unfixed code), 75 other tests green. GREEN confirmed after the fix: full scheduler suite 77/77 Ok, zero regressions -- including test_sim_e2e_original_motivation.c (the Recorded-Decision Reconciliation live-verify a FIRST, incorrect fix attempt this same session had broken and had to be reverted) and test_driver_translate.c, both untouched by this fix and green throughout.", @@ -26535,8 +13894,7 @@ "resync", "jobs-push", "allocation-loss", - "race-condition", - "fixed" + "race-condition" ], "related_bugs": [ "bug-1734", @@ -26544,15 +13902,20 @@ "bug-1739", "bug-1811" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1812.md" }, { "id": "bug-1813", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-15T08:35:00Z", - "error_message": "Follow-up to bug-1812's investigation (user-directed architectural fix, not a live-reproduced crash): while investigating why a scheduler reconnect/resync storm collides with in-flight array-job dispatches (bug-1812's own root cause), user asked to investigate the connection-loss trigger itself. Traced the scheduler's array-job receive path (handler.c's sched_on_job_array(), reached via scheduler_req_handler() on the shared vtaskmgr thread pool, default 8 threads) and found it does the FULL per-element expansion (dedup check via sched_get_job() + N independent deep clones via clone_job_submit_data()/clone_job_startinfo()) inline, synchronously, on whatever vtask thread handles the incoming push -- the exact 'long/unbounded work on a pool that must stay free for liveness-critical messages' anti-pattern .kiro/steering/ha-async-io.md's Thread-role contract warns about (the same pool is shared with the scheduler's own controller connection traffic). A prior, independent finding (2026-09-04 discovery, this project) already showed a burst of 6x50-element array pushes (300 job events, landing within ~2ms of each other) caused the scheduler to lose its connection to the primary controller within 245ms and enter a failover/reconnect loop -- directly consistent with this mechanism. User proposed the fix directly: the request-handling thread should just requeue the raw request, with the actual (expensive) work deferred to run alongside 'begin session' -- i.e. the round-processing thread's own xce_sched_driver_translate_pending_events()/xce_core_begin_session() sequence (main.c's xce_sched_driver_run_normal_mode_round(), driver.c), which carries no connection-liveness responsibility and already establishes this exact deferred-dispatch pattern for XCE_SCHED_OP_ADMIN_RECORD.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Follow-up to bug-1812's investigation (user-directed architectural fix, not a live-reproduced…\n\nFollow-up to bug-1812's investigation (user-directed architectural fix, not a live-reproduced crash): while investigating why a scheduler reconnect/resync storm collides with in-flight array-job dispatches (bug-1812's own root cause), user asked to investigate the connection-loss trigger itself. Traced the scheduler's array-job receive path (handler.c's sched_on_job_array(), reached via scheduler_req_handler() on the shared vtaskmgr thread pool, default 8 threads) and found it does the FULL per-element expansion (dedup check via sched_get_job() + N independent deep clones via clone_job_submit_data()/clone_job_startinfo()) inline, synchronously, on whatever vtask thread handles the incoming push -- the exact 'long/unbounded work on a pool that must stay free for liveness-critical messages' anti-pattern .kiro/steering/ha-async-io.md's Thread-role contract warns about (the same pool is shared with the scheduler's own controller connection traffic). A prior, independent finding (2026-09-04 discovery, this project) already showed a burst of 6x50-element array pushes (300 job events, landing within ~2ms of each other) caused the scheduler to lose its connection to the primary controller within 245ms and enter a failover/reconnect loop -- directly consistent with this mechanism. User proposed the fix directly: the request-handling thread should just requeue the raw request, with the actual (expensive) work deferred to run alongside 'begin session' -- i.e. the round-processing thread's own xce_sched_driver_translate_pending_events()/xce_core_begin_session() sequence (main.c's xce_sched_driver_run_normal_mode_round(), driver.c), which carries no connection-liveness responsibility and already establishes this exact deferred-dispatch pattern for XCE_SCHED_OP_ADMIN_RECORD.", "file": "src/scheduler/include/scheduler.h, src/scheduler/src/handler.c, src/scheduler/src/driver.c, src/scheduler/testsuite/test_job_array_expand.c, src/scheduler/testsuite/test_job_array_wire_e2e.c, src/scheduler/testsuite/test_driver_translate.c", "root_cause": "CONFIRMED via direct code reading (not yet live-reproduced end-to-end with gdb/ASAN under a real burst -- this fix is the same corrective direction bug-1714's own still-open investigation was heading, applied proactively here rather than waiting for a full live repro). handler.c's sched_on_job_array() (the XceData_data_jobArray case's handler, invoked by scheduler_req_handler() on a vtaskmgr thread) did the ENTIRE per-element expansion synchronously and inline: for each of up to array_size elements, a sched_get_job() dedup lookup plus a full deep clone (clone_job_submit_data()/clone_job_startinfo(), several calloc()+strdup() calls per element) before ever enqueueing anything. For a large concurrent burst (6 arrays x 50 elements landing within ~2ms, the 2026-09-04 finding), this ties up several of the vtaskmgr pool's 8 threads for the duration of 300 elements' worth of cloning -- the same pool the scheduler's own controller connection traffic depends on, a direct instance of ha-async-io.md's documented anti-pattern (a project-wide DESIGN INVARIANT, 'applies to ALL components: controller, worker, scheduler').", "fix": "FIXED (2026-09-15), TDD RED->GREEN, user-directed design. Added XCE_SCHED_OP_UPDATE_JOB_ARRAY (scheduler.h) mirroring XCE_SCHED_OP_ADMIN_RECORD's own established deferred-dispatch pattern, with a new xce_sched_update_request_t.u.jobarrayupdate union member carrying the raw, still-unexpanded xce_job_submit_t* descriptor plus (post-expansion) expanded_njobs/expanded_jobs for release_translated_payload()'s later cleanup. handler.c's sched_on_job_array() now does O(1) work only: validate the descriptor shape, wrap it in a request, enq_pend_update() -- no loop, no clone, no dedup check, ownership of the descriptor transferred to the queue on success (sched_handle_dataset()'s XceData_data_jobArray case updated to null dataset->rows on success instead of unconditionally freeing it). The full per-element expansion logic (clone_job_startinfo/clone_job_submit_data/make_array_element, moved verbatim) plus its dedup check now live in driver.c's translate_one_pending_update(), a new XCE_SCHED_OP_UPDATE_JOB_ARRAY case producing the same XCE_CORE_EVENT_JOBS_PUSH shape the old handler.c code did -- this runs on the round-processing thread (main.c's xce_sched_driver_run_normal_mode_round(), invoked before xce_core_begin_session() every round), which carries no connection-liveness responsibility. Side benefit: the dedup check's own pre-existing TOCTOU race (documented in the original code as bug-1714 follow-up 'Bug B', a genuine cross-thread race between the vtask thread's check and the driver thread's own core->jobs mutations) is now eliminated for the single-request case, since only the round-processing thread ever runs this check -- the remaining same-batch-double-delivery edge case is unchanged and was already documented as safe-by-design (sched_add_job()'s key_add() plus the JOBS_PUSH duplicate-merge logic, core_events.c, is the actual correctness boundary, untouched by this fix). release_translated_payload() (driver.c) gained a matching cleanup case: frees the original job_array descriptor unconditionally (never itself pushed into core) plus the N expanded clones (same unconditional free XCE_SCHED_OP_UPDATE_JOB's own jobupdate.jobs already gets). TDD: 4 new unit tests in test_driver_translate.c directly on translate_one_pending_update() (expands into N events with independent clones; skips already-existing elements against a REAL core->jobs kvmgr; all-duplicate redelivery returns 0/no-op; malformed payload returns 0) plus 1 new full drain-push-release end-to-end test (test_drain_translates_and_pushes_job_array_op) proving ownership correctness under real ASAN instrumentation. test_job_array_expand.c's and test_job_array_wire_e2e.c's own tests (which previously asserted the OLD inline-expansion behavior, including test_job_array_wire_e2e.c's tlog early-print-callback log-count capture, since the confirmation log line moved to driver.c) were revised to assert the NEW deferred-enqueue behavior instead -- redelivery dedup and log-firing-count coverage moved to test_driver_translate.c's own tests (log-firing-count itself was not re-added at the new location, a deliberate small scope trim, noted here rather than silently dropped). RED confirmed at each stage (xce-build.sh scheduler asan xce.build --incremental): first pass caught the 4 new driver-level unit test failures as expected plus 2 pre-existing test_job_array_wire_e2e.c failures this refactor had NOT yet been applied to (a test file this investigation initially missed, found only via the full suite run -- both asserted 'Expected 2 Was 11', i.e. the old handler-level XCE_SCHED_OP_UPDATE_JOB=2 check now correctly seeing the new XCE_SCHED_OP_UPDATE_JOB_ARRAY=11). GREEN confirmed after revising that file too: full scheduler suite 77/77 Ok, zero regressions, including test_sim_e2e_original_motivation.c (verified explicitly, given this same session's earlier unrelated mistake on bug-1811/1812's own investigation broke it once already) and all 10 new/revised job-array tests individually confirmed PASS via the raw test log.", @@ -26563,7 +13926,6 @@ "connection-stability", "job-array", "architecture", - "fixed", "user-directed" ], "related_bugs": [ @@ -26572,15 +13934,20 @@ "bug-1739", "bug-1714" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1813.md" }, { "id": "bug-1814", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-15T09:31:00Z", - "error_message": "User-reported, live replay session (/tmp/scheduler-replay-20260915-091847.replay, worker pc13v): `info worker pc13v` at session=2 (right after a burst of 19 JOB_UPDATE/JOB_RUSAGE event pairs for jobid[0]-[18]) showed `resource mem: value=22346629120.00 (20.81 GB) used=334251534843904.000000 (304.00 TB)` -- a used figure ~15000x larger than the worker's own total capacity, for a 22-slot/20.81GB host. Session 1 (before any JOB_RUSAGE events) showed a sane used=225280.000000 (220.00 KB) baseline.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported, live replay session (/tmp/scheduler-replay-20260915-091847.replay, worker pc13v)…\n\nUser-reported, live replay session (/tmp/scheduler-replay-20260915-091847.replay, worker pc13v): `info worker pc13v` at session=2 (right after a burst of 19 JOB_UPDATE/JOB_RUSAGE event pairs for jobid[0]-[18]) showed `resource mem: value=22346629120.00 (20.81 GB) used=334251534843904.000000 (304.00 TB)` -- a used figure ~15000x larger than the worker's own total capacity, for a 22-slot/20.81GB host. Session 1 (before any JOB_RUSAGE events) showed a sane used=225280.000000 (220.00 KB) baseline.", "file": "src/worker/src/xcecd_cgroups.c (collect_job_rusages() -- the fix), src/scheduler/src/resource.c (sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage() -- the consumer that sums the corrupted per-job value into w->resInUse, unmodified, already correct on its own terms), src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_RUSAGE handler -- confirmed the single, shared call site both normal-mode (driver.c) and sim/replay-mode (sim_loop.c) events funnel through, so this bug is not replay-specific)", "root_cause": "collect_job_rusages() queries 4 systemd cgroup resource counters (CPUUsageNSec, MemoryCurrent, MemorySwapCurrent, TasksCurrent) via get_unit_property()/sd_bus_get_property_trivial(). get_unit_property()'s success return (r>=0) means only that the D-Bus round trip itself worked -- systemd's own convention (undocumented in this codebase before now) is to report UINT64_MAX for a counter that isn't actually being tracked yet for a unit (accounting not yet active, or queried before the unit's cgroup is fully set up), NOT an error and NOT 0. The pre-fix code never checked for this sentinel. For MemoryCurrent specifically: (float)UINT64_MAX / (1024*1024) [MB unit conversion] rounds, under float32, to exactly 17592186044416.0 (2^44) -- a single job's fabricated 'memory usage'. sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage() (resource.c, unmodified and correct on its own terms -- resource-accounting.md's max(reserved,actual) policy) then charges this per-job figure into the worker's w->resInUse pool once per RUN job whose rusage tick reports it; the replay session had 19 such jobs report in the same burst, all apparently hitting the same not-yet-tracked-by-systemd condition (freshly-started array-job siblings), so the pool summed to exactly 19 * 2^44 = 334251534843904.0 -- matching the reported figure to the bit. Verified this arithmetic numerically (Python) before making any code change. sched_reconcile_host_resource_usage()'s own reconciliation math (resource.c:802-834) is not itself buggy -- garbage in, garbage charged, exactly as designed for a real (non-sentinel) report.", "fix": "FIXED (2026-09-15). Added a new static predicate sd_resource_counter_is_available(uint64_t val) (xcecd_cgroups.c, right after get_unit_property()) that returns false only for the exact UINT64_MAX sentinel, and gated all four collect_job_rusages() property reads (CPUUsageNSec/MemoryCurrent/MemorySwapCurrent/TasksCurrent) on `get_unit_property(...) >= 0 && sd_resource_counter_is_available(u64_val)` before trusting the value -- an unavailable counter is now silently skipped (the xce_rusage_t entry keeps its calloc-zeroed 0.0 default) rather than treated as a real reading. Applied uniformly to all four properties (not just MemoryCurrent) since systemd uses the identical UINT64_MAX-for-unavailable convention for each of them. Did NOT change memory_rss's existing MB unit convention (bug-1735's established wire contract, also relied on by the controller's own passthrough mem_usage/max_mem/avg_mem/used_mem display pipeline, misc.c/job.c) -- a candidate compensating fix in resource.c's sched_reconcile_job_resource_usage() (multiplying ru->value by 1024*1024 before charging, to match w->resInUse's apparent byte-native convention per resource-accounting.md) was drafted, then reverted: it would have broken test_resource_accounting.c's existing, currently-passing fixtures (test_reconcile_job_resource_usage_maps_memory_rss_to_mem_account and siblings), which assert memory_rss values flow through to w->resInUse with NO unit conversion -- contradicting the codebase's own current tested contract for this function. Left as an OPEN, FLAGGED, NOT-FIXED question (not resolved by this session): whether the scheduler's 'mem' resource job-level reserved/used accounting is actually MB-native end to end (matching the test fixtures and resource-accounting.md's own 'charge 4MB' worked example) or byte-native (matching w->resvals' real mem_free-in-bytes capacity reading) -- these two readings of the same document appear to conflict, and resolving it requires tracing what unit resreq's rusage[mem=...] parsing actually produces in production, which this session did not do. TDD: new src/worker/testsuite/test_xcecd_cgroups_rusage.c (white-box #include of xcecd_cgroups.c, same convention as hardware_monitor_test/resource_sync_test; a trivial make_job_sd_unit_name() stub satisfies the one linker dependency this file pulls in from xcecd_worker_utils.c that this test never actually calls) -- 5 tests covering the sentinel predicate (UINT64_MAX rejected, 0 and ordinary readings accepted, UINT64_MAX-1 accepted i.e. only the exact sentinel is special-cased) plus one test reproducing the exact pre-fix arithmetic (UINT64_MAX/1MB == 17592186044416.0, times 19 == 334251534843904.0) as a documented magnitude proof. The real D-Bus/systemd round trip inside collect_job_rusages() itself could not be live-verified in this sandbox -- no D-Bus/systemd daemon is reachable here (same environment limitation bug-1735 already established for this file); this is a static-analysis-plus-arithmetic-proof fix, needs live confirmation against a real cgroup-backed xcecd deployment. Wired into src/worker/testsuite/meson.build (new 'worker xcecd_cgroups rusage test' target). Full worker suite build+test in progress at time of this entry.\n\nADDENDUM (2026-09-15, same session, user's own multi-session replay trace): user reported that across multiple replay sessions, a job's corrupted 'used' figure NEVER self-corrects -- it stays elevated until the job finishes, not just at its first rusage tick. This resolves the open 'is this a one-time startup race or a persistent condition' question this entry originally left unanswered: sched_reconcile_host_resource_usage()'s delta-based math (resource.c) means a SUBSEQUENT poll returning a real, small value would compute a large NEGATIVE delta against the still-huge acct_entry->used, self-correcting the pool almost immediately -- that never happening across many sessions proves MemoryCurrent keeps returning the UINT64_MAX sentinel (or something producing an equally-unavailable reading) on EVERY 10-second poll (job_rusage_collect_interval, xcecd_worker.c:50) for that job's entire lifetime, not just once at unit creation. Confirmed MemoryAccounting=yes IS explicitly requested at transient-unit creation (xcecd_cgroups.c:270-291, start_systemd_job()) -- so this is not a 'forgot to enable accounting' bug; something about how MemoryCurrent is actually served back for these units on the live pc13v host (cgroup v1 vs v2, controller delegation, a systemd/D-Bus quirk) appears to be structurally broken, not transiently racy. Separately noted: CPUAccounting is NEVER explicitly requested (only TasksAccounting/MemoryAccounting are) -- CPUUsageNSec's availability depends entirely on that host's system-wide DefaultCPUAccounting= default, a related but distinct potential gap. NOT verified live (no D-Bus/systemd in this sandbox, same limitation as bug-1735) -- needs a check on the real pc13v host (`systemctl show -p MemoryCurrent -p MemoryAccounting` and/or `cat /sys/fs/cgroup//memory.current`) to distinguish a systemd-layer reporting bug from a kernel/cgroup-delegation-level gap. This fix's guard is still correct and necessary regardless of which it turns out to be -- it stops the garbage charge -- but the practical effect for affected jobs is that 'mem used' will sit at the flat reservation baseline for their whole runtime rather than tracking real usage, since real data genuinely never arrives for them; this is a host/cgroup-configuration question, out of this codebase's scope to resolve further.", @@ -26606,7 +13973,6 @@ "unit-mismatch-flagged-not-fixed", "found-live", "user-reported", - "fixed", "tdd", "environment-limitation", "needs-real-deployment-verification", @@ -26620,15 +13986,20 @@ "bug-1799", "bug-1801" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1814.md" }, { "id": "bug-1815", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-15T09:40:00Z", - "error_message": "Follow-up to bug-1814: user reported that tracing a job's 'mem used' figure across multiple live replay sessions showed it never self-corrects, staying elevated until the job finishes -- not a one-time blip. User also live-verified on the real pc13v host (`systemctl show -p MemoryCurrent -p MemoryAccounting`) that systemd genuinely reports correct, sane MemoryCurrent values for a healthy running job (MemoryCurrent=5525504, MemoryAccounting=yes) -- disproving the hypothesis that bug-1814's UINT64_MAX sentinel is a persistent/structural condition on this host. This left the 'why does it never self-correct' question open.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Follow-up to bug-1814: user reported that tracing a job's 'mem used' figure across multiple live…\n\nFollow-up to bug-1814: user reported that tracing a job's 'mem used' figure across multiple live replay sessions showed it never self-corrects, staying elevated until the job finishes -- not a one-time blip. User also live-verified on the real pc13v host (`systemctl show -p MemoryCurrent -p MemoryAccounting`) that systemd genuinely reports correct, sane MemoryCurrent values for a healthy running job (MemoryCurrent=5525504, MemoryAccounting=yes) -- disproving the hypothesis that bug-1814's UINT64_MAX sentinel is a persistent/structural condition on this host. This left the 'why does it never self-correct' question open.", "file": "src/scheduler/src/resource.c (sched_reconcile_host_resource_usage() -- the fix), src/scheduler/src/core_events.c (JOBS_PUSH resync-merge RUN branch, lines ~913-924 -- confirmed the account-object-replacement mechanism that interacts with this gap, not itself modified), src/scheduler/testsuite/test_resource_accounting.c (new regression test)", "root_cause": "sched_reconcile_host_resource_usage() (the JOB_RUSAGE-driven per-job mem-usage reconciliation path, resource.c) was missing the exact IS_WORKER_UNAVAIL/IS_WORKER_RECONCILING guard that bug-1801 (2026-09-11, same file) already added to sched_update_account() (the regular per-round charge/uncharge path). core_reset_workers() (bug-1751) deliberately FREEZES a RECONCILING/UNAVAIL worker's w->resInUse each round (skips the reset), on the understanding that NOTHING touches it until a clean per-round reset+resum can run once the worker's real state is confirmed; bug-1801 extended that freeze to sched_update_account() but this THIRD, independent resInUse-mutating path was never covered. Consequence: every JOB_RUSAGE tick that lands while a worker is RECONCILING keeps applying its computed delta to resInUse regardless, breaking the freeze. Compounding this: core_events.c's JOBS_PUSH resync-merge RUN branch (lines 913-924) can replace an already-tracked job's entire xce_job_account_t (existing->accounts = job->accounts, after free_job_account()-ing the old one) with a freshly-built object whose 'mem' xce_res_account_t has used=0.0 (make_res_account(), xcealloc.c) -- safe ONLY because the very next clean per-round rebuild (sched_account_jobs(), which fully re-derives resInUse from scratch each round) is supposed to correct any resulting discontinuity. That safety net is exactly what stays skipped while the worker remains RECONCILING. So a rusage tick landing after such a resync-merge, during the same still-RECONCILING window, computes its delta against the FRESH account's 0.0 baseline and adds the FULL usage on top of whatever was already frozen in the pool (potentially including bug-1814's own garbage charge, or simply a prior legitimate charge that now gets double-counted) -- with nothing left to ever subtract it back out until the worker escapes RECONCILING for good (letting a clean rebuild run) or the job reaches a terminal state (whose own uncharge path removes whatever the CURRENT account object holds). This produces exactly the observed 'used' figure that grows and never self-corrects for the job's entire remaining lifetime -- matching the user's live multi-session trace precisely, and consistent with this session's own bug-1812 finding that large array-job bursts cause repeated scheduler-to-controller connection loss (i.e. repeated/extended RECONCILING windows), a plausible trigger for this specific replay's affected jobs.", "fix": "FIXED (2026-09-15). Added `if (IS_WORKER_UNAVAIL(w->status) || IS_WORKER_RECONCILING(w->status)) { return 0; }` to sched_reconcile_host_resource_usage() (resource.c), immediately after the existing NULL-arg guard, before any resInUse/acct_entry mutation -- mirroring bug-1801's exact guard on sched_update_account() verbatim (same two status bits, same early-return-as-no-op semantics). Deliberately a full skip (neither the pool NOR acct_entry->used is touched) so the account's own tracked baseline stays consistent with whatever the still-frozen pool holds; once the worker's status clears, the very next rusage tick (or the next clean sched_account_jobs() rebuild) reconciles cleanly with no special-casing needed. TDD: added test_reconcile_host_resource_usage_skips_reconciling_or_unavail_worker (test_resource_accounting.c), mirroring test_charge_account_skips_reconciling_or_unavail_worker_host_charge's (bug-1801) own structure -- asserts a RECONCILING worker's resInUse AND acct_entry->used both stay untouched across a rusage tick, same for UNAVAIL, then confirms reconciliation resumes correctly (overage math verified) on the very next call once status clears. RED confirmed by temporarily disabling the new guard (`if (0 && (...))`) and rebuilding (`xce-build.sh scheduler asan xce.build --incremental`): exactly the 1 new test failed ('Expected 0 Was 5'), 76 other scheduler tests stayed green. GREEN confirmed after restoring the guard: full scheduler suite 77/77 Ok, zero regressions. NOT separately fixed here (out of scope, flagged only): the JOBS_PUSH resync-merge's own free-then-replace-without-uncharge pattern (core_events.c:913-924) remains as-is -- it is safe BY DESIGN as long as the following per-round rebuild is guaranteed to run, which is now consistently true again for all three resInUse-mutating paths (charge/uncharge/reconcile) once this fix closes the last gap; revisiting that pattern itself was judged unnecessary once this symmetry is restored.", @@ -26649,7 +14020,6 @@ "user-reported", "live-verified", "found-live", - "fixed", "tdd" ], "related_bugs": [ @@ -26660,15 +14030,20 @@ "bug-1812", "bug-1739" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1815.md" }, { "id": "bug-1816", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-09-15T13:05:00Z", - "error_message": "User request, following the bug-1814/1815 mem-accounting investigation: a live 3-node HA cluster on pc13v went through a repeated PreVote-failover storm (term climbing 2->6, standbys losing primary_ctrl_fd RDHUP repeatedly, xceadmin scheduler record status failing with 'scheduler not connected'/'too many redirects') while load was confirmed low (loadavg 0.5-1.6 on 22 cores). Root cause found live by the user: `ps axwww | grep control` on pc13v showed SIX xce_controller processes -- two full sets of ctrl-1/ctrl-2/ctrl-3 (PIDs 294848/294869/294905 and 610530/610546/610561) all bound to the identical ports (9901-9903) and identical --workdir/--certdir/--logfile paths under /tmp/xce-deploy-ha. The user's own hand-typed cleanup command (`ps axwww | grep xce-ha-deploy | ... | kill -9`) had a typo -- the actual workdir is /tmp/xce-deploy-ha/... ('xce-deploy-ha'), not 'xce-ha-deploy' (words transposed) -- so it matched nothing and silently failed to kill the first deploy's processes before a second deploy started on the same ports. User asked to enhance the xce-deploy-ha skill's shutdown-all mechanism so this class of mistake becomes structurally hard to repeat.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User request, following the bug-1814/1815 mem-accounting investigation: a live 3-node HA cluster…\n\nUser request, following the bug-1814/1815 mem-accounting investigation: a live 3-node HA cluster on pc13v went through a repeated PreVote-failover storm (term climbing 2->6, standbys losing primary_ctrl_fd RDHUP repeatedly, xceadmin scheduler record status failing with 'scheduler not connected'/'too many redirects') while load was confirmed low (loadavg 0.5-1.6 on 22 cores). Root cause found live by the user: `ps axwww | grep control` on pc13v showed SIX xce_controller processes -- two full sets of ctrl-1/ctrl-2/ctrl-3 (PIDs 294848/294869/294905 and 610530/610546/610561) all bound to the identical ports (9901-9903) and identical --workdir/--certdir/--logfile paths under /tmp/xce-deploy-ha. The user's own hand-typed cleanup command (`ps axwww | grep xce-ha-deploy | ... | kill -9`) had a typo -- the actual workdir is /tmp/xce-deploy-ha/... ('xce-deploy-ha'), not 'xce-ha-deploy' (words transposed) -- so it matched nothing and silently failed to kill the first deploy's processes before a second deploy started on the same ports. User asked to enhance the xce-deploy-ha skill's shutdown-all mechanism so this class of mistake becomes structurally hard to repeat.", "file": "src/tools/deploy-ha.sh (port pre-flight check, cleanup.sh generation fix, new stop-all.sh generation, start-all.sh wait_for_pidfile race fix), .claude/skills/xce-deploy-ha/SKILL.md (Step 2, Clean restart, Troubleshooting documentation updates)", "root_cause": "Two independent, compounding structural gaps let a hand-typed-command typo turn into a genuine HA failover storm instead of a clean, obvious failure: (1) deploy-ha.sh had NO pre-flight check that the target ports were actually free before starting a new cluster -- it would happily bind a second, fully independent 3-node Raft group to the exact same ports an existing, still-alive cluster already held, producing real cross-talk (mixed vote traffic, corrupted term/leader tracking between two unrelated groups) rather than a clean bind() failure with a clear error. (2) start.sh/start-all.sh already HAD a correct, working 'already running' guard (reads $TOP/run/*.pid, checks kill -0, refuses to start a duplicate) -- but the pre-existing cleanup.sh unconditionally did `rm -f $TOP/run/*.pid` as pure state cleanup, with NO check that the PID each file pointed to was actually dead first. Running cleanup.sh (or a fresh deploy-ha.sh invocation that regenerates it) without ALSO successfully killing the live processes first silently erased the ONLY signal that guard depends on, letting a stale process survive completely undetected by every existing safety net in the codebase. The user's specific typo was the proximate trigger, but the underlying design let a single hand-typed-command mistake cascade into two independent Raft clusters, sharing ports, cross-talking -- a much worse failure mode than the typo alone should have caused.", "fix": "FIXED (2026-09-15). Three changes to src/tools/deploy-ha.sh, live-verified end-to-end against real xce_controller/xceadmin/xcecd_virtual/xce_scheduler/xce_apiserver binaries (not just syntax-checked): (1) A port pre-flight check, added immediately after PEER_ARRAY is parsed and before ANY directory/cert/cluster.bin state is created -- for each peer's port, checks via `ss -tln` whether something is already listening; if so, prints the offending PID/process (via `ss -tlnp`, best-effort) and refuses to proceed with an actionable error (the same kill-by-name commands Step 2 documents), exiting before touching disk. Soft-skips (warns, doesn't fail) if `ss` isn't on PATH. Live-verified: a redeploy attempt against a still-running cluster's ports was correctly refused, identifying the exact blocking PIDs, and created no directory at all under the attempted $TOP. (2) cleanup.sh's generated body now stops every live PID in $TOP/run/*.pid (kill -9, with a warning to `sudo kill -9` if the real root-run worker doesn't die from a plain kill) BEFORE removing any pidfile or other state, closing the exact gap in root_cause (2) -- restoring the invariant that start.sh/start-all.sh's own guard depends on: no pidfile is ever removed while pointing at a still-alive process. stop-all.sh (new, generated alongside start.sh/cleanup.sh) is the same pidfile-based stop logic as a standalone script that leaves state untouched -- the canonical, typo-proof way to stop everything a specific deploy started, since it reads that deploy's own tracked PIDs instead of matching process names/paths by hand. (3) Found and fixed a related race while live-testing (1) and (2) together: start-all.sh launched worker/scheduler/apiserver via backgrounded `setsid nohup ... &` with no wait for each to actually write its own pidfile before declaring itself done -- running cleanup.sh/stop-all.sh immediately after start-all.sh returned could catch worker/scheduler BEFORE their pidfile existed, defeating the very fix in (2)/(3) for those two components specifically (reproduced live: cleanup.sh correctly stopped ctrl-1/2/3 but left worker-1/scheduler running with their pidfiles already wiped -- the identical zombie-process class this whole fix targets, via a narrower timing window instead of an operator typo). Added a small wait_for_pidfile() poll (up to 5s, 0.2s interval) after each of the three backgrounded launches; start-all.sh now only reports 'done' once every component it started has a live pidfile on disk (or warns by name if one didn't appear in time). Re-verified live after this fix: cleanup.sh run IMMEDIATELY after start-all.sh returned correctly stopped all 6 processes (3 controllers + worker + scheduler + apiserver), confirmed via `ps`/`ss` showing nothing left running and no listeners on the deployed ports. SKILL.md updated: Step 2 now prefers a known deploy's own stop-all.sh over the hand-typed blanket kill (documents exactly why, citing this incident), Clean restart section documents cleanup.sh's new stop-then-wipe behavior and stop-all.sh as the no-wipe alternative, and two new Troubleshooting rows cover the port-pre-flight error message and the 'inexplicable HA failover despite low load, multiple PIDs per role in logs' symptom pattern directly, matching this incident.", @@ -26689,22 +14064,26 @@ "user-requested", "live-verified", "found-live", - "fixed", "not-a-code-regression" ], "related_bugs": [ "bug-1814", "bug-1815" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1816.md" }, { "id": "bug-1817", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-09-15T19:20:00Z", - "error_message": "User-reported live core dump: xce_scheduler (build.dbg, HA-connected normal mode, --peers ctrl-1/2/3) crashed with SIGSEGV in its dedicated allocator thread (sched_allocator, scheduler.c:484). gdb backtrace: zarray_get_ex (lib/zarray.c:332, `data[i] = array->data[idx]`) <- zarray_get <- match_job_host_limit_ (plugins/limit.c:370) <- limit_prepare_allocation <- plugin_prepare_allocation <- sched_try_alloc_on_host_ (resource.c:528) <- parjob_try_plan_ <- my_alloc <- sched_make_parjob_allocation <- sched_make_primary_alloc_ <- sched_make_primary_allocation <- sched_make_allocation <- sched_allocator <- start_thread. gdb locals at the crash frame: i=0, array->data=0x0 (NULL), while the bounds check `idx (0) >= array->caps` had evidently passed (i.e. array->caps was nonzero) -- a zarray_t with data==NULL but caps>0, an internally inconsistent/torn state that should be structurally impossible for a correctly-initialized array.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported live core dump: xce_scheduler (build.dbg, HA-connected normal mode, --peers…\n\nUser-reported live core dump: xce_scheduler (build.dbg, HA-connected normal mode, --peers ctrl-1/2/3) crashed with SIGSEGV in its dedicated allocator thread (sched_allocator, scheduler.c:484). gdb backtrace: zarray_get_ex (lib/zarray.c:332, `data[i] = array->data[idx]`) <- zarray_get <- match_job_host_limit_ (plugins/limit.c:370) <- limit_prepare_allocation <- plugin_prepare_allocation <- sched_try_alloc_on_host_ (resource.c:528) <- parjob_try_plan_ <- my_alloc <- sched_make_parjob_allocation <- sched_make_primary_alloc_ <- sched_make_primary_allocation <- sched_make_allocation <- sched_allocator <- start_thread. gdb locals at the crash frame: i=0, array->data=0x0 (NULL), while the bounds check `idx (0) >= array->caps` had evidently passed (i.e. array->caps was nonzero) -- a zarray_t with data==NULL but caps>0, an internally inconsistent/torn state that should be structurally impossible for a correctly-initialized array.", "file": "src/lib/zarray.c (zarray_fini() -- primary fix), src/lib/xlist.c (xlist_fini() -- identical fix, same bug class), src/scheduler/src/schedobj.c (reset_sched_obj() -- the reachable production caller that exposed the bug, NOT itself modified: correct by construction once the library primitives are fixed), src/scheduler/src/plugins/limit.c (limit_job_election()/match_job_host_limit_() -- the call chain that reaches the torn array, not modified), src/lib/testsuite/test_zarray.c, src/lib/testsuite/test_xlist.c (new regression tests)", "root_cause": "zarray_fini(array) (lib/zarray.c) freed array->data and set it to NULL, but left array->caps/count/pos at whatever nonzero values they held before the call -- NOT reset to the safe (0,0,0) state zarray_init() establishes. xlist_fini() (lib/xlist.c) had the byte-for-byte identical gap (FREEUP(l->data) alone, l->caps/l->count left stale). Both functions' own bounds-checked readers (zarray_get_ex's `idx >= array->caps`, xlist_get's `idx >= l->caps`) treat a nonzero caps as a promise that array->data/l->data is a valid, sized allocation -- a promise zarray_fini()/xlist_fini() broke for any caller that reused the SAME struct after fini without an intervening re-init. schedobj.c's reset_sched_obj(xce_sched_obj_t* obj) does exactly that: `xlist_fini(&(obj->glimits)); zarray_fini(&(obj->hlimits));` with no matching re-init, called from plugins/limit.c's limit_job_election() every time a job's schedobj.configver falls behind the cluster's current core.configver (i.e. whenever the cluster config has been reloaded/pushed since this job's limits were last computed) -- `obj` here is the job's own PERSISTENT xce_sched_obj_t (job->schedobj, allocated once at job ingest, reused for the job's entire lifetime), not a disposable scratch object. Once at least one host-limit match has grown obj->hlimits' caps beyond 0 (zarray_put() inside match_job_host_limit_(), the very next line after the crash site) and the config version later advances again while that job is still PEND, limit_job_election()'s reset_sched_obj() call tears obj->hlimits down to (data=NULL, caps=stale-nonzero) -- and the next allocation attempt for that same job (in the same or a later scheduling round, from a completely different call path: the allocator thread's plugin_prepare_allocation -> limit_prepare_allocation -> match_job_host_limit_ -> zarray_get(&(obj->hlimits), worker->idx)) dereferences the freed array, since worker->idx (0 for a single-worker deployment) is below the stale caps. This exact crash class had already caused at least one earlier, never-successfully-root-caused crash near the same call site (memory observation from 2026-08-17, 'crash occurs in driver/decision processing after limit_job_election returns successfully' -- that investigation correctly localized the area but concluded the crash was downstream in the decision pipeline, not inside reset_sched_obj()/zarray_fini() itself, for lack of as clean a gdb backtrace as this session's core dump provided). reset_sched_obj() itself was ALSO nearly lost entirely in an earlier refactor (bug-1490, 're-added reset_sched_obj() verbatim (unchanged body)' after a wholesale schedobj.c rewrite dropped it) -- its buggy body has survived unchanged through at least that refactor.", "fix": "FIXED (2026-09-15). Fixed at the library-primitive level, not at reset_sched_obj()'s call site: zarray_fini() now resets array->count/array->caps/array->pos to 0 (matching zarray_init()'s own defaults) immediately after freeing/NULLing array->data; xlist_fini() now resets l->count/l->caps to 0 (matching xlist_init()) after FREEUP(l->data). This restores the invariant both functions' own doc comments now state explicitly: _fini() leaves the struct in the SAME safe, empty, immediately-reusable state _init() does, not just 'data freed, everything else however it happened to be.' schedobj.c's reset_sched_obj() and every other _fini()-without-reinit caller across the codebase are now correct by construction, with zero code changes needed there. Scoped the fix to the library primitives (not schedobj.c-side re-init calls) deliberately: audited every zarray_fini()/xlist_fini() call site outside testsuite/ (5 zarray_fini, ~25 xlist_fini, all in controller/lib) and confirmed every one either discards the struct immediately after (a stack-local scratch array/list, function returns right after) or is tearing down a struct being destroyed -- none relies on caps/count surviving fini, so this is a pure safety improvement for the whole 68+-caller zarray_init/all xlist_init blast radius, not a behavior change for any correct caller. TDD: added test_zarray_fini_resets_bounds_for_reuse (test_zarray.c) and test_xlist_fini_resets_bounds_for_reuse (test_xlist.c), both driving the raw zarray_init/zarray_put/zarray_fini/zarray_get and xlist_init/xlist_store/xlist_fini/xlist_get API directly (not the xzarray RTTI wrapper test_zarray.c's other tests use) since that's the exact API schedobj.c calls -- put an item (grows caps>0), fini WITHOUT re-init (exactly reset_sched_obj()'s pattern), assert caps/count reset to 0 and data is NULL, assert a get() for the previously-valid index now safely returns NULL instead of what used to be a NULL-pointer dereference, then assert the struct is genuinely reusable (a subsequent put/get round-trips correctly). RED confirmed by temporarily short-circuiting both fixes with `if (0) { ... }`: both new tests failed cleanly ('Expected 0 Was 4' / 'Expected 0 Was 2', the stale pre-fini caps value) -- Unity's assertion-failure longjmp aborts that one test function before reaching the actual NULL-deref line, so RED was captured as a clean, deterministic assertion failure rather than a process crash, letting the rest of the 300+-test suite still run and report cleanly around it. GREEN confirmed after restoring both fixes: full xcelib_test suite 314/314 individual test cases across all aggregated test files, zero regressions. Full scheduler suite also re-verified: 77/77, zero regressions (this library change is upstream of essentially all scheduler resource/allocation code). NOT investigated/fixed here, flagged only: a related, ALREADY-DOCUMENTED-BUT-UNAPPLIED bug (bug-1578, controller/sched.c's sched_worker_change()) has a DIFFERENT mechanism (a dangling ALIAS to a since-freed xlist_t backing array surviving on a separate struct field, not the same struct's own stale bookkeeping) that this fix does not touch or fix -- genuinely unrelated, noted only because it surfaced during this investigation's own zarray/xlist call-site audit.", @@ -26730,7 +14109,6 @@ "found-live", "core-dump", "gdb", - "fixed", "tdd", "library-primitive-fix", "recurring-bug-class", @@ -26741,15 +14119,20 @@ "bug-1578", "bug-1490" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1817.md" }, { "id": "bug-1818", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "harness", + "severity": "unknown", "timestamp": "2026-09-16T08:20:00Z", - "error_message": "Live 3-node HA E2E validation (controller-status spec, Requirement 12 delta, task 7.6): cmd_ctrlstatus (xce_test client) printed electedTerm: unknown / electedAt: unknown for the new primary-election-term/time fields on EVERY standby query, both before and after a real primary kill/failover (confirmed after ~4 heartbeat intervals, ruling out a convergence-timing gap) -- even though the same query correctly showed non-stale, converged term/committedLsn/appliedLsn values for the same standby at the same moment.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Live 3-node HA E2E validation (controller-status spec, Requirement 12 delta, task 7.6)…\n\nLive 3-node HA E2E validation (controller-status spec, Requirement 12 delta, task 7.6): cmd_ctrlstatus (xce_test client) printed electedTerm: unknown / electedAt: unknown for the new primary-election-term/time fields on EVERY standby query, both before and after a real primary kill/failover (confirmed after ~4 heartbeat intervals, ruling out a convergence-timing gap) -- even though the same query correctly showed non-stale, converged term/committedLsn/appliedLsn values for the same standby at the same moment.", "file": "src/harness/test.c (cmd_ctrlstatus -- fix), .kiro/specs/controller-status/design.md (Design Delta -- Requirement 12, Locked Decision 3 -- corrected)", "root_cause": "cmd_ctrlstatus gated the new electedTerm/electedAt print lines on NULLSTR(status->primaryName), reusing the same condition the pre-existing Primary: line already uses -- per the delta design's own Locked Decision 3, which assumed primaryName would be \"known\" on a standby that has learned the primary. But primaryName is UNCONDITIONALLY empty on every non-primary responder in this build: ctrlstatus_fill (src/controller/ctrlstatus.c) only self-reports the primary's own name when the responder IS the primary (ha->primary_host is vestigial/never written, a pre-existing behavior from Requirement 2 / commit ade0081, predating this delta). So Requirement 2.3's \"standby knows the primary\" branch is dead code via the plain ctrlstatus command, and gating the new fields on the same condition made them permanently unreachable on any standby -- even though status->elected_at_ms (populated unconditionally by ctrlstatus_fill, task 7.4, and correctly heartbeat-learned by task 7.3) was present and correct the whole time. Caught only by a real live cluster run; all unit tests (which test ctrlstatus_fill directly, not cmd_ctrlstatus's print gate) passed throughout, since the bug was entirely in the untested CLI display layer.", "fix": "FIXED (2026-09-16, controller-status Requirement 12 delta). Changed cmd_ctrlstatus's gate from NULLSTR(status->primaryName) to status->elected_at_ms == 0. This is both necessary and sufficient: elected_at_ms stays exactly 0 until a real promotion stamps it (either promotion road, ha_do_promote / raft_cb_become_leader) or a heartbeat carries a learned value from an actual primary (ha_consume_heartbeat_committed) -- unlike term, which can legitimately bump from mere vote/candidacy activity before any primary is actually elected, so term == 0 would have been the wrong gate (verified via a dedicated candidacy-in-progress test case: elected_at_ms == 0 but term != 0 must still print unknown). ctrlstatus_fill itself was NOT touched -- it correctly remained an unconditional, no-role-branch read (Locked Decision 2), unaffected by this bug. Verified via a standalone scratch program (against the real ctrlstatus_fmt.h header, 4 cases: standby/bug case, never-elected, candidacy-in-progress, primary/regression) plus a repeat live 3-node E2E run confirming the fix converges end-to-end after a real primary kill. design.md's Locked Decision 3 and its dependent traceability/service-contract text were corrected in the same commit to document this live-E2E finding, mirroring the spec's existing Req 10/11 delta convention for recording live findings.", @@ -26762,20 +14145,23 @@ "live-e2e-found", "display-layer", "dead-code-gate", - "primaryName", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "primaryName" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1818.md" }, { "id": "bug-1819", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-16T17:15:00Z", - "error_message": "controller-status spec Requirement 13 delta, task 8.6 (bug found live during task 8.5's optional E2E attempt): ctrlstatus_fill's primary-side peer-status loop (src/controller/ctrlstatus.c) scanned replicator_t.standbys[] for a host/port match and stopped at the FIRST match by slot index. When a standby was killed and later reconnected into a DIFFERENT replicator slot than the one it originally occupied (its old slot keeps host/port populated with ep==NULL after the kill), the lower-indexed stale slot won the scan and the peer was reported dead PERMANENTLY -- even minutes after it had genuinely reconnected and fully caught up. Task 8.5's live run observed this directly: primary log showed acked_lsn converging to the committed tip at a specific timestamp while ctrlstatus kept reporting dead from restart through +5m45s.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "controller-status spec Requirement 13 delta, task 8.6 (bug found live during task 8.5's optional…\n\ncontroller-status spec Requirement 13 delta, task 8.6 (bug found live during task 8.5's optional E2E attempt): ctrlstatus_fill's primary-side peer-status loop (src/controller/ctrlstatus.c) scanned replicator_t.standbys[] for a host/port match and stopped at the FIRST match by slot index. When a standby was killed and later reconnected into a DIFFERENT replicator slot than the one it originally occupied (its old slot keeps host/port populated with ep==NULL after the kill), the lower-indexed stale slot won the scan and the peer was reported dead PERMANENTLY -- even minutes after it had genuinely reconnected and fully caught up. Task 8.5's live run observed this directly: primary log showed acked_lsn converging to the committed tip at a specific timestamp while ctrlstatus kept reporting dead from restart through +5m45s.", "file": "src/controller/ctrlstatus.c (ctrlstatus_fill, primary-side peer-status matching loop), src/controller/testsuite/test_ctrlstatus.c", "root_cause": "The primary-side branch of ctrlstatus_fill's per-peer loop scanned replicator_t.standbys[] with a `for (j = 0; j < MAX_STANDBYS; j++) { if (host/port match) { ...; break; } }` pattern -- first host/port match wins, regardless of whether that slot is live (ep != NULL) or stale (ep == NULL). repl_add_standby's slot allocation is keyed by ep identity, not by configured-peer name, so a killed-then-reconnected standby's NEW connection can land in a different slot than its old, now-stale one (confirmed live this session via a related but distinct mechanism -- see bug-1474 cross-reference below); once that happens, the stale, lower-indexed slot permanently wins ctrlstatus_fill's first-match scan even though the peer is genuinely live and caught up in its new slot.", "fix": "FIXED (2026-09-16, TDD RED->GREEN). Changed the matching loop to scan ALL MAX_STANDBYS slots for a given configured peer (no `break` on the first match) and prefer any slot with ep != NULL over a slot with ep == NULL when multiple slots match the same host:port -- tracked via a `live_match` flag; the first slot with ep != NULL found during the scan determines the reported status (ok/catching-up per its own in_catch_up), and a stale (ep == NULL) match only sets the dead fallback if no live match has been seen yet. If more than one live slot somehow matches the same peer, any one of them is an acceptable source (design.md task 8.6 note) since they would all observe the same live peer. Scope: read-only, same MAX_STANDBYS==3 bounded scan and same repl->lock scope task 8.2 already established -- no lock-scope change (explicitly out of this task's scope per its own text). RED: 2 new unit tests (test_ctrlstatus_peer_status_prefers_live_slot_over_stale_ok, test_ctrlstatus_peer_status_prefers_live_slot_catching_up) added to test_ctrlstatus.c, each constructing a fixture with a lower-index stale slot (ep==NULL) and a higher-index live slot (ep!=NULL) matching the SAME configured peer's host:port -- failed pre-fix with 'Expected 1 Was 3' and 'Expected 2 Was 3' (the permanently-dead symptom, reproduced exactly). GREEN: same two tests pass post-fix; full controller_unit_test suite = 1216 Tests, 0 Failures (both the meson aggregate and the standalone controller_unit_test binary's own summary line cross-checked, per testing.md convention) -- no regressions. LIVE VERIFIED: fresh 3-node cluster (deploy-ha.sh, /tmp/xce-deploy-ctrlstatus86), killed standby ctrl-2, submitted an 80-job backlog while it was down (committedLsn advanced 1->161, confirming genuine WAL activity), restarted ctrl-2, polled ctrlstatus on the primary 8 times over ~24s -- ctrl-2 showed ok immediately on every poll after reconnect (never stuck at dead), while the primary's own log confirmed a genuine WAL catch-up stream (standby[1] acked 1->161) was still converging underneath, proving the fix reads the live registry state correctly rather than merely getting lucky on timing.", @@ -26790,21 +14176,25 @@ "stale-slot", "live-e2e-found", "task-8.6", - "fixed", "tdd" ], "related_bugs": [ "bug-1474" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1819.md" }, { "id": "bug-1820", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-09-17T02:45:00Z", - "error_message": "User report: after deploying a real (root, sudo-launched) xcecd worker via /xce-deploy-ha and stopping the cluster with scripts/stop-all.sh, xcecd keeps running -- ps shows a leftover xcecd process (a different PID than the one stop-all.sh reported stopping) still bound to the deploy dir's logs after stop-all.sh reported \"Stopping worker-1 (PID )\" with no warning.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User report: after deploying a real (root, sudo-launched) xcecd worker via /xce-deploy-ha and…\n\nUser report: after deploying a real (root, sudo-launched) xcecd worker via /xce-deploy-ha and stopping the cluster with scripts/stop-all.sh, xcecd keeps running -- ps shows a leftover xcecd process (a different PID than the one stop-all.sh reported stopping) still bound to the deploy dir's logs after stop-all.sh reported \"Stopping worker-1 (PID )\" with no warning.", "file": "src/tools/deploy-ha.sh (generated scripts/stop-all.sh and scripts/cleanup.sh kill loops; generated scripts/start-worker.sh's sudo branch)", "root_cause": "start-worker.sh's real-worker branch writes the pidfile via `echo $$ > \"$PIDFILE\"` and then does `exec sudo \"$WORKER_BIN\" ...` -- `exec` replaces the pidfile-writing shell's own process image with sudo IN PLACE (same PID), so the PID recorded in the pidfile is sudo's own PID, not the actual worker daemon's. sudo then forks a SEPARATE, root-owned child process that execs and runs the real xcecd daemon; sudo itself stays alive as a thin parent/monitor. stop-all.sh's (and cleanup.sh's, identical logic) kill loop did `kill -9 \"$PID\"` against the recorded (sudo) PID -- SIGKILL terminates sudo instantly but is never forwarded to its child (SIGKILL cannot be caught/relayed by the dying parent), so the real xcecd process is silently orphaned (reparented to init) and keeps running. The scripts' own pre-existing \"still alive after kill -9\" warning branch also never fired for this exact case, because it re-checks the SAME (sudo) PID, which genuinely is dead by then -- so the orphaned child produced zero warning output, making the bug fully silent.", "fix": "FIXED (2026-09-17). In both generated scripts/stop-all.sh and scripts/cleanup.sh (src/tools/deploy-ha.sh), before killing a pidfile'd PID, check whether that PID's own /proc/$PID/cmdline starts with \"sudo \" (the telltale sign left by `exec sudo ...`); if so, find its child process(es) via `pgrep -P \"$PID\"` and kill them first via `sudo kill -9` (they run as root), before killing the sudo PID itself. The post-kill \"still alive\" check was also extended to additionally test for the child PID's continued existence (via `/proc/$CHILD` presence, which needs no kill permission, since `kill -0` on a root-owned PID from a non-root caller returns EPERM regardless of whether the process is alive). Live-verified end-to-end: deployed a fresh 3-node cluster with XCE_REAL_WORKER=1 (forcing the sudo xcecd path), confirmed the exact sudo-parent/root-child PID pair via `ps axwww`, ran the fixed stop-all.sh and confirmed both processes died with the new \"is a sudo wrapper -- also stopping its child\" line printed and zero processes remaining afterward; repeated the same reproduction against cleanup.sh with the same clean result. No unit test exists for this bash tooling (deploy-ha.sh has no meson-registered test suite, consistent with other shell tooling in this project e.g. xce-build.sh); verification is this live end-to-end reproduction.", @@ -26818,21 +14208,25 @@ "orphaned-process", "real-worker", "sigkill", - "fixed", "live-verified" ], "related_bugs": [ "bug-1720" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1820.md" }, { "id": "bug-1821", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-09-17T04:10:00Z", - "error_message": "User-reported: xceadmin's --controller option (config push and all 3 scheduler record start/stop/status commands) accepted only a SINGLE host:port -- passing a comma-separated list of all configured controllers (the natural way to query an HA cluster without knowing in advance which node is alive/primary, e.g. `--controller \"localhost:9901,localhost:9902,localhost:9903\"` with 9901 dead) failed outright: `Error: cannot connect to localhost:9901,localhost:9902,localhost:9903: [Errno -2] Name or service not known` -- the whole comma-joined string was parsed as one bogus hostname.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported: xceadmin's --controller option (config push and all 3 scheduler record…\n\nUser-reported: xceadmin's --controller option (config push and all 3 scheduler record start/stop/status commands) accepted only a SINGLE host:port -- passing a comma-separated list of all configured controllers (the natural way to query an HA cluster without knowing in advance which node is alive/primary, e.g. `--controller \"localhost:9901,localhost:9902,localhost:9903\"` with 9901 dead) failed outright: `Error: cannot connect to localhost:9901,localhost:9902,localhost:9903: [Errno -2] Name or service not known` -- the whole comma-joined string was parsed as one bogus hostname.", "file": "src/xceadmin/src/xceadmin/discovery.py (new resolve_controllers/follow_redirects_any), src/xceadmin/src/xceadmin/push.py, src/xceadmin/src/xceadmin/scheduler.py (both migrated off their own duplicated _split_controller), src/xceadmin/src/xceadmin/cli.py (--controller help text)", "root_cause": "push.py and scheduler.py each had their own private _split_controller(controller: str) -> (host, port) that did a single controller.rsplit(':', 1) -- designed only ever to see one host:port. A comma-separated value was never split on ',' at all; it fell through as one literal (unparseable) hostname straight into socket.create_connection via xcecommon.wire.connect, which propagates the resulting socket.gaierror (a plain OSError subclass) unwrapped by design.", "fix": "FIXED (2026-09-17, TDD RED->GREEN). Added xceadmin.discovery.resolve_controllers(controller) -> list[(host,port)] (comma-split, whitespace-stripped, per-entry bare-host default port) and xceadmin.discovery.follow_redirects_any(candidates, send_fn) -> (host,port,reply_bytes), which tries each candidate in order via the existing follow_redirects(), catching per-candidate OSError (connection refused, DNS failure) and NoPrimaryError (discovery-level timeout) and moving to the next candidate -- but NOT catching ReplyDecodeError/TooManyRedirectsError, since a genuinely-reachable candidate's protocol-level failure isn't fixed by trying a different node. push.py/scheduler.py's own _split_controller removed; both now call resolve_controllers + follow_redirects_any. cli.py's --controller help text updated on all 4 commands to document the comma-list form. 12 new unit tests in test_discovery.py (single/list/whitespace parsing; first-reachable-wins; skip-refused/skip-DNS-failure/skip-timeout-then-succeed; all-fail raises NoPrimaryError naming every candidate; redirect-following composes correctly with failover; a reachable candidate's decode error does NOT fall through to the next). 2 existing tests per file (test_config_push_mock.py, test_scheduler.py) updated to call the new shared function instead of the removed private one. Full suite 304/304. Live-verified: deployed a 3-node cluster, killed the first-listed controller, confirmed `--controller \"dead,live1,live2\"` correctly skips the dead one and reaches a live one (both healthy-cluster and killed-primary scenarios).", @@ -26846,7 +14240,6 @@ "failover", "master-rotation", "user-reported", - "fixed", "tdd", "live-verified" ], @@ -26854,15 +14247,20 @@ "bug-1566", "bug-1822" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1821.md" }, { "id": "bug-1822", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "xceadmin", + "severity": "unknown", "timestamp": "2026-09-17T04:10:00Z", - "error_message": "Found live while verifying bug-1821's --controller failover-list fix: querying `xceadmin scheduler record status --controller \"host1,host2,host3\"` immediately after killing the primary hard-failed with `Error: no primary available: none of the configured controllers responded (...)` instead of riding out the cluster's own re-election, even though _send_admin_request() already has a bounded retry loop (bug-1713/1715) built for exactly this 'cluster still settling' condition.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "Found live while verifying bug-1821's --controller failover-list fix: querying `xceadmin scheduler…\n\nFound live while verifying bug-1821's --controller failover-list fix: querying `xceadmin scheduler record status --controller \"host1,host2,host3\"` immediately after killing the primary hard-failed with `Error: no primary available: none of the configured controllers responded (...)` instead of riding out the cluster's own re-election, even though _send_admin_request() already has a bounded retry loop (bug-1713/1715) built for exactly this 'cluster still settling' condition.", "file": "src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request)", "root_cause": "Right after a primary is killed, a standby does not even START a new election until its own heartbeat_timeout_ms elapses (ha.h: HA_DEFAULT_HEARTBEAT_TIMEOUT_MS = 10000, i.e. 10s) -- until then it keeps redirecting every schedAdmin request straight back to the now-dead old primary. follow_redirects_any() (bug-1821) correctly detects that dead redirect target is unreachable and tries the next configured candidate, but if EVERY candidate is in this same stale-redirect state, the whole candidate list is exhausted and follow_redirects_any raises NoPrimaryError -- an exception _send_admin_request()'s existing retry loop never caught. That loop only retried on a REPLY that arrived and said not-ready (RequestReply.status in _TRANSIENT_STATUSES, or SchedAdminReply.ok=false with a known-transient reason) -- a total connection failure (no reply at all) propagated immediately with zero retries, even though it is the identical 'cluster still settling' condition the existing retry logic was built to ride out, just signaled differently.", "fix": "FIXED (2026-09-17, TDD RED->GREEN). _send_admin_request()'s retry loop restructured from a single bounded for-loop sharing one attempt counter into a `while True` loop with TWO independent counters/budgets: the pre-existing status_attempt (_MAX_TRANSIENT_ATTEMPTS=4, _TRANSIENT_RETRY_DELAY_S=1.5s, unchanged, still shared by the reply.status and SchedAdminReply.ok=false branches exactly as before) and a new no_primary_attempt (_MAX_NO_PRIMARY_ATTEMPTS=8, _NO_PRIMARY_RETRY_DELAY_S=2.0s -- deliberately larger, since this failure mode's worst case is the ~10s heartbeat-timeout detection window plus real election time, categorically longer than the RPC-level 'already electing, replies not-ready' scenario the original budget was tuned for) that catches NoPrimaryError from follow_redirects_any and retries the whole candidate list fresh. 2 new unit tests (monkeypatching follow_redirects_any directly: retries-then-succeeds, and exhaustion-still-propagates-NoPrimaryError after exactly _MAX_NO_PRIMARY_ATTEMPTS attempts). Full suite 304/304. Live-verified: killed a settled cluster's primary, immediately queried the failover-list, confirmed (in one repro) the retry loop visibly cycled the candidate list 5+ times over ~15s while standbys still redirected to the dead primary, then successfully reached the genuinely newly-elected primary once the standbys' own election completed -- the query result correctly transitioned from NoPrimaryError to the new primary's real (if scheduler-daemon-absent-in-this-test-env) reply, proving the retry rode out the real HA re-election, not just a mocked delay.", @@ -26875,7 +14273,6 @@ "heartbeat_timeout_ms", "election", "live-verified", - "fixed", "tdd", "bug-1713", "bug-1715" @@ -26885,15 +14282,20 @@ "bug-1715", "bug-1821" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1822.md" }, { "id": "bug-1823", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "build", + "severity": "unknown", "timestamp": "2026-09-17T04:20:00Z", - "error_message": "User-reported (regression on top of bug-1820's own fix, live on the user's real host, 'pc13v'): after ./scripts/stop-all.sh reported 'Stopping worker-1 (PID N)' with no warning, the real xcecd worker was STILL running afterward as a different PID -- and a second `sudo ./scripts/stop-all.sh` run immediately reported 'Nothing was running for this deploy' (the pidfile had already been deleted by the first run despite the kill not actually succeeding).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (regression on top of bug-1820's own fix, live on the user's real host, 'pc13v')…\n\nUser-reported (regression on top of bug-1820's own fix, live on the user's real host, 'pc13v'): after ./scripts/stop-all.sh reported 'Stopping worker-1 (PID N)' with no warning, the real xcecd worker was STILL running afterward as a different PID -- and a second `sudo ./scripts/stop-all.sh` run immediately reported 'Nothing was running for this deploy' (the pidfile had already been deleted by the first run despite the kill not actually succeeding).", "file": "src/tools/deploy-ha.sh (generated scripts/stop-all.sh and scripts/cleanup.sh)", "root_cause": "bug-1820's own fix assumed sudo always has exactly ONE level of indirection (sudo forks exactly one direct child that IS the real target), and looked for that child via `pgrep -P $PID` (direct children of the pidfile'd PID only). On the user's real host, sudo interposes an ADDITIONAL 'monitor' process between the invoked sudo and the actual target (sudo -> monitor -> target, THREE processes, not two -- standard sudo 1.8.7+ architecture for proper signal/exit-status relay, not present in this sandbox's own simpler sudo build, which is why bug-1820's fix appeared to work here but not on the user's real box). `pgrep -P $PID` found and killed the MONITOR (a direct child of the pidfile'd PID) while the actual worker -- the monitor's OWN child, not a direct child of the pidfile'd PID -- survived completely untouched, silently orphaned exactly as in the original bug-1820 symptom. Confirmed and reproduced deterministically via a synthetic 3-level process tree (fake-outer -> fake-monitor -> fake-worker, no real sudo needed) built specifically to isolate the tree-walking logic from any particular sudo version's actual behavior.", "fix": "FIXED (2026-09-17). Replaced the parent/child tree-walking approach entirely with a depth-independent identity-based re-scan: after attempting to kill the pidfile'd PID, both generated scripts/stop-all.sh and scripts/cleanup.sh now search system-wide (`pgrep -f`) for ANY still-running process whose own cmdline matches the worker's distinguishing identity -- its `-n $NAME` flag plus this specific deploy's `-l $TOP/logs` flag, the same two args start-worker.sh's own 'already running' guard already checks -- and kills every match directly (both a plain `kill -9` and a `sudo kill -9`, since ownership can't be known in advance), regardless of how many hops deep in the process tree it landed. No longer needs to know or guess sudo's internal process-tree depth. A no-op for ctrl-N/scheduler/apiserver pidfiles, which use neither flag. Live-verified: (1) regression-checked against this sandbox's own real (2-level) sudo worker deploy -- still cleans up correctly; (2) built and killed a synthetic 3-level tree (fake-outer/fake-monitor/fake-worker, exact process-tree shape modeled on the user's live report) via the exact identity-pgrep mechanism the fix uses, confirming the deepest (target) process is found and killed directly, and the intermediate 'monitor' hop then exits on its own once its child is gone (it was just wait()-ing/relaying) -- proving the fix generalizes to arbitrary sudo process-tree depth rather than being tuned to one specific sudo version's behavior.", @@ -26907,23 +14309,26 @@ "real-worker", "sudo-monitor", "process-tree-depth", - "fixed", "live-verified", "synthetic-repro" ], "related_bugs": [ "bug-1820" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1823.md" }, { "id": "bug-1824", "status": "open", "status_updated": "2026-09-24", "status_note": "repl_info.c bounds check uses zarray count instead of caps", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-17T02:45:00Z", - "error_message": "NOT A CRASH -- pre-existing, out-of-scope latent bug discovered while implementing scheduler-sim-diagnostics task 3.1 (new repl_info_limit() live-usage display). repl_info_resource_one() (src/scheduler/src/repl_info.c ~line 996) bounds-checks a resource's zarray index via `(r->idx < 0) || (r->idx >= sim->core.sharedInst.count)` before calling zarray_get() -- but zarray_t.count tracks the number of POPULATED slots (incremented only when a previously-NULL slot becomes non-NULL via zarray_put_ex/zarray_store), not the array's allocated capacity (.caps, grown by zarray_ensure_items). A sparsely-populated zarray (e.g. only a high index ever zarray_put() -- realistic if resources are registered out of index order, or an index is skipped) can have count=1 while a LOWER, unpopulated index is itself < count and would incorrectly pass this check and be read as though populated, or a HIGHER populated index can exceed .count and be incorrectly rejected as absent even though zarray_get()'s own .caps-based check would have safely returned it (or correctly returned NULL for a genuinely-unpopulated-but-in-capacity slot, since zarray_ensure_items() NULL-initializes newly-grown slots). Net effect: repl_info_resource_one() can report a resource as having no value ('value=(none)') when a real, populated value exists at that index but the array is sparse, OR (less likely) skip a real bounds-safety net zarray_get_ex() would otherwise provide.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "out-of-scope latent bug discovered while implementing scheduler-sim-diagnostics task 3.1 (new…\n\nNOT A CRASH -- pre-existing, out-of-scope latent bug discovered while implementing scheduler-sim-diagnostics task 3.1 (new repl_info_limit() live-usage display). repl_info_resource_one() (src/scheduler/src/repl_info.c ~line 996) bounds-checks a resource's zarray index via `(r->idx < 0) || (r->idx >= sim->core.sharedInst.count)` before calling zarray_get() -- but zarray_t.count tracks the number of POPULATED slots (incremented only when a previously-NULL slot becomes non-NULL via zarray_put_ex/zarray_store), not the array's allocated capacity (.caps, grown by zarray_ensure_items). A sparsely-populated zarray (e.g. only a high index ever zarray_put() -- realistic if resources are registered out of index order, or an index is skipped) can have count=1 while a LOWER, unpopulated index is itself < count and would incorrectly pass this check and be read as though populated, or a HIGHER populated index can exceed .count and be incorrectly rejected as absent even though zarray_get()'s own .caps-based check would have safely returned it (or correctly returned NULL for a genuinely-unpopulated-but-in-capacity slot, since zarray_ensure_items() NULL-initializes newly-grown slots). Net effect: repl_info_resource_one() can report a resource as having no value ('value=(none)') when a real, populated value exists at that index but the array is sparse, OR (less likely) skip a real bounds-safety net zarray_get_ex() would otherwise provide.", "file": "src/scheduler/src/repl_info.c", "root_cause": "repl_info_resource_one()'s pre-check compares r->idx against sim->core.sharedInst.count (populated-slot count) instead of relying on zarray_get_ex()'s own bounds check, which is against .caps (allocated capacity) -- these two counters diverge for any sparsely-populated zarray. Confirmed by direct reading of src/lib/zarray.c: zarray_get_ex() (~line 341-362) checks only `idx >= array->caps` before dereferencing array->data[idx]; zarray_put_ex()/zarray_store() are the only writers of .count, incrementing it exclusively when filling a previously-NULL slot -- so .count is a 'high-water mark of contiguous-from-zero fill,' not a true capacity bound. Separately, and more seriously: neither the .count-based check NOR zarray_get_ex()'s own .caps check safely rejects a NEGATIVE idx -- `idx >= caps` is false for any negative idx when caps >= 0, so a negative idx falls through to `array->data[idx]`, a genuine out-of-bounds read. A negative idx is reachable in this codebase: add_vres() (src/lib/xcelimit.c ~407-435) unconditionally sets a newly-allocated xce_resource_t's ->idx to -1 before key_add() runs, only overwriting it with the real slot index on success -- a caller that reaches a resource-value read before/without that overwrite succeeding would pass a negative idx through. repl_info_resource_one()'s existing `r->idx < 0` half of its check does happen to guard this specific negative case correctly (by luck of ordering, not by understanding zarray's real contract) -- the count-vs-caps confusion is the part that is a genuine, distinct latent bug.", "fix": "NOT FIXED -- discovered and characterized during scheduler-sim-diagnostics task 3.1 (limit-usage display), out of that task's scope (task 3.1 only touches its OWN new code path, repl_print_limit_usage(), which was written from scratch to avoid this exact mistake: it omits the imprecise .count pre-check entirely, relies on zarray_get_ex()'s own .caps bound for the upper end, and adds an explicit idx<0 guard of its own -- verified correct and reviewed independently, see this session's git history for scheduler-sim-diagnostics task 3.1). repl_info_resource_one() itself was deliberately left untouched to keep task 3.1 in its declared boundary. Recommended fix when picked up: replace the `r->idx >= sim->core.sharedInst.count` half of the pre-check with reliance on zarray_get()'s own .caps-based bound (mirroring task 3.1's new repl_print_limit_usage() pattern), keeping the `r->idx < 0` guard. Likely benign in practice today only because resource registration order in this codebase's current production paths tends to be dense/contiguous from index 0, so .count and .caps rarely diverge for sim->core.sharedInst specifically -- but this is an incidental property of current callers, not a guarantee, and worth a regression test with a deliberately sparse fixture (zarray_put() at a high index with no lower indices ever populated) before relying on it.", @@ -26934,20 +14339,23 @@ "bounds-check", "out-of-bounds-read", "latent", - "unfixed", - "discovered-during-scheduler-sim-diagnostics", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "discovered-during-scheduler-sim-diagnostics" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1824.md" }, { "id": "bug-1825", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-17T07:10:00Z", - "error_message": "NOT A CRASH -- significant, verified architectural gap discovered while closing out scheduler-sim-diagnostics task 5 (final validation task for the new `info limit` REPL command). The limit plugin is never registered in ANY simulation-mode driver path, so `info limit ` will ALWAYS report \"no limits configured\"/\"no such limit\" for a real operator running `xce_scheduler -i`/`-s ` against a genuine recording -- even one whose original live run had limits configured and actively enforced. The entire feature built across scheduler-sim-diagnostics tasks 1-5 is correctly implemented, thoroughly tested, and read-only/deterministic as designed, but is currently INERT in real production usage: every positive-display test path (tasks 2-3, showing a limit's configured setting and live usage) only exercises the code via a test-only helper that hand-registers the plugin into core->plugins directly, which has no production equivalent in sim mode.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "verified architectural gap discovered while closing out scheduler-sim-diagnostics task 5 (final…\n\nNOT A CRASH -- significant, verified architectural gap discovered while closing out scheduler-sim-diagnostics task 5 (final validation task for the new `info limit` REPL command). The limit plugin is never registered in ANY simulation-mode driver path, so `info limit ` will ALWAYS report \"no limits configured\"/\"no such limit\" for a real operator running `xce_scheduler -i`/`-s ` against a genuine recording -- even one whose original live run had limits configured and actively enforced. The entire feature built across scheduler-sim-diagnostics tasks 1-5 is correctly implemented, thoroughly tested, and read-only/deterministic as designed, but is currently INERT in real production usage: every positive-display test path (tasks 2-3, showing a limit's configured setting and live usage) only exercises the code via a test-only helper that hand-registers the plugin into core->plugins directly, which has no production equivalent in sim mode.", "file": "src/scheduler/src/sim_picker.c", "root_cause": "register_limit() (plugins/limit.c:609) has exactly one call site in the entire codebase: scheduler.c:605, inside sched_register_plugins(), which is itself called only from scheduler.c:827 -- the normal-mode (controller-connected, live) driver's sched_init() path. Sim mode's own plugin-bootstrap mechanism, sim_bootstrap_picker() (sim_picker.c), exists specifically to register a picker plugin (builtin_fcfs, via plugin_register_core()) for simulation/replay mode, since sim mode deliberately does not run the full normal-mode init sequence -- but it only ever registers the PICKER plugin category. It has no equivalent step for the limit plugin (or, by the same reasoning, potentially other plugin categories registered by sched_register_plugins() that were never audited for this gap). Confirmed via full-codebase grep: register_limit()/sched_register_plugins() are never referenced anywhere under sim.c/sim_loop.c/sim_picker.c/driver.c/repl*.c, and no write into core->plugins occurs anywhere in the sim-mode driver files. This is the same class of gap scheduler-driver-equivalence.md already documents for OTHER driver-parity properties (e.g. bug-1613's sched_account_jobs() gap) -- a sim-mode driver feature silently missing something the normal-mode driver does, discovered only when a REPL feature tried to read state that assumes it.", "fix": "FIXED (2026-09-17, scheduler-sim-diagnostics task 6). Added limit_new_plugin() factory (extracted from register_limit(), mirroring builtin_fcfs_new_plugin()'s established pattern) and a new sim_bootstrap_limit_plugin() (new file src/scheduler/src/sim_limit.c, mirroring sim_picker.c's structure exactly): registers the limit plugin via plugin_register_core() and seeds it with sim->core.config (the snapshot's own borrowed-reference config, which already carries the original session's limit configuration -- no recording/replay format change needed) via a new limit_seed_config(plugin, xce_sched_core_t* core, config) entry point. Wired into sim_loop.c alongside the existing sim_bootstrap_picker(sim) call.\n\nFirst attempt (round 1) set plugin->sched = (void*)sim (a disguised xce_sched_sim_driver_t*) to reuse the existing config_hook path -- REJECTED on independent review: this repeats the exact pointer-to-first-member cross-cast pattern that already corrupted memory once in this codebase (see this bug's own root_cause note and core.h's xce_core_cache_clear_cb_t doc comment for that prior incident). Round 2 eliminated the cast entirely: limit_seed_config() takes xce_sched_core_t* core as an honest parameter (a real field both xce_scheduler_t and xce_sched_sim_driver_t genuinely have), delegating to a shared limit_seed_config_impl() also used by normal-mode limit_update_config(). No cast, no aliasing risk -- confirmed structurally sound by two independent reviewers, each reading the actual struct definitions and full call chain rather than trusting the report.\n\nAlso closed: sim_limit.c added to sched_sim_isolation_lib's source list (meson.build) so Req 5 AC 17's mechanical controller-I/O-isolation check covers it (round 1 omitted this; round 2 fixed it, sched_sim_isolation_test independently re-verified passing with the new file included).\n\nRipple: two waves of an identical, minimal link-satisfying stub pattern across the ~24 white-box test files that #include \"../src/sim_loop.c\" (one for limit_new_plugin(), one for limit_seed_config()) -- same established 'duplicate, don't share' idiom this file family already used for the earlier scheduler-sim-diagnostics tasks' repl_info.c ripple.\n\nTDD: RED->GREEN verified for both the config-seed test and the remediation itself (a temporary limit_seed_config() no-op stub reproduced the exact expected single-test failure, reverted cleanly). Full scheduler suite: 78/78 Ok, 0 Fail; sched_repl_limit_table_test 26/26; sched_limit_plugin_test 5/5 (zero regression to normal-mode registration); sched_sim_isolation_test 5/5 (genuinely covering the new file).\n\nDeliberately NOT fixed (separate, larger, tracked as a distinct follow-up item in .kiro/specs/scheduler-sim-diagnostics/tasks.md, not this bug): limit *matching* (limit_job_election()) is still never invoked during replay -- it's only reachable via plugin_job_election(), which is normal-mode-only (one caller, scheduler.c) and would itself need a core-safe variant before it could be wired into sim_loop.c's decision path. So 'info limit' now shows real CONFIGURED settings against a real replay, but live USAGE will still correctly show 'no active usage instance yet' until that separate, larger fix lands.", @@ -26958,21 +14366,24 @@ "limit-plugin", "plugin-bootstrap", "architectural-gap", - "unfixed", "discovered-during-scheduler-sim-diagnostics", - "driver-parity", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "driver-parity" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1825.md" }, { "id": "bug-1826", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-17T21:50:00Z", - "error_message": "NOT A CRASH -- known, deliberately out-of-scope follow-up discovered while fixing bug-1825 (sim-mode limit plugin bootstrap, scheduler-sim-diagnostics task 6). Bug-1825's fix makes the limit plugin's CONFIGURED SETTING visible in sim mode (info limit now shows a real replay's actual limit configuration). But limit *matching* -- the mechanism that actually populates live usage instances (xce_vres_t) as jobs get evaluated -- is still never invoked during replay. So info limit's live-usage display will correctly show 'no active usage instance yet' for every limit against any real replay, even one whose original live session actively enforced and charged against that limit.", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "deliberately out-of-scope follow-up discovered while fixing bug-1825 (sim-mode limit plugin…\n\nNOT A CRASH -- known, deliberately out-of-scope follow-up discovered while fixing bug-1825 (sim-mode limit plugin bootstrap, scheduler-sim-diagnostics task 6). Bug-1825's fix makes the limit plugin's CONFIGURED SETTING visible in sim mode (info limit now shows a real replay's actual limit configuration). But limit *matching* -- the mechanism that actually populates live usage instances (xce_vres_t) as jobs get evaluated -- is still never invoked during replay. So info limit's live-usage display will correctly show 'no active usage instance yet' for every limit against any real replay, even one whose original live session actively enforced and charged against that limit.", "file": "src/scheduler/src/plugin.c", "root_cause": "limit_job_election() (plugins/limit.c, the function that calls match_job_global_limit_()/match_job_host_limit_() and populates xce_vres_t usage instances) is only ever invoked via plugin_job_election() (plugin.c:554), which -- like plugin_config_update()/plugin_prepare_allocation() -- takes a `void* arg` cast unconditionally to `xce_scheduler_t*` and reads `sched->core.plugins`. plugin_job_election() has exactly ONE call site in the entire codebase: scheduler.c (normal-mode only). Nothing in sim_loop.c's replay decision path calls plugin_job_election() or any core-safe equivalent, so no limit ever gets matched against any job during simulation/replay, regardless of whether the limit plugin is registered (bug-1825 fixes registration+config, not this).", "fix": "FULLY FIXED (2026-09-19, scheduler-sim-live-allocation tasks 1.1-5.2). The core-safe plugin_job_election() variant this entry called for was built in task 1.2 (retyped to take xce_sched_core_t* directly, iterating core->plugins); it is now genuinely invoked in sim mode via two paths: (a) the unified sched_job_picker()/sched_allocator() thread pair (task 3.1, wired into sim mode in task 4.1) calls the full core-safe election/allocation chain for every job the live picker reaches, which includes limit matching via limit_job_election() (task 2.3, also fixed bug-1828, a pre-existing host-scoped-limit enforcement defect blocking this same path); (b) sim_reconcile_unaccounted_jobs() (task 4.2) additionally calls plugin_job_election() directly for any job only recovered via recorded-decision reconciliation, so a job never misses limit registration regardless of which path resolves it. Host-scoped limit fan-out during live seqjob/parjob dispatch specifically required a further fix (task 4.3, closing a throwaway-wrapper-registry seam gap) before it worked end-to-end. Proven via the full TDD + contention-fixture discipline this entry called for: bare-core contention fixtures (task 5.1, test_sim_contention_limit_driver_parity.c) and this entry concrete acceptance test -- a genuine real capnp-encoded .replay driven through the real, unmodified sched_sim_driver_run(), with both a global and a host-scoped limit configured and matched, both showing real non-zero usage via the same limit_get_usage_instances() accessor info limit reads (task 5.2, test_sim_e2e_limit_visibility.c) -- independently reviewed with a targeted revert-to-pre-task-4.3 RED-phase reproduction confirming the host-scoped half genuinely regresses without the fix.", @@ -26984,22 +14395,25 @@ "limit-matching", "plugin-job-election", "architectural-gap", - "unfixed", "discovered-during-scheduler-sim-diagnostics", "driver-parity", - "follow-up-to-bug-1825", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "follow-up-to-bug-1825" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1826.md" }, { "id": "bug-1827", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-18T07:15:00Z", - "error_message": "NOT A CRASH -- coverage gap found during an /xce-audit-invariants scheduler run (dimension 2: core module I/O isolation), while investigating driver state-equivalence for bug-1826. test_core_isolation.c's own header comment claims core_clock.c joins ONLY the nm(1) undefined-symbol enforcement layer, not the wall-clock text-scan check -- implying it is still covered by the OTHER two text-scan checks (forbidden #include list, forbidden symbol substrings). In reality, core_clock.c is absent from all three text-scan functions (test_core_never_includes_forbidden_headers(), test_core_never_calls_forbidden_symbols(), and the wall-clock scan), not just the wall-clock one -- no XCE_CORE_CLOCK_C_PATH macro is defined anywhere in test_core_isolation.c or testsuite/meson.build's c_args for that test.", + "last_seen": "2026-09-19T14:30:00Z", + "occurrences": 1, + "error_message": "), while investigating driver state-equivalence for bug-1826. test_core_isolation.c's own header…\n\nNOT A CRASH -- coverage gap found during an /xce-audit-invariants scheduler run (dimension 2: core module I/O isolation), while investigating driver state-equivalence for bug-1826. test_core_isolation.c's own header comment claims core_clock.c joins ONLY the nm(1) undefined-symbol enforcement layer, not the wall-clock text-scan check -- implying it is still covered by the OTHER two text-scan checks (forbidden #include list, forbidden symbol substrings). In reality, core_clock.c is absent from all three text-scan functions (test_core_never_includes_forbidden_headers(), test_core_never_calls_forbidden_symbols(), and the wall-clock scan), not just the wall-clock one -- no XCE_CORE_CLOCK_C_PATH macro is defined anywhere in test_core_isolation.c or testsuite/meson.build's c_args for that test.", "file": "src/scheduler/testsuite/test_core_isolation.c", "root_cause": "core_clock.c was added to the mechanically-enforced sched_core_isolation_lib archive (meson.build) and given a documented single exception (the wall-clock text scan, for its one sanctioned clock_gettime() call), but the two OTHER text-scan functions (forbidden-headers, forbidden-symbols) were never extended to also read core_clock.c's source text -- only core.h/core.c/core_events.c/core_session.c are read by those two checks (via XCE_CORE_H_PATH/_C_PATH/_EVENTS_C_PATH/_SESSION_C_PATH macros; no clock-file variant exists). The nm(1) undefined-symbol layer (which DOES include core_clock.c in the archive) only catches an actual undefined-symbol reference at link time -- it cannot catch a bare forbidden #include (e.g. client.h/netio.h/xceapi.h/delay.h/idgen.h/xce/cluster.h) that resolves to no undefined symbol (a macro-only or inline-only header), so the doc comment's implied coverage is narrower in practice than stated. No evidence found that core_clock.c's actual current content violates isolation today -- this is a gap in what the mechanism would CATCH if it did, not a confirmed live violation.", "fix": "FIXED (2026-09-19). Added XCE_CORE_CLOCK_C_PATH (testsuite/meson.build c_args for sched_core_isolation_test, mirroring the existing XCE_CORE_H_PATH/_C_PATH/_EVENTS_C_PATH/_SESSION_C_PATH pattern) and extended both test_core_never_includes_forbidden_headers() and test_core_never_calls_forbidden_symbols() (test_core_isolation.c) to also scan core_clock.c -- it remains excluded ONLY from the wall-clock scan (test_core_never_calls_wallclock_functions), for the same sanctioned-single-clock_gettime()-call reason as before. TDD: confirmed RED by temporarily inserting #include \"netio.h\" into core_clock.c and rebuilding -- the extended test correctly failed with the message \"core_clock.c directly includes forbidden header '#include \"netio.h\"'\" (test_core_isolation.c:226) while every other test stayed green (84/85); reverted (git diff confirms core_clock.c byte-identical to HEAD) and reconfirmed GREEN (85/85). No live violation existed in core_clock.c -- this closes the coverage gap the mechanism itself had, matching what its own header/doc comments already claimed.", @@ -27010,23 +14424,26 @@ "test_core_isolation", "core_clock", "audit-finding", - "discovered-during-xce-audit-invariants", - "fixed" + "discovered-during-xce-audit-invariants" ], - "occurrences": 1, - "last_seen": "2026-09-19T14:30:00Z" + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1827.md" }, { "id": "bug-1828", "status": "fixed", "status_updated": "2026-09-24", - "error_message": "IMPACT CORRECTED 2026-09-18 (independent review of scheduler-sim-live-allocation task 2.3): this is NOT just a display/accounting gap -- it meant host-scoped resource limits were NEVER ACTUALLY ENFORCED, in ANY mode (normal or simulation), for this function's entire history. Originally discovered while writing scheduler-sim-live-allocation spec task 2.3's own required TDD test (a host-scoped limit match, via the simulation-mode-registered limit plugin, must create a real usage instance verifiable through limit_get_usage_instances()). limit_prepare_allocation()/match_job_host_limit_() consistently returned rc=-2 (-ENOENT) instead of >= 0, and no usage instance was ever created, even though match_limit() itself genuinely matched (rc=1) and is_host_based_limit() correctly identified the limit as host-based. Tracing the call chain (plugin.c's plugin_prepare_allocation() hook loop -> sched_fill_limit_requests()) shows the missing usage instance meant a host-scoped limit's vres was never added to obj->hlimits, so sched_fill_limit_requests() always iterated an empty list -- the limit's numeric cap was silently never applied to any job's resource request, in production, since this code existed. This does not corrupt the dispatch/pend decision machinery itself (plugin_prepare_allocation()'s hook-calling loop discards the hook's return value, so the bug never fed into a control-flow branch) -- it is a dormant-limit-never-binds defect, but a real enforcement gap, not merely a cosmetic one.", + "status_note": "", + "component": "scheduler", + "severity": "unknown", + "timestamp": null, + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "IMPACT CORRECTED 2026-09-18 (independent review of scheduler-sim-live-allocation task 2.3): this…\n\nIMPACT CORRECTED 2026-09-18 (independent review of scheduler-sim-live-allocation task 2.3): this is NOT just a display/accounting gap -- it meant host-scoped resource limits were NEVER ACTUALLY ENFORCED, in ANY mode (normal or simulation), for this function's entire history. Originally discovered while writing scheduler-sim-live-allocation spec task 2.3's own required TDD test (a host-scoped limit match, via the simulation-mode-registered limit plugin, must create a real usage instance verifiable through limit_get_usage_instances()). limit_prepare_allocation()/match_job_host_limit_() consistently returned rc=-2 (-ENOENT) instead of >= 0, and no usage instance was ever created, even though match_limit() itself genuinely matched (rc=1) and is_host_based_limit() correctly identified the limit as host-based. Tracing the call chain (plugin.c's plugin_prepare_allocation() hook loop -> sched_fill_limit_requests()) shows the missing usage instance meant a host-scoped limit's vres was never added to obj->hlimits, so sched_fill_limit_requests() always iterated an empty list -- the limit's numeric cap was silently never applied to any job's resource request, in production, since this code existed. This does not corrupt the dispatch/pend decision machinery itself (plugin_prepare_allocation()'s hook-calling loop discards the hook's return value, so the bug never fed into a control-flow branch) -- it is a dormant-limit-never-binds defect, but a real enforcement gap, not merely a cosmetic one.", + "file": "src/scheduler/src/plugins/limit.c (match_job_host_limit_), src/scheduler/testsuite/test_repl_limit_table.c (new test)", "root_cause": "match_job_host_limit_() (src/scheduler/src/plugins/limit.c) is the host-scoped sibling of match_job_global_limit_() and has the same defect CLASS bug-1786 fixed in that function alone (misinterpreting key_find()'s negative \"not found\" return as fatal, rather than the correct not-yet-instantiated signal) -- not byte-for-byte identical wording (bug-1786's own guard was `if (rc <= 0) { return rc; }`; this sibling's was `if (rc < 0) { return rc; }`), same underlying mistake. Right after a successful match_limit(), it does `rc = key_find(privdata->instances, vresname, &vres); if (rc < 0) { return rc; }` before the `if (vres == NULL) { instantiate_limit(...) }` block below it. On the FIRST-EVER lookup of a given vresname (always true the first time a host-scoped limit ever matches), key_find() signals \"not found\" as a NEGATIVE rc (-ENOENT), not 0 -- this codebase's established kvmgr convention (get_group(), is_host_based_limit(), and bug-1786/bug-1495's own root causes) -- so this early-return unconditionally caught the expected miss and returned before instantiate_limit() could ever run. bug-1786 (2026-09-07) fixed this exact defect class in match_job_global_limit_() but never touched its host-scoped sibling, which retained the bug because no test exercised match_job_host_limit_()/limit_prepare_allocation() with real matching until this task added one (verified via grep: zero prior test call sites for either function anywhere in the test suite).", "fix": "FIXED (2026-09-18, scheduler-sim-live-allocation task 2.3). Removed the erroneous `if (rc < 0) { return rc; }` guard in match_job_host_limit_() entirely, mirroring match_job_global_limit_()'s own bug-1786 fix exactly: `vres` (calloc'd NULL, only ever reassigned by a successful key_find) is already the correct, sufficient signal for \"needs instantiation\" via the existing `if (vres == NULL)` block immediately below. TDD: new test test_limit_prepare_allocation_matches_host_limit_via_sim_registered_plugin (test_repl_limit_table.c) drives the real limit_prepare_allocation()/match_job_host_limit_() path against a plugin registered via plugin_register_core() and a real percent-based host-scoped limit; RED (rc=-2, confirmed via temporary debug instrumentation tracing is_host_based_limit()=1, match_limit()=1, but limit_prepare_allocation()=-2) before this fix, GREEN (rc>=0, usage instance created and visible via limit_get_usage_instances()) after. Full scheduler suite: 79/79, zero regressions.", - "files": [ - "src/scheduler/src/plugins/limit.c (match_job_host_limit_)", - "src/scheduler/testsuite/test_repl_limit_table.c (new test)" - ], "tags": [ "scheduler", "limit-plugin", @@ -27036,31 +14453,26 @@ "match_job_host_limit_", "sibling-of-bug-1786", "scheduler-sim-live-allocation", - "task-2.3", - "fixed" + "task-2.3" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1828.md" }, { "id": "bug-1829", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-19T04:00:00Z", - "error_message": "AddressSanitizer: SEGV (NULL pointer read) in sched_make_seqjob_allocation (src/scheduler/src/scheduler.c:59, `plugin->job_alloc_fn` where `plugin = core->seqjob_allocator` is NULL), called from sched_make_primary_alloc_ -> sched_make_primary_allocation -> sched_make_allocation -> sched_allocator (sched_election.c) -> the real pthread spawned by sched_sim_driver_run() (sim_loop.c). Discovered while writing the mandated TDD test for scheduler-sim-live-allocation task 4.1 (its own Observable: 'a simulation-mode session ... produces a genuine dispatch or pend decision for a job based on its actual resource availability, instead of unconditionally pending it') -- the crash is 100% reproducible for ANY job reaching primary resource allocation in simulation mode, regardless of resreq shape or dispatch/pend outcome (sched_make_parjob_allocation has the identical NULL-core->parjob_allocator crash for maxTasks>1 jobs).", + "last_seen": "2026-09-19T09:00:00Z", + "occurrences": 1, + "error_message": "AddressSanitizer: SEGV (NULL pointer read) in sched_make_seqjob_allocation…\n\nAddressSanitizer: SEGV (NULL pointer read) in sched_make_seqjob_allocation (src/scheduler/src/scheduler.c:59, `plugin->job_alloc_fn` where `plugin = core->seqjob_allocator` is NULL), called from sched_make_primary_alloc_ -> sched_make_primary_allocation -> sched_make_allocation -> sched_allocator (sched_election.c) -> the real pthread spawned by sched_sim_driver_run() (sim_loop.c). Discovered while writing the mandated TDD test for scheduler-sim-live-allocation task 4.1 (its own Observable: 'a simulation-mode session ... produces a genuine dispatch or pend decision for a job based on its actual resource availability, instead of unconditionally pending it') -- the crash is 100% reproducible for ANY job reaching primary resource allocation in simulation mode, regardless of resreq shape or dispatch/pend outcome (sched_make_parjob_allocation has the identical NULL-core->parjob_allocator crash for maxTasks>1 jobs).", + "file": "src/scheduler/include/plugin.h (seqjob_allocator_new_plugin()/parjob_allocator_new_plugin() factory declarations, task 4.3), src/scheduler/src/plugins/seqjob_allocator.c (task 4.3: init_fn + prepare_allocation_hook + job_alloc_fn retyped to read plugin->core; new seqjob_allocator_new_plugin() factory -- genuinely modified, no longer just the root-cause file), src/scheduler/src/plugins/parjob_allocator.c (task 4.3: same retyping + new parjob_allocator_new_plugin() factory), src/scheduler/src/sim_alloc.c (task 4.1: new file, initial bootstrap via throwaway wrapper; task 4.3: rewritten to register directly against sim->core via plugin_register_core(), no wrapper in the registration/fan-out path), src/scheduler/src/sim_loop.c (task 4.1: sched_sim_driver_run() call sites -- unmodified by task 4.3), src/scheduler/meson.build (task 4.1: scheduler_srcs, sched_sim_isolation_lib -- unmodified by task 4.3), src/scheduler/testsuite/test_sim_live_allocation.c (task 4.1: new file, live-dispatch RED/GREEN proof), src/scheduler/testsuite/test_sim_alloc_limit_fanout.c (task 4.3: new file, host-scoped-limit-fan-out RED/GREEN proof), src/scheduler/testsuite/test_seqjob_allocator.c, test_seqjob_rewrap_equivalence.c, test_parjob_combinatorial.c (task 4.3: white-box fixture ripple -- plugin instances built directly in these tests needed their own .core field set alongside .sched)", "root_cause": "core->seqjob_allocator/core->parjob_allocator (xce_sched_core_t fields, core.h) are never populated for the simulation driver, anywhere. Normal mode populates them via sched_init()'s sched_register_plugins() -> register_seqjob_allocator()/register_parjob_allocator() (plugins/seqjob_allocator.c, plugins/parjob_allocator.c), whose init_fn does `((xce_scheduler_t*)plugin->sched)->core.seqjob_allocator = plugin` -- REQUIRING a genuine, live xce_scheduler_t*. sched_sim_driver_start() (sim.c) registers no plugin at all (sim.h's own documented 'Known scope boundary'), and xce_sched_sim_driver_t deliberately has no xce_scheduler_t*/xce_scheduler_t field of its own (Req 5 AC 17, simulation isolation). This is NOT limited to the init_fn: every one of these two plugins' RUNTIME hooks (seqjob_prepare_allocation()/my_alloc(), parjob_prepare_allocation()/my_alloc(), and several internal static helpers taking xce_scheduler_t* sched as a parameter) ALSO read `plugin->sched->core.*` (resmgr for the 'slots'/rusage lookups, workers for host_iter, and sched_make_allocs()/sched_try_alloc_on_host_() in scheduler.c/resource.c which ALSO take xce_scheduler_t*/need `sched->core.resmgr`) on every single allocation attempt, not just at registration. This gap was already discovered and explicitly deferred by scheduler-sim-live-allocation task 3.1 (see test_sched_election_sim_fixture.c's own header comment: 'a pre-existing, tracked, OUT-OF-SCOPE gap ... NOT one of task 3.1's five permitted changes -- fixing it would be scope creep into a different, not-yet-assigned task') -- task 3.1's own fixture never reached this crash only because IT registers seqjob_allocator directly against a real, throwaway xce_scheduler_t wrapper whose OWN .core IS the core under test (single-core fixture), a shape simulation mode itself can never have (its core lives inside xce_sched_sim_driver_t, never xce_scheduler_t). Task 4.1 is the first task whose OWN production change (spawning the real sched_job_picker()/sched_allocator() thread pair against simulation mode's own core, replacing the old always-PEND sim_picker_thread_fn() placeholder that never called sched_make_allocation() at all) makes this code path reachable in simulation mode for the first time ever.", "fix": "FULLY FIXED (2026-09-19, scheduler-sim-live-allocation task 4.3, closing this bug's own tracked follow-up). Root cause: seqjob_allocator_init()/parjob_allocator_init() (plugins/seqjob_allocator.c, plugins/parjob_allocator.c) and every runtime hook these plugins register (seqjob_prepare_allocation()/my_alloc(), parjob_prepare_allocation()/my_alloc()/parjob_try_plan_()) now read the plugin's own honestly-typed `core` field (task 1.2's xce_sched_plugin_t.core) instead of deriving an xce_scheduler_t* from plugin->sched -- mirroring task 2.3's plugins/limit.c migration exactly. Two new factory functions, seqjob_allocator_new_plugin()/parjob_allocator_new_plugin() (plugin.h, mirroring limit_new_plugin()/builtin_fcfs_new_plugin()), let sim_alloc.c's sim_bootstrap_seqjob_allocator()/sim_bootstrap_parjob_allocator() register the allocator plugin DIRECTLY against &(sim->core) via the existing plugin_register_core() -- the same call sim_bootstrap_picker()/sim_bootstrap_limit_plugin() already use -- so plugin->core honestly points at sim->core and every later plugin_prepare_allocation() fan-out (my_alloc() -> sched_try_alloc_on_host_() -> plugin_prepare_allocation(), resource.c/plugin.c) reaches the SAME core->plugins registry the limit plugin (and picker) are registered in. Task 4.1's throwaway xce_scheduler_t wrapper is gone from the plugin-registry/fan-out path entirely -- no separate registry stands in for it anymore. One narrow, necessary exception remains: my_alloc() still keeps a small, long-lived, heap-allocated xce_scheduler_t wrapper (never freed, same accepted-leak convention) whose ONLY purpose is satisfying sched_make_allocs()'s (scheduler.c, out of this task's file boundary, unmodified) own real xce_scheduler_t* requirement for the FINAL allocation record of an already-decided winning host -- that function's body only ever reads sched->core.resmgr/lock/nextid, so the wrapper's .core.resmgr is ALIASED (a `var`, not copied) to sim->core's own; it is set on plugin->sched ONLY AFTER plugin_register_core() has already run (init_fn no longer reads plugin->sched at all) and is never used for the plugin registry or any prepare_allocation_hook fan-out, so it cannot reintroduce the separate-registry gap this fix closes. Verified live: new test_sim_alloc_limit_fanout.c (real scheduler_srcs link, bare xce_sched_sim_driver_t fixture, real sched_job_picker()/sched_allocator() thread pair) registers a host-scoped limit directly against sim->core (sim_bootstrap_limit_plugin()) alongside the seqjob allocator plugin (sim_bootstrap_seqjob_allocator()), drives a genuine live dispatch to a real DISPATCH decision (JOB_STAT_RUN), and confirms the limit's usage instance is created (limit_get_usage_instances()) -- RED before this fix (job dispatches correctly but zero usage instances; the fan-out never reached the limit plugin), GREEN after. Full scheduler suite: 82/82, zero regressions -- three white-box test files that construct an xce_sched_plugin_t directly without going through plugin_register()/plugin_register_core() (test_seqjob_allocator.c, test_seqjob_rewrap_equivalence.c, test_parjob_combinatorial.c) needed their fixture plugin instances' own `.core` field set alongside the pre-existing `.sched` field, plus parjob_try_plan_()'s one direct white-box call site (test_parjob_combinatorial.c) updated from &sched to &(sched.core) for its now-honest xce_sched_core_t* signature -- the same established ripple-fix pattern this feature's own tasks.md Implementation Notes already document for prior signature migrations.", - "files": [ - "src/scheduler/include/plugin.h (seqjob_allocator_new_plugin()/parjob_allocator_new_plugin() factory declarations, task 4.3)", - "src/scheduler/src/plugins/seqjob_allocator.c (task 4.3: init_fn + prepare_allocation_hook + job_alloc_fn retyped to read plugin->core; new seqjob_allocator_new_plugin() factory -- genuinely modified, no longer just the root-cause file)", - "src/scheduler/src/plugins/parjob_allocator.c (task 4.3: same retyping + new parjob_allocator_new_plugin() factory)", - "src/scheduler/src/sim_alloc.c (task 4.1: new file, initial bootstrap via throwaway wrapper; task 4.3: rewritten to register directly against sim->core via plugin_register_core(), no wrapper in the registration/fan-out path)", - "src/scheduler/src/sim_loop.c (task 4.1: sched_sim_driver_run() call sites -- unmodified by task 4.3)", - "src/scheduler/meson.build (task 4.1: scheduler_srcs, sched_sim_isolation_lib -- unmodified by task 4.3)", - "src/scheduler/testsuite/test_sim_live_allocation.c (task 4.1: new file, live-dispatch RED/GREEN proof)", - "src/scheduler/testsuite/test_sim_alloc_limit_fanout.c (task 4.3: new file, host-scoped-limit-fan-out RED/GREEN proof)", - "src/scheduler/testsuite/test_seqjob_allocator.c, test_seqjob_rewrap_equivalence.c, test_parjob_combinatorial.c (task 4.3: white-box fixture ripple -- plugin instances built directly in these tests needed their own .core field set alongside .sched)" - ], "tags": [ "scheduler", "sim-mode", @@ -27072,29 +14484,27 @@ "plugin-sched-vs-plugin-core", "scheduler-sim-live-allocation", "task-4.1", - "partially-fixed", "follow-up-tracked", "task-4.3", - "fully-fixed", - "fixed", - "closed-by-scheduler-sim-live-allocation" + "fully-fixed" ], "related_bugs": [ "bug-1828" ], - "occurrences": 1, - "last_seen": "2026-09-19T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1829.md" }, { "id": "bug-1830", - "gitea_synced_status": "cannot-reproduce", - "gitea_issue": 53, "status": "cannot-reproduce", "status_updated": "2026-09-24", "status_note": "downgraded 07b5c989; user's own retest showed mem limit usage correctly", - "repro_attempts": "user retest 2026-09-20, not reproduced", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-20T09:00:00Z", - "error_message": "CORRECTED 2026-09-20 (user's own independent retest) -- original framing was too strong. A mem-based limit (limits2, from the real src/configs/limits.yaml) DID correctly populate a live usage instance in a user-run retest: `info limit limits2` against a real replay showed `usage limit=limits2/u=rshen/q=xce/p= host=pc13v: accounted=18022400.000000 (17.19 MB) capacity=4635344384.000000 (4.32 GB)` -- fully populated and correctly human-formatted on both accounted and capacity. This directly contradicts this entry's original claim that mem-based limits 'never' populate a usage instance via replay. ORIGINAL FINDING (retained for context, now understood to be narrower than stated): an earlier live-verification session (agent-run, single short-lived deploy+capture cycle) saw every mem-based limit show 'no active usage instance yet' while a slots-based limit in the same replay populated correctly. Given the user's retest succeeded, the real condition is likely session-shape/timing-dependent (e.g. elapsed replay rounds, WORKERS_PUSH arrival timing relative to job dispatch, or how long the recording ran) rather than a categorical mem-vs-slots failure.", + "last_seen": "2026-09-20T09:30:00Z", + "occurrences": 1, + "error_message": "CORRECTED 2026-09-20 (user's own independent retest) -- original framing was too strong\n\nCORRECTED 2026-09-20 (user's own independent retest) -- original framing was too strong. A mem-based limit (limits2, from the real src/configs/limits.yaml) DID correctly populate a live usage instance in a user-run retest: `info limit limits2` against a real replay showed `usage limit=limits2/u=rshen/q=xce/p= host=pc13v: accounted=18022400.000000 (17.19 MB) capacity=4635344384.000000 (4.32 GB)` -- fully populated and correctly human-formatted on both accounted and capacity. This directly contradicts this entry's original claim that mem-based limits 'never' populate a usage instance via replay. ORIGINAL FINDING (retained for context, now understood to be narrower than stated): an earlier live-verification session (agent-run, single short-lived deploy+capture cycle) saw every mem-based limit show 'no active usage instance yet' while a slots-based limit in the same replay populated correctly. Given the user's retest succeeded, the real condition is likely session-shape/timing-dependent (e.g. elapsed replay rounds, WORKERS_PUSH arrival timing relative to job dispatch, or how long the recording ran) rather than a categorical mem-vs-slots failure.", "file": "src/scheduler/src/plugins/limit.c", "root_cause": "Suspected, not confirmed: match_job_global_limit_()/match_job_host_limit_() (plugins/limit.c) key_find() the limit's constrained resource by name against core->resmgr at match time. For `slots`, this resource is always already present in core->resmgr from session start. For `mem`, the resource may only become known to core->resmgr via a WORKERS_PUSH replay event reporting the worker's live mem value -- if that push event is decoded/applied AFTER the job's own dispatch decision is replayed (or never lands in the resmgr in a way key_find() can see before the match attempt), the mem-based limit's match silently misses, leaving no xce_vres_t/usage instance ever created for it. test_sim_e2e_limit_visibility.c's own acceptance test (scheduler-sim-live-allocation task 5.2) only proves this works when the mem resource is hand-baked directly into the snapshot's resmgr BEFORE any replay event is processed -- it does not cover the ordinary real-deployment shape (resource known only via a live WORKERS_PUSH event that arrives as part of the replay stream itself, not pre-seeded in the snapshot). This needs direct tracing of match timing vs. WORKERS_PUSH application order in sim_loop.c's real round sequence to confirm.", "fix": "STATUS DOWNGRADED to investigation-needed, not a confirmed defect. The original agent-run live test's single short capture window may not be representative -- the user's own longer/different retest shows mem-based limit usage population DOES work under real replay conditions. If this is ever seen to fail again, the reproduction needs to capture the SPECIFIC session shape that fails (replay round count, elapsed time between record-start and the job's dispatch, WORKERS_PUSH cadence) since a bare 'mem-based limits don't populate' framing has now been shown to be incorrect as a categorical claim. Not fixed because there is no longer a confirmed, reproducible defect to fix -- downgrading rather than closing, since the ORIGINAL single agent-run observation was real and unexplained, just not reproduced by the user's own retest.", @@ -27117,17 +14527,23 @@ "bug-1825", "bug-1826" ], - "occurrences": 1, - "last_seen": "2026-09-20T09:30:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1830.md", + "repro_attempts": "user retest 2026-09-20, not reproduced", + "gitea_issue": 53, + "gitea_synced_status": "cannot-reproduce" }, { "id": "bug-1831", - "gitea_synced_status": "fixed", - "gitea_issue": 56, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-20T10:00:00Z", - "error_message": "Live-verification finding (user's own retest of scheduler-sim-diagnostics Requirement 6): a correctly-configured 20% memory limit (limits2, resname: mem, value: 0.2, percent: true) displayed as 'mem = 0.200000%' in `info limit` -- reading as 0.2 percent to a human, not the 20 percent it actually computes and enforces.", + "last_seen": "2026-09-20T10:00:00Z", + "occurrences": 1, + "error_message": "Live-verification finding (user's own retest of scheduler-sim-diagnostics Requirement 6): a…\n\nLive-verification finding (user's own retest of scheduler-sim-diagnostics Requirement 6): a correctly-configured 20% memory limit (limits2, resname: mem, value: 0.2, percent: true) displayed as 'mem = 0.200000%' in `info limit` -- reading as 0.2 percent to a human, not the 20 percent it actually computes and enforces.", "file": "src/lib/xcelimit.c", "root_cause": "xce_limit_val_t.val is genuinely a 0-1 fraction when percent != 0 (confirmed end-to-end: no /100 or *100 scaling exists anywhere in the config->capnp->runtime pipeline -- src/xceadmin/src/xcecommon/mapping.py, controller.capnp's LimitValue, its C encode/decode, build_limit(), and update_limit_res_value() (plugins/limit.c) all pass the value through unchanged, multiplying it DIRECTLY against the resource's current value for capacity computation -- confirmed authoritative by the original hand-written C literal this codebase's oldest percent-based limit was migrated from, `.val = 0.2` for a documented '20% mem cap', sched.c's limit2_config, commit ecf1916d). fmt_limit()/fmt_limit_units() (src/lib/xcelimit.c) formatted the percent branch as a bare `snprintf(..., \"%f%%\", limit->val.val)` -- echoing the raw 0-1 fraction with a literal '%' appended, instead of converting to conventional 0-100 percent notation first.", "fix": "FIXED (2026-09-20). fmt_limit_units()'s percent branch now formats `limit->val.val * 100.0` instead of the raw `limit->val.val`, so a correctly-configured 0.2 (20%) limit displays as '20.000000%'. This is display-only -- update_limit_res_value()'s actual capacity computation was already correct and is untouched; only what a human reads changed. fmt_limit() (a thin XCE_UNITS_RAW wrapper around fmt_limit_units()) inherits this fix for its own percent-branch output too -- a deliberate, intentional behavior change from its pre-fix output for percent-based limits specifically (its 5 pre-existing direct unit tests only assert `rc > 0`, never the exact percent string, so this was safe). Updated test_limit.c's test_fmt_limit_units_percent_limit_ignores_units_mode fixture from an unrealistic `.val = 80.0` to a realistic `.val = 0.8` (80%) -- assertion text needed zero change since 0.8*100=80 either way. Updated test_repl_limit_table.c's test_info_limit_named_percent_limit_own_percentage_value_never_scaled_by_units_mode to expect the shared add_configured_percent_limit() fixture's (deliberately unchanged, .val=50.0) raw display as '5000.000000%' -- mathematically correct given that fixture's arbitrary test constant was never meant to represent a realistic percent value. RED confirmed by grepping the raw Unity testlog for :FAIL: (the aggregate meson summary masks this, a known Unity-harness quirk in this project) before the fix; GREEN confirmed after, both lib (0 Unity FAILs) and scheduler (85/85, 0 Unity FAILs) suites.", @@ -27139,24 +14555,28 @@ "percent-limit", "info-limit", "requirement-6", - "fixed", "discovered-during-live-retest", "related-to-bug-1832" ], "related_bugs": [ "bug-1832" ], - "occurrences": 1, - "last_seen": "2026-09-20T10:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1831.md", + "gitea_issue": 56, + "gitea_synced_status": "fixed" }, { "id": "bug-1832", - "gitea_synced_status": "fixed", - "gitea_issue": 58, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "config", + "severity": "unknown", "timestamp": "2026-09-20T10:00:00Z", - "error_message": "src/configs/limits.yaml's night_mem_pct limit (resname: mem, value: 50, percent: true) is functionally dead -- it can never bind or deny any job, since its actual computed capacity is 50x (5000%) the resource's current value, not 50% of it. Surfaced while investigating bug-1831's percent-display discrepancy.", + "last_seen": "2026-09-20T10:00:00Z", + "occurrences": 1, + "error_message": "src/configs/limits.yaml's night_mem_pct limit (resname: mem, value: 50, percent: true) is…\n\nsrc/configs/limits.yaml's night_mem_pct limit (resname: mem, value: 50, percent: true) is functionally dead -- it can never bind or deny any job, since its actual computed capacity is 50x (5000%) the resource's current value, not 50% of it. Surfaced while investigating bug-1831's percent-display discrepancy.", "file": "src/configs/limits.yaml", "root_cause": "xce_limit_val_t.val is a 0-1 fraction convention when percent is set (see bug-1831's root_cause for full pipeline trace -- no /100 scaling exists anywhere). night_mem_pct was authored directly in YAML for a docs/example config (commit a1478510, 'docs(admin-config-tool): commit spec docs and example configs') with no C-literal ancestry to cross-check against, unlike limits2 (migrated verbatim from a hand-written C literal `.val = 0.2` for '20% mem cap', sched.c's limit2_config) -- so it used the human-intuitive but incorrect 0-100 scale convention ('50' meaning 50%) instead of the pipeline's real 0-1 fraction convention.", "fix": "FIXED (2026-09-20, user-directed). Changed limits.yaml's night_mem_pct value from 50 to 0.5, matching the 0-1 fraction convention -- it now correctly computes and enforces a genuine 50% cap on the night queue's mem usage instead of a non-binding 5000% one. This is a production config behavior change, not merely a display fix: night_mem_pct will now actually constrain job admission for the first time in any real deployment using this config, where previously it silently did nothing.", @@ -27167,7 +14587,6 @@ "percent-limit", "night_mem_pct", "config-authoring-bug", - "fixed", "discovered-during-live-retest", "related-to-bug-1831", "behavior-change" @@ -27175,17 +14594,22 @@ "related_bugs": [ "bug-1831" ], - "occurrences": 1, - "last_seen": "2026-09-20T10:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1832.md", + "gitea_issue": 58, + "gitea_synced_status": "fixed" }, { "id": "bug-1833", - "gitea_synced_status": "fixed", - "gitea_issue": 59, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "config", + "severity": "crash", "timestamp": "2026-09-20T11:00:00Z", - "error_message": "NOT A CRASH -- proactive hardening, user-requested, following bug-1831/bug-1832. Nothing in xceadmin's config validation caught a percent-based limit's value being outside the valid 0.0-1.0 fraction range (e.g. 'value: 50, percent: true', meant to express 50% but computing as 5000% of the resource) -- both night_mem_pct in the real src/configs/limits.yaml (bug-1832) and, discovered while adding this validation, an identical stale value in the xceadmin test suite's own checked-in golden fixture (src/xceadmin/tests/fixtures/full/configs/limits.yaml) had this exact defect.", + "last_seen": "2026-09-20T11:00:00Z", + "occurrences": 1, + "error_message": "user-requested, following bug-1831/bug-1832. Nothing in xceadmin's config validation caught a…\n\nNOT A CRASH -- proactive hardening, user-requested, following bug-1831/bug-1832. Nothing in xceadmin's config validation caught a percent-based limit's value being outside the valid 0.0-1.0 fraction range (e.g. 'value: 50, percent: true', meant to express 50% but computing as 5000% of the resource) -- both night_mem_pct in the real src/configs/limits.yaml (bug-1832) and, discovered while adding this validation, an identical stale value in the xceadmin test suite's own checked-in golden fixture (src/xceadmin/tests/fixtures/full/configs/limits.yaml) had this exact defect.", "file": "src/configs/schema/limits.schema.json", "root_cause": "limits.schema.json's Limit definition had `\"value\": {\"type\": \"number\"}` with no range constraint at all, and no conditional (if percent then range) constraint existed anywhere in this codebase's 7 bundled JSON schemas -- confirmed no if/then usage exists in any of them prior to this fix.", "fix": "FIXED (2026-09-20). Added an if/then JSON Schema (draft 2020-12) conditional constraint to the Limit definition: `if percent == true, then value must be a number in [0.0, 1.0]`. Empirically verified against the real jsonschema==4.26.0 library this repo uses -- a violation surfaces with the exact same file/json_pointer/constraint contract validate_yaml()'s existing ValidationError already provides (constraint='maximum'/'minimum'), so zero changes were needed to validation.py's own error-reporting code. This is now enforced at real `xceadmin config build` time (deploy-ha.sh's actual cluster.bin generation path), not merely a unit-test-only check. IMPORTANT BUILD GOTCHA discovered while implementing this: this repo has TWO copies of every bundled JSON schema -- src/configs/schema/*.schema.json (the REAL canonical source) and src/xceadmin/src/xcecommon/schema_files/json/ (a checked-in mirror). meson-build.sh:104 ALWAYS overwrites the xceadmin copy from src/configs/schema/ during any real build (`cp \"$SRC_DIR/configs/schema/\"*.schema.json \"$SCHEMA_DST/json/\"`) -- editing ONLY the xceadmin-tree copy is silently discarded on the next build. Both copies were edited to stay in sync. Also fixed the same night_mem_pct value: 50 -> 0.5 defect the new constraint caught in the xceadmin test suite's own checked-in 'full' golden fixture, and regenerated its checked-in expected.bin golden artifact to match (via build.build_cluster_config()). Added 3 new tests to test_validation.py covering: reject out-of-range percent value, accept in-range percent value, and confirm a non-percent value is never range-checked regardless of magnitude.", @@ -27196,7 +14620,6 @@ "limit-plugin", "percent-limit", "proactive-hardening", - "fixed", "related-to-bug-1831", "related-to-bug-1832", "build-gotcha" @@ -27205,17 +14628,22 @@ "bug-1831", "bug-1832" ], - "occurrences": 1, - "last_seen": "2026-09-20T11:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1833.md", + "gitea_issue": 59, + "gitea_synced_status": "fixed" }, { "id": "bug-1834", - "gitea_synced_status": "fixed", - "gitea_issue": 61, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-20T09:35:00Z", - "error_message": "User-identified design gap (not yet reproduced as a live symptom, code-traced and confirmed): for a percent-based limit whose `resname` is a dynamic/consumable resource (e.g. `mem`, `swap`), the limit's CAPACITY is computed as percent * the resource's current LIVE value, not percent * the resource's static total. This means the same limit's enforced cap drifts up and down as unrelated memory pressure on the host changes, rather than staying pinned to a stable fraction of total capacity (e.g. 'this queue may use up to 20% of the host's total memory' should not shrink because some other job is currently using more memory).", + "last_seen": "2026-09-20T10:15:00Z", + "occurrences": 1, + "error_message": "User-identified design gap (not yet reproduced as a live symptom, code-traced and confirmed): for…\n\nUser-identified design gap (not yet reproduced as a live symptom, code-traced and confirmed): for a percent-based limit whose `resname` is a dynamic/consumable resource (e.g. `mem`, `swap`), the limit's CAPACITY is computed as percent * the resource's current LIVE value, not percent * the resource's static total. This means the same limit's enforced cap drifts up and down as unrelated memory pressure on the host changes, rather than staying pinned to a stable fraction of total capacity (e.g. 'this queue may use up to 20% of the host's total memory' should not shrink because some other job is currently using more memory).", "file": "src/scheduler/src/plugins/limit.c", "root_cause": "update_limit_res_value() (limit.c:142-165) computes `val = get_resvals(array, 1, &(vres->res->idx), &val); val = val * vres->limit->val.val;` for any percent!=0 limit -- `vres->res` is resolved in instantiate_limit() (xcelimit.c:574) via a single key_find(resmgr, limit->val.resname, ...), i.e. exactly the resource named by the limit's OWN `resname` field. `resname` today serves two roles that this code conflates: (1) selecting the accounting/consumable resource the limit's virtual resource (`vres->vres`) is keyed against for usage tracking (correctly `mem`, since jobs consume `mem`), and (2) supplying the live base value multiplied by percent to derive capacity (currently ALSO `mem`, the dynamic worker-reported load index per .kiro/steering/resource-accounting.md -- 'mem is not maxmem... mem is a dynamic load index... refreshed on the worker's regular load-report cadence'). Role (2) should read a static/non-consumable resource instead (`maxmem` for `mem`, `maxswap` for `swap`) -- both already exist as registered per-host resources (src/configs/resources.yaml:108 `maxmem`, :112 `maxswap`, 'self-reported once by the worker at join time and never refreshed' per resource-accounting.md) so no new resource type is needed, only a change to which resource the percent multiplication reads. xce_limit_val_t/xce_limit_config_t (types.h:800-818) have no field today to name this second resource explicitly -- resname is single-purpose in the struct.", "fix": "FIXED (2026-09-20). Added xce_vres_t.percent_res (src/include/xce/types.h) -- a resolved xce_resource_t* pointer for the percent-branch capacity read, kept separate from `res` (which stays the limit's own accounting resource, unchanged) so the two roles are no longer conflated. instantiate_limit() (src/lib/xcelimit.c) resolves it ONCE at instantiation time via a small table-driven static lookup (percent_capacity_resource_map[]: mem->maxmem, swap->maxswap) -- table-driven per user request, so a future dynamic/static pair (e.g. tmp/maxtmp) is a one-line addition. A missing/unregistered counterpart is NOT a hard failure: percent_res stays NULL and update_limit_res_value() (src/scheduler/src/plugins/limit.c) falls back to `res`, preserving today's behavior for any resource with no known static pair -- this is what keeps every pre-existing percent-limit test passing unchanged (they never register 'maxmem' in their resmgr fixtures, and none of them exercise update_limit_res_value()'s real computation anyway -- its existing white-box harness, test_repl_limit_table.c, stubs get_resvals()/put_resvals() as no-ops). New coverage: 4 unit tests in src/lib/testsuite/test_limit.c proving percent_res resolves to maxmem/maxswap for mem/swap, stays NULL for percent=0, and stays NULL (graceful fallback, not -ENOENT) for a percent-based limit on a resource with no known static counterpart (e.g. ncpus). Extended the shared src/lib/testsuite/test_resmgr_fixture.c/.h with maxmem/swap/maxswap test resources for this. Full lib (314 tests) and scheduler (85 test binaries) suites pass with zero regressions -- confirmed via raw testlog grep for ':FAIL:', not just the meson aggregate summary.", @@ -27233,24 +14661,28 @@ "needs-owner", "related-to-bug-1831", "related-to-bug-1832", - "night_mem_pct", - "fixed" + "night_mem_pct" ], "related_bugs": [ "bug-1831", "bug-1832" ], - "occurrences": 1, - "last_seen": "2026-09-20T10:15:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1834.md", + "gitea_issue": 61, + "gitea_synced_status": "fixed" }, { "id": "bug-1835", - "gitea_synced_status": "fixed", - "gitea_issue": 64, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "lib", + "severity": "crash", "timestamp": "2026-09-21T03:47:10Z", - "error_message": "Live 3-node HA cluster crash: xce_controller SIGSEGV in __memcpy_avx_unaligned_erms, called from capnp's copy_ptr/capn_setp, called from write_CtrlHelloRequest (lib/controller.capnp.c:22821) via encode_CtrlHelloRequest_ptr (lib/controller.capnp.c:22935), via xce_encode/xce_request, via ha_send_ctrlhello_to_all_standbys (controller/cluster.c:2432), via raft_cb_become_leader (controller/ha.c:2113), via raft_on_vote_reply -> vote_reply_dispatch -> peer_ctrl_msg_handler_inline -> peer_ctrl_vtask_entry, on the vcpu thread handling the winning vote reply. Core confirmed built from commit 1e43345b (xce v0.0.1-1e43345b, 2026-09-21T03:10:39Z); ctrl-1 had just won its Raft election (term 2, votes_granted=2) and crashed while broadcasting CtrlHello to its 2 standbys.", + "last_seen": "2026-09-21T03:47:10Z", + "occurrences": 1, + "error_message": "Live 3-node HA cluster crash: xce_controller SIGSEGV in __memcpy_avx_unaligned_erms, called from…\n\nLive 3-node HA cluster crash: xce_controller SIGSEGV in __memcpy_avx_unaligned_erms, called from capnp's copy_ptr/capn_setp, called from write_CtrlHelloRequest (lib/controller.capnp.c:22821) via encode_CtrlHelloRequest_ptr (lib/controller.capnp.c:22935), via xce_encode/xce_request, via ha_send_ctrlhello_to_all_standbys (controller/cluster.c:2432), via raft_cb_become_leader (controller/ha.c:2113), via raft_on_vote_reply -> vote_reply_dispatch -> peer_ctrl_msg_handler_inline -> peer_ctrl_vtask_entry, on the vcpu thread handling the winning vote reply. Core confirmed built from commit 1e43345b (xce v0.0.1-1e43345b, 2026-09-21T03:10:39Z); ctrl-1 had just won its Raft election (term 2, votes_granted=2) and crashed while broadcasting CtrlHello to its 2 standbys.", "file": "src/schema/controller.capnp", "root_cause": "CtrlHelloRequest.key was declared `Data` in controller.capnp -- the exact same latent bug class as bug-1504 (SnapshotData.internalState) and bug-1505 (WalCatchupEntry.payload), and explicitly named as a known-but-unfixed instance of it in bug-1505's own writeup ('Known/suspected other Data fields with the same latent gap: ... CtrlHelloRequest.key'). This project's capnp-c backend silently no-ops the encoding of any Data-typed field inside the generated encode_XXX() composite helper (see encode_CtrlHelloRequest's ' /* key key */' stub, lib/controller.capnp.c) without ever zero-initializing the caller's local `struct CtrlHelloRequest d;` first (encode_CtrlHelloRequest_ptr, lib/controller.capnp.c:22926-22938) -- so d.key.p is left as genuinely uninitialized stack garbage. write_CtrlHelloRequest (lib/controller.capnp.c:22821) then unconditionally calls capn_setp(p.p, 1, s->key.p) with that garbage capn_ptr, and capn_setp's copy_ptr dereferences the garbage segment/data pointer, SIGSEGV'ing in memcpy. ha_send_ctrlhello_to_all_standbys (controller/cluster.c:2419-2424) and ha.c:9495 both always set ctrl_hello_req_t.key = NULL (the field has never had a real producer anywhere in the codebase -- confirmed via project-wide grep, no call site ever sets it to non-NULL nor reads it back), so this is genuine undefined behavior whose manifestation depends entirely on whatever garbage bytes happen to occupy that vtask thread's stack slot at each call -- exactly the same 'lucky vs unlucky stack layout' nondeterminism documented for bug-1504/1505, which is why this had not crashed before despite the gap being known since 2026-08-07. Also confirmed the C struct itself compounded the risk: ctrl_hello_req_s.key (typemap.h) was typed `xce_data_t*` -- the local-Data-struct-shadowing wrapper type from bug-1461's own root cause -- rather than a real byte-blob type, even though it was never actually used as one.", "fix": "FIXED (2026-09-21), same pattern as bug-1504/bug-1505: changed CtrlHelloRequest.key from `Data` to `List(UInt8)` in src/schema/controller.capnp (which this capnp-c backend DOES correctly auto-generate real, non-no-op encode/decode/free for -- proven precedent: FetchFileReply.content/bytesRead, SnapshotData.internalState, WalCatchupEntry.payload), with a new keyLen (@4, UInt32) companion field as its maplistcount. Updated typemap.h's ctrl_hello_req_s.key from `xce_data_t*` to `uint8_t*` and added a new `key_len` (uint32_t) field. No consumer-code changes were needed: project-wide grep confirmed .key is never set to non-NULL nor read anywhere in production (ha_send_ctrlhello_to_all_standbys and ha.c's hello builder both always pass NULL), so the fix is purely structural -- it eliminates the uninitialized-capn_ptr class of UB for this field rather than papering over one occurrence. Added regression test test_ctrl_hello_request_key_round_trip (controller/testsuite/test_e2e_capnp.c), covering both the actual production shape (key==NULL/key_len==0, the exact shape that used to SIGSEGV -- now proven to encode/decode cleanly via the real generic codec) and a real non-NULL key payload (proves content genuinely round-trips now, which the old no-op'd Data codec never did). A deterministic RED test reproducing the SIGSEGV itself was not attempted -- by its own nature as reads of uninitialized stack memory, this bug class is not ASAN-catchable and its manifestation is stack-layout-dependent (bug-1504's own writeup makes the identical observation); the fix instead structurally eliminates the uninitialized-read entirely rather than chasing a flaky repro. Verified: full controller_unit_test suite (5/5 meson test binaries, 0 failures, including the new round-trip test and the pre-existing test_ctrl_hello_req_t_layout/test_ha_ctrlhello.c suite) via a clean ASAN rebuild (`xce-build.sh controller asan`) that also regenerated controller.capnp.c/.h from the updated schema with no codegen errors.", @@ -27270,25 +14702,29 @@ "live-crash", "related-to-bug-1504", "related-to-bug-1505", - "related-to-bug-1461", - "fixed" + "related-to-bug-1461" ], "related_bugs": [ "bug-1504", "bug-1505", "bug-1461" ], - "occurrences": 1, - "last_seen": "2026-09-21T03:47:10Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1835.md", + "gitea_issue": 64, + "gitea_synced_status": "fixed" }, { "id": "bug-1836", - "gitea_synced_status": "fixed", - "gitea_issue": 65, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-20T22:00:00Z", - "error_message": "User-reported live over-dispatch in xce_scheduler -i -s : replaying a captured session (JOB_PUSH of a 50-element array job) against a real single worker with 22 slots dispatched 44 concurrent single-slot jobs (`assigned jobs: 0-3-1789963203-0[0-43]`), exactly double the host's real capacity, while `info worker` simultaneously showed `resource slots: value=22.00 used=22.00` -- the accounting display looked self-consistent (used == value) even though double the job count the display's own 'used' figure implied was actually running. User confirmed normal mode (the live controller-connected driver) does not exhibit this under equivalent conditions, isolating the defect to simulation/replay mode specifically.", + "last_seen": "2026-09-21T05:46:00Z", + "occurrences": 1, + "error_message": "User-reported live over-dispatch in xce_scheduler -i -s : replaying a captured session…\n\nUser-reported live over-dispatch in xce_scheduler -i -s : replaying a captured session (JOB_PUSH of a 50-element array job) against a real single worker with 22 slots dispatched 44 concurrent single-slot jobs (`assigned jobs: 0-3-1789963203-0[0-43]`), exactly double the host's real capacity, while `info worker` simultaneously showed `resource slots: value=22.00 used=22.00` -- the accounting display looked self-consistent (used == value) even though double the job count the display's own 'used' figure implied was actually running. User confirmed normal mode (the live controller-connected driver) does not exhibit this under equivalent conditions, isolating the defect to simulation/replay mode specifically.", "file": "src/scheduler/src/sim_loop.c", "root_cause": "sim_reconcile_unaccounted_jobs() (the 'Recorded-Decision Reconciliation' mechanism, scheduler-sim-decision-parity/scheduler-sim-eager-reconciliation specs) unconditionally force-applied any recorded DISPATCH decision it found in decoded_session->decisions for a job with accounts==NULL -- setting job->state=JOB_STAT_RUN and job->allocs/job->accounts straight from the recording -- with ZERO call to sched_check_host_satisfy()/sched_check_shared_satisfy() or any other admission/capacity check. This directly contradicted the scheduler-sim-decision-parity spec's own design.md assumption underpinning its Requirement 2 ('capacity awareness'): 'sched_check_host_satisfy_()/sched_check_shared_satisfy_() already read resInUse/sharedInUse unconditionally; once those fields are correctly populated in simulation mode, the existing (unmodified) capacity-check code path produces the same outcome normal mode would' -- an assumption that only holds for jobs decided via the LIVE picker/allocator path (sched_election.c's sched_allocator(), which DOES call these checks via sched_try_alloc_on_host_()), not for jobs captured via this SEPARATE reconciliation backstop, which bypasses that code path entirely. Confirmed via the round sequence in sched_sim_driver_run(): xce_core_push_events() creates all 50 freshly-pushed jobs (PEND, accounts==NULL) BEFORE sim_reconcile_unaccounted_jobs() runs for that same round, so ALL 50 are eligible reconciliation candidates the moment the function starts -- for however many of them (44, in the live report) the session's OWN originally-recorded decisions said DISPATCH, reconciliation force-applied all of them with no regard for the CURRENT sim snapshot's actual worker capacity (22 slots), before the live picker/allocator (which runs afterward, per the fixed round order) ever got a chance to independently admission-check any of them -- explaining both the 44 dispatches and the 6 correctly-PENDed remainder (jobs whose OWN recorded decision was PEND, never matched by reconciliation's ReplayDecision_body_dispatch-only filter). This is a distinct mechanism from bug-1611/bug-1613 (both already fixed/superseded by scheduler-sim-live-allocation and scheduler-sim-eager-reconciliation) -- those closed a TIMING gap in when reconciliation's job->state transition happens; this bug is a CAPACITY-VALIDATION gap in whether it happens at all, independent of timing. Uncovered via /xce-audit-invariants scheduler, dimension 1 (driver state-equivalence).", "fix": "FIXED (2026-09-21). Added sim_reconciled_alloc_fits_capacity() (sim_loop.c, static) -- re-validates a candidate recorded allocation's full resource footprint (every per-host allocation plus any shared/global allocation) against LIVE resInUse/sharedInUse using the SAME sched_check_host_satisfy()/sched_check_shared_satisfy() admission-check functions (resource.c) the live picker/allocator path already calls, resolving each allocation's resnames against core->resmgr fresh (the decoded allocation carries only resnames/values, not resolved indices). sim_reconcile_unaccounted_jobs()'s FOUND branch now calls this BEFORE taking ownership of a matched entry; a capacity miss is handled exactly like 'no matching recorded decision' (job left unaccounted this round, entry NOT consumed/freed, naturally retried later or correctly PENDed by the live picker once it runs). A newly-admitted candidate's usage is charged into the LIVE resInUse/sharedInUse immediately via sched_charge_account() (mirroring sched_commit_decision()'s own immediate charge for a live dispatch) so a LATER candidate in the SAME reconciliation pass correctly sees an EARLIER accepted candidate's usage -- essential for the exact multi-job-contention scenario this bug exercises. To make the capacity check see genuinely prior-round (already-running) jobs' usage too, added sim_reconcile_toggle_prior_charges() (sim_loop.c, static) -- seeds resInUse/sharedInUse from every already-RUN+accounted job at the start of sim_reconcile_unaccounted_jobs(), then fully undoes that seed plus every incremental charge made during the pass at the end, leaving the function's net effect on the real accounting arrays at exactly zero on return; the caller's own unchanged, immediately-following sched_account_jobs() call remains the sole, non-double-counting authoritative charge for the round (unchanged architecture/ordering). Deliberately NOT implemented via sched_account_jobs()/a new sched_unaccount_jobs() exported pair (tried first, then reverted): that exact by-name function is separately instrumented and call-count-asserted by test_sim_contention_parity.c's g_round_idx mechanism ('sched_account_jobs() called exactly once per round' is itself a protected invariant there) -- doubling its call count broke 2 of that file's tests (masked at first by a coincidental g_round_resinuse[8] array-capacity cap that hid the true +5/+10 call-count delta behind an apparent '+2' pattern; properly diagnosed by reading the array declaration, not guessed). sched_check_host_satisfy()/sched_check_shared_satisfy() were relocated from scheduler.h to core.h (pure declaration move, bodies unchanged in resource.c, following the exact sched_account_jobs()/sched_resolve_job_picker() precedent already established for this exact purpose) since sim_loop.c must never include scheduler.h (pulls in client.h/xce/cluster.h, forbidden by test_sim_isolation.c's Req 5 AC 17 enforcement). New regression test test_reconcile_respects_host_capacity_under_contention (test_sim_reconcile_unaccounted.c) -- two PEND jobs each recorded-DISPATCH to the same 1-slot-capacity host; asserts exactly one is admitted (not both, not neither), the rejected one's entry is not consumed, and post-round resInUse reads exactly 1.0 (never 2.0). Fixing this surfaced 12 pre-existing tests (9 in test_sim_reconcile_unaccounted.c, 3 in test_sim_verify.c) whose fixtures named a host in a recorded allocation without ever registering a real xce_sched_worker_t of that name -- harmless before this fix (nothing checked capacity), now correctly rejected by the new gate; fixed by registering a real (or, for test_sim_verify.c, which never white-box-includes resource.c, a bare link-satisfying) worker for each such fixture, generous capacity since none of those 12 tests are actually about capacity limits. Verified: full scheduler suite 85/85 Ok, 0 Fail (asan profile, xce-build.sh scheduler asan), independently re-confirmed by running sched_sim_reconcile_unaccounted_test (22/22) and sched_sim_verify_test (17/17) directly with ASAN_OPTIONS=detect_leaks=0 (this test-file family's own established convention, unrelated pre-existing leak reports otherwise printed by these binaries when run standalone outside meson's own suite-level ASAN_OPTIONS).", @@ -27304,8 +14740,7 @@ "capacity-check", "xce-audit-invariants", "user-reported", - "found-live", - "fixed" + "found-live" ], "related_bugs": [ "bug-1611", @@ -27315,17 +14750,22 @@ "bug-1759", "bug-1779" ], - "occurrences": 1, - "last_seen": "2026-09-21T05:46:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1836.md", + "gitea_issue": 65, + "gitea_synced_status": "fixed" }, { "id": "bug-1837", - "gitea_synced_status": "fixed", - "gitea_issue": 67, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-20T22:05:00Z", - "error_message": "Found during /xce-audit-invariants scheduler (dimension 3, jbuf capacity-1 rendezvous audit), not user-reported and not a live symptom: sim_reconcile_unaccounted_jobs() (sim_loop.c) calls picker->picker_iface->on_dispatch(picker, job->jobid) directly from the sim driver thread (sched_sim_driver_run()'s own thread, between xce_core_begin_session() and xce_core_signal_go()), but plugin.h documents on_dispatch()/on_pend() as 'Post-allocation hooks (MUST) -- allocator thread, after each allocation attempt' -- a MUST-level threading contract this call site violates by construction (driver-thread code calling an allocator-thread-only hook).", + "last_seen": "2026-09-21T05:00:00Z", + "occurrences": 1, + "error_message": "Found during /xce-audit-invariants scheduler (dimension 3, jbuf capacity-1 rendezvous audit), not…\n\nFound during /xce-audit-invariants scheduler (dimension 3, jbuf capacity-1 rendezvous audit), not user-reported and not a live symptom: sim_reconcile_unaccounted_jobs() (sim_loop.c) calls picker->picker_iface->on_dispatch(picker, job->jobid) directly from the sim driver thread (sched_sim_driver_run()'s own thread, between xce_core_begin_session() and xce_core_signal_go()), but plugin.h documents on_dispatch()/on_pend() as 'Post-allocation hooks (MUST) -- allocator thread, after each allocation attempt' -- a MUST-level threading contract this call site violates by construction (driver-thread code calling an allocator-thread-only hook).", "file": "src/scheduler/src/sim_loop.c", "root_cause": "bug-1759/bug-1779's own fix added this on_dispatch() call to sim_reconcile_unaccounted_jobs() specifically to prevent sim_picker_thread_fn() from re-offering (and re-PENDing) an already-reconciled job on a later round -- a real, necessary fix for that narrower symptom (a duplicate PEND report) -- but the fix's own header comment justifies it only in those terms ('the picker never re-decides an already-resolved job'), never addressing thread-safety of calling an allocator-thread-only hook from the driver thread instead. It currently causes no observable corruption only because the sole real (non-no-op) picker plugin, builtin_fcfs.c, happens to guard its entire candidate queue with an internal lock (priv->lock, taken in every hook including end_session()) -- an implementation detail plugin.h's own documented contract does not require picker plugins to provide (\"the picker thread holds no lock the allocator thread needs\"). A future or alternate picker plugin implemented per that documented (lock-free) contract would race sim_reconcile_unaccounted_jobs()'s driver-thread on_dispatch() call against the picker thread's own end_session()/next_job() tail-end activity from the prior round, unsynchronized.", "fix": "FIXED (2026-09-21, TDD). Chose the fuller of the two directions the audit finding itself proposed: moved the on_dispatch() call onto the real allocator thread instead of documenting an exception to plugin.h's contract. Since the scheduler-sim-live-allocation spec, sim mode's sched_sim_driver_run() (sim_loop.c) spawns the SAME unmodified sched_job_picker()/sched_allocator() thread pair (sched_election.c) normal mode uses, so a real allocator thread now genuinely exists to route through in production. Mechanism: two new dedicated cbufs on xce_sched_core_t (core.h) -- reconcile_req (jobid) and reconcile_ack (rc) -- deliberately SEPARATE from jbuf, not a widened jbuf item type, since jbuf's item size/capacity are a pinned determinism invariant (test_core_jbuf_capacity_pin.c, Req 5 AC 7) that a byte-for-byte source-text-scanning regression test forbids changing. A new reserved sentinel, XCE_SCHED_RECONCILE_WAKE (== UINT64_MAX, core.h; same reserved-value idiom this codebase already uses for SCHED_SIM_START_TS_LATEST, sim.h, and jbuf's own pre-existing 0 end-of-session marker), is what the driver pushes onto jbuf itself to wake the allocator thread out of its cbuf_get(&core->jbuf) block -- never a real jobid, so it can never be mistaken for one by sched_get_job(). sim_reconcile_unaccounted_jobs()'s on_dispatch call site (sim_loop.c) now goes through a new sim_reconcile_invoke_on_dispatch() helper: pushes jobid onto reconcile_req, THEN pushes XCE_SCHED_RECONCILE_WAKE onto jbuf (ordering matters -- cbuf_put()'s own mutex gives happens-before, so reconcile_req is guaranteed populated by the time the allocator's cbuf_get(&core->jbuf) returns the sentinel), then blocks on reconcile_ack for the result -- one request in flight at a time, matching jbuf's own capacity-1 rendezvous discipline without touching jbuf itself. sched_election.c's sched_allocator() gained a matching branch: on XCE_SCHED_RECONCILE_WAKE, drain reconcile_req, call on_dispatch() (no plugin_job_election()/sched_make_allocation()/decision -- reconciliation already did its own accounting/charging on the driver thread), ack via reconcile_ack, continue -- a true no-op for normal mode, which never populates reconcile_req. A new xce_sched_sim_driver_t.allocator_thread_live bool (sim.h) tells sim_reconcile_invoke_on_dispatch() whether to route through the relay at all: sched_sim_driver_run() sets it true immediately after successfully spawning the allocator thread (same thread, program order -- no atomics needed, no startup race, since both the write and every later read happen on the driver thread only); it stays false for a bare xce_sched_sim_driver_t that never spawned any thread pair at all (test_sim_reconcile_unaccounted.c's own documented Testing Strategy: ~23 existing tests call sim_reconcile_unaccounted_jobs() directly, standalone, with no picker/allocator threads running), in which case a direct on_dispatch() call is safe by construction -- there is provably only one thread touching picker state, so plugin.h's allocator-thread-only contract has nothing to violate. This let all ~23 pre-existing direct-call tests in that file pass unchanged. Blast radius (grepped the full src+testsuite tree for core->jbuf and every hand-rolled white-box sched_allocator()/sched_job_picker() stub before assuming any of this was safe, per this project's own established discipline): 4 other test files drive sim_reconcile_unaccounted_jobs()'s success path through the REAL sched_sim_driver_run() round loop with their own local sched_allocator() stub (a simplified PEND-only stand-in, never the real sched_election.c symbol) -- test_sim_verify.c, test_sim_round_boundary_reconciliation.c, test_sim_contention_parity.c, test_sim_e2e_original_motivation.c. Confirmed empirically, not just by static grep: a first full-suite build+test run with the fix applied but before patching these four hung all four at meson's 30s per-test timeout (SIGTERM) -- exactly the deadlock a stub that never answers reconcile_ack would cause. Added the identical relay branch to each of their local sched_allocator() stubs; re-ran the full suite and all four passed. TDD: new test_reconcile_invokes_on_dispatch_on_allocator_thread_not_caller (test_sim_reconcile_unaccounted.c) uses a minimal test-owned picker plugin (not builtin_fcfs, whose own internal locking was exactly what made the pre-fix violation dormant/unobservable) whose on_dispatch() records pthread_self(), plus a one-shot relay_one_reconcile_request() thread standing in for sched_allocator(); asserts the recorded tid equals the spawned relay thread's tid and NOT the calling (test) thread's own tid. RED confirmed by temporarily forcing sim_reconcile_invoke_on_dispatch() back to an unconditional direct call and rebuilding: exactly this new test hung (the relay thread's pthread_join() never returned, since nothing was ever pushed onto jbuf/reconcile_req) and was killed at meson's 30s timeout, all 84 other tests still passed. GREEN confirmed after restoring the real fix: full scheduler suite 85/85 Ok, zero regressions, matching the exact pre-fix baseline count.", @@ -27339,7 +14779,6 @@ "xce-audit-invariants", "related-to-bug-1759", "related-to-bug-1779", - "fixed", "tdd", "allocator-thread-relay", "user-directed" @@ -27349,17 +14788,22 @@ "bug-1779", "bug-1836" ], - "occurrences": 1, - "last_seen": "2026-09-21T05:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1837.md", + "gitea_issue": 67, + "gitea_synced_status": "fixed" }, { "id": "bug-1838", - "gitea_synced_status": "fixed", - "gitea_issue": 70, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-21T00:30:00Z", - "error_message": "Follow-up architectural correction after bug-1836 (found during user discussion, not a separately-reported live symptom): even with bug-1836's capacity gate in place, sim_reconcile_unaccounted_jobs() still unnecessarily PREEMPTED the live picker/allocator for any freshly-PEND job with a matching recorded DISPATCH decision, rather than letting the live path -- now genuinely capacity-checked, per scheduler-sim-live-allocation -- decide it on its own the way normal mode always has. User specifically asked why sim mode diverged from normal mode given they 'share the same core', then proposed the architecturally correct framing directly: the simulator should run the identical decision-making core (minus controller/worker I/O) and use the recording only to verify/reconstruct, not to short-circuit live decisions.", + "last_seen": "2026-09-21T02:40:00Z", + "occurrences": 1, + "error_message": "Follow-up architectural correction after bug-1836 (found during user discussion, not a…\n\nFollow-up architectural correction after bug-1836 (found during user discussion, not a separately-reported live symptom): even with bug-1836's capacity gate in place, sim_reconcile_unaccounted_jobs() still unnecessarily PREEMPTED the live picker/allocator for any freshly-PEND job with a matching recorded DISPATCH decision, rather than letting the live path -- now genuinely capacity-checked, per scheduler-sim-live-allocation -- decide it on its own the way normal mode always has. User specifically asked why sim mode diverged from normal mode given they 'share the same core', then proposed the architecturally correct framing directly: the simulator should run the identical decision-making core (minus controller/worker I/O) and use the recording only to verify/reconstruct, not to short-circuit live decisions.", "file": "src/scheduler/src/sim_loop.c", "root_cause": "sim_reconcile_unaccounted_jobs()'s trigger condition was widened from JOB_IS_RUN-only to state-independent (PEND-or-RUN) by the scheduler-sim-eager-reconciliation spec (2026-09-11), to fix a real problem at the time: a job's recorded DISPATCH decision only exists in the ONE round's decoded_session it was captured in (freed at round end), so a RUN-only gate meant reconciliation could never find it once the job's SEPARATE, later-replayed RUN-confirmation event finally made it RUN. That widening was the only available fix at the time because sim mode's own live picker/allocator path had no working capacity accounting (bug-1613) and so could not be trusted to independently re-derive the SAME decision the original recording made -- reconciliation's job was to force in the recording's answer since the live path's own answer might diverge or be wrong. scheduler-sim-live-allocation (completed 2026-09-19) closed that gap for real: sim mode now spawns the SAME real, capacity-checked sched_job_picker()/sched_allocator() threads (sched_election.c) normal mode does, with sched_account_jobs() correctly wired into its round sequence -- its own design.md explicitly states a 'Live/Recorded Authority Rule' (live decision wins when the live path reaches a job; reconciliation is a narrow safety net, not a first-resort source of truth) but the actual per-round ORDERING (reconciliation runs unconditionally before the live picker/allocator gets a turn each round) never enforced that rule -- so the PEND-inclusive trigger left in place after scheduler-sim-live-allocation landed kept preempting jobs the live path was now fully capable of deciding correctly and verifiably on its own. This was the deeper root cause underlying bug-1836's exact reproduction (44 jobs force-dispatched via reconciliation before the live picker/allocator ever got a turn); bug-1836's capacity gate made that force-application SAFE (never exceeds real capacity) but did not stop it from happening. Also confirmed (via direct investigation): a snapshot-loaded job (snapshot_load.c's xce_sched_core_load_snapshot()) is registered with the live picker via on_job_push() for every restored job unconditionally, RUN or PEND -- so there is no 'invisible to the picker' case a RUN-only trigger could miss; every legitimate reconciliation candidate today is also, structurally, a job the live picker/allocator can independently resolve. The sole remaining genuine case: a job already RUN before this replay's own observation window (dispatched too far in the past for its DISPATCH decision to still be in any currently-decoded session, and never offered by next_job() since RUN jobs are never picker candidates) -- for that job alone, the recording remains the sole source of truth.", "fix": "FIXED (2026-09-21). Narrowed sim_reconcile_unaccounted_jobs()'s trigger back to JOB_IS_RUN(job->state)-only (undoing the 2026-09-11 widening), so a freshly-PEND job now always flows through the same real, capacity-checked live picker/allocator normal mode uses -- verified via the pre-existing --sim-verify generate-then-compare mechanism -- rather than being force-applied from history. bug-1836's capacity gate (sim_reconciled_alloc_fits_capacity()/sim_reconcile_toggle_prior_charges()) remains in place unchanged as defense-in-depth for the narrower remaining RUN-only case. Fixing this correctly broke 6 test binaries whose fixtures depended on the old PEND-inclusive behavior; investigation of each was more nuanced than initially guessed: test_sim_verify.c's and test_sim_contention_parity.c's failures were NOT missing-worker-registration issues (the initial hypothesis) but rather that those two files each stub sched_job_picker()/sched_allocator() to a deliberately PEND-only pair (used to keep their OTHER, unrelated tests decoupled from real dispatch logic) -- under the new design a PEND job in those fixtures genuinely can never be dispatched by anything, so the correct fix was inverting each test's assertions to lock in the new, correct behavior (a PEND job stays PEND, and --sim-verify correctly reports the resulting MISMATCH against a history that shows it dispatched, since these minimal harnesses have no real capacity-tracked path to reproduce that decision) rather than trying to make the stub dispatch something it structurally cannot. The 5 directly-affected tests in test_sim_reconcile_unaccounted.c were each individually re-examined for their TRUE intent and fixed accordingly: one inverted/renamed to lock in 'PEND is now always ignored regardless of a matching decision' (test_reconcile_ignores_pend_state_job_even_when_dispatch_recorded), three converted from a PEND-starting to a RUN-starting fixture job (still meaningfully testing the same underlying resInUse-timing/retry/multi-round mechanics, now via the only state that's still eligible), and one (the bug-1759/1779 on_dispatch-retirement regression test) kept its job PEND-built-then-force-set-to-RUN specifically to reproduce the pathological 'RUN job the picker was never told about' state reconciliation's on_dispatch() safety net still needs to guard against, since no real production path can reach that shape anymore post-bug-1838. A sweep for other coincidentally-passing tests found and fixed one more: bug-1836's own new regression test (test_reconcile_respects_host_capacity_under_contention) also used two PEND jobs, converted to RUN with its 'loser' assertion corrected to match. Added a new live-dispatch-vs-recording round-trip test (test_sim_live_allocation.c) to restore the coverage inverting test_sim_verify.c's test removed (that a genuinely live-dispatched decision correctly reproduces a matching historical recording when both the job and the resources it wants are real). scheduler-sim-eager-reconciliation's own requirements.md/design.md got dated 'Correction (2026-09-21, bug-1838)' notes in place, following the exact precedent format its own 2026-09-11 correction already established, superseding (not deleting) the original PEND-inclusive Acceptance Criterion 1 text. Verified: full scheduler suite 85/85 Ok, 0 Fail (asan profile); independently re-confirmed by running sched_sim_reconcile_unaccounted_test (22/22), sched_sim_verify_test (17/17), sched_sim_contention_parity_test (2/2), and sched_sim_live_allocation_test (2/2) directly.", @@ -27374,7 +14818,6 @@ "architecture", "user-directed", "xce-audit-invariants", - "fixed", "related-to-bug-1836" ], "related_bugs": [ @@ -27386,17 +14829,22 @@ "bug-1779", "bug-1804" ], - "occurrences": 1, - "last_seen": "2026-09-21T02:40:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1838.md", + "gitea_issue": 70, + "gitea_synced_status": "fixed" }, { "id": "bug-1839", - "gitea_synced_status": "fixed", - "gitea_issue": 71, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-09-22T02:30:00Z", - "error_message": "Live 3-node HA cluster crash: xce_controller SIGSEGV in peer_ctrl_msg_handler_inline (ha.c:7035, `ha_state_t *ha = vctx->ha;`). gdb analysis of the actual core dump (build cd9ecb41) confirmed vctx == NULL (i.e. ep->data == NULL) while the endpoint itself was fully alive and in active use: ep->refcnt=191964, ep->tearing_down=0, ep->heap_alloc=1, ep->msg_handler=peer_ctrl_msg_handler, ep->err_handler=standby_ep_err_handler_primary, ep->fsm_out.state=FSM_STATE_ERROR. The crash fired on the vtask-pool path (vcpu_entry -> run_task -> peer_ctrl_vtask_entry -> peer_ctrl_msg_handler_inline); g_drive_async_peerctrl_dispatch_enabled defaults to 1 (ON, the production path).", + "last_seen": "2026-09-22T02:30:00Z", + "occurrences": 1, + "error_message": "Live 3-node HA cluster crash: xce_controller SIGSEGV in peer_ctrl_msg_handler_inline (ha.c:7035…\n\nLive 3-node HA cluster crash: xce_controller SIGSEGV in peer_ctrl_msg_handler_inline (ha.c:7035, `ha_state_t *ha = vctx->ha;`). gdb analysis of the actual core dump (build cd9ecb41) confirmed vctx == NULL (i.e. ep->data == NULL) while the endpoint itself was fully alive and in active use: ep->refcnt=191964, ep->tearing_down=0, ep->heap_alloc=1, ep->msg_handler=peer_ctrl_msg_handler, ep->err_handler=standby_ep_err_handler_primary, ep->fsm_out.state=FSM_STATE_ERROR. The crash fired on the vtask-pool path (vcpu_entry -> run_task -> peer_ctrl_vtask_entry -> peer_ctrl_msg_handler_inline); g_drive_async_peerctrl_dispatch_enabled defaults to 1 (ON, the production path).", "file": "src/controller/ha.c", "root_cause": "standby_ep_err_handler_primary (the err_handler for an outbound standby/heartbeat replication channel, registered in ha_connect_prep_peer) synchronously did `free(sctx); ep->data = NULL;` on whichever thread handle_error() called it from (the io_worker thread), with zero coordination with the vtask pool. If a peer-control vtask for the SAME ep was already queued or in flight (entirely plausible -- the endpoint is actively receiving replication ACKs/heartbeats via peer_ctrl_msg_handler -> peer_ctrl_dispatch_to_pool -> the vtask pool, and can independently hit a write/SSL error on that same connection), that vtask later ran peer_ctrl_msg_handler_inline, read ep->data as NULL, and crashed with a NULL dereference (not a use-after-free -- ep->data was explicitly nulled, not freed-and-reused memory). This is exactly the hazard this codebase already identified and fixed for the sibling vote_ep_ctx_t case: vote_ep_err_handler (ha.c:7787) and pre_vote_ep_err_handler (ha.c:7825) both explicitly do NOT free their context, with an inline comment ('Do NOT free vctx here -- peer_ctrl_msg_handler may still reference ep->data.'). standby_ep_err_handler_primary was never updated to the same discipline when the vtask-pool dispatch machinery (raft-drive-async, bug-1197/bug-1298) was introduced and retrofitted onto the other err handlers -- it slipped through because it's a distinct, older handler for the ongoing standby-heartbeat connection rather than the vote-time connection. Audited the other ~6 `ep->data = NULL` sites in ha.c (ha_connect_prep_peer/ha_connect_finish_peer/ha_connect_to_peer) and confirmed none of them share this bug class -- each frees an endpoint that either was never registered with iomgr at all, or whose iomgr_monitor() call itself just failed, so no vtask could ever have been dispatched against it.", "fix": "FIXED (2026-09-22, TDD, user-reported live crash). Reused the established, already-tested vote_ep_teardown_async(ha, ep) mechanism (ha.c:7232) instead of inventing a new one: it calls endpoint_mark_tearing_down(ep) (sets the flag + drains dispatch_inflight, closing the TOCTOU window against peer_ctrl_msg_handler's late-event gate), iomgr_unmonitor() if monitored, then enqueues a VTASK_IO teardown task bound to this ep via vtask_enq_classified -- the per-endpoint FIFO guarantees this teardown vtask runs strictly after any earlier-queued or in-flight same-ep peer-control vtask. Its body (vote_ep_teardown_vtask_entry) is fully generic over what ep->data holds -- it just endpoint_unref()s the endpoint, and endpoint_fini()'s own FREEUP(ep->data) frees whatever ep->data currently points to once the last reference truly releases. standby_ep_err_handler_primary's `free(sctx); ep->data = NULL;` tail was replaced with a single `vote_ep_teardown_async(ha, ep);` call (its ha->standby_endpoints removal and repl_remove_standby bookkeeping stayed unchanged, since those only read sctx's still-valid fields before the deferred free). Added an `if (endpoint_is_tearing_down(ep)) return 0;` idempotency guard at the top of the handler: the old `if (!sctx) return 0` accidentally doubled as a reentry guard (a second call after the first had already nulled ep->data would bail), but that protection disappears once ep->data is no longer nulled synchronously, and handle_error() firing this handler more than once for the same ep (e.g. a read error and a write error both observed before iomgr_unmonitor takes effect) would otherwise double-drop the endpoint's owner reference. Also found and fixed a second-order regression this exposed: on_standby_connected's stale-duplicate-channel eviction path (ha.c ~9461-9476) called standby_ep_err_handler_primary(stale_same_peer) and THEN did its own iomgr_unmonitor + endpoint_unref(stale_same_peer) afterward -- correct under the OLD contract (the handler used to only free ep->data, leaving ref-dropping to the caller) but now a double-unref/use-after-free once the handler owns the full teardown itself. Removed the caller's now-redundant iomgr_unmonitor/endpoint_unref calls; the err handler's own vote_ep_teardown_async call handles everything. Caught by ASAN (heap-use-after-free in endpoint_unref via on_standby_connected, immediately on the first full-suite rebuild after the primary fix) rather than by design review -- confirms the value of running the full suite, not just the new tests, after this kind of shared-contract change. Also updated 3 pre-existing tests in test_ha_liveness.c (test_reconnect_resolves_dual_channel_same_peer, test_reconnect_resolves_dual_channel_promoted_vote_ep, test_reconnect_preserves_other_peer_channel) that called standby_ep_err_handler_primary_for_test(ep) followed by a manual endpoint_fini(ep)/free(ep) -- with ha.pool == NULL in these unit fixtures, vote_ep_teardown_async now takes its documented no-pool inline fallback (WARN + endpoint_unref, refcnt 1->0, fini+free) inside the handler itself, so the tests' own manual cleanup became a double-free; removed it. TDD: two new tests in test_ha_vep_teardown.c using a real standby_ep_ctx_t (standby_ep_ctx_build_for_test) and the real standby_ep_err_handler_primary_for_test entry point (not the generic vote_ep_ctx_t stand-in test (a) in that file uses): test_standby_err_handler_defers_teardown_until_inflight_completes proves the deferred-teardown ordering (mirrors test_vep_teardown_runs_after_inflight's gate/sequence-ticket technique -- parks an in-flight same-ep vtask, calls the err handler, asserts teardown does NOT run while the vtask is parked, then releases it and asserts teardown runs strictly after); test_standby_err_handler_idempotent_against_double_invocation proves calling the handler twice in a row on the same (still-alive, tearing-down) ep is a safe no-op with exactly one teardown vtask ever running. RED confirmed against the pre-fix handler: both new tests failed with 'ep must be marked tearing_down...' (endpoint_is_tearing_down never became true, since the old code never called vote_ep_teardown_async at all) -- 1219 tests, 2 failures, zero other regressions. GREEN after the fix plus the on_standby_connected correction: full controller suite 1219/1219 Ok, zero failures.", @@ -27413,26 +14861,28 @@ "related-to-bug-1298", "live-crash", "user-reported", - "fixed", "tdd" ], "related_bugs": [ "bug-1197", "bug-1298" ], - "occurrences": 1, - "last_seen": "2026-09-22T02:30:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1839.md", + "gitea_issue": 71, + "gitea_synced_status": "fixed" }, { "id": "bug-1840", - "repro_attempts": "0/1 ASAN run, 2026-09-25: 3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' (2400 elements) on the current ASAN build -- no AddressSanitizer report from the scheduler; plus ~10 ASAN HA soaks today (sequential/chaotic, all with scheduler replay-verify) with no scheduler heap error", - "gitea_synced_status": "cannot-reproduce", - "gitea_issue": 74, "status": "cannot-reproduce", "status_updated": "2026-09-25", "status_note": "no ASAN repro found. Strongest code-level candidate fixed 2026-09-25: bug-1769 (sched_make_alter_allocation wrote obj1->alloc 136 bytes into a 24-byte heap chunk for any ALTER '{..}||{..}' job -- exactly the kind of out-of-bounds write that corrupts glibc fastbin metadata and surfaces later in an unrelated malloc_consolidate). Reopen to investigating if it recurs, ideally captured on an ASAN build.", + "component": "scheduler", + "severity": "crash", "timestamp": "2026-09-22T02:35:00Z", - "error_message": "Live 3-node HA cluster crash (same soak run as bug-1839): xce_scheduler SIGABRT, glibc `malloc_consolidate(): unaligned fastbin chunk detected`, surfacing during an OpenSSL TLS handshake's MAC-buffer growth: abort -> malloc_printerr -> malloc_consolidate -> _int_malloc -> __libc_malloc2 -> CRYPTO_malloc -> CRYPTO_clear_realloc -> BUF_MEM_grow_clean -> mem_write -> bwrite_conv -> BIO_write -> ssl3_finish_mac -> tls_get_message_body -> state_machine -> handle_ssl_handshake (netio.c:548) -> io_handler (netio.c:588) -> io_worker_dispatch_batch (netio.c:648) -> io_worker (netio.c:689).", + "last_seen": "2026-09-22T02:35:00Z", + "occurrences": 1, + "error_message": "Live 3-node HA cluster crash (same soak run as bug-1839): xce_scheduler SIGABRT, glibc…\n\nLive 3-node HA cluster crash (same soak run as bug-1839): xce_scheduler SIGABRT, glibc `malloc_consolidate(): unaligned fastbin chunk detected`, surfacing during an OpenSSL TLS handshake's MAC-buffer growth: abort -> malloc_printerr -> malloc_consolidate -> _int_malloc -> __libc_malloc2 -> CRYPTO_malloc -> CRYPTO_clear_realloc -> BUF_MEM_grow_clean -> mem_write -> bwrite_conv -> BIO_write -> ssl3_finish_mac -> tls_get_message_body -> state_machine -> handle_ssl_handshake (netio.c:548) -> io_handler (netio.c:588) -> io_worker_dispatch_batch (netio.c:648) -> io_worker (netio.c:689).", "file": "unknown -- heap corruption detected far from its cause", "root_cause": "NOT YET DIAGNOSED -- heap corruption detected far from its cause; a corrupted fastbin chunk was only found during an unrelated later malloc_consolidate() call, inside an OpenSSL TLS handshake on the scheduler's io_worker thread. The actual out-of-bounds write that corrupted the allocator's freelist could have happened in any thread, any amount of time earlier, and is not reconstructable from a non-ASAN build.dbg core dump -- there is no shadow memory, no poisoning, nothing to localize it to a specific allocation or write site. Confirmed directly via gdb against the real core file (/tmp/core.xce_scheduler, build cd9ecb41): the crashing frames' locals (malloc_consolidate's av/fb/p) are all optimized out / no debug symbols available for glibc or libcrypto/libssl in this environment, so there is nothing further to extract from the core itself beyond the symbol-only backtrace already captured. Needs an ASAN reproduction of the same 3-node HA deploy soak scenario to catch the actual out-of-bounds write with a real stack trace.", "fix": "NOT FIXED -- awaiting an ASAN repro. Recommended next step: `/xce-build scheduler asan` (or `bash src/tools/xce-build.sh scheduler asan`), then rerun the same HA deploy scenario that produced /tmp/core.xce_scheduler until ASAN aborts at the real corruption site, which will pinpoint the actual overflow/write instead of just its downstream symptom.", @@ -27445,24 +14895,28 @@ "live-crash", "user-reported", "needs-asan-repro", - "not-fixed", - "unfixed", "needs-diagnosis" ], "related_bugs": [ "bug-1839" ], - "occurrences": 1, - "last_seen": "2026-09-22T02:35:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1840.md", + "repro_attempts": "0/1 ASAN run, 2026-09-25: 3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' (2400 elements) on the current ASAN build -- no AddressSanitizer report from the scheduler; plus ~10 ASAN HA soaks today (sequential/chaotic, all with scheduler replay-verify) with no scheduler heap error", + "gitea_issue": 74, + "gitea_synced_status": "cannot-reproduce" }, { "id": "bug-1841", - "gitea_synced_status": "fixed", - "gitea_issue": 75, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-22T09:00:00Z", - "error_message": "User-reported (xce_scheduler -i -s /tmp/scheduler-replay-20260922-071101.replay, worker pc13v, 22 real slots), also reported to occur in normal (non-sim) mode: after worker pc13v rejoined mid-replay (session~162, multiple WORKERS_PUSH+JOB_PUSH bursts, several `JOBS_PUSH: refusing resync state 0x1 for jobid ... existing job is RUN with a real allocation on record` WARNs -- bug-1812's guard correctly refusing a stale PEND resync), every one of that worker's jobs eventually legitimately finished (`info jobs` at session=423 shows all of jobids [0-22] DONE, [23-59] PEND, zero RUN), yet `info worker pc13v` still showed `resource slots: value=22.00 used=22.00` for the rest of the run (session 162 through 423, ~18 simulated minutes) -- the worker was permanently starved even though nothing was actually running on it.", + "last_seen": "2026-09-22T09:00:00Z", + "occurrences": 1, + "error_message": "User-reported (xce_scheduler -i -s /tmp/scheduler-replay-20260922-071101.replay, worker pc13v, 22…\n\nUser-reported (xce_scheduler -i -s /tmp/scheduler-replay-20260922-071101.replay, worker pc13v, 22 real slots), also reported to occur in normal (non-sim) mode: after worker pc13v rejoined mid-replay (session~162, multiple WORKERS_PUSH+JOB_PUSH bursts, several `JOBS_PUSH: refusing resync state 0x1 for jobid ... existing job is RUN with a real allocation on record` WARNs -- bug-1812's guard correctly refusing a stale PEND resync), every one of that worker's jobs eventually legitimately finished (`info jobs` at session=423 shows all of jobids [0-22] DONE, [23-59] PEND, zero RUN), yet `info worker pc13v` still showed `resource slots: value=22.00 used=22.00` for the rest of the run (session 162 through 423, ~18 simulated minutes) -- the worker was permanently starved even though nothing was actually running on it.", "file": "src/controller/resync.c (resync_resolve_round() -- the fix), src/controller/persist.c/src/include/persist.h (new persist_notify_worker_status_updated(), mirroring persist_notify_job_updated_already_persisted()/bug-1679), src/controller/testsuite/test_worker_resync.c (new regression test)", "root_cause": "The worker-rejoin-resync feature's resync_arm_round() (resync.c) sets XCE_WORKER_RECONCILING on the controller's local worker_t.host.status and (via the HOOK_DETAIL_WORKER_JOIN hook already firing on rejoin) pushes it to the scheduler over WORKERS_PUSH. On the scheduler side, sched_reset_worker_()/sched_update_account() (bug-1799/bug-1801, 2026-09-11) deliberately FREEZE that worker's resInUse -- skip both the per-round zero-reset and the recharge -- while IS_WORKER_RECONCILING is set, by design, so in-flight reconciliation bookkeeping isn't clobbered mid-flight; bug-1799's own fix comment documents that normal reset+resum 'resumes and self-corrects on the very next round' once the status bit clears. But resync_resolve_round() (called from both the report-completion path, resync_apply_report -> resync_maybe_resolve, and the Req-5.3 timeout path, resync_on_timeout) only cleared XCE_WORKER_RECONCILING on the CONTROLLER's own local worker_t -- it never fired HOOK_DETAIL_WORKER_UPDATE or otherwise pushed the corrected status to the scheduler (contrast mark_worker_back()/worker.c:878-888, which does exactly that for the sibling XCE_WORKER_UNAVAIL bit). core_upsert_worker() (scheduler/src/core_events.c) only ever learns a worker's status from whatever WORKERS_PUSH it last received (`worker->status = w->status`, a bare unconditional copy, no other refresh path) -- so the scheduler's copy of the worker's status kept XCE_WORKER_RECONCILING set forever after the controller-side round resolved, permanently freezing resInUse at whatever it was when reconciliation started, even though every job on that worker had long since finished DONE. Reproduces identically in both sim/replay and live/normal mode since both drive the identical xce_core_begin_session() -> sched_reset_worker_()/sched_update_account() freeze logic; sim just replays whatever WORKERS_PUSH events the live controller actually sent, and the live controller never sent the correcting one either. test_worker_resync.c thoroughly tested that RECONCILING clears on the controller's OWN worker_t after resync_resolve_round(), but nothing tested that the scheduler-facing notification actually fires -- a gap noted separately during controller-peers-env-fallback design research (Sep 15 observation: 'full resync path lacks test coverage').", "fix": "FIXED (2026-09-22). Added persist_notify_worker_status_updated(cluster_t* c, worker_t* w) (persist.c, declared in persist.h next to persist_worker_hook_fun) -- the worker-side sibling of persist_notify_job_updated_already_persisted() (bug-1679): fires XCE_CONTROLLER_HOOK_WORKER's HOOK_DETAIL_WORKER_UPDATE AFTER_EVENT under the existing t_apply_seam_notifying guard so persist_worker_hook_fun (also subscribed to this hook, unconditionally WAL-appends on AFTER_EVENT) does NOT durably persist a WAL_WORKER_CTRL fact for this fire -- correct, since XCE_WORKER_RECONCILING/resync_state_t is intentionally ephemeral, re-derived fresh per rejoin, never replayed from WAL. resync_resolve_round() (resync.c) now calls this immediately after clearing XCE_WORKER_RECONCILING and detaching w->resync, guarded so it only fires when a round was genuinely outstanding (never on the already-documented idempotent no-op case). Verified safe under all three call sites' shared precondition (caller already holds w's write lock): confirmed neither HOOK_DETAIL_WORKER_UPDATE subscriber (sched_worker_change/apiserver_worker_change) nor persist_worker_hook_fun (guarded out entirely by the flag before touching w) locks w itself, so no deadlock risk. TDD: added test_resync_resolve_round_notifies_worker_update (test_worker_resync.c) -- a HOOK_DETAIL_WORKER_UPDATE spy hook proves resync_resolve_round() fires it exactly once per genuine resolve and not at all for the idempotent already-resolved no-op. RED confirmed first (Expected 1 Was 0) against the unfixed code, then GREEN after the fix. Full controller_unit_test suite: 1220 Tests, 0 Failures, 0 Ignored, OK (debug profile) -- no regressions. Note: the user's original .replay file is a frozen recording of the pre-fix controller's actual live traffic and will still show the frozen state when replayed, since sim mode only replays already-recorded events; the fix takes effect for new live runs/recordings going forward, not retroactively on old captures.", @@ -27474,8 +14928,7 @@ "resInUse", "scheduler", "controller", - "hook-notification", - "fixed" + "hook-notification" ], "related_bugs": [ "bug-1679", @@ -27485,17 +14938,22 @@ "bug-1812", "bug-1751" ], - "occurrences": 1, - "last_seen": "2026-09-22T09:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1841.md", + "gitea_issue": 75, + "gitea_synced_status": "fixed" }, { "id": "bug-1842", - "gitea_synced_status": "fixed", - "gitea_issue": 77, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-22T19:35:00Z", - "error_message": "User-reported: replaying a real captured .replay file (scheduler-replay-20260922-095207.replay) via --sim --sim-verify logged a MISMATCH for every single DISPATCH decision, even though the printed 'expected:'/'actual:' summaries (jobid, nhosts) were byte-for-byte identical on both sides -- e.g. 'expected: DISPATCH jobid=0-13-1790071174-0[44] nhosts=1 -- actual: DISPATCH jobid=0-13-1790071174-0[44] nhosts=1' still reported as MISMATCH. User asked whether the comparison should stop using a timestamp as part of its check condition.", + "last_seen": "2026-09-22T19:35:00Z", + "occurrences": 1, + "error_message": "User-reported: replaying a real captured .replay file (scheduler-replay-20260922-095207.replay)…\n\nUser-reported: replaying a real captured .replay file (scheduler-replay-20260922-095207.replay) via --sim --sim-verify logged a MISMATCH for every single DISPATCH decision, even though the printed 'expected:'/'actual:' summaries (jobid, nhosts) were byte-for-byte identical on both sides -- e.g. 'expected: DISPATCH jobid=0-13-1790071174-0[44] nhosts=1 -- actual: DISPATCH jobid=0-13-1790071174-0[44] nhosts=1' still reported as MISMATCH. User asked whether the comparison should stop using a timestamp as part of its check condition.", "file": "src/scheduler/src/sim_verify.c (sim_verify_alloc_eq, the fix), src/lib/xcealloc.c (make_alloc, the field's origin), src/scheduler/testsuite/test_sim_verify.c (new regression test), src/scheduler/testsuite/test_sim_live_allocation.c (stale-comment cleanup)", "root_cause": "sim_decision_eq()/sim_log_mismatch() (sim_verify.c) never compare the decision's own `ts` field (only log it for diagnostics) -- the user's timestamp hypothesis was right in spirit but aimed at the wrong field. The REAL culprit is buried one level deeper: sim_verify_alloc_eq() compares xce_alloc_t field-by-field including `from`, and make_alloc() (xcealloc.c:30) unconditionally stamps `alloc->from = time(NULL)` -- the real OS wall clock, not core.c's virtualized sim clock -- every time an allocation is built. `expected->from` (decoded from the .replay file) is whatever real second the ORIGINAL recording ran in; `actual->from` (freshly rebuilt by the live seqjob_allocator.c/parjob_allocator.c -> sched_make_allocs() -> make_alloc() chain during THIS replay) is whatever real second THIS replay runs in. Two independent real-world timestamps that essentially never coincide for any replay of a real recording, even though the decision itself (job/host/resource/value/duration) is exactly what was recorded. Confirmed empirically: a clean run of xce_scheduler --sim --sim-verify against the user's own replay file mismatched 120/120 DISPATCH decisions, 100% on this field, zero relation to real-time-vs-batch simulation speed (reproduced identically at --sim-speed=0, 1, and 20). The codebase already had a documented workaround for the same footgun elsewhere: dup_alloc() (xcealloc.c:179) has to re-stamp ->from/->duration from its source AFTER calling make_alloc(), specifically because make_alloc() clobbers them with the current wall time otherwise. A prior session (2026-09-21, see this same discovery independently reached and left unfixed) had already isolated this exact mechanism via test_sim_verify_no_false_mismatch_after_live_dispatch_and_later_confirmation's own header comment, which explicitly (and accurately, until this fix) documented it as accepted 'practically negligible' same-second-window flakiness.", "fix": "FIXED (2026-09-22). Dropped the `a->from == b->from` clause from sim_verify_alloc_eq() (sim_verify.c) with a doc comment explaining why: `from` is reservation-start bookkeeping (real wall-clock, for expiry/accounting purposes), not part of a DISPATCH decision's semantic identity -- unlike `duration`, which is a genuine request-derived value and stays part of the comparison. TDD: new test test_dispatch_decisions_differing_from_timestamp_still_match (test_sim_verify.c) builds two structurally-identical dispatch allocations with deliberately far-apart `from` values (1790000000 vs 1) and asserts sim_decision_eq() still reports a match. Also updated test_sim_live_allocation.c's test_sim_verify_no_false_mismatch_after_live_dispatch_and_later_confirmation header comment, which previously described this as accepted same-second-window flakiness -- now accurately states the flakiness is eliminated by construction. Full scheduler suite: 85/85 Ok, zero regressions (xce-build.sh scheduler asan --incremental). Live-verified against the user's own real replay file: DISPATCH-vs-DISPATCH false mismatches dropped from 120/120 to 0/120 (--sim-speed=0, --sim-verify, --logfile capture, before/after diff). A separate, distinct issue remains in the same file (61 'expected: -- actual: PEND ... main_reason=0' mismatches) -- see bug-1843, not touched by this fix.", @@ -27511,7 +14969,6 @@ "decision-verification", "false-positive", "user-reported", - "fixed", "tdd" ], "related_bugs": [ @@ -27521,17 +14978,22 @@ "bug-1759", "bug-1838" ], - "occurrences": 1, - "last_seen": "2026-09-22T19:35:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1842.md", + "gitea_issue": 77, + "gitea_synced_status": "fixed" }, { "id": "bug-1843", - "gitea_synced_status": "fixed", - "gitea_issue": 78, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-22T19:35:00Z", - "error_message": "Same user report as bug-1842: replaying scheduler-replay-20260922-095207.replay via --sim --sim-verify also logs a small number of 'expected: -- actual: PEND jobid=... main_reason=0' mismatches -- live decisions with no recorded counterpart at all (beyond decoded_session->n_decisions), distinct from bug-1842's DISPATCH-vs-DISPATCH false positives (already fixed). 61 occurrences total across the full 643-session replay file (0.02% of the 3431 live PEND decisions produced).", + "last_seen": "2026-09-23T08:00:00Z", + "occurrences": 1, + "error_message": "Same user report as bug-1842: replaying scheduler-replay-20260922-095207.replay via --sim…\n\nSame user report as bug-1842: replaying scheduler-replay-20260922-095207.replay via --sim --sim-verify also logs a small number of 'expected: -- actual: PEND jobid=... main_reason=0' mismatches -- live decisions with no recorded counterpart at all (beyond decoded_session->n_decisions), distinct from bug-1842's DISPATCH-vs-DISPATCH false positives (already fixed). 61 occurrences total across the full 643-session replay file (0.02% of the 3431 live PEND decisions produced).", "file": "src/scheduler/src/sim_loop.c (sched_sim_driver_run()'s decision-drain loop, sim_reconcile_unaccounted_jobs()), src/scheduler/src/sched_election.c (sched_job_picker()/sched_allocator())", "root_cause": "CONFIRMED (2026-09-23), and the leading hypothesis from the prior session's investigation (event-batching-granularity in sim_loop.c's replay driver -- 'a job's first appearance lands one round earlier during replay than the session it was recorded in') is REFUTED by direct measurement, not just re-reasoned about: added temporary per-round diagnostic logging to sched_sim_driver_run() (sim_loop.c, reverted after use -- not part of the fix) and replayed the exact scheduler-replay-20260922-095207.replay capture this bug was originally filed against with --sim --sim-verify --sim-speed=0. For every one of the 61 mismatches, the LIVE (replay) side deterministically re-offers the full, correct set of already-known PEND candidates every round (e.g. round begin_round=...789016869984 produced 38 PEND decisions for jobs [22]..[59], with ZERO new events pushed into that round -- core state was provably identical to the immediately preceding round, which also produced 38). The RECORDED side, for that exact same round, has only 32 decisions (jobs [22]..[53]) -- missing exactly the newest-added cohort, jobs [54]..[59] -- while the immediately preceding and following recorded rounds both have all 38. This is the true shape of the bug: an isolated, self-healing gap in the ORIGINAL LIVE RECORDING, not a replay-side batching/discretization artifact (sim_loop.c's event-buffer-then-push-before-begin_session mechanism was verified, by direct code reading of both sim_loop.c's replay-side push loop and driver.c's live-side xce_sched_driver_translate_pending_events()/sched_record_write_events() call order, to be a byte-for-byte-faithful reproduction of the recorder's own file ordering -- there is no live-vs-replay discretization gap to close there). True root cause: sched_record_session_add_decision() (recording_session.c) grows its per-round ctx->decisions array via realloc(), doubling from an initial capacity of 8 (8 -> 16 -> 32 -> 64...). If that realloc() call fails, the existing code silently drops the CURRENT decision (free_record_decision(rd); return;) and leaves ctx->n/ctx->cap exactly where they were -- every SUBSEQUENT decision that round hits the same n==cap growth check and is ALSO silently dropped, so the round effectively truncates at whichever doubling boundary (8/16/32/64) the failing realloc() landed on (n=32 here -- exactly one of those boundaries, and exactly matching the empirically observed truncation point). Nothing marks the round as incomplete: sched_record_session_finalize() (recording_session.c) then serializes and writes ctx's now-short decisions array as an ordinary, apparently-complete ReplaySession -- indistinguishable on replay from a genuinely smaller live round. This is the ONE inconsistent failure-handling path in this file: every OTHER failure in the same recording subsystem (write_capnp_message() failures, XCE_DECISION_PREEMPT/NONE cases) is either propagated as a hard error (sched_record_abort_on_error()) or logged loudly by design -- only this one growth failure was silent-and-partial. The live SCHEDULING round itself was never affected either way (on_decision() still fires for every decision drained from core.decisions regardless of recording state) -- only the .replay FILE's own record of that one round silently lost its tail, which --sim-verify's replay comparison then had no way to distinguish from a genuinely smaller live round.", "fix": "FIXED (2026-09-23, TDD). recording_session.c: added a `bool truncated` field to sched_record_session_ctx_s; sched_record_session_add_decision()'s realloc-failure branch now sets ctx->truncated = true (in addition to its existing ERROR log) before dropping the decision. sched_record_session_finalize() now checks ctx->truncated FIRST and, if set, frees everything and returns -ENOMEM WITHOUT encoding/writing anything -- routing through the exact same sched_record_abort_on_error() cleanup path (via sched_record_session_end(), driver.c) that every other recording failure in this subsystem already uses, instead of silently writing a truncated-looking-complete ReplaySession. sim_loop.c's own --sim-output path (sched_record_session_finalize() called directly) already treated any negative return as fatal, so it is covered for free with zero changes there. recording.h's doc comments for sched_record_session_add_decision()/sched_record_session_end()/sched_record_session_finalize() updated to describe the new contract. TDD: added test_decision_batch_growth_failure_drops_incomplete_session() (test_record_lifecycle.c) -- this codebase had no existing malloc/realloc fault-injection harness, so added `-Wl,--wrap=realloc` linker interposition scoped to ONLY this test executable's meson target (same established `-Wl,--wrap=` idiom already used elsewhere in this test suite for xcecli_*/sched_meet_host_selection, meson.build), with a `__wrap_realloc()` that passes through to `__real_realloc()` unless the one new test arms it. The test forces a REAL, deterministic realloc() failure on the 9th decision's array growth (8 -> 16) and asserts sched_record_session_end() returns an error and recording is cleanly disabled (matching test_write_failure_disables_recording_and_driver_keeps_running()'s existing assertion style). Verified genuine red->green: temporarily reverted just the production fix and reran -- scheduler record lifecycle test FAILED (exit 1) without the fix, confirming the new test actually exercises the bug; restored the fix and reran -- full 85/85 scheduler suite green. NOTE: this fix prevents FUTURE recordings from losing decisions this way -- it cannot retroactively repair the already-captured scheduler-replay-20260922-095207.replay file, which will continue to show the same 61 --sim-verify mismatches when replayed (now fully explained, not a live bug) unless/until it is re-captured.", @@ -27544,7 +15006,6 @@ "recording_session.c", "realloc-failure", "silent-data-loss", - "fixed", "tdd", "wrap-realloc", "fault-injection" @@ -27556,17 +15017,22 @@ "bug-1779", "bug-1759" ], - "occurrences": 1, - "last_seen": "2026-09-23T08:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1843.md", + "gitea_issue": 78, + "gitea_synced_status": "fixed" }, { "id": "bug-1844", - "gitea_synced_status": "fixed", - "gitea_issue": 81, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-23T09:15:00Z", - "error_message": "User-reported follow-up to bug-1843: replaying a fresh capture (scheduler-replay-20260923-085356.replay, built after bug-1843's fix) with --sim --sim-verify still shows a huge volume of MISMATCH lines (4881 in one run) on DISPATCH decisions whose printed description is IDENTICAL on both sides -- e.g. 'expected: DISPATCH jobid=...[40] nhosts=1 -- actual: DISPATCH jobid=...[40] nhosts=1' still flagged as a mismatch.", + "last_seen": "2026-09-23T09:15:00Z", + "occurrences": 1, + "error_message": "User-reported follow-up to bug-1843: replaying a fresh capture…\n\nUser-reported follow-up to bug-1843: replaying a fresh capture (scheduler-replay-20260923-085356.replay, built after bug-1843's fix) with --sim --sim-verify still shows a huge volume of MISMATCH lines (4881 in one run) on DISPATCH decisions whose printed description is IDENTICAL on both sides -- e.g. 'expected: DISPATCH jobid=...[40] nhosts=1 -- actual: DISPATCH jobid=...[40] nhosts=1' still flagged as a mismatch.", "file": "src/scheduler/src/sim_verify.c (sim_verify_alloc_eq())", "root_cause": "CONFIRMED via temporary field-by-field diagnostic instrumentation in sim_verify_alloc_eq() (added, verified, then reverted -- not part of the fix) replayed against the real captured .replay file: `instid` was the ONLY field ever differing (value/duration/resname/reserve matched exactly every single time, e.g. recorded instid=826 vs replay instid=1, both otherwise byte-identical). `instid` is stamped from `sched->core.nextid++` (scheduler.c's sched_get_id()) at allocation-build time -- a bare per-process monotonic counter tracking how many allocation instances THIS core has built over its own entire execution history, not a semantic property of what got allocated. The ORIGINAL recording's counter (running, and building allocations, for potentially far longer before this capture window started) and a fresh replay's own counter (starting from whatever its loaded snapshot's nextid happens to be) have no reason to reach the same value for 'the same' dispatch. Exact same class of bug as bug-1838-followup/bug-1842's `from` field (also non-deterministic bookkeeping, already excluded) -- this comparator simply had a second field of the same kind left in.", "fix": "FIXED (2026-09-23, TDD). Dropped the `a->instid == b->instid` clause from sim_verify_alloc_eq() (sim_verify.c), same treatment as bug-1842's `from` exclusion. Updated that function's doc comment to document the exclusion and cite the real captured-file evidence (826 vs 1). Added test_dispatch_decisions_differing_instid_still_match() (test_sim_verify.c), mirroring the existing test_dispatch_decisions_differing_from_timestamp_still_match() pattern -- two allocations identical in every field except instid (826 vs 1, matching the real observed divergence) must still compare equal. Verified genuine red->green: temporarily re-added the instid comparison and reran -- scheduler sim verify test FAILED (exit 1); restored the fix and reran -- full 85/85 scheduler suite green. NOTE: this fix does NOT address the second, separate anomaly found in the same replay file during this investigation (see bug-1845): a genuine one-round-early DISPATCH timing difference for one job (a resource-accounting/capacity divergence between live and replay, not a field-comparison issue) -- open, tracked separately.", @@ -27579,7 +15045,6 @@ "nextid", "monotonic-counter", "non-deterministic-field", - "fixed", "tdd" ], "related_bugs": [ @@ -27588,17 +15053,22 @@ "bug-1843", "bug-1845" ], - "occurrences": 1, - "last_seen": "2026-09-23T09:15:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1844.md", + "gitea_issue": 81, + "gitea_synced_status": "fixed" }, { "id": "bug-1845", - "gitea_synced_status": "fixed", - "gitea_issue": 83, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-23T10:00:00Z", - "error_message": "Follow-up to bug-1843/bug-1844: replaying a fresh capture (scheduler-replay-20260923-085356.replay) with --sim --sim-verify still showed a large volume of MISMATCH lines on PEND decisions where recorded and replay agreed on jobid/main_reason but the deep host/resource reason-detail comparison disagreed (e.g. recorded {main_reason=0, nhost_reasons=0, nres_reasons=0} vs replay {main_reason=0, nhost_reasons=0, nres_reasons=2}), live-inspected via gdb by the user.", + "last_seen": "2026-09-23T10:00:00Z", + "occurrences": 1, + "error_message": "Follow-up to bug-1843/bug-1844: replaying a fresh capture…\n\nFollow-up to bug-1843/bug-1844: replaying a fresh capture (scheduler-replay-20260923-085356.replay) with --sim --sim-verify still showed a large volume of MISMATCH lines on PEND decisions where recorded and replay agreed on jobid/main_reason but the deep host/resource reason-detail comparison disagreed (e.g. recorded {main_reason=0, nhost_reasons=0, nres_reasons=0} vs replay {main_reason=0, nhost_reasons=0, nres_reasons=2}), live-inspected via gdb by the user.", "file": "src/scheduler/src/sim_verify.c (sim_verify_job_preason_eq())", "root_cause": "CONFIRMED (2026-09-23) as a documented, intentional architectural asymmetry, not a defect: core.h states outright that 'a simulation-mode session never touches the real cache implementation at all' (simulation isolation). core->unsatisfiable_lookup_cb/unsatisfiable_populate_cb (alloc_cache.c's xce_alloc_cache_lookup_cb/populate_cb) are registered only by main.c's normal-mode branch -- sim_loop.c never registers them. sched_make_primary_alloc_() (scheduler.c) consults that cache FIRST; on a hit it records reason detail via sched_pend_from_cached_reason_(), which sets ONLY main_reason (no host/resource detail at all -- skipping that expensive per-host computation is the entire point of the cache). Replay, having no cache, always falls through to the full per-host computation (sched_try_alloc_on_host_(), resource.c) and therefore always produces full detail. Two independent, architecturally-legitimate detail levels for the exact same semantic outcome (this job is PEND, for this main reason), not a decision correctness signal. Separately verified the cache mechanism itself is SOUND (not a distinct bug): its key (alloc_cache.c's build_key_(): resreq_fp + user + queue + project, no host/capacity dimension) is safe within a round because capacity only decreases monotonically as earlier jobs dispatch and the cache is cleared fresh every round (xce_alloc_cache_clear_cb() at begin_session) -- a cached 'unsatisfiable' verdict from an earlier job this round remains valid for a later job sharing the same key.", "fix": "FIXED (2026-09-23, TDD). sim_verify_job_preason_eq() (sim_verify.c) now compares ONLY main_reason, dropping the nhost_reasons/nres_reasons/host_based_reasons/res_based_reasons deep comparison entirely (that deep check was added as an optional 'cheap to add' enhancement, never proven necessary by a dedicated test, and is provably wrong given the cache-isolation asymmetry above). Doc comment updated to explain why. Added test_pend_decisions_differing_reason_detail_still_match() (test_sim_verify.c) -- two PEND decisions with the same jobid/main_reason but one with empty (cache-hit-shaped) and one with populated (full-computation-shaped) host/resource reason detail must still compare equal. Verified genuine red->green: temporarily re-added the deep comparison and reran -- scheduler sim verify test FAILED; restored the fix and reran -- full 85/85 scheduler suite green.", @@ -27611,7 +15081,6 @@ "unsatisfiable-cache", "alloc_cache", "simulation-isolation", - "fixed", "tdd" ], "related_bugs": [ @@ -27620,18 +15089,22 @@ "bug-1842", "bug-1846" ], - "occurrences": 1, - "last_seen": "2026-09-23T10:00:00Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1845.md", + "gitea_issue": 83, + "gitea_synced_status": "fixed" }, { "id": "bug-1846", - "gitea_synced_status": "fixed", - "gitea_issue": 84, "status": "fixed", "status_updated": "2026-09-24", "status_note": "77f047f2, a9feded8, ac033f47", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-23T10:05:00Z", - "error_message": "OPEN, NOT YET ROOT-CAUSED. Same replay session as bug-1844/1845 (scheduler-replay-20260923-085356.replay): after excluding instid (bug-1844) and reason-detail (bug-1845), --sim-verify still shows one job (jobid=0-8-1790154325-0[39] in the observed capture) genuinely DISPATCHING one round earlier in replay than in the original live recording -- not a false-positive field comparison, a real positional divergence: replay's round begin_round=...436582151823 dispatches a 13-job batch (jobs [27]..[39]) while the SAME round in the original recording only dispatched 12 (jobs [27]..[38]); job [39]'s recorded dispatch instead lands in the NEXT round, cascading into ~40+ downstream positional MISMATCH lines for that round pair alone.", + "last_seen": "2026-09-23T10:05:00Z", + "occurrences": 1, + "error_message": "Same replay session as bug-1844/1845 (scheduler-replay-20260923-085356.replay): after excluding…\n\nOPEN, NOT YET ROOT-CAUSED. Same replay session as bug-1844/1845 (scheduler-replay-20260923-085356.replay): after excluding instid (bug-1844) and reason-detail (bug-1845), --sim-verify still shows one job (jobid=0-8-1790154325-0[39] in the observed capture) genuinely DISPATCHING one round earlier in replay than in the original live recording -- not a false-positive field comparison, a real positional divergence: replay's round begin_round=...436582151823 dispatches a 13-job batch (jobs [27]..[39]) while the SAME round in the original recording only dispatched 12 (jobs [27]..[38]); job [39]'s recorded dispatch instead lands in the NEXT round, cascading into ~40+ downstream positional MISMATCH lines for that round pair alone.", "file": "src/scheduler/src/plugins/limit.c (match_job_host_limit_); src/scheduler/src/handler.c + main.c + scheduler.c (config push deferral)", "root_cause": "CONFIRMED (2026-09-23, via temporary per-decision diagnostics in the --sim-verify drain loop, since reverted) -- two distinct things. (1) ROOT-CAUSED + FIXED: match_job_host_limit_() (limit.c) runs on EVERY per-host allocation attempt (limit_prepare_allocation() via plugin_prepare_allocation()) and unconditionally xlist_store()d the matched vres into obj->hlimits[worker->idx]; obj->hlimits is only rebuilt by limit_job_election() on a configver bump. A job that kept pending on the same host accumulated one extra copy per attempt; sched_fill_limit_requests() staged every copy into ctx->alloc_req.host and seqjob's make_host_alloc() built a host alloc carrying all of them. In the captured replay the LIVE recording itself shows it (pc13v host allocs with mem+slots+70/79/94/112/136/175 identical limit=limits2/u=rshen/q=xce/p= entries, resetting only on a config push) -- i.e. a PRODUCTION bug, not sim-only. make_job_account() makes one res_account per entry and sched_update_res_accounts() charges each, so the host limit was charged (and later released) N-fold. Replay had the same accumulation with different counts, which is what --sim-verify flagged. (2) ROOT-CAUSED (2026-09-23): the 9 live rounds cut off after 11-12 decisions were each ended by a mid-round scheduler REJOIN. sched_publish_retry_loop() (scheduler.c) calls sched_check_pending_rejoin() after every successful publish -- i.e. inside the driver's decision-drain loop -- and after a client-library reconnect that issues xcecli_join(). The controller answers every join (on_scheduler_join(), controller/sched.c) with xcesched_push_configs() + xcesched_push_workers() + xcesched_push_jobs(). handler.c applied the config push synchronously on the handler thread (sched_init_config() -> sched_init_queues() -> builtin_fcfs on_queue_update()), which nulls priv->cursor when it reinserts the queue the cursor rests on; next_job() then returns NULL and the round ends. Evidence from the capture (temporary event-stream diagnostics, reverted): every truncated round is followed by a workersPush + full 50-job jobsPush (ntotal=50, pos=0, the join's resync -- e.g. jobs just dispatched reported back as PEND), every full-length dispatching round has none, and the config push itself never appears because it bypassed the pending-update queue the recorder writes from. Rejoins came roughly every ~100s (the configver bump also explains the periodic reset of part (1)'s duplicate counts). The underlying connection churn is a separate issue (see bug-1812/1813's pc13v reconnect/resync storm).", "fix": "PARTIAL (2026-09-23, TDD). limit.c: new hlimits_contains_() pointer-identity check (vres is interned per name in privdata->instances) -- match_job_host_limit_() now stores a matched vres into obj->hlimits[worker] only once. New regression test test_limit_host_dedup.c (sched_limit_host_dedup_test, meson suite 'scheduler'): 5 plugin_prepare_allocation() attempts for one job on one worker must leave hlimits[worker]->count==1 and exactly one staged copy of the vres in alloc_req.host -- verified RED before the fix (Expected 1 Was 2 on attempt 2) and GREEN after. Replay re-run after the fix: replay-side host allocs are now always 3 entries; remaining DISPATCH alloc mismatches are the recording's own inflated entries (recorded with the buggy binary) and will disappear only with a fresh capture. PART (2) FIXED (2026-09-23, TDD, user-chosen design 'defer to round edge'): handler.c's configpushmsg case now calls new sched_stash_config() (parks the Conf4Sched in new xce_scheduler_t.pending_config under pending_config_lock, freeing any older unapplied push) instead of sched_init_config(); main.c's loop calls new sched_apply_pending_config() before each xce_sched_driver_run_normal_mode_round(), which runs the unchanged sched_init_config() strictly between sessions; sched_fini() frees a never-applied push. Same deferral direction as bug-1813's array-job fix. Tests: test_handlers.c's test_config_push_is_stashed_not_applied_on_handler_thread (RED: sched_init_config called 1x on the handler thread; GREEN after) and new test_config_push_deferred.c (sched_config_push_deferred_test: real scheduler_req_handler() + real builtin_fcfs mid-session -- RED: next_job() returned NULL right after the config push, job B never considered; GREEN: session walks B and C, configver unchanged until sched_apply_pending_config() applies it once). Known remaining gap (pre-existing, not addressed): a config push is still not written to a recording, so a replay spanning a config CHANGE won't see it.", @@ -27643,9 +15116,7 @@ "resource-accounting", "resInUse", "driver-equivalence", - "unfixed", "needs-diagnosis", - "open", "limit", "hlimits", "host-limit", @@ -27659,17 +15130,26 @@ "bug-1611", "bug-1613" ], - "occurrences": 1, - "last_seen": "2026-09-23T10:05:00Z" + "fix_commits": [ + "77f047f2", + "a9feded8", + "ac033f47" + ], + "record": ".wolf/bugs/bug-1846.md", + "gitea_issue": 84, + "gitea_synced_status": "fixed" }, { "id": "bug-1847", - "gitea_synced_status": "fixed", - "gitea_issue": 290, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-24T05:30:00Z", - "error_message": "bug-1846 follow-up: a scheduler config push (sent by the controller on every scheduler join) was never written to a .replay recording, so a replay spanning a rejoin or a genuine config change never saw it.", + "last_seen": "2026-09-24T05:30:00Z", + "occurrences": 1, + "error_message": "bug-1846 follow-up: a scheduler config push (sent by the controller on every scheduler join) was…\n\nbug-1846 follow-up: a scheduler config push (sent by the controller on every scheduler join) was never written to a .replay recording, so a replay spanning a rejoin or a genuine config change never saw it.", "file": "src/scheduler/src/scheduler.c (sched_apply_pending_config)", "root_cause": "handler.c applied config pushes synchronously via sched_init_config(), bypassing the pending-update queue that xce_sched_driver_translate_pending_events() records from. bug-1846's deferral moved application to a round boundary on the driver thread (main.c -> sched_apply_pending_config()), making it recordable at the right point.", "fix": "FIXED (2026-09-24, TDD). sched_apply_pending_config() now writes a CONFIG_PUSH core event via the existing sched_record_write_events() after sched_init_config() succeeds (no-op when not recording), ahead of the round's other recorded events -- matching the order they take effect. Test: test_config_push_deferred.c's test_applied_config_push_is_recorded (RED: record_event_count stayed 0; GREEN: 1 event written, stashing alone writes nothing). Known fidelity limit: replay applies configPush through core_events.c's CONFIG_PUSH path (queues/resources/resvals/configver), which does not re-run groupmgr_load_configs()/plugin_config_update()/sched_init_sharedInst() like sched_init_config() does -- a replayed config CHANGE to limits/groups still won't take effect in sim (pre-existing driver-equivalence gap, not introduced here).", @@ -27679,16 +15159,24 @@ "replay", "config-push", "driver-equivalence" - ] + ], + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1847.md", + "gitea_issue": 290, + "gitea_synced_status": "fixed" }, { "id": "bug-1848", - "gitea_synced_status": "fixed", - "gitea_issue": 291, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "leak", "timestamp": "2026-09-24T05:30:00Z", - "error_message": "Leak: xce_sched_obj_t.hlimits was created with a NULL zarray item killer, so reset_sched_obj() (limit_job_election()'s config-version rebuild) and schedobj teardown leaked the per-worker xlist_t that match_job_host_limit_() callocs, one per worker a job had tried. Visible as part of the large LeakSanitizer report on every --sim replay run.", + "last_seen": "2026-09-24T05:30:00Z", + "occurrences": 1, + "error_message": "Leak: xce_sched_obj_t.hlimits was created with a NULL zarray item killer, so reset_sched_obj()…\n\nLeak: xce_sched_obj_t.hlimits was created with a NULL zarray item killer, so reset_sched_obj() (limit_job_election()'s config-version rebuild) and schedobj teardown leaked the per-worker xlist_t that match_job_host_limit_() callocs, one per worker a job had tried. Visible as part of the large LeakSanitizer report on every --sim replay run.", "file": "src/scheduler/src/schedobj.c, core_events.c, snapshot_load.c (hlimits zarray_init)", "root_cause": "zarray_fini() only frees items through the zarray's killer; all three schedobj constructors passed NULL.", "fix": "FIXED (2026-09-24, TDD). New kill_hlimits_list() (schedobj.c, declared in sched/types.h) frees one per-worker list (xlist_fini + free; never its vres entries, owned by the limit plugin's instance table); installed at all three zarray_init(&(obj->hlimits), ...) sites -- schedobj.c uses kill_hlimits_list(); core_events.c and snapshot_load.c get identical static local copies (core_hlimits_list_killer / snapshot_hlimits_list_killer) because both deliberately don't share helpers with schedobj.c and several test binaries (sched_core_events_test, sched_core_full_cycle_test, sched_snapshot_save_load_test, sched_sim_startup_test) link them without schedobj.c -- a shared symbol broke those links. zarray_fini() preserves the killer, so reset_sched_obj() leaves the zarray reusable. Test: test_limit_host_dedup.c's test_hlimits_zarray_releases_its_lists (RED: killer NULL; GREEN). A direct __lsan_do_recoverable_leak_check() test was tried and rejected -- the fixture's own resreq parsing (parse_select strdup) leaks independently, so the baseline is never leak-free.", @@ -27697,16 +15185,24 @@ "limit", "hlimits", "memory-leak" - ] + ], + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1848.md", + "gitea_issue": 291, + "gitea_synced_status": "fixed" }, { "id": "bug-1849", - "gitea_synced_status": "fixed", - "gitea_issue": 292, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-24T05:30:00Z", - "error_message": "Every --sim run logged 'WARN xce_sched_core_load_snapshot failed to resolve job picker plugin (rc=-2); picker rebuild will be skipped for this load' -- misleading; flagged as a suspect during the bug-1846 investigation.", + "last_seen": "2026-09-24T05:30:00Z", + "occurrences": 1, + "error_message": "Every --sim run logged 'WARN xce_sched_core_load_snapshot failed to resolve job picker plugin…\n\nEvery --sim run logged 'WARN xce_sched_core_load_snapshot failed to resolve job picker plugin (rc=-2); picker rebuild will be skipped for this load' -- misleading; flagged as a suspect during the bug-1846 investigation.", "file": "src/scheduler/src/snapshot_load.c (xce_sched_core_load_snapshot)", "root_cause": "Benign ordering, not a fault: the only caller (sim.c's sim_locate_and_load_snapshot()) loads the snapshot before any picker is registered; sim_bootstrap_picker() (sim_picker.c, bug-1567) then replays the full on_queue_update + on_job_push walk against the restored core (builtin_fcfs has no on_config, and on_job_push filters non-PEND jobs itself), so the picker ends up fully rebuilt.", "fix": "FIXED (2026-09-24). Downgraded to TRACE with a comment explaining the deferral; header comment updated. No test (log-level-only change).", @@ -27715,17 +15211,24 @@ "sim", "snapshot", "logging" - ] + ], + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1849.md", + "gitea_issue": 292, + "gitea_synced_status": "fixed" }, { "id": "bug-1850", - "gitea_synced_status": "fixed", - "gitea_issue": 274, "status": "fixed", "status_updated": "2026-09-25", "status_note": "6b1862f8; soaks 2026-09-25: 0 NOT_READY-triggered rotations, 1 live same-controller retry (worker, recovered in 1.2s), all PASS", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-24T05:30:00Z", - "error_message": "OPEN, NOT FIXED -- root trigger of bug-1846 part 2's mid-round rejoins: the live scheduler reconnected/re-joined roughly every ~100s in scheduler-replay-20260923-085356.replay (each rejoin = full config/worker/job re-push from on_scheduler_join()).", + "last_seen": "2026-09-24T05:30:00Z", + "occurrences": 1, + "error_message": "root trigger of bug-1846 part 2's mid-round rejoins: the live scheduler reconnected/re-joined…\n\nOPEN, NOT FIXED -- root trigger of bug-1846 part 2's mid-round rejoins: the live scheduler reconnected/re-joined roughly every ~100s in scheduler-replay-20260923-085356.replay (each rejoin = full config/worker/job re-push from on_scheduler_join()).", "file": "src/lib/client_failover.c (xcecli_retry_next / xce_failover_classify), src/lib/xceapi.c (10s reply deadline)", "root_cause": "UNCONFIRMED (code reading only; the live scheduler.log for the capture window isn't available here). The controller never closes a scheduler connection proactively (only handle_disconnect() on a socket error sets XCE_CTX_ERR). But xcecli_retry_next() calls xce_failover_rotate() -> xce_ctx_reconnect() on EVERY retry, and xce_failover_classify() treats ETIMEDOUT (any request whose reply takes >10s, cbuf_deadline_10s in xceapi.c), E_XCE_NOT_READY and E_XCE_REDIRECT/STANDBY as retryable. So a slow or 'not ready' reply to a dispatch/pend-reason publish from a HEALTHY primary rotates the scheduler to the next master (typically a standby, which redirects back), and each reconnect fires the rejoin callback -> sched_check_pending_rejoin() -> xcecli_join() -> full re-push, which adds load and can self-reinforce. Also: a reply arriving after its 10s deadline lands in the capacity-1 wbuf and can be consumed by the NEXT request (xcecli_ctx_recover() drains only one stale entry).", "fix": "NOT FIXED. Next step: grep the live scheduler.log for 'publish retry-budget exhausted', 'xcecli failover', 'pending rejoin succeeded' around each truncated round to identify the triggering rc. Any fix changes libxce HA client failover semantics (e.g. not rotating away from a primary on ETIMEDOUT/NOT_READY) -> spec workflow per CLAUDE.md, not a drive-by.", @@ -27736,16 +15239,26 @@ "reconnect", "ha", "scheduler" - ] + ], + "related_bugs": [], + "fix_commits": [ + "6b1862f8" + ], + "record": ".wolf/bugs/bug-1850.md", + "gitea_issue": 274, + "gitea_synced_status": "fixed" }, { "id": "bug-1851", - "gitea_synced_status": "fixed", - "gitea_issue": 293, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "controller", + "severity": "crash", "timestamp": "2026-09-24T07:40:00Z", - "error_message": "Live sequential HA soak (run-ha-soak.sh --sequential 6, round 5): newly-promoted primary ctrl-3 (term 10) died ~8s after promotion with ASAN heap-use-after-free READ in strdup <- collect_resval (dataex.c:71) <- apiserver_worker_change (apiserver.c:450) <- apiserver_worker_hook_fun <- xce_run_hooks <- persist_notify_worker_status_updated (persist.c:7997) <- resync_resolve_round (resync.c:418) <- resync_arm_round (resync.c:315) <- apply_join_state (worker.c:351) <- rejoin_worker (worker.c:444) <- handle_worker_join. Freed by the same thread in rejoin_worker (worker.c:434, xstring_dereference of the old hostmodel). Soak surfaced it as 'FAIL: round 5: no new primary elected after kill (no serving primary at term>9)' -- the new primary crashed rather than failing to elect.", + "last_seen": "2026-09-24T07:40:00Z", + "occurrences": 1, + "error_message": "Live sequential HA soak (run-ha-soak.sh --sequential 6, round 5): newly-promoted primary ctrl-3…\n\nLive sequential HA soak (run-ha-soak.sh --sequential 6, round 5): newly-promoted primary ctrl-3 (term 10) died ~8s after promotion with ASAN heap-use-after-free READ in strdup <- collect_resval (dataex.c:71) <- apiserver_worker_change (apiserver.c:450) <- apiserver_worker_hook_fun <- xce_run_hooks <- persist_notify_worker_status_updated (persist.c:7997) <- resync_resolve_round (resync.c:418) <- resync_arm_round (resync.c:315) <- apply_join_state (worker.c:351) <- rejoin_worker (worker.c:444) <- handle_worker_join. Freed by the same thread in rejoin_worker (worker.c:434, xstring_dereference of the old hostmodel). Soak surfaced it as 'FAIL: round 5: no new primary elected after kill (no serving primary at term>9)' -- the new primary crashed rather than failing to elect.", "file": "src/controller/worker.c", "root_cause": "update_val_res() (worker.c:~99) memcpy()s the xce_val_t into w->host.resvals, so the 'type'/'model' string resources seeded by add_hostattrs_resvals() ALIAS the interned xstrings owned by w->host.attrs->hosttype/hostmodel (no reference taken). rejoin_worker() dereferences (frees, at refcount 0 -- even when the new value is the same string, since it dereferences before makeString) the old hosttype/hostmodel, then apply_join_state() calls resync_arm_round() BEFORE add_hostattrs_resvals() re-seeds resvals. When the resync round resolves synchronously inside arm (resync_resolve_round -> persist_notify_worker_status_updated -> worker hooks), apiserver_worker_change -> collect_resval strdup()s the dangling svalue. Latent since the alias was introduced; made reachable by worker-rejoin-resync's synchronous resolve+notify path on rejoin (bug-1841 area).", "fix": "FIXED (2026-09-24, TDD). update_val_res() (worker.c) now stores a private strdup() for Value_u_svalue (copy taken before freeing the previous one, so val may alias v) and frees an old string when a slot changes type; the owning arrays now free their entries: w->host.resvals gets kill_resval (lib/wobj.c, replacing the FIXME NULL killer) and c->sharedinst's resval_obj_killer (cluster.c, previously a no-op) frees svalue + the xce_val_t -- both previously also leaked every xce_val_t. Also closes the same alias hazard for worker-reported string resvals (update_resval_list aliased the decoded request) and config values (mark_host_res). Regression test test_worker_join.c::test_rejoin_worker_string_resvals_survive_outstanding_round: first join arms a round (non-terminal job on host), second rejoin_worker with new attrs resolves it and fires a registered XCE_CONTROLLER_HOOK_WORKER hook that reads the type/model resvals -- RED reproduced the live stack exactly (heap-use-after-free, freed by rejoin_worker worker.c:432 xstring_dereference); GREEN after. Full meson test 109/109 OK. Note: get_resval() still hands out a borrowed svalue pointer (opget.c), now into the resval's own copy -- same lifetime contract as before, not widened.", @@ -27757,16 +15270,24 @@ "ha-soak", "resvals", "xstring" - ] + ], + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1851.md", + "gitea_issue": 293, + "gitea_synced_status": "fixed" }, { "id": "bug-1852", - "gitea_synced_status": "fixed", - "gitea_issue": 294, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-24T08:10:00Z", - "error_message": "Scheduler replay-parity soak gate (run-ha-soak.sh --sequential 6, post bug-1851): 1 of 7 clean recordings failed --sim-verify with 3 MISMATCHes, all in its first decision round: live 'DISPATCH jobid=... nhosts=1' vs sim 'PEND ... main_reason=0' (sim: unsatisfy_ unmet resource 0 (slots) shortfall 1). Sim trace: snapshot at record start restored the worker with 0/0 resvals; the recording's only WORKERS_PUSH 'carries 10 resval(s)' but core_apply_worker_resvals 'applied 0' -- every recorded NameValue.val was NULL.", + "last_seen": "2026-09-24T08:10:00Z", + "occurrences": 1, + "error_message": "Scheduler replay-parity soak gate (run-ha-soak.sh --sequential 6, post bug-1851): 1 of 7 clean…\n\nScheduler replay-parity soak gate (run-ha-soak.sh --sequential 6, post bug-1851): 1 of 7 clean recordings failed --sim-verify with 3 MISMATCHes, all in its first decision round: live 'DISPATCH jobid=... nhosts=1' vs sim 'PEND ... main_reason=0' (sim: unsatisfy_ unmet resource 0 (slots) shortfall 1). Sim trace: snapshot at record start restored the worker with 0/0 resvals; the recording's only WORKERS_PUSH 'carries 10 resval(s)' but core_apply_worker_resvals 'applied 0' -- every recorded NameValue.val was NULL.", "file": "src/scheduler/src/driver.c", "root_cause": "The recorder encoded each event batch AFTER applying it: driver.c's xce_sched_driver_translate_pending_events() called xce_core_push_events() and only then sched_record_write_events() on the same xce_core_event_t payloads. Applying is destructive -- core_apply_worker_resvals() MOVE_PTRs each resval (nv->val -> worker->resvals), core_build_job MOVE_PTRs a JOBS_PUSH's allocs, JOB_RUSAGE transfers rusages, and core_apply_config_resvals() MOVE_PTRs a CONFIG_PUSH's shared resvals (bug-1847's new config-push recording also recorded after sched_init_config). So recordings silently lost exactly those fields: a worker's resval names were written with NULL values. Masked whenever the worker's values were already in the record-start snapshot; exposed when a recording opened after a scheduler restart, before the worker's first value push. Field-completeness violation (persistence.md). bug-1688's 'a repeat WORKERS_PUSH carries 20 resval names with nv->val==NULL' was traced from a replay file and was very likely this same stripping, not something the controller sends.", "fix": "FIXED (2026-09-24, TDD). driver.c records the batch BEFORE xce_core_push_events(); the periodic SnapshotEntry checkpoint was split out of sched_record_write_events() into new sched_record_checkpoint_after_apply() (recording.h/recording_events.c), called AFTER the push so a checkpoint still reflects the events ahead of it in the stream. sched_apply_pending_config() (scheduler.c) likewise records the CONFIG_PUSH before sched_init_config() and checkpoints after. Stubs added to the 4 driver test units that stub the recorder. Regression test test_record_lifecycle.c::test_recorded_workers_push_keeps_resval_values (RESOURCES_PUSH 'slots' + WORKERS_PUSH hostA slots=22 through the real driver with recording on; decodes the recorded WORKERS_PUSH) -- RED (recorded val NULL), GREEN after; 3 periodic-checkpoint tests updated to the write-then-checkpoint contract. Full meson test 109/109 OK.", @@ -27778,16 +15299,24 @@ "field-completeness", "MOVE_PTR", "driver" - ] + ], + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1852.md", + "gitea_issue": 294, + "gitea_synced_status": "fixed" }, { "id": "bug-1853", - "gitea_synced_status": "fixed", - "gitea_issue": 295, "status": "fixed", "status_updated": "2026-09-24", + "status_note": "", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-09-24T08:20:00Z", - "error_message": "Test-harness false positive, not a controller safety bug. 10-round chaotic soak (run-ha-soak.sh --chaotic 10): VERDICT committed_divergence=1 committed_job_loss=0 result=FAIL. Single hit, round 3, mid-loop walcompare with boundary=auto: 'WALCMP_DIVERGENCE lsn=58 class=committed compared=3 present=3'. Final settled sweep: committed prefix consistent, all 33 jobs present.", + "last_seen": "2026-09-24T08:20:00Z", + "occurrences": 1, + "error_message": "Test-harness false positive, not a controller safety bug\n\nTest-harness false positive, not a controller safety bug. 10-round chaotic soak (run-ha-soak.sh --chaotic 10): VERDICT committed_divergence=1 committed_job_loss=0 result=FAIL. Single hit, round 3, mid-loop walcompare with boundary=auto: 'WALCMP_DIVERGENCE lsn=58 class=committed compared=3 present=3'. Final settled sweep: committed prefix consistent, all 33 jobs present.", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", "root_cause": "LSN 58 was never committed. ctrl-3 (term-4 primary) appended lsn=58 at 08:06:22, its only standby NACKed it ('WAL NACK from standby lsn=58 success=0'), acks never passed acked_lsn=57; the soak then killed ctrl-3. New primary ctrl-1 (term 5) correctly appended its own-term noop at lsn=58 (08:06:35). The round-3 walcheck fell back to boundary=auto (live_committed_boundary unavailable right after the election) and auto-derive (min CRC-intact tip across ALL members' on-disk WALs, including the just-killed ex-primary's) put lsn 58 inside the 'committed' window, so the dead ex-primary's uncommitted term-4 tail vs the new term-5 entry was classified as class=committed. ctrl-3 truncated/overwrote it on rejoin; settled sweep consistent. Same family as the transient-artifact false positives documented at live_committed_boundary()/bug-1487.", "fix": "FIXED (2026-09-24, harness only). walcheck_count() takes an optional 2nd arg 'midloop', passed only by run_chaotic_rounds' per-round check: it takes the serving primary's own committedLsn (probe_primary, up to 3 tries; live_committed_boundary() needs 3-of-3 convergence, which a chaotic round never reaches -- first cut retried it and always fell back to auto) before falling back to auto, and compares only live members guaranteed to hold the committed prefix -- the serving primary plus live standbys whose node_probe appliedLsn >= boundary (all live members under auto); with fewer than 2 such members the mid-loop check is logged as skipped. Every other caller, including the settled final sweep, keeps the unchanged full 3-member comparison, so an excluded member is still checked once it has caught up. Log lines now name members=. SKILL.md chaotic-verdict section updated. Candidate (c), re-check-after-settle, rejected: a real divergence can 'heal' onto wrong content. Validated: 10-round chaotic soak PASS (committed_divergence=0, job_loss=0).", @@ -27797,17 +15326,24 @@ "walcompare", "false-positive", "test-harness" - ] + ], + "related_bugs": [], + "fix_commits": [], + "record": ".wolf/bugs/bug-1853.md", + "gitea_issue": 295, + "gitea_synced_status": "fixed" }, { "id": "bug-1854", - "gitea_synced_status": "fixed", - "gitea_issue": 275, "status": "fixed", "status_updated": "2026-09-25", "status_note": "663ea63c", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-24T08:35:00Z", - "error_message": "OPEN, NOT FIXED, low priority. Every HA soak run logs 'ERROR [sched_init_resources] sched_init_resources add resource failed' for EVERY configured resource (~33 lines), once per scheduler process, ~30s after it joined -- i.e. when a primary failover makes the scheduler re-join and the controller re-pushes its config (231/264/429 lines in 3 soak runs today). Scheduling is unaffected in those runs (all soaks PASS, replay parity 0 mismatches).", + "last_seen": "2026-09-24T08:35:00Z", + "occurrences": 1, + "error_message": "Every HA soak run logs 'ERROR [sched_init_resources] sched_init_resources add resource …\n\nOPEN, NOT FIXED, low priority. Every HA soak run logs 'ERROR [sched_init_resources] sched_init_resources add resource failed' for EVERY configured resource (~33 lines), once per scheduler process, ~30s after it joined -- i.e. when a primary failover makes the scheduler re-join and the controller re-pushes its config (231/264/429 lines in 3 soak runs today). Scheduling is unaffected in those runs (all soaks PASS, replay parity 0 mismatches).", "file": "src/scheduler/src/scheduler.c", "root_cause": "sched_init_config() is written as a first-time initializer but runs on every config push (sched_apply_pending_config, including each rejoin's push). sched_init_resources() calls makeResource() for each config->resources[i], which fails when the name is already registered in core.resmgr (key_add of an existing key) and only logs. Consequences: (1) ERROR log noise on every rejoin; (2) latent: a rejoin/upgrade config push that CHANGES a resource definition (flags, valtype, order) never updates the scheduler's copy. Pre-existing -- the same call ran on the handler thread before bug-1846's deferral; not introduced by it. sched_init_queues() re-adding queues on each push is worth checking under the same lens.", "fix": "NOT FIXED. Candidate: on an existing name, update the registered xce_resource_t in place (or treat identical definitions as a no-op and log at DEBUG), mirroring core_events.c's core_upsert_* upsert posture for RESOURCES_PUSH; decide separately whether a changed definition needs picker/accounting refresh.\n\nUPDATE (2026-09-25): 2026-09-25: same defect also at the core CONFIG_PUSH path -- core_events.c core_apply_config_resources() calls makeResource() per pushed resource; makeResource_() (lib/resobj.c) returns NULL/EEXIST for any already-registered name and the caller logs ERROR 'core_apply_config_resources add resource failed'. Seen as 33 ERROR lines in data/scheduler-replay-20260925-133708.replay (sim --sim-verify, b63bb581): the scheduler rejoined at 13:39:27 and the controller re-pushed CONFIG (plus WORKERS/JOB_PUSH) onto a core whose resources were already registered. Verify 0 mismatches, all 60 jobs DONE -- noise only in that run. A fix must cover both sites (sched_init_resources and core_apply_config_resources): identical definition -> no-op at DEBUG; changed definition -> decide update-in-place vs loud WARN.", @@ -27817,18 +15353,27 @@ "rejoin", "resources", "log-noise" - ] + ], + "related_bugs": [], + "fix_commits": [ + "663ea63c" + ], + "record": ".wolf/bugs/bug-1854.md", + "gitea_issue": 275, + "gitea_synced_status": "fixed" }, { "id": "bug-1855", - "gitea_synced_status": "fixed", - "gitea_issue": 88, - "status_note": "commit 'feat(buglog): status field, buglog skill, status-driven gitea-sync' (2026-09-24)", "status": "fixed", "status_updated": "2026-09-24", + "status_note": "commit 'feat(buglog): status field, buglog skill, status-driven gitea-sync' (2026-09-24)", + "component": "tooling", + "severity": "unknown", "timestamp": "2026-09-24T09:39:50Z", - "error_message": "gitea-sync mirrored unfixed bugs as closed + bug:fixed: an issue was closed whenever the buglog entry's fix text was non-empty, including fix texts reading 'NOT FIXED', 'PENDING' or 'PARTIALLY FIXED' (e.g. bug-1450 #25, bug-1451 #35, bug-1744 #47)", - "file": "scripts/gitea-sync.py,.claude/skills/gitea-sync/SKILL.md", + "last_seen": "2026-09-24T09:39:50Z", + "occurrences": 1, + "error_message": "gitea-sync mirrored unfixed bugs as closed + bug:fixed: an issue was closed whenever the buglog…\n\ngitea-sync mirrored unfixed bugs as closed + bug:fixed: an issue was closed whenever the buglog entry's fix text was non-empty, including fix texts reading 'NOT FIXED', 'PENDING' or 'PARTIALLY FIXED' (e.g. bug-1450 #25, bug-1451 #35, bug-1744 #47)", + "file": "scripts/gitea-sync.py, .claude/skills/gitea-sync/SKILL.md", "root_cause": "buglog.json had no status field, so scripts/gitea-sync.py and the gitea-sync skill inferred open/closed from fix being non-empty. Same script also: crashed on entries whose fix is a list (.strip() on list); replaced an issue's whole label set with just bug:fixed on close (wiping bug/category/user labels); wrote gitea_issue back only after the whole loop via a full json.dumps rewrite (a mid-run failure lost issue numbers -> duplicate issues next run, and the rewrite churned ~1,200 unrelated lines); read approvals.implementation, which no spec.json has, so spec implementation/validation issues and milestones never closed; never reopened an issue whose bug came back.", "fix": "Added a status field to every buglog entry (backfilled 2026-09-24) and made status the only open/closed source: STATUS_MAP -> state + one bug: label; entries sharing an issue take the most-open status; only status labels are swapped (other labels kept); reopen supported; gitea_synced_status tracks what was mirrored so unchanged entries make no API calls; per-entry surgical write-back via scripts/buglog.py set_fields; spec issues close on DONE_PHASES (implementation-complete/implemented/superseded). Skill doc updated to match.\n\nUPDATE (2026-09-24): Found during the first live run: DELETE/204 replies have an empty body (json.loads crashed the run after the first label removal), requests had no timeout (a stalled call hung the run), spec.json write-back reflowed 15 hand-formatted files, and the implementation-issue body showed the tasks approval flag as 'Tasks generated'. All fixed; first live run: bugs 20 created, 2 reopened (#25, #35), 14 relabelled; specs 65 milestones + 195 issues (158 closed at creation).", "tags": [ @@ -27839,21 +15384,25 @@ "related_bugs": [ "bug-1677" ], - "occurrences": 1, - "last_seen": "2026-09-24T09:39:50Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1855.md", + "gitea_issue": 88, + "gitea_synced_status": "fixed" }, { "id": "bug-1856", - "gitea_synced_status": "open", - "gitea_issue": 89, "status": "open", "status_updated": "2026-09-24", "status_note": "upstream OpenWolf defect; mitigated locally by openwolf.buglog.auto_detect=false; scripts/buglog.py add uses max id + 1; report upstream", + "component": "unknown", + "severity": "unknown", "timestamp": "2026-09-24T09:39:50Z", - "error_message": "OpenWolf 2.5.0 post-write hook would create a duplicate bug id: autoDetectBugFix numbers new entries bug- (next: bug-1194), but ids run to bug-1854 and bug-1194 already exists", - "file": ".wolf/hooks/post-write.js,.wolf/config.json", + "last_seen": "2026-09-24T09:39:50Z", + "occurrences": 1, + "error_message": "OpenWolf 2.5.0 post-write hook would create a duplicate bug id: autoDetectBugFix numbers new…\n\nOpenWolf 2.5.0 post-write hook would create a duplicate bug id: autoDetectBugFix numbers new entries bug- (next: bug-1194), but ids run to bug-1854 and bug-1194 already exists", + "file": ".wolf/hooks/post-write.js, .wolf/config.json", "root_cause": "Upstream OpenWolf hook (.wolf/hooks/post-write.js autoDetectBugFix, nextId = bugLog.bugs.length + 1) assumes ids are dense and never renumbered; this buglog's ids are sparse after renumbering/consolidation (see bug-1630). The hook also never sets the project's status field.", - "fix": "", + "fix": "_Not recorded (migrated from buglog.json)._", "tags": [ "openwolf", "buglog", @@ -27862,19 +15411,23 @@ "related_bugs": [ "bug-1630" ], - "occurrences": 1, - "last_seen": "2026-09-24T09:39:50Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1856.md", + "gitea_issue": 89, + "gitea_synced_status": "open" }, { "id": "bug-1857", - "gitea_synced_status": "fixed", - "gitea_issue": 276, "status": "fixed", "status_updated": "2026-09-24", "status_note": "3cc2f6e3", + "component": "config", + "severity": "unknown", "timestamp": "2026-09-24T13:02:14Z", - "error_message": "xce-test-multihost FAIL: job never reached RUN within 30s (last observed state: PEND); scheduler.log: 'unsatisfy_ unmet resource 33 (limit=limit1) shortfall 30.000000'", - "file": ".claude/skills/xce-test-multihost/run-multihost-verify.sh,src/tools/deploy-ha.sh,src/configs/limits.yaml", + "last_seen": "2026-09-24T13:02:14Z", + "occurrences": 1, + "error_message": "xce-test-multihost FAIL: job never reached RUN within 30s (last observed state: PEND)…\n\nxce-test-multihost FAIL: job never reached RUN within 30s (last observed state: PEND); scheduler.log: 'unsatisfy_ unmet resource 33 (limit=limit1) shortfall 30.000000'", + "file": ".claude/skills/xce-test-multihost/run-multihost-verify.sh, src/tools/deploy-ha.sh, src/configs/limits.yaml", "root_cause": "run-multihost-verify.sh deploys with the repo's default src/configs/limits.yaml, whose limit1 caps all users except rshen at a shared 3 slots. xce_test submits as root, so the 33-task job is capped. The fixture passed 2026-09-06 only because global limits did not yet count a job's full task total; the bug-1790 fix (9caf1424, 2026-09-08) correctly started doing so, leaving the fixture stale. Allocator itself is fine: passes with limit1 raised to 100.", "fix": "UPDATE (2026-09-24): Fix (uncommitted): deploy-ha.sh gains optional XCE_DEPLOY_CONFIG_DIR (default unchanged: src/configs); run-multihost-verify.sh copies src/configs into $TOP/configs, replaces limits.yaml with 'limits: []', and deploys with that override. Verified 2026-09-24: multihost e2e PASS (plain 33-task job + array element both split across 2 hosts, DONE), 0 limit1 rejections; default deploy path still builds from repo configs; bad override fails with 'config dir not found'.", "tags": [ @@ -27884,19 +15437,25 @@ "multihost" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-24T13:02:14Z" + "fix_commits": [ + "3cc2f6e3" + ], + "record": ".wolf/bugs/bug-1857.md", + "gitea_issue": 276, + "gitea_synced_status": "fixed" }, { "id": "bug-1858", - "gitea_synced_status": "fixed", - "gitea_issue": 277, "status": "fixed", "status_updated": "2026-09-24", "status_note": "9e9f2798", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-24T13:12:53Z", - "error_message": "AddressSanitizer: attempting free on address which was not malloc()-ed in core_upsert_queue (core_events.c:383) <- core_apply_config_queues <- xce_core_push_events <- sched_sim_driver_run; address 56 bytes inside a 57-byte makeString instance. Seen as HA soak (sequential) FAIL: scheduler replay-parity SIMVERIFY errors=1 (simverify-3, rc=1)", - "file": "src/scheduler/src/core_events.c,src/scheduler/src/schedobj.c", + "last_seen": "2026-09-24T13:12:53Z", + "occurrences": 1, + "error_message": "AddressSanitizer: attempting free on address which was not malloc()-ed in core_upsert_queue…\n\nAddressSanitizer: attempting free on address which was not malloc()-ed in core_upsert_queue (core_events.c:383) <- core_apply_config_queues <- xce_core_push_events <- sched_sim_driver_run; address 56 bytes inside a 57-byte makeString instance. Seen as HA soak (sequential) FAIL: scheduler replay-parity SIMVERIFY errors=1 (simverify-3, rc=1)", + "file": "src/scheduler/src/core_events.c, src/scheduler/src/schedobj.c", "root_cause": "q->resreq is an interned, refcounted xstring from makeString() (pointer past the object header), but core_upsert_queue releases the old value with FREEUP()/free() instead of dereference(). Only fires when a CONFIG_PUSH updates an existing queue whose resreq is non-NULL (e.g. a queue restored from snapshot via snapshot_load.c:313). Same bad free(q->resreq) exists in the normal-mode path, schedobj.c:603 (sched_update_existing_queue). Convention elsewhere: qobj.c:69 dereference(q->queue.resreq).", "fix": "UPDATE (2026-09-24): Fix (uncommitted): core_events.c core_upsert_queue and schedobj.c sched_update_existing_queue now dereference() the old interned q->resreq (NULL-guarded) instead of FREEUP()/free(). Regression tests added (red first, both hit the exact bad-free): test_core_events.c test_queue_update_replaces_existing_interned_resreq, test_job_update.c test_update_existing_queue_replaces_interned_resreq (the existing queue-update test had deliberately skipped the resreq branch). Verified 2026-09-24: scheduler suite 87/87 (ASAN); the crashing HA-soak recording scheduler-replay-20260924-130653.replay now replays with rc=0, 0 ASAN errors, 0 mismatches.", "tags": [ @@ -27907,19 +15466,25 @@ "soak" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-24T13:12:53Z" + "fix_commits": [ + "9e9f2798" + ], + "record": ".wolf/bugs/bug-1858.md", + "gitea_issue": 277, + "gitea_synced_status": "fixed" }, { "id": "bug-1859", - "gitea_synced_status": "fixed", - "gitea_issue": 278, "status": "fixed", "status_updated": "2026-09-25", "status_note": "7c98c056", + "component": "scheduler", + "severity": "unknown", "timestamp": "2026-09-24T13:45:31Z", - "error_message": "xce-test-ha --chaotic FAIL: chaotic: 1/4 worker-resync job(s) never reconciled after the loop settled: 0-2-1790256202-546579(last=PEND). scheduler.log: 'JOBS_PUSH: refusing resync state 0x1 for jobid ... -- existing job is RUN with a real allocation on record (existing state 0x2 kept)'", - "file": "src/scheduler/src/core_events.c,src/controller/job.c", + "last_seen": "2026-09-24T13:45:31Z", + "occurrences": 1, + "error_message": "xce-test-ha --chaotic FAIL: chaotic: 1/4 worker-resync job(s) never reconciled after the loop…\n\nxce-test-ha --chaotic FAIL: chaotic: 1/4 worker-resync job(s) never reconciled after the loop settled: 0-2-1790256202-546579(last=PEND). scheduler.log: 'JOBS_PUSH: refusing resync state 0x1 for jobid ... -- existing job is RUN with a real allocation on record (existing state 0x2 kept)'", + "file": "src/scheduler/src/core_events.c, src/controller/job.c", "root_cause": "Not yet confirmed. Timeline (2026-09-24 chaotic soak, 6 rounds): job dispatched on primary ctrl-1 (term 6) at 13:23:23.74, commit_advance 80->81 at 13:23:23.79; worker started it and finished it at 13:23:43 (retained outcome exit_status=1); ctrl-1 killed in round 5, overlapping kills follow; after settle the cluster reports the job PEND while the scheduler keeps RUN+alloc and refuses the PEND resync (core_events.c:984), so neither side converges. Also: round 6 'scheduler restart did not confirm join within 20s (bug-1760 workaround)'. Candidates: dispatch/RUN not in the surviving committed prefix while the scheduler already applied it, or worker's retained finish never re-delivered to the new primary (cf. bug-1741 fire-and-forget finish notify). Same chaotic soak passed earlier the same day (21 jobs, 6 resync jobs reconciled), so intermittent.", "fix": "UPDATE (2026-09-24): Frequency 2026-09-24: 1 failure in 3 chaotic 6-round soaks (pass 12:5x, FAIL 13:2x, pass 13:5x on 9e9f2798). Failing run's logs kept at scratchpad ha-chaos-bug1859-run1 (session-local).\n\nUPDATE (2026-09-24): ROOT CAUSE (from preserved logs of the failing run): job 546579 dispatched+committed (LSN 81) on ctrl-1 term 6; worker finished it 13:23:43 and sent the fire-and-forget finish notify (xcecli_notify = single-shot send, rc ignored: 'FIXME: add error handling when msg send failure') into a dead connection during overlapping failovers; worker could not reach any primary for 84s; its retained finished outcome expired at finish_time + 60s (default window; xce.yaml sets no worker_resync_window_ms, controller pushes 0 -> default) DURING the outage; at rejoin (13:25:07) ctrl-2's resync query (op 43) got 'unrecognized' -> record_job_reconcile_pending_fact requeued it PEND while the scheduler was disconnected; scheduler rejoined 13:25:11 and its JOBS_PUSH bug-1812 guard refused the PEND (RUN+alloc on record) -> permanent divergence. FIX (uncommitted): worker retention window no longer counts outage time -- xcecd_on_reconnect() records the reconnect time and expiry is measured from max(finish_time, last_reconnect); nothing expires while checking_connection()'s hello probe reports unhealthy. Reported finish_time_ms unchanged. Tests: test_finished_job_retention.c test_reconnect_restarts_window_for_outcome_finished_before_outage (84s outage) + test_unhealthy_connection_suspends_expiry, both red first; worker suite 10/10. The scheduler-side divergence (controller requeue while scheduler disconnected is refused by bug-1812 guard) logged separately.", "tags": [ @@ -27931,21 +15496,27 @@ "intermittent" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-24T13:45:31Z" + "fix_commits": [ + "7c98c056" + ], + "record": ".wolf/bugs/bug-1859.md", + "gitea_issue": 278, + "gitea_synced_status": "fixed" }, { "id": "bug-1860", - "gitea_synced_status": "fixed", - "gitea_issue": 279, "status": "fixed", "status_updated": "2026-09-27", "status_note": "c69705ae: controller is ground truth on rejoin; controller refuses non-PEND dispatch and re-sends state; scheduler adopts controller allocation on RUN update", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-24T14:38:03Z", - "error_message": "Scheduler keeps a job RUN (with allocation) forever after the controller requeued it to PEND while the scheduler was disconnected: scheduler.log 'JOBS_PUSH: refusing resync state 0x1 for jobid ... -- existing job is RUN with a real allocation on record' on rejoin; job never redispatched (seen in bug-1859's chaotic soak)", - "file": "src/scheduler/src/core_events.c,src/controller/sched.c,src/controller/resync.c", + "last_seen": "2026-09-24T14:38:03Z", + "occurrences": 1, + "error_message": "Scheduler keeps a job RUN (with allocation) forever after the controller requeued it to PEND while…\n\nScheduler keeps a job RUN (with allocation) forever after the controller requeued it to PEND while the scheduler was disconnected: scheduler.log 'JOBS_PUSH: refusing resync state 0x1 for jobid ... -- existing job is RUN with a real allocation on record' on rejoin; job never redispatched (seen in bug-1859's chaotic soak)", + "file": "src/scheduler/src/core_events.c, src/controller/sched.c, src/controller/resync.c", "root_cause": "core_events.c JOBS_PUSH bug-1812 guard refuses any non-terminal resync state that would drop a real allocation, to protect against a committed snapshot racing the scheduler's own fresh dispatch. It cannot tell that race apart from a genuine, committed controller-side requeue (e.g. resync 'unrecognized' -> record_job_reconcile_pending_fact) that happened while the scheduler was disconnected, and JOBS_PUSH carries no per-job freshness (commit LSN/version) to decide. Needs a design decision (e.g. per-job last-modified LSN in the push).", - "fix": "", + "fix": "_Not recorded (migrated from buglog.json)._", "tags": [ "scheduler", "controller", @@ -27955,21 +15526,27 @@ "design" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-24T14:38:03Z" + "fix_commits": [ + "c69705ae" + ], + "record": ".wolf/bugs/bug-1860.md", + "gitea_issue": 279, + "gitea_synced_status": "fixed" }, { "id": "bug-1861", - "gitea_synced_status": "fixed", - "gitea_issue": 280, "status": "fixed", "status_updated": "2026-09-25", "status_note": "b5d59253", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-24T15:05:56Z", - "error_message": "config_repl.c do_upgrade_config_with_bytes: temp decode file /tmp/xce_upgrade_cfg..tmp -- the name is identical across processes (parallel controller_unit_test instances failed handle_upgrade_config with -EINVAL, 'a majority-acked upgrade must succeed') and is opened O_CREAT|O_TRUNC without O_EXCL in world-writable /tmp", + "last_seen": "2026-09-24T15:05:56Z", + "occurrences": 1, + "error_message": "config_repl.c do_upgrade_config_with_bytes: temp decode file…\n\nconfig_repl.c do_upgrade_config_with_bytes: temp decode file /tmp/xce_upgrade_cfg..tmp -- the name is identical across processes (parallel controller_unit_test instances failed handle_upgrade_config with -EINVAL, 'a majority-acked upgrade must succeed') and is opened O_CREAT|O_TRUNC without O_EXCL in world-writable /tmp", "file": "src/controller/config_repl.c", "root_cause": "pthread_self() is a thread-descriptor address, not a unique id: the main thread's value repeats across processes (esp. with ASLR off under ASAN), so two processes on one host share and clobber the file; a predictable name opened without O_EXCL in /tmp is also a symlink-attack target (write/truncate through an attacker's link).", - "fix": "", + "fix": "_Not recorded (migrated from buglog.json)._", "tags": [ "controller", "security", @@ -27977,19 +15554,25 @@ "config-upgrade" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-24T15:05:56Z" + "fix_commits": [ + "b5d59253" + ], + "record": ".wolf/bugs/bug-1861.md", + "gitea_issue": 280, + "gitea_synced_status": "fixed" }, { "id": "bug-1862", - "gitea_synced_status": "fixed", - "gitea_issue": 281, "status": "fixed", "status_updated": "2026-09-25", "status_note": "c29518b2", + "component": "worker", + "severity": "crash", "timestamp": "2026-09-24T15:24:57Z", - "error_message": "xcecd_virtual ASAN heap-use-after-free in find_worker_by_jobid (xcecd_worker_utils.c:702) <- xcecd_resync_classify_job (xcecd_resync.c:33) <- xcecd_resync_build_report <- handle_jobstatequery (controller_connector.c:1662) on a vtask thread; freed by remove_worker (xcecd_worker_utils.c:283) <- cleanup_worker_list <- main loop (main.c:652). Worker aborted mid chaotic HA soak (2026-09-24, run 1 of 3), so later rounds lost worker-resync coverage.", - "file": "src/worker/src/xcecd_resync.c,src/worker/src/xcecd_worker_utils.c,src/worker/src/controller_connector.c,src/worker/src/main.c", + "last_seen": "2026-09-24T15:24:57Z", + "occurrences": 1, + "error_message": "xcecd_virtual ASAN heap-use-after-free in find_worker_by_jobid (xcecd_worker_utils.c:702) <…\n\nxcecd_virtual ASAN heap-use-after-free in find_worker_by_jobid (xcecd_worker_utils.c:702) <- xcecd_resync_classify_job (xcecd_resync.c:33) <- xcecd_resync_build_report <- handle_jobstatequery (controller_connector.c:1662) on a vtask thread; freed by remove_worker (xcecd_worker_utils.c:283) <- cleanup_worker_list <- main loop (main.c:652). Worker aborted mid chaotic HA soak (2026-09-24, run 1 of 3), so later rounds lost worker-resync coverage.", + "file": "src/worker/src/xcecd_resync.c, src/worker/src/xcecd_worker_utils.c, src/worker/src/controller_connector.c, src/worker/src/main.c", "root_cause": "The worker's live job list is mutated by the main loop (add_worker / cleanup_worker_list -> remove_worker) with no lock, while the controller's OPCode_jobStateQuery is answered on a vtask thread (myreq_handler) that walks the same list. Any rejoin resync overlapping a job reap reads a freed node.", "fix": "UPDATE (2026-09-25): Fix (uncommitted): xcecd_worker_utils.c gains a worker-list mutex (xcecd_worker_list_lock/unlock). add_worker takes it around linking a fully-built entry; remove_worker unlinks under it and frees after. Readers on vtask threads: handle_jobstatequery holds it from reading the list head through the whole query page (report build walks the list several times; the send is a non-blocking xce_request); the job-control handler copies job_status under it and re-finds the entry under it to set killed_signal; remote task start strdup's the five fields it sends under it and frees them after xcecli_runtask. The finished-job retention store (appended by the main loop, read by resync) gets its own internal mutex via public wrappers around *_locked_ bodies. Main-thread-only traversals (uv_ipc.c find_worker_by_pid, cleanup/update) need no lock. Test: test_xcecd_resync.c test_build_report_safe_against_concurrent_list_mutation (mutator thread add/remove/retain vs 20000 locked report builds) -- heap-use-after-free in find_worker_by_jobid 3/3 with no-op lock stubs, clean 5/5 with the fix; worker suite 10/10.", "tags": [ @@ -28001,21 +15584,27 @@ "ha" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-24T15:24:57Z" + "fix_commits": [ + "c29518b2" + ], + "record": ".wolf/bugs/bug-1862.md", + "gitea_issue": 281, + "gitea_synced_status": "fixed" }, { "id": "bug-1863", - "gitea_synced_status": "fixed", - "gitea_issue": 282, "status": "fixed", "status_updated": "2026-09-25", "status_note": "aea34738", + "component": "scheduler", + "severity": "leak", "timestamp": "2026-09-25T13:07:56Z", - "error_message": "A COMPOUND resreq with 2+ primaries (e.g. '1*{A} + 2*{B}') keeps only the LAST primary's allocation: seqjob/parjob my_alloc assign ctx->alloc = make_job_alloc(...) per primary, overwriting (and leaking) the previous primary's job alloc, while every primary's hosts were already debited into ctx->uncomitted -- the dispatched allocation omits earlier primaries' hosts and accounting diverges from the decision.", - "file": "src/scheduler/src/scheduler.c,src/scheduler/src/plugins/seqjob_allocator.c,src/scheduler/src/plugins/parjob_allocator.c", + "last_seen": "2026-09-25T13:07:56Z", + "occurrences": 1, + "error_message": "A COMPOUND resreq with 2+ primaries (e.g\n\nA COMPOUND resreq with 2+ primaries (e.g. '1*{A} + 2*{B}') keeps only the LAST primary's allocation: seqjob/parjob my_alloc assign ctx->alloc = make_job_alloc(...) per primary, overwriting (and leaking) the previous primary's job alloc, while every primary's hosts were already debited into ctx->uncomitted -- the dispatched allocation omits earlier primaries' hosts and accounting diverges from the decision.", + "file": "src/scheduler/src/scheduler.c, src/scheduler/src/plugins/seqjob_allocator.c, src/scheduler/src/plugins/parjob_allocator.c", "root_cause": "sched_make_compound_alloc_() (scheduler.c) loops primaries on one ctx; each allocator's success path replaces ctx->alloc instead of merging host allocs (and shared allocs) across primaries. Latent until bug-1768 (fixed 2026-09-25) because nested primaries previously requested nothing.", - "fix": "", + "fix": "_Not recorded (migrated from buglog.json)._", "tags": [ "scheduler", "allocation", @@ -28023,19 +15612,25 @@ "accounting" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-25T13:07:56Z" + "fix_commits": [ + "aea34738" + ], + "record": ".wolf/bugs/bug-1863.md", + "gitea_issue": 282, + "gitea_synced_status": "fixed" }, { "id": "bug-1864", - "gitea_synced_status": "fixed", - "gitea_issue": 283, "status": "fixed", "status_updated": "2026-09-25", "status_note": "0d3bb965", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-25T14:26:45Z", - "error_message": "Standby controllers NACK in-order WAL catch-up entries (ae_reject_logmismatch, prevIdx one ahead of their log) ~576 times in a 45-min run (data/logs, 2026-09-25): e.g. ctrl-2 21:37:50.115 handled walCatchupEntry seq 258 (prevIdx=149) BEFORE seq 257 (prevIdx=148). With both standbys NACKing, the primary marks them catch-up, loses its write quorum and answers jobdispatch/join with E_XCE_NOT_READY (-10013) (ctrl-1 21:39:24.494); the scheduler client fails over (bug-1850), rejoins, and gets a JOBS_PUSH built from lagging committed state -> 'refusing resync state 0x1 ... existing job is RUN' WARNs for 14 jobs.", - "file": "src/lib/vtaskmgr.c,src/lib/dataq.c,src/lib/netio.c", + "last_seen": "2026-09-25T14:26:45Z", + "occurrences": 1, + "error_message": "Standby controllers NACK in-order WAL catch-up entries (ae_reject_logmismatch, prevIdx one ahead…\n\nStandby controllers NACK in-order WAL catch-up entries (ae_reject_logmismatch, prevIdx one ahead of their log) ~576 times in a 45-min run (data/logs, 2026-09-25): e.g. ctrl-2 21:37:50.115 handled walCatchupEntry seq 258 (prevIdx=149) BEFORE seq 257 (prevIdx=148). With both standbys NACKing, the primary marks them catch-up, loses its write quorum and answers jobdispatch/join with E_XCE_NOT_READY (-10013) (ctrl-1 21:39:24.494); the scheduler client fails over (bug-1850), rejoins, and gets a JOBS_PUSH built from lagging committed state -> 'refusing resync state 0x1 ... existing job is RUN' WARNs for 14 jobs.", + "file": "src/lib/vtaskmgr.c, src/lib/dataq.c, src/lib/netio.c", "root_cause": "vtaskmgr's dispatcher (pick_eligible_task -> dataq_deq(taskq, vtask_eligible)) evaluates each queued task's eligibility one item at a time while holding only the taskq lock; the per-endpoint busy flag it reads (endpoint_is_task_blocked) is cleared by the vcpu finishing the previous message WITHOUT that lock. Within one scan, task N (seq 257) is checked while the endpoint is still busy and skipped, the endpoint unblocks, and task N+1 (seq 258) for the SAME endpoint is then found eligible and dequeued first. endpoint_block_tasks gives mutual exclusion per endpoint but not FIFO order. Log proof (ctrl-2 21:37:50.113-.118): '04b650 not eligible' -> 'endpoint not blocked' -> '03cae0 eligible' in one scan; 03cae0=seq 258, 04b650=seq 257; tasks enqueued in arrival order by the single netio thread (vtask_handle_msg).", "fix": "UPDATE (2026-09-25): Fix (uncommitted): dataq gains dataq_deq_ctx (per-scan matcher context); vtaskmgr's scan is now vtask_deq_eligible(taskq, is_blocked) with a per-scan set of endpoints found busy -- every later task of such an endpoint stays ineligible for the rest of that scan (overflow past 64 distinct busy endpoints makes all later IO tasks ineligible), so per-endpoint FIFO holds without taking any new lock (bug-1282 ABBA). Test lib/testsuite/test_vtask_order.c: is_blocked injected to flip ep busy->free mid-scan; reproduced the reorder (N+1 dequeued) before, NULL after; other-endpoint progress and FIFO/generic cases pass. Suites: lib 1/1, controller 5/5, worker 10/10, scheduler 87/87, harness 5/5. Reorder signature (reject prevIdx=K then accept K-1): user run 26 (-> 576 NACKs incl. cascades), pre-fix chaotic soaks 2 and 5.", "tags": [ @@ -28048,21 +15643,27 @@ "controller" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-25T14:26:45Z" + "fix_commits": [ + "0d3bb965" + ], + "record": ".wolf/bugs/bug-1864.md", + "gitea_issue": 283, + "gitea_synced_status": "fixed" }, { "id": "bug-1865", - "gitea_synced_status": "fixed", - "gitea_issue": 284, "status": "fixed", "status_updated": "2026-09-25", "status_note": "910b43cd; soaks 2026-09-25 (2 chaotic + 1 sequential) fdatasync_failed=0, all PASS", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-25T14:43:27Z", - "error_message": "Standby's WAL fd closed out from under it at failover: chaotic HA soak 2026-09-25 (ha-chaos-o1), ctrl-3 at 14:39:55.574 'liveness poll: no primary activity for 11s -> failover' / 'primary_ctrl_fd RDHUP', and 3 ms later 'flusher fallback fdatasync FAILED (errno=9 Bad file descriptor) -- durable_lsn held at 90', then errno=22 EINVAL once the number was reused by a socket; 2305 fdatasync failures, durable_lsn stuck; soak still PASSed (counted as excluded bug924) but one round's jobsub failed EIO.", - "file": "src/controller/ha.c,src/controller/config_repl.c", + "last_seen": "2026-09-25T14:43:27Z", + "occurrences": 1, + "error_message": "Standby's WAL fd closed out from under it at failover: chaotic HA soak 2026-09-25 (ha-chaos-o1)…\n\nStandby's WAL fd closed out from under it at failover: chaotic HA soak 2026-09-25 (ha-chaos-o1), ctrl-3 at 14:39:55.574 'liveness poll: no primary activity for 11s -> failover' / 'primary_ctrl_fd RDHUP', and 3 ms later 'flusher fallback fdatasync FAILED (errno=9 Bad file descriptor) -- durable_lsn held at 90', then errno=22 EINVAL once the number was reused by a socket; 2305 fdatasync failures, durable_lsn stuck; soak still PASSed (counted as excluded bug924) but one round's jobsub failed EIO.", + "file": "src/controller/ha.c, src/controller/config_repl.c", "root_cause": "config_repl.c:2192 stores the netio endpoint's raw fd (c->ha.primary_ctrl_fd = ep->fd) and ha_on_primary_ctrl_fd_rdhup (ha.c:8648, three branches) later close()s it directly. netio owns that socket and closes it on endpoint teardown, so by then the number may have been reused -- here by the standby's WAL segment (lowest free fd) -- and HA's close() closes the WAL instead. Double close / stale fd. Once a socket takes the number, WAL fdatasync gets EINVAL and pwrite could write WAL bytes into a socket.", - "fix": "", + "fix": "_Not recorded (migrated from buglog.json)._", "tags": [ "controller", "ha", @@ -28073,39 +15674,51 @@ "durability" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-25T14:43:27Z" + "fix_commits": [ + "910b43cd" + ], + "record": ".wolf/bugs/bug-1865.md", + "gitea_issue": 284, + "gitea_synced_status": "fixed" }, { "id": "bug-1866", - "gitea_synced_status": "fixed", - "gitea_issue": 285, "status": "fixed", "status_updated": "2026-09-27", - "status_note": "e84a8d43: remove_worker now also frees launch_type and q_pre/q_post/job_pre/job_post_exec (5 strings leaked per finished job, not just launch_type)", + "status_note": "", + "component": "worker", + "severity": "leak", "timestamp": "2026-09-25T15:42:02Z", - "error_message": "LeakSanitizer at xcecd_virtual exit (burst run 2026-09-25): 'Direct leak of 12 byte(s) in 12 object(s)' from strndup in add_worker (xcecd_worker_utils.c:140) <- xcecd_process_command <- handle_message_from_queue", + "last_seen": "2026-09-25T15:42:02Z", + "occurrences": 1, + "error_message": "xcecd_virtual leaks up to five strings per finished job (LeakSanitizer at exit)\n\n**Observed:**\n```\nDirect leak of 12 byte(s) in 12 object(s) allocated from:\n strndup add_worker (xcecd_worker_utils.c:140)\n <- xcecd_process_command <- handle_message_from_queue\n```\n**Expected:** no leaks reported when xcecd_virtual exits.\n**Impact:** one leaked string per job that carries a launch type, plus one per queue/job pre- or post-exec command; grows with the number of finished jobs for the life of the daemon.\n**Found by:** LeakSanitizer at xcecd_virtual exit after a job burst run, 2026-09-25.", "file": "src/worker/src/xcecd_worker_utils.c", - "root_cause": "add_worker strndup's startinfo->launch_type into worker->launch_type, but remove_worker frees exechome/execcwd/execuser/job_cmd/pids/job_exdata and never launch_type -- one string leaked per finished job that carried a launch type.", - "fix": "e84a8d43: remove_worker frees launch_type, q_pre_exec, q_post_exec, job_pre_exec, job_post_exec. Regression test test_remove_worker_frees_every_startinfo_copy (ASAN live-heap byte delta across add/remove; RED leaked 58 B).", + "root_cause": "add_worker() strndup()s launch_type, q_pre_exec, q_post_exec, job_pre_exec and job_post_exec out of the job's start info, but remove_worker() only freed exechome, execcwd, execuser, job_cmd, pids and job_exdata. LeakSanitizer reported only the launch_type copy because the burst's jobs had no pre/post-exec commands. Pre-existing, origin unknown. remove_worker() is the only place a worker entry is freed, so no other path is affected.", + "fix": "remove_worker() also frees launch_type, q_pre_exec, q_post_exec, job_pre_exec and job_post_exec (src/worker/src/xcecd_worker_utils.c).\n\nVerification:\n- Test: test_remove_worker_frees_every_startinfo_copy in src/worker/testsuite/test_finished_job_retention.c compares ASAN's live-heap bytes across an add/remove round trip (the suite runs with detect_leaks=0): RED before the fix leaked 58 B, the five strings; GREEN after, 0 B\n- Suites: worker 12/12, asan\n- Live: n/a; the leak is in a pure unlink/free path the unit test covers completely", "tags": [ "worker", "leak", "asan" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-25T15:42:02Z" + "fix_commits": [ + "e84a8d43" + ], + "record": ".wolf/bugs/bug-1866.md", + "gitea_issue": 285, + "gitea_synced_status": "fixed" }, { "id": "bug-1867", - "gitea_synced_status": "fixed", - "gitea_issue": 286, "status": "fixed", "status_updated": "2026-09-26", "status_note": "e4bb492a: capnp Job.exitInfo (JobExitInfo) -- checkpoint encode/decode/restore carry exit_info. Live force-snapshot soak blocked by pre-existing bug-1870.", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-26T03:33:15Z", - "error_message": "Controller job snapshot/checkpoint drops job.exit_info: the capnp Job struct (controller.capnp) has no exit-info field, so capnp_encode_job() never writes it and a job restored from a snapshot (rather than WAL replay) has exit_info NULL for both DONE/EXIT (FINISH) and failed (FAIL) jobs", + "last_seen": "2026-09-26T03:33:15Z", + "occurrences": 1, + "error_message": "Controller job snapshot/checkpoint drops job.exit_info: the capnp Job struct (controller.capnp)…\n\nController job snapshot/checkpoint drops job.exit_info: the capnp Job struct (controller.capnp) has no exit-info field, so capnp_encode_job() never writes it and a job restored from a snapshot (rather than WAL replay) has exit_info NULL for both DONE/EXIT (FINISH) and failed (FAIL) jobs", "file": "src/schema/controller.capnp", "root_cause": "struct Job in src/schema/controller.capnp has finishtime/runtime/pidinfo/pendreason but no exit code; xce_job_t.exit_info (job_exitinfo_t*) has no capnp mapping. bug-1689 and bug-1754 made WAL_JOB_FINISH/WAL_JOB_FAIL carry the exit code, but anything restored from a checkpoint/entity snapshot after WAL truncation loses it (persistence.md field-completeness).", "fix": "UPDATE (2026-09-26): live-verified by the bug-1870 force-snapshot soak (real snapshot install, 63fd3d2a)", @@ -28120,21 +15733,27 @@ "bug-1689", "bug-1754" ], - "occurrences": 1, - "last_seen": "2026-09-26T03:33:15Z" + "fix_commits": [ + "e4bb492a" + ], + "record": ".wolf/bugs/bug-1867.md", + "gitea_issue": 286, + "gitea_synced_status": "fixed" }, { "id": "bug-1868", - "gitea_synced_status": "fixed", - "gitea_issue": 288, "status": "fixed", "status_updated": "2026-09-26", "status_note": "7864a3da: xcecd_create_job_output() unlinks then creates with O_EXCL|O_NOFOLLOW|O_CLOEXEC.", + "component": "worker", + "severity": "security", "timestamp": "2026-09-26T07:40:17Z", - "error_message": "uv_proc_exec (worker/src/uv_proc.c) creates each task's output file /tmp/job__task.out as root with O_WRONLY|O_CREAT|O_TRUNC and no O_NOFOLLOW/O_EXCL: a local user who pre-plants a symlink at a predictable future job's output name makes the root worker truncate and write job output into the symlink target", + "last_seen": "2026-09-26T07:40:17Z", + "occurrences": 1, + "error_message": "uv_proc_exec (worker/src/uv_proc.c) creates each task's output file /tmp/job__task.out…\n\nuv_proc_exec (worker/src/uv_proc.c) creates each task's output file /tmp/job__task.out as root with O_WRONLY|O_CREAT|O_TRUNC and no O_NOFOLLOW/O_EXCL: a local user who pre-plants a symlink at a predictable future job's output name makes the root worker truncate and write job output into the symlink target", "file": "src/worker/src/uv_proc.c", "root_cause": "Predictable name in world-writable /tmp opened by root following symlinks. Only mitigated by the host's fs.protected_symlinks / fs.protected_regular sysctls, not by the code. Sibling of bug-1508 (read side, fixed in 7723a4f4).", - "fix": "", + "fix": "_Not recorded (migrated from buglog.json)._", "tags": [ "security", "worker", @@ -28145,21 +15764,27 @@ "related_bugs": [ "bug-1508" ], - "occurrences": 1, - "last_seen": "2026-09-26T07:40:17Z" + "fix_commits": [ + "7864a3da" + ], + "record": ".wolf/bugs/bug-1868.md", + "gitea_issue": 288, + "gitea_synced_status": "fixed" }, { "id": "bug-1869", - "gitea_synced_status": "fixed", - "gitea_issue": 289, "status": "fixed", "status_updated": "2026-09-26", "status_note": "5df33e69: xcecd_open_owned_file()/xcecd_owned_file_exists() (xcecd_tmpfile.h) for the worker log, xcecd_worker.log and restart flag; fchmod instead of chmod(path).", + "component": "worker", + "severity": "security", "timestamp": "2026-09-26T07:55:23Z", - "error_message": "Other root-owned opens of fixed /tmp names in the worker daemon follow symlinks: process_worker_manager.c opens /tmp/xcecd_worker.log O_WRONLY|O_CREAT|O_APPEND (xcecd_worker's stdout/stderr appended through a planted symlink into any file), and xcecd_prepare_restart() (xcecd.c) opens /tmp/.xcecd.restarting O_CREAT|O_WRONLY (creates an empty root-owned file at a dangling symlink's target)", + "last_seen": "2026-09-26T07:55:23Z", + "occurrences": 1, + "error_message": "Other root-owned opens of fixed /tmp names in the worker daemon follow symlinks…\n\nOther root-owned opens of fixed /tmp names in the worker daemon follow symlinks: process_worker_manager.c opens /tmp/xcecd_worker.log O_WRONLY|O_CREAT|O_APPEND (xcecd_worker's stdout/stderr appended through a planted symlink into any file), and xcecd_prepare_restart() (xcecd.c) opens /tmp/.xcecd.restarting O_CREAT|O_WRONLY (creates an empty root-owned file at a dangling symlink's target)", "file": "src/worker/src/process_worker_manager.c, src/worker/src/xcecd.c", "root_cause": "Same class as bug-1868: fixed, predictable names in world-writable /tmp opened by root without O_NOFOLLOW/O_EXCL; only the host's fs.protected_symlinks sysctl mitigates. Found while fixing bug-1868; left out of that fix's scope.", - "fix": "", + "fix": "_Not recorded (migrated from buglog.json)._", "tags": [ "security", "worker", @@ -28169,18 +15794,24 @@ "related_bugs": [ "bug-1868" ], - "occurrences": 1, - "last_seen": "2026-09-26T07:55:23Z" + "fix_commits": [ + "5df33e69" + ], + "record": ".wolf/bugs/bug-1869.md", + "gitea_issue": 289, + "gitea_synced_status": "fixed" }, { "id": "bug-1870", - "gitea_synced_status": "fixed", - "gitea_issue": 296, "status": "fixed", "status_updated": "2026-09-26", "status_note": "63fd3d2a: checkpoints no longer write the LSN-consuming WAL_CHECKPOINT marker; force-snapshot, config-upgrade-wal-trim, sequential soaks PASS.", + "component": "controller", + "severity": "unknown", "timestamp": "2026-09-26T09:55:09Z", - "error_message": "xce-test-ha --force-snapshot FAIL: 'could not submit 110 jobs to trigger a checkpoint' -- writes stall right after the first max-wal-records checkpoint. The primary's WAL holds a CHECKPOINT entry (plen 0) at the checkpoint LSN that no standby ever receives (primary ctrl-1: 109 JOB_SUBMIT, 110 CHECKPOINT, 111 JOB_SUBMIT; standbys stop at 109). Every later AppendEntries carries prevIdx=, standbys reply ae_reject_logmismatch, the primary marks catch-up and re-sends the last common entry forever: 'commit: async continuation deadline expired without commit', 'HA quorum lost: connected=0'.", + "last_seen": "2026-09-26T09:55:09Z", + "occurrences": 1, + "error_message": "xce-test-ha --force-snapshot FAIL: 'could not submit 110 jobs to trigger a checkpoint' -- writes…\n\nxce-test-ha --force-snapshot FAIL: 'could not submit 110 jobs to trigger a checkpoint' -- writes stall right after the first max-wal-records checkpoint. The primary's WAL holds a CHECKPOINT entry (plen 0) at the checkpoint LSN that no standby ever receives (primary ctrl-1: 109 JOB_SUBMIT, 110 CHECKPOINT, 111 JOB_SUBMIT; standbys stop at 109). Every later AppendEntries carries prevIdx=, standbys reply ae_reject_logmismatch, the primary marks catch-up and re-sends the last common entry forever: 'commit: async continuation deadline expired without commit', 'HA quorum lost: connected=0'.", "file": "src/controller/persist.c (checkpoint marker append), src/controller/repl.c / config_repl.c (broadcast / catch-up)", "root_cause": "Not yet root-caused. The checkpoint marker is appended to the local WAL (and consumes an LSN in the replicated log) without being broadcast to / streamed to standbys, so the log has a hole standbys can never fill. Reproduced on a565779d (baseline, 2026-09-26) and on the bug-1867 working tree -- independent of both. Repro: nix develop ./src --command bash .claude/skills/xce-test-ha/run-ha-soak.sh --force-snapshot (max-wal-records=100, wal-segment-bytes=4096).", "fix": "UPDATE (2026-09-27): follow-up cb1415d7: load_snapshot seeded current_lsn = checkpoint_lsn (off by one, hidden by the removed marker) -- broke --config-upgrade soak; fixed", @@ -28194,21 +15825,27 @@ "livelock" ], "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-09-26T09:55:09Z" + "fix_commits": [ + "63fd3d2a" + ], + "record": ".wolf/bugs/bug-1870.md", + "gitea_issue": 296, + "gitea_synced_status": "fixed" }, { "id": "bug-1871", - "gitea_synced_status": "investigating", - "gitea_issue": 297, "status": "investigating", "status_updated": "2026-09-27", "status_note": "found while soak-verifying the HA group; logs in /home/claude/tmp/xce-ha-final-force-snapshot", + "component": "lib", + "severity": "unknown", "timestamp": "2026-09-27T07:03:48Z", - "error_message": "HA snapshot transfer to a rejoining standby stalls mid-stream: primary queues all chunks at once (528KB, 9 chunks); the standby reads a few (1 or 5 of 9 in two runs), its endpoint is unblocked after the last processed chunk, and then no further input event arrives on that connection for 3s+ although the primary has more chunks queued -- --force-snapshot soak's single post-convergence check saw no install (2 of 4 runs, 2026-09-27)", + "last_seen": "2026-09-27T07:03:48Z", + "occurrences": 1, + "error_message": "HA snapshot transfer to a rejoining standby stalls mid-stream: primary queues all chunks at once…\n\nHA snapshot transfer to a rejoining standby stalls mid-stream: primary queues all chunks at once (528KB, 9 chunks); the standby reads a few (1 or 5 of 9 in two runs), its endpoint is unblocked after the last processed chunk, and then no further input event arrives on that connection for 3s+ although the primary has more chunks queued -- --force-snapshot soak's single post-convergence check saw no install (2 of 4 runs, 2026-09-27)", "file": "src/lib/netio.c (endpoint_block_tasks/unblock, handle_input), src/controller/ha.c ha_handle_snapshot_chunk", "root_cause": "Not root-caused. Suspect: edge-triggered epoll input + per-endpoint task blocking (netio endpoint_block_tasks): data that arrives while the endpoint is blocked is not re-signalled after endpoint_unblock_tasks, so reading resumes only on the next write from the peer (e.g. the next heartbeat). Likely delays, not loses, the transfer; the soak check was changed to poll 30s.", - "fix": "", + "fix": "_Not recorded (migrated from buglog.json)._", "tags": [ "netio", "epoll", @@ -28220,8 +15857,10 @@ "related_bugs": [ "bug-1870" ], - "occurrences": 1, - "last_seen": "2026-09-27T07:03:48Z" + "fix_commits": [], + "record": ".wolf/bugs/bug-1871.md", + "gitea_issue": 297, + "gitea_synced_status": "investigating" } ] } diff --git a/.wolf/bugs/bug-006.md b/.wolf/bugs/bug-006.md new file mode 100644 index 00000000..2bad8d1e --- /dev/null +++ b/.wolf/bugs/bug-006.md @@ -0,0 +1,38 @@ +--- +id: "bug-006" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-05-21T08:30:20.719Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c"] +tags: ["md", "double-free", "asan", "ha_connect_to_peer", "endpoint_fini", "integration-test"] +related: [] +legacy: true +--- + +# bug-006: AddressSanitizer: attempting double-free on sctx in ha_connect_to_peer + +## Description +AddressSanitizer: attempting double-free on sctx in ha_connect_to_peer + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +endpoint_fini calls FREEUP(ep->data) which frees sctx, then ha_connect_to_peer also calls free(sctx) — double-free at lines 3594-3595 and 3600-3601 + +## Fix +Set ep->data = NULL before endpoint_fini so FREEUP becomes a no-op; then free(sctx) explicitly in the error paths + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-007.md b/.wolf/bugs/bug-007.md new file mode 100644 index 00000000..ecb63996 --- /dev/null +++ b/.wolf/bugs/bug-007.md @@ -0,0 +1,38 @@ +--- +id: "bug-007" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-05-21T12:53:13.789Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c"] +tags: ["guard-clause", "c", "heap-use-after-free", "asan", "liveness_poll_fn", "ha_arm_heartbeat_timeout_vtask", "race-condition", "integration-test"] +related: [] +legacy: true +--- + +# bug-007: AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 — reading ctx->cancelled… + +## Description +AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 — reading ctx->cancelled after ha_arm_heartbeat_timeout_vtask freed the old ctx + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_arm_heartbeat_timeout_vtask freed the old liveness_poll_ctx_t immediately after setting cancelled=1, but the delaymgr task was already in flight reading ctx->cancelled=0 and then continued to read ctx->ha after free + +## Fix +ha_arm_heartbeat_timeout_vtask no longer frees old ctx; liveness_poll_fn now frees ctx when it fires with cancelled=1 (guarded by #ifndef XCE_TESTING to avoid bad-free on stack-allocated test ctxs) + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1013.md b/.wolf/bugs/bug-1013.md new file mode 100644 index 00000000..ab34991c --- /dev/null +++ b/.wolf/bugs/bug-1013.md @@ -0,0 +1,38 @@ +--- +id: "bug-1013" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-22T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/commit.c"] +tags: ["ha", "commit", "raft", "term-safety", "starvation", "commit-ceiling", "optA", "liveness", "catch-up"] +related: ["bug-991", "bug-661", "bug-851"] +legacy: true +--- + +# bug-1013: Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes… + +## Description +Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes — STRICT_TERM live failover gate fails 'committed_lsn>=5' + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha-term-safety task 2.4 added a min(durable-majority, raft_commit_index) ceiling in majority_candidate_locked. raft_commit_index is fed only via the library ACK path (ha_raft_on_repl_ack -> raft_advance_commit -> on_commit_advanced), which advances from match_index[] driven by the asynchronous catch-up streamer and routinely LAGS the durable majority. When delivery goes via catch-up, raft_commit_index stays low and PINS committed_lsn below the safe durable watermark (a redundant, starvation-prone gate). The durable-majority barrier was already current-term-safe under single-writer + the own-term NOOP writable gate, making the ceiling redundant. + +## Fix +Option A: removed the raft_commit_index fold from majority_candidate_locked (commit.c). The durable-majority barrier + own-term writable_lsn gate are now authoritative for commit-advance. commit_set_raft_commit_index / ce->raft_commit_index made INERT-ADVISORY (recorded for observability/conformance-oracle only; still fed by on_commit_advanced so the library half keeps a sink). Kept durable self-cap (persist_durable_lsn), persist-before-ACK, and assert(cand<=persist_durable_lsn). Verified live: commit advances 4->5 via durable barrier (commit.c:184); walcompare OK committed prefix byte-identical across 3 nodes up to LSN 5. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-991, bug-661, bug-851 diff --git a/.wolf/bugs/bug-1023.md b/.wolf/bugs/bug-1023.md new file mode 100644 index 00000000..0728450a --- /dev/null +++ b/.wolf/bugs/bug-1023.md @@ -0,0 +1,38 @@ +--- +id: "bug-1023" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-23T03:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["ha", "raft", "failover", "leader-completeness", "bug-828", "bug-795", "durability", "truncate", "election", "durable_lsn"] +related: ["bug-661", "bug-795", "bug-828", "bug-851", "bug-854", "bug-339"] +legacy: true +--- + +# bug-1023: Clean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite… + +## Description +Clean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite a standby being genuinely durable at LSN 5; walcompare reports committed-prefix divergence at LSN 2 (jobs gone on the new leader). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_enter_real_election() PRE-TRUNCATED the candidate's WAL above ha->last_known_committed_lsn BEFORE the vote. last_known_committed_lsn is the LOCALLY-LEARNED commit watermark, which LAGS the true cluster commit point (the primary commits an entry on a durable majority, then dies before a heartbeat carries that committed LSN to the standby). persist_truncate_wal_above clamps durable_lsn down to the cutoff (clamp_durable_lsn_locked). So a standby durably holding committed entries [committed+1..tip] discarded them, then advertised the stale-low cutoff as its durable floor (ha_candidate_advertise_lsn / ha_vote_self_freshness both read persist_durable_floor). Every voter doing the same collapsed safe-committed-from-votes to the stale watermark; the AUTHORITATIVE post-election truncate (raft_cb_become_leader -> persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale prefix, losing committed jobs. Live trace (DBG828) PROVED the standbys wrote+fdatasync'd LSN 2-5 (durable_lsn=5) and only LOST them at the pre-election truncate clamp — NOT a durability/ack-before-fsync violation. Final bug-828/795 slice. + +## Fix +Removed the pre-election persist_truncate_wal_above(last_known_committed_lsn) in ha_enter_real_election (src/controller/ha.c). Raft model: a candidate does NOT truncate its log before campaigning — it keeps its full durable log and advertises its real durable tip so voters holding more committed entries deny a colder candidate and the winner has every committed entry (Leader Completeness). Divergent UNCOMMITTED tails are removed only by the authoritative post-election safe-commit truncate (above the majority-held point) and by the library's per-entry (term,lsn) reconcile (on_truncate_from) during AppendEntries. RED: live STRICT_TERM harness safe=1 + walcompare divergence at LSN 2. GREEN: every run now logs safe_committed_lsn=5 (from 2 granted voters) and walcompare 'committed prefixes identical'. Unit: new test_repl_catchup_durable_floor_survives_failover (test_repl_apply.c) + 177/177 unit GREEN. NOTE: live EXIT still 1 due to a PRE-EXISTING libxce client-failover bug (bug: 'client op fails to reach live primary when a DEAD controller is FIRST in the masters list') surfacing now that the durability bug no longer masks the post-walcompare expect-job/wait-primary step — committed jobs are physically present on disk (walcompare proves it). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-661, bug-795, bug-828, bug-851, bug-854, bug-339 diff --git a/.wolf/bugs/bug-1028.md b/.wolf/bugs/bug-1028.md new file mode 100644 index 00000000..65af3aeb --- /dev/null +++ b/.wolf/bugs/bug-1028.md @@ -0,0 +1,38 @@ +--- +id: "bug-1028" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-23" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c (ha_dispatch_repl_action_for_standby / catch-up streamer on the NEW term)", "config_repl.c (ha_stream_wal_to_standby)", "persist.c (standby apply) -- NOT data loss; committed prefix is byte-identical on disk (walcompare OK)."] +tags: ["failover", "availability", "liveness", "catch-up", "new-primary", "not-serving", "ha-commit-durability", "NOT-FIXED", "FIXED", "catch-up", "ha_stream_wal_to_standby", "persist_entry_term", "logOk"] +related: ["bug-828", "bug-854", "bug-795", "bug-991"] +legacy: true +--- + +# bug-1028: Post-failover AVAILABILITY: after a clean failover the new primary never becomes available to… + +## Description +Post-failover AVAILABILITY: after a clean failover the new primary never becomes available to serve clients (wait-primary times out). New primary logs 'ha_dispatch_repl_action_for_standby: standby[0] lsn=5 < committed_lsn=6 -> stream WAL batches' every ~3s indefinitely; the lagging standby never advances to the new primary's own-term tip, and the client cannot reach a serving primary. Surfaced only AFTER bug-828 was fixed (the committed-job-loss previously masked this by failing the harness earlier at walcompare). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_stream_wal_to_standby (config_repl.c) stamped each streamed entry's term and the AppendEntries prev_log_term from raw on-disk hdr.term (the WAL header), but the standby's logOk (raft_on_append_entries_request) and the library's own send path both source terms from persist_entry_term (the authoritative in-memory per-LSN term index), which maps a legacy-0 entry to WAL_LEGACY_TERM(1). So for a legacy-0 predecessor entry (raw hdr.term==0), the streamer sent prev_log_term=0 while the standby computed its own term as 1 → logOk mismatch → the new-term NOOP (the post-promotion first entry) is perpetually rejected → no WalAck → applied_lsn stuck at pre-promotion value → the heartbeat state report advertises the stale lsn → the primary re-streams [lsn+1..committed] every heartbeat cycle. The committed prefix stays byte-identical because only term metadata diverges, not data. CONFIRMED: baseline walcompare shows committed-prefix DIVERGENCE (term mismatch → CRC-chain breaks on some members); with the fix the committed prefix is IDENTICAL. + +## Fix +FIXED (2026-06-23). ha_stream_wal_to_standby gained a const persist_t* p parameter (decl in config_repl.h). All three hdr.term source sites in the streamer body (entry->term, prev_streamed_term on the main path, prev_streamed_term on the OOM-skip path) replaced with p ? persist_entry_term(p, hdr.lsn) : hdr.term (NULL-p path for count-only unit tests). Callers (ha_dispatch_repl_action_for_standby in ha.c, catchup_stream_range in config_repl.c) pass &c->persist. Unit test (test_wal_stream_term_from_index_not_raw_hdr) legacy-appends LSN 1 at term 0 + LSN 2 at term 3; streams standby_lsn=1,committed=2; asserts streamed entry term==3 and prev_log_term==WAL_LEGACY_TERM(1). RED→GREEN proven (Expected 1 Was 0 → PASS). Live: standby converge + committed prefix identical. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-828, bug-854, bug-795, bug-991 diff --git a/.wolf/bugs/bug-1060.md b/.wolf/bugs/bug-1060.md new file mode 100644 index 00000000..7d62968c --- /dev/null +++ b/.wolf/bugs/bug-1060.md @@ -0,0 +1,38 @@ +--- +id: "bug-1060" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-06-24T12:05:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (raft_cb_become_leader / safe_committed_lsn)", "src/controller/cluster.c (OPCode_preVoteRequest)"] +tags: ["prevote", "deadlock", "safe_committed_lsn", "truncation", "leader-completeness", "data-loss", "coverage-maneuver", "FIXED"] +related: ["bug-795", "bug-828"] +legacy: true +--- + +# bug-1060: After soak test coverage maneuver, ctrl-1 and ctrl-2 enter permanent prevote deadlock: ctrl-1… + +## Description +After soak test coverage maneuver, ctrl-1 and ctrl-2 enter permanent prevote deadlock: ctrl-1 denies ctrl-2 (freshness: LSN=45 < floor=47), ctrl-2 denies ctrl-1 (log_stale: cand_llt=12 < local_last_log_term=13). Neither can ever win. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ctrl-3 computed safe_committed_lsn=44 from voter peerLastLogLsn (ctrl-3=47, ctrl-2=44 → majority at 44) and truncated committed entries 45-47 (term=12, acked by ctrl-3 itself as standby under ctrl-1). ctrl-3 then wrote leadership NOP at LSN=45 (term=13) and replicated it to ctrl-2. Result: ctrl-1 has LSN=45=JOB_SUBMIT/term=12, ctrl-2 has LSN=45=NOOP/term=13 — divergent WALs. The safe_committed_lsn formula ignores the candidate's own durable knowledge of committed entries, violating Leader Completeness: the new leader must never truncate entries it has that were committed by a prior leader. + +## Fix +IMPLEMENTED (2026-06-24): added pure helper ha_promotion_truncate_floor(safe_lsn, durable_tip)=max(safe_lsn,durable_tip) in ha.c/ha.h; raft_cb_become_leader now truncates at max(safe_committed_lsn, persist_durable_floor(&c->persist)) instead of bare safe_committed_lsn. Since the cutoff is never below the leader's own durable tip, persist_truncate_wal_above is a no-op on the leader's durable log — the new leader NEVER truncates entries it holds (Raft §5.4 Leader Completeness). safe_committed_lsn is still the commit lower-bound and promotion-gate HWM (NOT clamped — clamping it would defeat the behind-node fence). The leadership NOP now appends at the true tip (current_lsn) so replication converges peers to the leader's WAL with zero job loss. Unit test test_promotion_never_truncates_own_committed_entries in test_ha_election.c (GREEN). Steering src/controller/CLAUDE.md updated (Leader Completeness section + Post-Election Safe Commit Point + verification checklist). Recovery of an already-diverged cluster still requires restarting the over-truncated node (loses the cut jobs); this fix prevents the divergence. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-795, bug-828 diff --git a/.wolf/bugs/bug-1067.md b/.wolf/bugs/bug-1067.md new file mode 100644 index 00000000..c870d724 --- /dev/null +++ b/.wolf/bugs/bug-1067.md @@ -0,0 +1,38 @@ +--- +id: "bug-1067" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-24T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c", "src/controller/ha.c"] +tags: ["catch-up", "replication", "raft", "ae_reject_stale", "ae_reject_logmismatch", "walCatchupEntry", "cross-term", "FIXED"] +related: ["bug-1028", "bug-991"] +legacy: true +--- + +# bug-1067: Cross-term WAL catch-up permanently stalls: ctrl-2 stuck at lsn=8, all walCatchupEntry rejected… + +## Description +Cross-term WAL catch-up permanently stalls: ctrl-2 stuck at lsn=8, all walCatchupEntry rejected (ae_reject_stale + ae_reject_logmismatch) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +AE term conflation: entry->term is the WAL entry's stored term (e.g. 4 for entries written by ctrl-1 at term=4), but ha_consume_append_entries uses it as the AE header ae_term. For live appends this is correct (leader appends at current term), but for catch-up streaming of historical entries from prior terms, ae_term = entry_term < leader.current_term. The Raft AE stale-term check (if term < s->term → ae_reject_stale) correctly rejects, but it should never fire — the AE header term must always be the leader's current term. There is no separate leader_term field in wal_catchup_entry_t. Consequence: historical entries at entry_term < raft.term are permanently rejected; standby_reported_lsn never advances; streamer retransmits forever. + +## Fix +IMPLEMENTED (2026-06-24): separated the AppendEntries HEADER term (leader's CURRENT term) from the per-entry STORED term. Added wire field WalCatchupEntry.leaderCurrentTerm @8 (controller.capnp) + C field wal_catchup_entry_t.leader_current_term (typemap.h, hand-defined struct — capnp generates encode/decode against it). Encoder (request.c wal_entry_to_wire + top container) writes it; decode auto-populates (generated, scalar mapname field). The catch-up streamer ha_stream_wal_to_standby gained a leader_term param (callers pass atomic_load(&c->ha.term)) and stamps entry->leader_current_term=leader_term on every streamed entry. Receiver ha_consume_append_entries now uses ae_term = e->leader_current_term ? e->leader_current_term : entry_term (single-entry branch) — the leader term rides on the sub-entry e by reference through ha_apply_term_entry_via_library, so no extra threading needed. entry->term (@4) remains the per-entry stored term for logOk/on_persist_entry. 0 = unset (legacy/live sender) → falls back to stored term (live appends have entry_term==leader_term, so unchanged). repl.c live paths UNTOUCHED. Tests (test_term_safety_adapter.c): test_intake_crossterm_catchup_uses_leader_current_term (historical entry@term4 streamed by leader@term5 to follower@term5 is ACCEPTED, not stale-rejected) + test_intake_stale_leader_term_still_rejected (leader_current_term=3 < follower term 5 still REJECTED — safety preserved). GREEN: 185 PASS term-safety+config-repl, 6 PASS wal_wire wire round-trip, 80 PASS failover/repl/durability, 0 failures. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1028, bug-991 diff --git a/.wolf/bugs/bug-1077.md b/.wolf/bugs/bug-1077.md new file mode 100644 index 00000000..b4c6bc6e --- /dev/null +++ b/.wolf/bugs/bug-1077.md @@ -0,0 +1,38 @@ +--- +id: "bug-1077" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-06-24T17:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/xcectx.c"] +tags: ["asan", "use-after-return", "union-type-confusion", "ha", "failover", "ctrlstatus", "libxce", "memory-safety", "regression-61e1391", "VERIFIED"] +related: ["bug-646", "bug-1060", "bug-1067"] +legacy: true +--- + +# bug-1077: AddressSanitizer: SEGV / stack-use-after-return in free_CtrlStatus (lib/controller.capnp.c:11638)… + +## Description +AddressSanitizer: SEGV / stack-use-after-return in free_CtrlStatus (lib/controller.capnp.c:11638) via handle_reply -> free_CtrlStatus_ptr (lib/xcectx.c:91) on `ctrlstatus` after failover; READ derefs the dead-stack `getreply` of xce_get_with_condition (xceapi.c) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ctx->reply.u is an UNTAGGED union shared across opcodes (debugmsg/get/jobid/clusterconf/ctrlstatus alias the same 8 bytes). Commit 61e1391 ('fix bug-1060 bug-1067', unrelated Raft work) added speculative free-guards `if (ctx->reply.u.ctrlstatus) free_CtrlStatus_ptr(&ctx->reply.u.ctrlstatus)` (and the clusterconf twin). A non-NULL slot does NOT imply a CtrlStatus*: a failover-aborted reslist/joblist leaves ctx->reply.u.get == &getreply (a dead STACK pointer) because xce_get_with_condition/xce_resval only NULL the slot on the success path, not the xce_get_req<0 / cbuf_get<0 early-returns. The next ctrlstatus reply (worker thread) sees the foreign non-NULL slot and free_CtrlStatus walks it -> stack-use-after-return SEGV. + +## Fix +1) Revert the two unsafe free-guards in handle_reply (untagged union slot can never be safely freed by type) — restores prior MOVE_PTR-overwrite behavior. 2) Clear ctx->reply.u.get = NULL on ALL early-return paths of xce_get_with_condition and xce_resval (kill the dangling-stack-pointer leak at its source; also prevents a late OPCode_get reply writing through the dead stack via handle_get_reply). 3) Defense-in-depth: handle_get_reply drops a reply when ctx->reply.u.get is NULL. Regression test: test_handle_reply_ctrlstatus_does_not_free_foreign_slot (exposes xce_handle_reply, proves a foreign heap value in the shared union is not freed). VERIFIED 2026-06-24 (Opus): re-confirmed 3-layer fix present in working tree; RED proven by re-injecting the ctrlstatus free-guard -> ASan double-free at test_ctrlstatus.c:384 (free_CtrlStatus_ptr at controller.capnp.c:11892), then restored -> GREEN. Full libxce suite 291 Tests / 0 Failures / 0 ASan errors under ASAN_OPTIONS=detect_leaks=0. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-646, bug-1060, bug-1067 diff --git a/.wolf/bugs/bug-1095.md b/.wolf/bugs/bug-1095.md new file mode 100644 index 00000000..9e8ec986 --- /dev/null +++ b/.wolf/bugs/bug-1095.md @@ -0,0 +1,42 @@ +--- +id: "bug-1095" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-24" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/lib/raft/raft.c"] +tags: ["raft", "vote-reply", "idempotency", "election-safety", "split-brain", "conformance", "needs-library-fix", "bug-851-family", "dedup", "stale-buglog-entry"] +related: ["bug-1096", "bug-851"] +fix_commits: ["b5b7e01"] +legacy: true +gitea_issue: 29 +gitea_synced_status: "fixed" +--- + +# bug-1095: Election Safety violation (two leaders at same term) surfaced by raft-sim-conformance step driver… + +## Description +Election Safety violation (two leaders at same term) surfaced by raft-sim-conformance step driver (task 4.2): a duplicated VOTE_RSP grant double-counts a single peer's vote + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +raft_on_vote_reply tallies grants with a COUNTER (s->votes_granted++) and explicitly ignores from_idx ((void)from_idx;), so DuplicateMessage of one peer's grant counts that peer twice and a candidate reaches quorum with a single real vote. Diverges from raft.tla where votesGranted is a SET (idempotent). Proven: 3-node, single peer grant duplicated -> candidate becomes LEADER with votes_granted=2. + +## Fix +FIXED (verified 2026-07-18 by validate-impl for raft-sim-conformance): src/lib/raft/raft.c raft_on_vote_reply now has a per-peer vote_replies[from_idx] idempotency guard (dedupes a DuplicateMessage grant) plus a term < s->term DropStaleResponse guard, landed together with bug-1096 in commit b5b7e01 (fix(raft): vote-reply per-peer dedup + stale-term drop). This buglog entry was stale (said NOT FIXED) even though the fix has been live since 2026-06-24 per tasks.md raft-sim-conformance Implementation Notes and bug-1096.fix; corrected here for consistency. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1096, bug-851 +- Gitea: #29 diff --git a/.wolf/bugs/bug-1096.md b/.wolf/bugs/bug-1096.md new file mode 100644 index 00000000..dc21e4f7 --- /dev/null +++ b/.wolf/bugs/bug-1096.md @@ -0,0 +1,38 @@ +--- +id: "bug-1096" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-24" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/raft/raft.c"] +tags: ["raft", "vote-reply", "stale-term", "election-safety", "split-brain", "conformance", "needs-library-fix", "stale-report", "bug-851-family", "stale-response"] +related: ["bug-1095", "bug-851"] +legacy: true +--- + +# bug-1096: raft_on_vote_reply counts a stale-term vote-reply (term < currentTerm) toward the current election… + +## Description +raft_on_vote_reply counts a stale-term vote-reply (term < currentTerm) toward the current election instead of dropping it + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +No DropStaleResponse: a reply with term < s->term fell through to replies_received++/votes_granted++, corrupting quorum accounting (raft.tla DropStaleResponse discards mterm < currentTerm). + +## Fix +FIXED (verified 2026-07-02): src/lib/raft/raft.c:832 has `if (reply_term != s->term)` guard before tallying, with an on_trace("vote_reply_stale_drop", ...) at raft.c:455 — exactly the fix this bug's own second entry described. Landed under raft-sim-conformance / raft-commit-safety (both implemented, 18/18 + 23/23 tasks). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1095, bug-851 diff --git a/.wolf/bugs/bug-1112.md b/.wolf/bugs/bug-1112.md new file mode 100644 index 00000000..3acc4b65 --- /dev/null +++ b/.wolf/bugs/bug-1112.md @@ -0,0 +1,39 @@ +--- +id: "bug-1112" +status: "fixed" +status_updated: "2026-09-24" +status_note: "b1f289d2" +component: "lib" +severity: "unknown" +timestamp: "2026-06-24T15:15:03.882281Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/raft/raft_sim.c"] +tags: ["raft-sim-conformance", "link-error", "task-5.3", "undefined-reference", "pre-existing"] +related: [] +fix_commits: ["b1f289d2"] +legacy: true +--- + +# bug-1112: ld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 — lib test… + +## Description +ld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 — lib test binary fails to link + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Commit e445404 (task 5.3 drop/dup/reorder) added the raft.h declaration + the test_sim_5_3 test but FORGOT to add the raft_sim_scenario_drop_dup_reorder implementation to raft_sim.c. Symbol is declared+referenced but never defined, breaking the link for the whole xcelib_test binary. + +## Fix +Pre-existing, outside task 5.4 boundary. Surfaced while validating task 5.4 (figure8). Needs task 5.3 implementation added to raft_sim.c. NOT fixed by 5.4. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1128.md b/.wolf/bugs/bug-1128.md new file mode 100644 index 00000000..3030a6fa --- /dev/null +++ b/.wolf/bugs/bug-1128.md @@ -0,0 +1,38 @@ +--- +id: "bug-1128" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-25T03:20:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/testsuite/test_demotion_convergence.c"] +tags: ["test-flake", "demotion-convergence", "persist", "overlayfs", "ha-convergence-recovery", "quiescence", "wal-truncate", "sandbox", "commit-engine-race", "deterministic", "c"] +related: ["bug-343", "bug-704", "bug-795", "bug-828", "bug-9799", "bug-1060"] +legacy: true +--- + +# bug-1128: test_demotion_replay_yields_committed_only flakes ~16% on overlayfs: boot recovery over surviving… + +## Description +test_demotion_replay_yields_committed_only flakes ~16% on overlayfs: boot recovery over surviving WAL must yield the committed prefix ONLY + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +TWO compounding non-determinism sources, both fixture artifacts not product bugs: (1) overlayfs metadata incoherence — a freshly-opened persist_t fd over a just-ftruncate'd WAL intermittently reads the STALE pre-truncate size, so the cross-context fresh recovery replays the already-removed suffix; (2) DEEPER, fs-independent: the fixture wires a LIVE commit engine and on a single node (n_peers=1) the persist durable-advance callback (fsync worker -> on_durable_advance) commits the whole durable log, so persist_flush asynchronously raced committed_lsn UP from the test-set committed_boundary (5) to the appended tip (8) AFTER the test pinned it; the demotion seam reads committed_lsn at task time -> truncated to 8 not 5 -> current_lsn=9 > committed_boundary+1. DIAG proof: committed_boundary=5 live_committed=8 live_tip=9 on failing runs. + +## Fix +Test-only, two parts. (a) Detach the durable-advance commit wake (g_dc.persist.on_durable_advance = NULL) and re-pin committed_lsn = committed_boundary right before driving the demotion task, killing the commit-engine race deterministically (helps BOTH fs paths). (b) Hybrid gate by fs type via static dc_path_is_overlayfs() (statfs f_type == OVERLAYFS_SUPER_MAGIC 0x794c7630, defaults to real-fs path on statfs error): on a real fs keep the strong cross-context fresh boot-recovery assertion (fresh->current_lsn <= committed_boundary+1); on overlayfs skip that fd-incoherent step and assert the LIVE-context truncate effect (g_dc.persist.current_lsn <= committed_boundary+1, coherent under p->lock). Both branches prove the same property. Production persist.c truncate (ftruncate+fdatasync) untouched. + +## Verification +Full controller suite 20/20 runs: 0 demotion_replay failures, 0 total failures. Group-only 60/60: 0 failures. Baseline (pre-fix) was 4/20. /tmp confirmed overlayfs (statfs magic 0x794c7630), so the overlayfs branch is the one exercised here. + +## References +- Related: bug-343, bug-704, bug-795, bug-828, bug-9799, bug-1060 diff --git a/.wolf/bugs/bug-1147.md b/.wolf/bugs/bug-1147.md new file mode 100644 index 00000000..309122b4 --- /dev/null +++ b/.wolf/bugs/bug-1147.md @@ -0,0 +1,38 @@ +--- +id: "bug-1147" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-26T03:46:38.904419+00:00" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c:1374 (trim_wal_segments)"] +tags: ["durability", "permanent-gap", "stale-report", "trim", "bug-828", "raft-drive-async", "latent", "catch-up", "wal", "snapshot", "bug-795"] +related: ["bug-795", "bug-828", "bug-1060"] +legacy: true +--- + +# bug-1147: WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor →… + +## Description +WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor → lagging/empty-rejoin standby loses its needed log range with no snapshot backfill (permanent committed-prefix gap) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +trim_wal_segments bounds deletion ONLY by active_seg_lsn (the currently-open segment): `if (seg_start_lsn < active_seg_lsn) unlink()`. It consults no standby acked_lsn, no committed_lsn, no in-flight snapshot boundary. Strategy-A (cursor-bounded retention) discipline is absent. Strategy-B (entity-bearing snapshot install) is also not live — write_snapshot (persist.c:1247) writes only a 16-byte P0 sentinel, not materialized state. So a standby that lags by more than ~one open segment (down across segment rolls, slow link, >64MB lag, or empty rejoin from cursor=0) hits a log range that exists nowhere and cannot catch up. + +## Fix +FIXED (verified 2026-07-02): src/controller/persist.c trim_wal_segments() computes `floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s)` (min of active segment, all standby acked_lsn, disconnected-slot guards) and only unlinks segments below that floor — comment inline explicitly cites 'bug-1147 guard'. Landed under raft-drive-async task 3.2 (all leaf tasks complete). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-795, bug-828, bug-1060 diff --git a/.wolf/bugs/bug-1197.md b/.wolf/bugs/bug-1197.md new file mode 100644 index 00000000..89db534b --- /dev/null +++ b/.wolf/bugs/bug-1197.md @@ -0,0 +1,39 @@ +--- +id: "bug-1197" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-26T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/ha.c"] +tags: ["endpoint-quiescence", "c", "dispatch_inflight", "task-3.4a", "tearing_down", "wrong-reference", "raft-drive-async", "task-3.1", "use-after-free", "flag-on-hazard", "deferred", "review-finding"] +related: ["bug-1198"] +fix_commits: ["6e05dc1"] +legacy: true +--- + +# bug-1197: raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint… + +## Description +raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint UAF — raft_cb_become_follower runs on a pool worker and endpoint_fini()s all peer vote-eps (frees ep->data) while the single io_worker can concurrently read ep_j->data in peer_ctrl_msg_handler_inline for a same-batch queued event + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Moving peer-control off the sequential io_worker onto the pool co-locates teardown (frees ep->data) and inbound delivery (reads ep->data) on different threads with no unmonitor-before-free ordering. Safe today only because g_drive_async_peerctrl_dispatch_enabled defaults OFF. + +## Fix +SUPERSEDED 2026-06-30 by bug-1298 endpoint reference counting (commit 6e05dc1). The pool-reachable teardown-frees-peer-ep UAF class is now governed generally by endpoint_ref/endpoint_unref (refcnt + heap_alloc gate) acquired per ep-bound VTASK_IO in vtask_enq and released in run_task/taskq_item_killer — a queued vtask holds a ref so the io_worker cannot free the ep out from under the dispatcher. Re-validate when raft-drive-async flips g_drive_async_peerctrl_dispatch ON. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1198 diff --git a/.wolf/bugs/bug-1198.md b/.wolf/bugs/bug-1198.md new file mode 100644 index 00000000..2a8e2b9b --- /dev/null +++ b/.wolf/bugs/bug-1198.md @@ -0,0 +1,38 @@ +--- +id: "bug-1198" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-06-26T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["task-3.1", "review-finding", "stale-report", "raft-drive-async", "deferred", "flag-on-hazard", "deadlock-freedom", "liveness-admission"] +related: ["bug-1197"] +legacy: true +--- + +# bug-1198: raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work… + +## Description +raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work VTASK_LIVENESS, but the CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT->persist_checkpoint (blocking fdatasync) inline — a blocking op under the never-gated liveness reservation breaks deadlock-freedom + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl_ack_decode_and_dispatch -> ha_dispatch_repl_action_for_standby can run the snapshot send inline. LIVENESS is only correct once that long op is offloaded off the pool. + +## Fix +FIXED/CLOSED (verified 2026-07-02): src/controller/ha.c:5647 carries an explicit in-code 'CLOSED' banner comment — the bug-1197 endpoint_ref/endpoint_unref bracket around the lane dispatch means the repl-ack path's snapshot send is ALWAYS offloaded before peer_ctrl_classify can tag it VTASK_LIVENESS, so bug-1198 cannot recur. Both flags (peerctrl dispatch + snapshot offload) default ON since bug-1376 was fixed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1197 diff --git a/.wolf/bugs/bug-1199.md b/.wolf/bugs/bug-1199.md new file mode 100644 index 00000000..630823f2 --- /dev/null +++ b/.wolf/bugs/bug-1199.md @@ -0,0 +1,38 @@ +--- +id: "bug-1199" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "leak" +timestamp: "2026-06-26T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/vtaskmgr.c"] +tags: ["raft-drive-async", "task-3.1", "memory-leak", "vtaskmgr", "packet-ownership", "review-finding"] +related: ["bug-1077"] +legacy: true +--- + +# bug-1199: raft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its… + +## Description +raft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its io_packet_t (only the executed entry transfers ownership via iod->pkt=NULL) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +taskq_item_killer called vtask_fini (frees task->data wrapper only), never the carried io_packet_t. The generic killer could not free the pkt unconditionally because the libxce client path (xcectx.c vtask_handle_msg) does NOT clear ep->fsm_in.pkt — its pkt lifetime is the endpoint's, so an unconditional free would double-free. + +## Fix +Added int owns_io_pkt to vtask_t (default 0 in vtask_init). taskq_item_killer frees the pkt ONLY when owns_io_pkt && vtype==VTASK_IO && pkt!=NULL. Controller peer_ctrl_dispatch_to_pool sets owns_io_pkt=1. libxce path keeps owns_io_pkt=0 -> unchanged, no double-free. Verified: leak count 808 unchanged vs baseline, 0 double-free under ASan. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1077 diff --git a/.wolf/bugs/bug-1210.md b/.wolf/bugs/bug-1210.md new file mode 100644 index 00000000..f9a1ac52 --- /dev/null +++ b/.wolf/bugs/bug-1210.md @@ -0,0 +1,38 @@ +--- +id: "bug-1210" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-26" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c", "src/controller/ha.h"] +tags: ["raft-drive-async", "task-3.2", "snapshot", "slow-lane", "UAF", "feature-flag", "bug-1197", "bug-1198", "liveness"] +related: ["bug-1197", "bug-1198", "bug-1147"] +legacy: true +--- + +# bug-1210: raft-drive-async task 3.2 snapshot slow-lane offload flag g_drive_async_snapshot_offload defaulted… + +## Description +raft-drive-async task 3.2 snapshot slow-lane offload flag g_drive_async_snapshot_offload defaulted ON (=1); header doc also said default ON + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Snapshot offload borrows a standby endpoint shared with the io_worker epoll loop (not owned, unlike reconnect_lane_fn). With the flag ON the send runs on a slow-lane thread while io_worker can RDHUP-free that endpoint => cross-thread UAF (bug-1197 family). Defaulting ON would ship that UAF; the borrowed-endpoint lifetime discipline is not established until task 3.4. + +## Fix +Default the flag OFF (dormant landing, same protocol as task 3.1 peerctrl dispatch). Inline path runs on io_worker, serialized with teardown -> no overlap. Rewrote both the ha.c definition comment and the ha.h accessor doc to state OFF is the production default and 3.4 is the gate that flips it ON. bug-1198 stays open by design until then. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1197, bug-1198, bug-1147 diff --git a/.wolf/bugs/bug-1224.md b/.wolf/bugs/bug-1224.md new file mode 100644 index 00000000..ad7bbcae --- /dev/null +++ b/.wolf/bugs/bug-1224.md @@ -0,0 +1,38 @@ +--- +id: "bug-1224" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-06-26" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_ha_dispatch_integration.c"] +tags: ["raft-drive-async", "task-3.4b", "test", "flaky", "vtaskmgr_fini", "pthread_cancel", "teardown-race", "deadlock"] +related: ["bug-924"] +legacy: true +--- + +# bug-1224: raft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently… + +## Description +raft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently — controller_unit_test never completes (timeout-killed) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Test teardown race: P1/P2/P3 released a parked vtask body (gate_open) then immediately called vtaskmgr_fini, which tears down workers via pthread_cancel. The tests polled a run-counter incremented MID-body, so fini raced the just-released body still unwinding (esp. P2 REQ_BLOCK bodies in vtaskmgr_blocking_release racing blk_lock teardown) → flaky deadlock. slowlane_fini is graceful (join) so the lane side was safe; vtaskmgr_fini (cancel) was the hazard. + +## Fix +Add end-of-body completion counters (g_pc_completed on both pc_probe paths incl. the park path; g_block_done at the end of block_task_entry) and poll them to the full expected count AFTER gate_open and BEFORE vtaskmgr_fini, so every pool body has fully returned (worker idle in cbuf_get) before the cancel-based fini. Verified deterministic 5/5 isolated runs + full suite EXIT=0, ASan-clean. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-924 diff --git a/.wolf/bugs/bug-1225.md b/.wolf/bugs/bug-1225.md new file mode 100644 index 00000000..a8414fff --- /dev/null +++ b/.wolf/bugs/bug-1225.md @@ -0,0 +1,38 @@ +--- +id: "bug-1225" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-06-26" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".claude/skills/xce-test-ha/run-ha-soak.sh (wait_for_full_convergence)"] +tags: ["ASAN", "baseline-confirmed", "bug-851", "false-negative", "not-a-regression", "raft-drive-async", "reconverge", "soak", "task-4.2", "xce-test-ha"] +related: ["bug-1042", "bug-924"] +legacy: true +--- + +# bug-1225: /xce-test-ha live failover soak (ASAN build, real fs ~/tmp): FAIL round 4 "cluster did not… + +## Description +/xce-test-ha live failover soak (ASAN build, real fs ~/tmp): FAIL round 4 "cluster did not reconverge to 3-of-3 after restarting ctrl-2" + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Pre-existing HA-core instability under 6-round CHAOTIC repeated failover, NOT a raft-drive-async regression. BASELINE-CONFIRMED: the same soak against pre-Phase-3 binaries (commit 21e26a1, release/non-ASAN) ALSO fails at round 4 (baseline symptom: no-new-primary election stall; current symptom: round-4 reconverge + committed-prefix divergence at LSN 9). Both builds preserve committed JOBS (zero job loss in all completed rounds). The committed-prefix divergence is the documented open bug-851/795 family (chaotic-failover silent divergence; separate HA-core spec). raft-drive-async Phase 3 flags are default-OFF (inert); the live trim-guard only preserves more WAL. + +## Fix +Not a raft-drive-async bug (baseline-confirmed). 4.2 regression sweep PASSES for this spec: no NEW failure introduced. Pre-existing bug-851 chaotic-failover divergence + bug-1042 reconverge flakiness remain open for a separate HA-core effort. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1042, bug-924 diff --git a/.wolf/bugs/bug-1272.md b/.wolf/bugs/bug-1272.md new file mode 100644 index 00000000..f994158f --- /dev/null +++ b/.wolf/bugs/bug-1272.md @@ -0,0 +1,39 @@ +--- +id: "bug-1272" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-30T02:25:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c (election/PreVote path)"] +tags: ["prevote", "ha", "liveness", "chaotic-soak", "environment", "election", "bug-924", "corrected", "7.2", "fdatasync", "raft", "5.3", "in_catch_up"] +related: ["bug-924", "bug-851", "bug-1042", "bug-1060"] +fix_commits: ["bf88681"] +legacy: true +--- + +# bug-1272: raft-ha-core-safety 7.2 chaotic soak: round-4 'NO new primary within window (possible election… + +## Description +raft-ha-core-safety 7.2 chaotic soak: round-4 'NO new primary within window (possible election stall)' after 3 overlapping kills + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +After 3 overlapping kills the only fully-up-to-date node is killed; the two survivors are both restarted-and-lagging (tips 15 and 11; leader was 19). Both hold the full committed prefix [1..11] (safe candidates) but their PreVote rounds repeatedly time out with no grant quorum (prospective terms frozen, never bumped). Distinct from the 5.2 spurious-failover flap (that is fixed and confirmed). Contributors: freshness mutual-denial between lagging survivors, in_catch_up-denies-all-votes (Req 2.5) with no live primary to catch up from, and/or peer vote connection not established between two nodes that only held a control channel to the dead primary. + +## Fix +FIXED at unit level by task 5.3 (commit bf88681): gate the in_catch_up PreVote deny in raft_cb_vote_eligible on a LIVE primary (primary_ctrl_fd>=0 && ha_heartbeat_is_fresh), so a catching-up survivor with a dead primary can grant/win a fresh PreVote (logOk preserves Leader Completeness). CORRECTION: this in_catch_up deadlock is a real latent defect (unit RED→GREEN) but is NOT the soak's round-4 blocker. The soak stall is bug-924 (WAL fdatasync→EINVAL wedges a survivor so it stops answering PreVotes; presents as an election stall). 7.2 live acceptance is blocked on that env defect (out of scope; owned by ha-commit-durability), not on consensus. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-924, bug-851, bug-1042, bug-1060 diff --git a/.wolf/bugs/bug-1281.md b/.wolf/bugs/bug-1281.md new file mode 100644 index 00000000..5f9a1fac --- /dev/null +++ b/.wolf/bugs/bug-1281.md @@ -0,0 +1,38 @@ +--- +id: "bug-1281" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "crash" +timestamp: "2026-06-30T03:25:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller (restart/recovery/election path — exact site unknown", "no backtrace)"] +tags: ["7.2", "iomgr", "gdb", "election", "crash", "5.4", "restart", "recovery", "heisenbug", "accept-hang", "restart-stability", "segfault", "chaotic-soak"] +related: ["bug-1272", "bug-1282", "bug-924"] +legacy: true +--- + +# bug-1281: Restart-stability bug: a restarted xce_controller becomes unable to participate at chaotic round… + +## Description +Restart-stability bug: a restarted xce_controller becomes unable to participate at chaotic round 4. Release build SIGSEGVs ('Segmentation fault (core dumped) --myname ctrl-3'); under gdb the SAME corruption manifests as a WEDGED inbound-accept/iomgr path (last accept 03:42:03 while the election timer keeps firing) so two restarted survivors can't exchange PreVotes -> permanent round-N election stall. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED via gdb SIGQUIT all-thread bt: an ABBA lock-order deadlock in src/lib async dispatch. io_worker (netio.c:672) holds ep->lock then blocks on q->lock (dataq_enq); vdispatcher dataq_deq (dataq.c:65) holds q->lock then blocks on ep->lock via the vtask_eligible->endpoint_is_task_blocked (netio.c:1001) match callback. io_worker dies -> node stops accepting/answering PreVotes -> round-4 stall. In release the same ep/q race corrupts -> SIGSEGV. See bug-1282 (the actual dispatch defect). + +## Fix +The crash/hang IS bug-1282 (UPSTREAM raft-drive-async dispatch deadlock). raft-ha-core-safety 7.2 stays blocked until bug-1282 is fixed upstream, then re-run the gate. No raft-ha-core-safety code change. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1272, bug-1282, bug-924 diff --git a/.wolf/bugs/bug-1282.md b/.wolf/bugs/bug-1282.md new file mode 100644 index 00000000..7ecac02e --- /dev/null +++ b/.wolf/bugs/bug-1282.md @@ -0,0 +1,39 @@ +--- +id: "bug-1282" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-06-30T04:17:08.329Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/lib/dataq.c:65 (dataq_deq holds q->lock across match cb) + src/lib/vtaskmgr.c:255 (vtask_eligible) + src/lib/netio.c:1001 (endpoint_is_task_blocked takes ep->lock) + src/lib/netio.c:672 (io_worker holds ep->lock across vtask_enq->dataq_enq)"] +tags: ["deadlock", "lock-order", "abba", "vtaskmgr", "dataq", "netio", "io_worker", "vdispatcher", "raft-drive-async", "async-io", "upstream", "async-fix", "md"] +related: ["bug-1281"] +fix_commits: ["25e741d"] +legacy: true +--- + +# bug-1282: ABBA lock-order deadlock under chaotic churn: io_worker (ep->lock then dataq_enq wants q->lock) vs… + +## Description +ABBA lock-order deadlock under chaotic churn: io_worker (ep->lock then dataq_enq wants q->lock) vs vdispatcher dataq_deq (q->lock then vtask_eligible->endpoint_is_task_blocked wants ep->lock). Wedges io_worker -> controller stops accepting connections; in release the race corrupts -> SIGSEGV. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +dataq_deq runs its match callback while holding q->lock, and the callback (vtask_eligible->endpoint_is_task_blocked) acquires ep->lock — inverting the ep->lock->q->lock order io_worker uses. Two-thread ABBA deadlock, deterministic under round-4 chaotic churn. Confirmed via gdb all-thread backtrace on two independently-wedged nodes. + +## Fix +FIXED (commit 25e741d): promoted endpoint_t.busy from a 1-bit bitfield to _Atomic int; endpoint_block_tasks/endpoint_unblock_tasks/endpoint_is_task_blocked now lockless (release/acquire), removing the ep->lock acquire from the vtask-eligibility path that ran under q->lock. Verified: post-fix gdb soak has ZERO __lll_lock_wait threads + no endpoint_is_task_blocked in any lock path; controller_unit_test all green. Mirrors dispatch_inflight (bug-1197). NOTE: a SEPARATE release-only restart segfault (bug-1283) remains and still blocks 7.2. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1281 diff --git a/.wolf/bugs/bug-1286.md b/.wolf/bugs/bug-1286.md new file mode 100644 index 00000000..e1adcde9 --- /dev/null +++ b/.wolf/bugs/bug-1286.md @@ -0,0 +1,39 @@ +--- +id: "bug-1286" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-06-30T05:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/cluster.c:251 (myreq_handler) + src/controller/ha.c:3194 (ha_dispatch_gate)"] +tags: ["segfault", "null-deref", "ep-owner", "myreq_handler", "cluster.c", "ha_dispatch_gate", "restart", "7.2", "ROOT-CAUSED"] +related: ["bug-1282", "bug-1281", "bug-1197"] +fix_commits: ["5db734a"] +legacy: true +--- + +# bug-1286: xce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart — Thread received… + +## Description +xce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart — Thread received SIGSEGV at ha.c:3194 (atomic_load &ha->role) with ha=0x3ff0 (garbage). The release-only crash that blocks raft-ha-core-safety 7.2; DISTINCT from the bug-1282 deadlock (which is fixed). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NULL ep->owner. myreq_handler does `cluster_t* c = (cluster_t*)ep->owner;` then `ha_dispatch_gate(&c->ha, opcode)`. On a chaotic restart an inbound endpoint dispatches a walCatchupEntry (op 37) vtask while ep->owner==NULL, so c=NULL and &c->ha = NULL+offsetof(cluster_t,ha) = 0x3ff0 — a garbage pointer ha_dispatch_gate dereferences (atomic_load &ha->role, ha.c:3194) -> SIGSEGV. Backtrace (debug -O0 -g3 under gdb): ha_dispatch_gate(ha=0x3ff0,opcode=37) <- myreq_handler cluster.c:251 <- vtask_entry cluster.c:693 <- run_task/vcpu_entry. ep->owner is NULL because the accept/dispatch path runs a request before owner is wired (or owner cleared mid-flight during teardown). + +## Fix +FIXED 2026-06-30 (commit 5db734a). NULL-guard `c = ep->owner` in myreq_handler (cluster.c): a chaotic-restart walCatchupEntry (op 37) vtask ran with ep->owner==NULL → &c->ha==0x3ff0 deref crash at ha.c:3194. Guard returns early when owner unset. Verified: round-4 restart SIGSEGV gone; controller_unit_test green. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1282, bug-1281, bug-1197 diff --git a/.wolf/bugs/bug-1296.md b/.wolf/bugs/bug-1296.md new file mode 100644 index 00000000..7066440f --- /dev/null +++ b/.wolf/bugs/bug-1296.md @@ -0,0 +1,39 @@ +--- +id: "bug-1296" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-06-30T13:21:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["raft-ha-core-safety", "election-liveness", "prevote-cross-denial", "dueling-candidates", "task-5.6", "chaotic-soak", "round-7-stall"] +related: ["bug-1281", "bug-1282", "bug-1286", "bug-1292"] +fix_commits: ["81cb09a", "74784b2"] +legacy: true +--- + +# bug-1296: 10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP→PreVote→timeout→repeat livelock — zero… + +## Description +10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP→PreVote→timeout→repeat livelock — zero pre_vote_reply_in events (not denials, silence). ctrl-3 sent 20+ PreVote rounds with NO reply received; ctrl-1 and ctrl-2 same loop. Rounds 1-6 PASS clean (all elections + job-verification succeeded); round 7 kills ctrl-2 (term=9 primary), restarts ctrl-3 (no convergence wait — chaotic). 2 surviving nodes (ctrl-1, ctrl-3) both start PreVote; neither processes the other's request → mutual timeout → RDHUP fires again → repeat forever. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +TASK 5.6 dueling-candidate PreVote cross-denial — DISTINCT from the earlier PreVote-WINS + real-election split-vote (which had both nodes winning PreVote then deadlocking in RequestVote). Here PreVote itself starves: each node's PreVote handler likely drops/ignores inbound PreVote requests while in its own PreVote round (no reply sent, not even denial). The task 5.6 RDHUP guard (commit 25e741d+) only prevents restarting failover while in CTRL_ELECTING — it does NOT fix the core problem where two nodes enter simultaneous PreVote rounds and each starves the other's reply. The priority stagger (bug-606) helps when 3 nodes are alive (one grants to break symmetry), but with only 2 survivors (the third restarting without convergence in chaotic mode), both deny each other and the stagger is insufficient. + +## Fix +FIXED 2026-06-30 (subsumed by tasks 5.6 commit 81cb09a + 5.7 commit 74784b2). The RDHUP->PreVote->timeout liveloc was the same fixed-watchdog resonance as bug covered by 5.7: per-campaign election watchdog re-randomized to [T,2T] (Ongaro 3.4) + don't-preempt-in-flight-election (5.6). CONFIRMED by a 10-round chaotic soak: 10/10 rounds elect a new primary in a single term increment (4->13), zero term escalation, round 7+ (the old liveloc point) all clean. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1281, bug-1282, bug-1286, bug-1292 diff --git a/.wolf/bugs/bug-1298.md b/.wolf/bugs/bug-1298.md new file mode 100644 index 00000000..957fd253 --- /dev/null +++ b/.wolf/bugs/bug-1298.md @@ -0,0 +1,39 @@ +--- +id: "bug-1298" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-06-30T13:36:00-07:00" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/netio.c / src/lib/vtaskmgr.c / src/controller/cluster.c"] +tags: ["netio", "vtaskmgr", "use-after-free", "asan", "endpoint-teardown", "concurrency", "raft-ha-core-safety", "soak", "quorum", "bug-1282-family", "FIXED"] +related: ["bug-1282", "bug-1197", "bug-1286"] +fix_commits: ["6e05dc1"] +legacy: true +--- + +# bug-1298: AddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask… + +## Description +AddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask dispatcher T3) — endpoint freed by endpoint_killer cluster.c:813 (io_worker T2); ctrl-3 ABORTs mid-chaotic-soak round 3→4, dropping the cluster to 1/3 (no quorum) so round 4 'NO new primary within window' + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Teardown-vs-dispatch race: io_worker (T2) tears down a connection on error — handle_endpoint_error → zarray_del → endpoint_killer free()s the endpoint_t — while the vtask dispatcher (T3) is concurrently scanning the dataq (dataq_deq_first_match → vtask_eligible → endpoint_is_task_blocked) and reads ep->busy on that just-freed endpoint. The endpoint is freed while still referenced by a pending vtask in the dataq; nothing drains/guards the ep's queued vtasks before the free, and no refcount keeps it alive across the dispatcher read. Same netio surface as bug-1282 (ABBA, ep->busy made lockless) and the bug-1197 dispatch_inflight precedent; the bug-1282 lockless read removed the (incidental) lock contention so the dispatcher reaches the read faster, exposing the latent UAF. + +## Fix +FIXED (commit 6e05dc1): added endpoint_t reference count (refcnt _Atomic + heap_alloc flag). endpoint_get starts refcnt=1 (owner); vtask_enq takes a per-vtask ref before publishing the task to the dispatcher, run_task/taskq_item_killer release it; every heap-ep teardown site drops its owner ref via endpoint_unref (last unref fini+frees). The dispatcher can now read ep->busy on a queued vtask without the io_worker freeing it underneath. Embedded eps (heap_alloc=0) never freed by unref. Validated: 182 unit tests/0 fail; 6-round ASAN chaotic soak ZERO ASan reports, ctrl-3 survives round 4, all rounds elect. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1282, bug-1197, bug-1286 diff --git a/.wolf/bugs/bug-1306.md b/.wolf/bugs/bug-1306.md new file mode 100644 index 00000000..68f9fbe5 --- /dev/null +++ b/.wolf/bugs/bug-1306.md @@ -0,0 +1,38 @@ +--- +id: "bug-1306" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-30T14:31:00-07:00" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/hook.c (xce_run_hooks); job.c:603 gate is correct"] +tags: ["ha-commit-durability", "pre-existing", "not-a-regression", "soak", "task-5.9", "committed-job-loss", "commit-durability", "term-boundary-divergence", "catch-up-gap", "failover", "hook-rc-clobber", "ack-before-majority", "data-loss"] +related: ["bug-795", "bug-828", "bug-851", "bug-1298"] +legacy: true +--- + +# bug-1306: 6-round chaotic-failover soak FAIL: committed_job_loss=4 committed_divergence=7 (rounds 4-6) + +## Description +6-round chaotic-failover soak FAIL: committed_job_loss=4 committed_divergence=7 (rounds 4-6). Committed job 0-0-1782829181-1 permanently absent on reachable primary from round 4 on, across 3 elected primaries; rounds 1-3 clean. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +SHARPENED 2026-06-30 (waldump+log bisect of the 10-round soak): NOT the catch-up prefix-gap/entity-snapshot mechanism. xce_run_hooks (src/lib/hook.c:134-157) returns only the LAST-registered hook's rc (rc overwritten each loop iter; break is BEFORE_EVENT-only). The durable-majority gate works: a term-9 leader's job at LSN 30 timed out commit (peers mid-restart, repl_wait_majority_ack -> commit_submit -> -ETIMEDOUT), persist_job_hook_fun returned -ETIMEDOUT, but a later-registered JOB AFTER_EVENT hook (sched/apiserver) returned 0 and CLOBBERED it -> handle_jobsubmit saw hook_rc==0 -> replied 'jobsubmit ok' for an uncommitted entry -> term-10 leader legally overwrote LSN 30 (term-10 NOOP) -> committed-job-loss. Violates the Commit Durability invariant (reply-success => majority-durable). Evidence: all 3 WALs gapless/CRC-intact/full payloads; lost job only on ctrl-1@LSN30 term9 vs term10 NOOP on majority; ctrl-1 log has the WAIT-deadline-expired WARNs and NO 'rejecting job' line. + +## Fix +FIXED 2026-06-30 (raft-apply-decoupling item #1 down payment). Extracted the WAL append + majority-commit barrier out of persist_job_hook_fun's WAIT branch into an explicit persist_submit_wal_and_commit(p, jlist); handle_jobsubmit now calls it directly and reads the commit rc DIRECTLY (rejects on <0 before notifying), so the -ETIMEDOUT can no longer be clobbered by a later notify hook's 0 in xce_run_hooks. The JOB_NEW AFTER hook is now notification-only. PROVEN: 10-round chaotic soak committed_job_loss 5 -> 0 (same election code, A/B isolates the durability fix). Unit: +test_job_hook_new_is_notify_only; persist_hooks 19/0, e2e_hooks 3/0, commit_barrier_reply 22/0, commit_durability 38/0. Remaining soak FAIL is committed_divergence (transient catch-up lag, present=2) + a separate round-4 election escalation tail (5.7 class) — neither is job loss. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-795, bug-828, bug-851, bug-1298 diff --git a/.wolf/bugs/bug-1307.md b/.wolf/bugs/bug-1307.md new file mode 100644 index 00000000..2ad6ae6e --- /dev/null +++ b/.wolf/bugs/bug-1307.md @@ -0,0 +1,38 @@ +--- +id: "bug-1307" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "data-loss" +timestamp: "2026-06-30T15:05:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["data-safety-regression", "task-5.7", "chaotic-soak", "leader-completeness-violation", "confirmed-via-soak", "committed-job-loss", "election-liveness", "bug-1296-followup", "committed-divergence"] +related: ["bug-1296", "bug-1281", "bug-1282", "bug-1286", "bug-851"] +legacy: true +--- + +# bug-1307: Task 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic… + +## Description +Task 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic soak shows 7 committed divergences (LSNs 8,12,16,19) + 4 committed job losses (job 0-0-1782831451-1). Previous run (pre-5.7, commit 81cb09a) had ZERO divergences and ZERO job losses through 6 rounds. LSN 19 divergence is present=3 (ALL 3 members have different content at same LSN — silent divergence, textbook bug-851). Election liveness improved (all 6 rounds complete vs stalled at round 7), but the jitter defeats the election-safety balance: faster-starting node wins before catching up on committed prefix → Leader Completeness violation. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Task 5.7 jittered watchdog ([T,2T] randomized) lets a less-complete node start and win an election before catching up on the full committed prefix from the dead primary. With the original fixed watchdog, both survivors fired RDHUP simultaneously → cross-denied → the library's priority stagger resolved the tie safely (the fresher node won after deterministic desynchronization). With jitter, one node can fire much earlier (near T) and the other much later (near 2T) → the early node campaigns with an incomplete log → voters grant based on stale view → winner truncates committed entries it never held. The 5.3 in_catch_up release (which drops the vote-deny when the primary is dead) opens a window where a still-catching-up candidate can win — with the new jitter timings this window is hit frequently. + +## Fix +CONFIRMED FIXED via live re-soak (2026-07-03): ran the exact reproducing scenario — 6-round chaotic soak (overlapping kill-the-primary, real ext4 at ~/tmp, ASAN controller binary) — that originally showed 7 committed divergences + 4 job losses. This run: 21 jobs submitted, 0 committed_divergence, 0 committed_job_loss, PASS across all 6 rounds (terms 3->9, all 3 nodes rotated through primary). Matches the earlier circumstantial evidence (raft-ha-core-safety Phase 5 fully complete + prior soak-clean sessions) with a fresh, direct confirmation run — no residual doubt. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1296, bug-1281, bug-1282, bug-1286, bug-851 diff --git a/.wolf/bugs/bug-1376.md b/.wolf/bugs/bug-1376.md new file mode 100644 index 00000000..309a38ca --- /dev/null +++ b/.wolf/bugs/bug-1376.md @@ -0,0 +1,38 @@ +--- +id: "bug-1376" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-07-02T05:10:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["raft-drive-async", "cold-start", "vtask-pool", "self-deadlock"] +related: ["bug-1197", "bug-1298", "bug-1310"] +legacy: true +--- + +# bug-1376: raft-drive-async: cluster hangs silently (no crash, no ASAN report) during a --force-snapshot soak… + +## Description +raft-drive-async: cluster hangs silently (no crash, no ASAN report) during a --force-snapshot soak with g_drive_async_snapshot_offload + g_drive_async_peerctrl_dispatch_enabled both ON: a freshly-wiped empty node cold-starts, self-elects, then goes silent after 'peer_ctrl_dispatch_to_pool: vtask pool not wired -- running inline' is immediately followed by cluster_init_phase2 wiring the pool while raft_cb_become_follower's deferred-teardown vtasks (bug-1197's mechanism) are being enqueued -- the process never logs another line and is eventually killed by the test harness timeout. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +peer_ctrl_msg_handler (ha.c) held ep->dispatch_inflight incremented across the ENTIRE peer_ctrl_dispatch_to_pool call, including its inline fallback (the common cold-start shape when ha->pool is not yet wired). That inline body can reach raft_cb_become_follower, which -- now that cluster_init_phase2 ran synchronously earlier in the SAME call chain and wired the pool -- tears down its own loser vote-ep (the very ep whose reply triggered this call) via the deferred path (vote_ep_teardown_async), whose endpoint_mark_tearing_down spins waiting for dispatch_inflight==0 on that SAME ep. That count is held only by this exact call, on this exact thread, and cannot reach 0 until the spin itself returns -- a permanent self-deadlock. Confirmed live via a gdb thread backtrace (GDB_WRAP debug seam added to run-ha-soak.sh, launching the target node directly under `gdb --batch` so it becomes ptrace-legal as gdb's own direct child, then SIGINT to gdb once the log went stale for a bounded interrupt + full backtrace): frame `peer_ctrl_dispatch_to_pool` showed pool=0x0 at dispatch time; the same ep pointer appeared in the nested vote_ep_teardown_async -> endpoint_mark_tearing_down frame, spinning on sched_yield(). + +## Fix +Moved the ep->dispatch_inflight release into peer_ctrl_dispatch_to_pool itself, at the exact right point per exit path: released BEFORE calling the inline body on every fallback path (pool==NULL, OOM, enqueue failure) -- safe because io_worker is single-threaded and the inline path never enqueues anything, so there is no pool-worker race left to protect against once inline execution is committed to -- and only AFTER the (non-blocking) enqueue completes on the success path, preserving the original protection for the genuinely concurrent pool-worker case. New unit test test_vep_self_teardown_does_not_deadlock (test_ha_vep_teardown.c) uses a bounded-timeout pthread_timedjoin_np so a regression fails loud instead of hanging the suite; RED confirmed via a temporary manual revert (Expected 0 Was 110 == ETIMEDOUT) before restoring the fix. Live-verified with both g_drive_async_snapshot_offload and g_drive_async_peerctrl_dispatch_enabled flipped ON: --force-snapshot soak (the exact repro shape) completes without hanging across repeated runs, confirmed real snapshot installs, committed_divergence=0, committed_job_loss=0; a 6-round --chaotic soak with both flags ON also passed clean; full controller suite green throughout. Both flags now default ON in production (raft-drive-async tasks 3.1/3.2 closed). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1197, bug-1298, bug-1310 diff --git a/.wolf/bugs/bug-1381.md b/.wolf/bugs/bug-1381.md new file mode 100644 index 00000000..c925a859 --- /dev/null +++ b/.wolf/bugs/bug-1381.md @@ -0,0 +1,38 @@ +--- +id: "bug-1381" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-02T00:35:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c"] +tags: ["raft-drive-async", "configpush", "endpoint-lifetime", "uaf-suspect"] +related: ["bug-1197", "bug-1298", "bug-1376"] +legacy: true +--- + +# bug-1381: raft-drive-async task 3.3 (config-push slow-lane offload) blocked: push_config_transport_only has… + +## Description +raft-drive-async task 3.3 (config-push slow-lane offload) blocked: push_config_transport_only has a pre-existing endpoint-lifetime race, independent of the g_drive_async_configpush_offload flag -- it grabs raw endpoint_t* pointers for every registered standby under c->ha.lock, releases the lock, then spawns one pthread per standby (push_thread) that reads/writes those endpoints unprotected for up to CONF_PUSH_TIMEOUT_MS. If a standby RDHUP-disconnects during that window, the io_worker's teardown path (endpoint_killer -> endpoint_unref, the bug-1298 refcount system) can free the endpoint while a push_thread is still using it -- a UAF. This is NOT specific to the lane offload: the transport fan-out spawns its own independent OS threads regardless of which caller (inline drive-thread or lane worker) invokes it, so the documented 'runs inline on io_worker so teardown and push are serialized on one thread' rationale for keeping the flag OFF does not actually hold -- the race exists in the inline path today too. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +push_config_transport_only (config_repl.c ~line 1366) takes c->ha.lock only long enough to copy standby ep pointers into a push_ctx_t array, then releases the lock before pthread_create-ing one push_thread per standby. Those threads read/write the raw endpoint_t* for the full duration of the send + wait_all_with_timeout(CONF_PUSH_TIMEOUT_MS), with no ref taken on the endpoints and no coordination with concurrent teardown (handle_endpoint_error -> zarray_del -> endpoint_killer -> endpoint_unref). Unlike the snapshot-lane fix (bug-1197/bug-1376 session), no endpoint_ref/endpoint_unref bracket protects this fan-out. + +## Fix +FIXED. push_config_transport_only (config_repl.c) now takes endpoint_ref() on each standby ep while the pointer is snapshotted under c->ha.lock, mirroring the snapshot-lane fix (bug-1197/bug-1376). Every ref has exactly one matching endpoint_unref(): from push_thread after it finishes using ctx->ep (new close in push_thread itself), or from push_config_transport_only inline if pthread_create fails for that standby, or from a cleanup loop if the threads[] allocation fails entirely. A concurrent RDHUP teardown (endpoint_killer -> endpoint_unref) can now only drop the OWNER reference while a push is in flight -- it can no longer free ep out from under push_thread. New regression test test_configpush_transport_holds_endpoint_ref_across_send (test_config_repl.c) uses a new push_thread_send_hook test seam (same pattern as xce_wal_stream_request_hook) to park a push_thread mid-send, simulate the owner teardown, and assert ep survives with refcnt intact. RED->GREEN verified via temporary revert (Expected 2 Was 1 with the ref-take removed); full controller suite green (230/230) after restore. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1197, bug-1298, bug-1376 diff --git a/.wolf/bugs/bug-1386.md b/.wolf/bugs/bug-1386.md new file mode 100644 index 00000000..b0623e44 --- /dev/null +++ b/.wolf/bugs/bug-1386.md @@ -0,0 +1,38 @@ +--- +id: "bug-1386" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-02T08:15:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c"] +tags: ["raft-drive-async", "snapshot-transfer", "silent-failure", "force-snapshot-soak"] +related: ["bug-1197", "bug-1376", "bug-1381"] +legacy: true +--- + +# bug-1386: /xce-test-ha --force-snapshot round: after a wiped ctrl-2 rejoins empty and the primary sends a… + +## Description +/xce-test-ha --force-snapshot round: after a wiped ctrl-2 rejoins empty and the primary sends a full-checkpoint snapshot, ctrl-2 sometimes never logs a completed install ("HA: snapshot loaded/legacy" line absent) though the cluster still reconverges to 3-of-3 -- OR the cluster fails to reconverge to 3-of-3 at all. Reproduced with g_drive_async_configpush_offload BOTH ON (0/3 pass) and OFF (1/2 pass) on the same freshly-built xce.build binary, so it is independent of that flag -- discovered while validating the bug-1381 configpush flip. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_send_snapshot_to_standby (config_repl.c, ~line 2596) loops over up to 12 65536-byte chunks calling xce_snapshot_chunk_req_raw(ep, &chunk, buf, chunk_len) WITHOUT checking or propagating its return value. xce_snapshot_chunk_req_raw -> endpoint_enq can return -ECONNRESET (e.g. if the standby endpoint is mid-teardown or reconnecting during the empty-rejoin window) but the loop keeps incrementing offset/chunk_index regardless, and the function still logs "transfer complete" and returns rc=0 at the end. Confirmed via ctrl-2 log evidence: primary logs start+complete for a 735546-byte (12-chunk) snapshot within ~2ms, but ctrl-2 only ever receives a handful of leading chunks (2 in one run, 5 in another) before the stream silently stops -- no ERROR on either side. + +## Fix +FIXED 2026-07-02 (config_repl.c): ha_send_snapshot_to_standby's chunk loop now captures the return value of xce_snapshot_chunk_req_raw (production path) and xce_snapshot_chunk_hook (XCE_TESTING path) into send_rc, breaks the loop on send_rc != 0, and propagates it as the function's rc instead of silently continuing. The 'transfer complete' INFO log is now strictly gated on rc==0; a failure path now logs a WARN with checkpoint_lsn/total_size/sent-offset/chunk_index/rc for observability. No receive-side timeout/re-request mechanism was added (out of scope, no existing seam) — the existing per-peer snapshot_inflight[] flag already clears unconditionally on send-thread exit (snapshot_lane_fn, ha.c) and the next heartbeat's SEND_SNAPSHOT decision naturally re-triggers a fresh transfer to a peer whose applied_lsn never advanced, so the existing retry cadence recovers once the endpoint is healthy again. New RED->GREEN unit test test_snapshot_transfer_stops_and_propagates_send_failure (test_config_repl.c) injects a mid-loop -ECONNRESET via the existing xce_snapshot_chunk_hook seam (new snaptest_fail_after_hook) and asserts rc == -ECONNRESET and exactly 2 chunks attempted (not the false-'complete' 3). RED confirmed via temporary revert ('Expected -104 Was 0'); GREEN restored, full controller_unit_test suite clean (0 failures across all groups). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1197, bug-1376, bug-1381 diff --git a/.wolf/bugs/bug-1389.md b/.wolf/bugs/bug-1389.md new file mode 100644 index 00000000..fc730a43 --- /dev/null +++ b/.wolf/bugs/bug-1389.md @@ -0,0 +1,38 @@ +--- +id: "bug-1389" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-07-02T09:05:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/request.c (suspected: xce_snapshot_chunk_encode_raw capnp Data encode) or src/controller/ha.c (ha_handle_snapshot_chunk per-chunk append)"] +tags: ["raft-drive-async", "snapshot-transfer", "data-corruption", "capnp", "force-snapshot-soak"] +related: ["bug-1386", "bug-847", "bug-1197"] +legacy: true +--- + +# bug-1389: /xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on… + +## Description +/xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on true completion), the force-snapshot soak still FAILs. ctrl-2's log shows all 12 chunks of a 735546-byte snapshot received, then `ha_handle_snapshot_chunk: persist_load_snapshot failed rc=-5` / `handle_snapshot_transfer: chunk 11 rc=-5` (rc=-5=-EIO). ctrl-1 (sender) correctly logs 'transfer complete' both times — the send genuinely succeeded, full byte count. This is NOT bug-1386 (that was a chunk DROP producing a truncated/short transfer). Byte-comparing the primary's source checkpoint file against the exact bytes ctrl-2 assembled into snap.part -> snap-{lsn}.bin (same run, two separate checkpoint LSNs 0x75 and 0x76, both reproduced) shows: identical total size (735546==735546) but DIFFERENT CONTENT — the received file diverges from the sender's file starting exactly at a SNAPSHOT_CHUNK_SIZE (65536-byte) chunk boundary, then the content is a byte-SHIFTED copy of the correct data (same repeating job-frame strings, same occurrence count, just offset) that re-synchronizes with the source at a later offset. The shift direction/magnitude differs between the two independent attempts (-912 bytes resyncing at 262144 vs +4684 bytes resyncing at ~458752) — nondeterministic, not a fixed structural bug. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT YET PINNED to an exact line — root cause is a chunk-CONTENT-corruption bug in the OPCode_snapshotTransfer chunk pipeline, most likely in the capnp raw-Data encode/decode path for the 64KB `payload :Data` field (xce_snapshot_chunk_encode_raw, request.c:630-671, uses the same capn_new_list8/capn_setv8 pattern as bug-847 — 'WAL Catch-Up Streaming Silently Corrupts Payloads at capnp Segment Boundary' — a proven prior case of this exact capnp pattern corrupting raw-byte payloads at a segment boundary). Ruled out as NOT the cause: (1) receiver-side concurrent chunk processing — vtaskmgr's per-endpoint FIFO (endpoint_block_tasks/endpoint_unblock_tasks) guarantees chunk N+1 cannot dispatch until chunk N's ha_handle_snapshot_chunk (including its fopen/fwrite/fclose) fully returns; (2) sender buf-reuse race — xce_snapshot_chunk_encode_raw copies bytes into the capnp segment synchronously before ha_send_snapshot_to_standby's loop re-fread()s the shared buf; (3) fixed-size packet-pool truncation — alloc_packet (netio.c:416-432) mallocs fresh per packet, sized to capn_size(&c), no shared buffer to overflow into. persist_load_snapshot (persist.c:2440 -> load_snapshot at 2325 -> load_snapshot_entities at 2222) is NOT the bug — it correctly rejects the corrupted file via one of its length-sanity -EIO guards (most likely persist.c:2240-2241, the job-frame payload_len-vs-file_size guard); format_version/job_count field-order misparse was considered and ruled out (load_snapshot consumes and validates format_version at persist.c:2399-2413 before ever calling load_snapshot_entities). + +## Fix +FIXED 2026-07-02 (ha.c, ha.h). Root cause pinned via live crc32 instrumentation (TRACE logs at both ha_send_snapshot_to_standby's send point and ha_handle_snapshot_chunk's receive point, correlated by snapshot_lsn+chunk_index): every chunk's CONTENT crossed the wire byte-for-byte intact (crc32 matched exactly for all 12 chunks) — the corruption was never in capnp encode/decode/transport at all. The real bug: OPCode_snapshotTransfer chunks of one transfer are dispatched across the vtask pool and are NOT guaranteed to arrive/execute in chunk_index order (live soak log showed arrival order 0,1,3,4,2,5,6,7,8,9,10,11 across cpu#0-3 workers), but ha_handle_snapshot_chunk appended every chunk's bytes to snap.part in ARRIVAL order (fopen "ab"/fwrite) instead of at the chunk's own chunk_offset (a field already carried on the wire but never used for anything). Reordered chunks landed at the wrong file position: same total byte count, wrong content from the first reordered chunk's offset onward — exactly the observed 'byte-shifted, resyncing later, nondeterministic magnitude' signature. Fix: (1) write each chunk via pwrite(fd, payload, len, chunk->chunk_offset) instead of append — order-independent and safe for concurrent writers (positioned writes to disjoint byte ranges of the same fd don't race). (2) Track completion via a new ha_state_t.snapshot_recv_count/snapshot_recv_total pair (guarded by the existing ha->lock) instead of finalizing on 'this call happens to carry is_last' — under out-of-order arrival, is_last can be processed before an earlier chunk; finalize now fires only once every chunk 0..N-1 has actually been written, and snapshot_recv_total is claimed (reset to 0) by whichever thread crosses the completion threshold so a late straggler chunk can never double-finalize. chunk_index==0 still resets both the stale-file unlink and the new counters (Req 4.4 unchanged). New RED->GREEN unit test test_snapshot_recv_out_of_order_chunks_assemble_correctly (test_config_repl.c) replays the exact non-sequential delivery order and asserts byte-identical assembly; RED confirmed via temporary revert (Expected 0 Was -22, job_count field corrupted by the misordered chunk boundary); GREEN restored, full controller suite clean (0 failures). Also added 2 new deterministic full-size (65536-byte) capnp encode/decode round-trip tests (test_wal_wire.c) that definitively cleared the capnp/transport layer before the real cause was found — permanent regression coverage, the prior test only covered a 300-byte payload. New xce_test snapverify/snapdiff subcommands (built as part of this investigation) were instrumental in diagnosing this live. Live-validated: 3/3 --force-snapshot rounds PASS (genuine 'HA: snapshot loaded ... full job/queue entity restoration' installs confirmed, zero divergence/loss) + 6-round --chaotic PASS. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1386, bug-847, bug-1197 diff --git a/.wolf/bugs/bug-1396.md b/.wolf/bugs/bug-1396.md new file mode 100644 index 00000000..964b7e8d --- /dev/null +++ b/.wolf/bugs/bug-1396.md @@ -0,0 +1,39 @@ +--- +id: "bug-1396" +status: "cannot-reproduce" +status_updated: "2026-09-24" +status_note: "extra committed job (20 vs 19) never reproduced after 9318f997 harness hardening" +component: "build" +severity: "unknown" +timestamp: "2026-07-02T14:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/tools/test-commit-durability.sh (harness) or controller job-recovery path (unconfirmed which)"] +tags: ["ha-commit-durability", "low-priority", "cold-restart-all", "job-count-mismatch", "harness-instrumented", "unconfirmed", "related-fix-applied"] +related: ["bug-661", "bug-1422"] +repro_attempts: "2 fresh repro attempts x3 rounds (2026-07-09), 0 reproduced" +legacy: true +--- + +# bug-1396: ha-commit-durability task 6.1 (cold-restart-all real-fs reproduction… + +## Description +ha-commit-durability task 6.1 (cold-restart-all real-fs reproduction, src/tools/test-commit-durability.sh on a genuine ext4 deploy dir, ~/tmp not overlay): run 5 of 5 reported `expect-jobs: expected 19, actual 20` after a full cold-stop-all + cold-start-all cycle — the cluster had ONE MORE committed job than the harness's own ACKED_FILE (deduplicated via `sort -u`) tracked. All 19 tracked job IDs individually verified present (`expect-job` loop all `ok`); the 20th job's identity was not captured (cluster had already been torn down by the script's EXIT trap by the time this was investigated). This is the OPPOSITE of bug-661/795's symptom (loss) — an unexplained EXTRA job, not a missing one. 4 of the other 5 runs (and every prior round within run 5 itself, rounds 1-8) showed correct exact-count durability with zero anomalies. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +- Attempts: 2 fresh repro attempts x3 rounds (2026-07-09), 0 reproduced + +## Root cause +NOT INVESTIGATED — deprioritized because it doesn't match the loss scenario task 6.1 was probing for, and reproducing it requires re-running with a 'joblist' capture at the exact expect-jobs failure point (the script doesn't dump the full list on a count mismatch, only the count). Two live hypotheses, unconfirmed: (a) harness-side race — ACKED_FILE's write (submit_n) and run_cold_restart_recovery's read (`committed=$(sort -u ACKED_FILE | grep -c .)`) are not synchronized against a submission whose ack arrives asynchronously after the harness moves on, so a genuinely-committed job simply never got recorded in ACKED_FILE — a test-harness bookkeeping gap, not a controller bug; (b) a genuine job-duplication bug on the controller's recovery/replay path creating an extra job entry with a fresh ID. Also observed in this run's ctrl-1.log: repeated unrelated `handle_jobsubmit fail to lock job 63-15--1-1015808[32751]` errors and one `job not durable (rc=-110) — rejecting submission` for a DIFFERENTLY-FORMATTED job ID (not this harness's `0-0--0` convention) — almost certainly a stray unrelated process hitting the same cluster ports in the shared sandbox, not this test's own traffic, but flagged here in case it's connected. + +## Fix +STILL NOT ROOT-CAUSED for the original exact symptom (successful query, count=20 vs expected=19) — 2 fresh repro attempts (2026-07-09, 3 rounds each, real ext4 fs under ~/tmp) did not reproduce an extra job; both runs showed the tracked count exactly matched after the fix below. HOWEVER, both attempts DID hit a DIFFERENT, related bug in the exact same assert_job_count() check: right after the cold-restart-all election, the aggregate expect-jobs/joblist query itself FAILED outright with 'xce_joblist: Unknown error 10014' / 'could not determine job count' (a query FAILURE, not a wrong-but-successful count) — this is the identical client-side race as bug-1422 (libxce's failover retry loop, client_failover.c, can exhaust its 20-attempt budget in a rapid post-election retry storm and surface a stale E_XCE_STANDBY). Confirmed as a false alarm both times: the very next per-id assert_no_loss check (individual expect-job calls) found every job present. Fixed assert_job_count() with the same bounded-retry pattern used for bug-1422 (3 attempts, 2s settle) — re-run after the fix: 7 passed, 0 failed, clean. This closes the QUERY-FAILURE false-negative class in this check, and makes the harness reliable enough to actually catch the ORIGINAL extra-job symptom if it recurs (previously a query failure and a genuine extra-job count were indistinguishable at a glance; now the retry-exhausted case self-heals and no longer masks a real mismatch). The original 'extra job' phenomenon itself remains open/unconfirmed — needs a longer run (original was 1-of-5 full 8-round soaks) to have a realistic chance of firing again. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-661, bug-1422 diff --git a/.wolf/bugs/bug-1400.md b/.wolf/bugs/bug-1400.md new file mode 100644 index 00000000..a1b593cc --- /dev/null +++ b/.wolf/bugs/bug-1400.md @@ -0,0 +1,38 @@ +--- +id: "bug-1400" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-07-02T14:20:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c (cmd_jobsub", "~line 1748)"] +tags: ["stale-report", "low-priority", "test-client-only", "input-validation", "crash", "cmd_jobsub", "xce_test"] +related: [] +legacy: true +--- + +# bug-1400: xce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` — positional args instead of required getopt… + +## Description +xce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` — positional args instead of required getopt flags (-u/-q/-c). getopt silently ignores unrecognized positional args, leaving submit.command/queue etc. NULL, and something downstream derefs a NULL field. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED STALE for the reported SEGV: git blame shows NULLSTR guards for submit.user/submit.queue/submit.start_info->cmdline were added 2025-11-05 (commit 41f56fa6), long before this bug was logged 2026-07-02 — 'jobsub sleep 1' getopt-returns -1 immediately on the non-option 'sleep' token, then hits the -EINVAL guard at test.c:1838 before any NULL deref. No SEGV path exists in current code; the buglog entry was stale (fix landed under a different/untracked change, buglog never updated). A REAL, separate defect was found in the same function: submit.start_info (calloc'd test.c:1767) was never freed on any of the 4 early-return validation paths (getopt default case + the 3 NULLSTR checks) — a ~128-byte leak per malformed jobsub invocation. + +## Fix +SEGV claim: not applicable, already fixed pre-existing. Leak: refactored cmd_jobsub (test.c) to `goto cleanup` on all 4 early-return validation failures instead of a bare `return -EINVAL`, reusing the existing FREEUP(submit.user/queue/resreq/start_info->cmdline/exdata/start_info) block via a new `cleanup:` label — mirrors the `goto clean` idiom already used in queue_ctrl() (queue.c). Verified: controller ASAN build + full controller_unit_test suite green (0 failures) after the change. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1401.md b/.wolf/bugs/bug-1401.md new file mode 100644 index 00000000..89345253 --- /dev/null +++ b/.wolf/bugs/bug-1401.md @@ -0,0 +1,38 @@ +--- +id: "bug-1401" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "leak" +timestamp: "2026-07-02T14:25:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/controller.capnp.c (decode_CtrlStatus_ptr", "generated codec) or its caller in request.c/xcecli"] +tags: ["asan", "low-priority", "controller-status", "ctrlstatus", "memory-leak"] +related: [] +legacy: true +--- + +# bug-1401: ASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr… + +## Description +ASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr (lib/controller.capnp.c:11873) — observed during ha-heartbeat-committed-propagation task 4.1 live validation: 120 bytes/1 alloc after a single ctrlstatus call, 12360 bytes/103 allocs after ~a dozen calls (roughly proportional to call count). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via codegraph + generated codec trace: src/include/client.h:119 defined `xce_free_ctrlstatus(s)` as `free_CtrlStatus(s)` — the generated free_CtrlStatus() only frees the FIELDS INSIDE the xce_ctrlstatus_t (clusterName/primaryName/responderName/configChecksum/peers list), matching decode_CtrlStatus(). But the real client reply path (xce_handle_reply -> MOVE_PTR from the decoded RequestReply -> xce_ctrlstatus) populates the struct via decode_CtrlStatus_PTR(), which calloc's the xce_ctrlstatus_t itself (controller.capnp.c:11873) before calling decode_CtrlStatus() to fill it. Only free_CtrlStatus_ptr() (which calls free_CtrlStatus() THEN free()s the struct and nulls the caller's pointer) matches that allocation. free_RequestReply/free_Request/MOVE_PTR were all confirmed symmetric and correct — the bug was isolated to this one macro definition. Existing unit tests never caught it because they call free_CtrlStatus(&out) directly on a stack struct, never exercising the xce_free_ctrlstatus() macro used by real client code. + +## Fix +src/include/client.h:119 changed to `#define xce_free_ctrlstatus(s) free_CtrlStatus_ptr(&(s))`. Added regression test test_xce_free_ctrlstatus_frees_the_struct_itself (src/lib/testsuite/test_ctrlstatus.c), which decodes via decode_CtrlStatus_ptr (the real allocation path, not the stack-struct helper the other roundtrip tests use) and frees via the xce_free_ctrlstatus macro. RED confirmed: reverting the fix reproduces the exact reported leak signature (LeakSanitizer: 120 bytes in 1 object, decode_CtrlStatus_ptr calloc). GREEN after fix: 285 lib tests / 0 failures / 0 leaks. Controller ASAN build + controller_unit_test also green post-fix (client.h is shared). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1406.md b/.wolf/bugs/bug-1406.md new file mode 100644 index 00000000..05472d10 --- /dev/null +++ b/.wolf/bugs/bug-1406.md @@ -0,0 +1,38 @@ +--- +id: "bug-1406" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-07-03T08:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/testsuite/test_ha_vep_teardown.c"] +tags: ["testsuite", "controller", "segv", "ssl", "iomgr", "pre-existing", "raft-apply-decoupling"] +related: [] +legacy: true +--- + +# bug-1406: controller_unit_test SIGSEGV (full run, no --run filter): Thread 1 crashes in SSL_CTX_free()… + +## Description +controller_unit_test SIGSEGV (full run, no --run filter): Thread 1 crashes in SSL_CTX_free() called from iomgr_fini() during test_ha_vep_teardown.c:271 (test_vep_teardown_runs_after_inflight), which runs at controller_test.c main() TEST(test_ha_vep_teardown). Kills the whole binary before any later-registered TEST() can run, so a bare `controller_unit_test` invocation with no --run filter never reaches a final summary and the exit code is 139 (SIGSEGV), not a Unity pass/fail count. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED 2026-07-09: iomgr_init() (src/lib/netio.c:1168-1184) only ever initialized iomgr_t's `lock` and `epoll_fd` fields -- it never set `ssl_ctx` (or `worker`). Every production caller embeds iomgr_t inside a calloc'd/static context struct (cluster_t.iomgr, xce_ctx_t.iomgr), which masked the gap. test_ha_vep_teardown.c stack-allocates `iomgr_t iomgr;` at 4 call sites (lines 235, 294, 349, 476) WITHOUT a memset -- unlike the sibling libxce tests (test_netio_arm_failure.c:52, test_request_send_propagation.c:34) which explicitly `memset(&iomgr, 0, sizeof(iomgr))` before iomgr_init, establishing the real (undocumented) contract. Left un-zeroed, `ssl_ctx` holds whatever garbage bit pattern occupies that stack slot -- non-deterministic across runs/process history (confirmed intermittent: some full-suite runs exit 0, others SIGSEGV, matching the reported non-strictly-deterministic behavior). iomgr_fini()'s `if (iomgr->ssl_ctx != NULL) SSL_CTX_free(...)` guard then passes on that garbage pointer, and SSL_CTX_free() dereferences a wild address -> SIGSEGV. Proven deterministically with a standalone repro (poisoned an iomgr_t with 0xAB, called the real iomgr_init()+iomgr_fini() from libxce.a) producing an identical backtrace (SSL_CTX_free <- iomgr_fini netio.c:1262 <- caller). gdb backtrace from the original report: SSL_CTX_free (libssl.so.3) <- iomgr_fini (netio.c:1262) <- test_vep_teardown_runs_after_inflight (test_ha_vep_teardown.c:271) <- UnityDefaultTestRun <- test_ha_vep_teardown (test_ha_vep_teardown.c:630) <- main (controller_test.c). Not a double-free/UAF and not a cross-test state leak -- iomgr_set_ssl_ctx() is never called anywhere in test_ha_vep_teardown.c, so no real SSL_CTX ownership is even in play; it is purely uninitialized-memory read. + +## Fix +FIXED 2026-07-09: src/lib/netio.c iomgr_init() now explicitly sets `iomgr->ssl_ctx = NULL;` and `iomgr->worker = 0;` before touching lock/epoll_fd, so iomgr_init() fully establishes iomgr_t's invariants itself instead of assuming the caller pre-zeroed the struct -- mirrors the same file's do_endpoint_init(), which already fully initializes every endpoint_t field rather than relying on calloc. Verified safe for all production callers (xcectx.c, myapp.c, cluster.c): every one calls iomgr_set_ssl_ctx() AFTER iomgr_init(), never before, so clearing ssl_ctx in iomgr_init cannot clobber a caller-supplied value. No test file was modified -- the library-level fix protects every current and future caller, including the 4 unguarded stack declarations in test_ha_vep_teardown.c. Verified: (1) standalone ASAN repro poisoning the stack before iomgr_init now shows ssl_ctx == NULL post-init and iomgr_fini() returns cleanly instead of crashing; (2) rebuilt controller_unit_test and ran the full suite (no --run filter) repeatedly under ASAN (detect_leaks=0) with zero SIGSEGVs across all runs -- one full run captured 265 Tests / 0 Failures / 0 Ignored, with all 6 tests in the test_ha_vep_teardown group (including test_vep_teardown_runs_after_inflight) passing their own assertions, not just failing to crash. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1407.md b/.wolf/bugs/bug-1407.md new file mode 100644 index 00000000..bb50c5f0 --- /dev/null +++ b/.wolf/bugs/bug-1407.md @@ -0,0 +1,39 @@ +--- +id: "bug-1407" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-03T08:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 5 +files: ["src/controller/testsuite/test_config_repl.c"] +tags: ["testsuite", "controller", "commit-engine", "build-environment", "false-positive", "raft-apply-decoupling"] +related: ["bug-924"] +fix_commits: ["3ecb724"] +legacy: true +--- + +# bug-1407: controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence FAILS ("Expected… + +## Description +controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence FAILS ("Expected TRUE Was FALSE") when running --run test_config_repl (RUN_TEST site: test_config_repl.c:4008). NOTE: Unity's own printed file:line prefix for this failure is misleading/stale (shows test_repl_unit.c:1543 or controller_test.c:1543 depending on which other suites ran earlier in the same process) -- Unity.TestFile is apparently not re-set per test_xxx(void) group in this harness, only Unity.CurrentTestLineNumber is accurate. The true source is test_config_repl.c. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +DEFINITIVELY ROOT-CAUSED 2026-07-03: this was never a source-code bug. It was a stale/corrupted incremental-build artifact in the main worktree's shared `build/` directory, which had been incrementally `ninja -C build`-rebuilt across 5 sequential tasks (1.1-1.5) without ever a fresh `meson setup`. Proof (four independent measurements, isolated via separate `git worktree add` + fresh `meson setup` + `ninja`, never mutating the main worktree in place): (1) pre-feature commit 17b210c, fresh build: 0 failures, 3/3 runs. (2) commit 9b4b805 (task-1.4 HEAD, IDENTICAL source to the failing main-worktree binary), fresh build in a separate worktree: 0 failures, 3/3 runs. (3) main worktree's stale incrementally-rebuilt `build/`, same 9b4b805 commit: 100% failure, 8/8 runs across two investigation sessions. (4) main worktree after `rm -rf build && meson setup --buildtype=debug -Db_sanitize=address && ninja -C build` (full clean rebuild, same source, same commit): 0 failures, 3/3 runs. This conclusively isolates the failure to the incremental build state, not the code. This is a DIFFERENT gotcha from bug-924 (which is about the `meson test` driver hanging on overlayfs fdatasync) -- this one is `ninja -C build`'s incremental compilation silently producing a wrong/stale binary despite reporting success, likely a header-dependency tracking gap on this overlayfs-backed worktree. The prior 'CLOSED' (task 1.4, 70/70) and 'REOPENED' (task 1.5, 10/10 fail) statuses were BOTH real, contradictory, correct measurements of the SAME source code in DIFFERENT incremental-build states -- neither implementer did anything wrong; the build itself was non-deterministic relative to source content. + +## Fix +CLOSED 2026-07-03 (high confidence, this time backed by cross-checked fresh-build isolation, not just an in-place re-run). No source-code fix needed -- `test_commit_advance_retry_clears_push_before_commit_fence` has apparently been correct since at least commit 3ecb724 (pre-feature). ACTION ITEM for all future tasks in this series: do not trust `ninja -C build` incremental results alone when a test result looks surprising or contradicts a very recent prior measurement on the same commit -- rebuild with `rm -rf build && meson setup build --buildtype=debug -Db_sanitize=address && ninja -C build` (or isolate via a throwaway `git worktree add` + fresh setup) before concluding a regression exists. Added to `.wolf/cerebrum.md` Do-Not-Repeat. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-924 diff --git a/.wolf/bugs/bug-1408.md b/.wolf/bugs/bug-1408.md new file mode 100644 index 00000000..106ea4a1 --- /dev/null +++ b/.wolf/bugs/bug-1408.md @@ -0,0 +1,38 @@ +--- +id: "bug-1408" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "security" +timestamp: "2026-07-03T21:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c"] +tags: ["raft-apply-decoupling", "task-3.1", "handle_jobsubmit", "array-job-submit", "notification", "commit-barrier", "toctou", "round-2-remediation", "test-longjmp-past-teardown"] +related: [] +legacy: true +--- + +# bug-1408: raft-apply-decoupling task 3.1 round-1 handle_jobsubmit(): when an array job submission has >1… + +## Description +raft-apply-decoupling task 3.1 round-1 handle_jobsubmit(): when an array job submission has >1 element and ONE element fails post-commit construction (create_job()/makeJob_() returns NULL, e.g. a jobid collision that slipped past jobid_range_exists() pre-check TOCTOU, or a plain calloc failure), the function returned the construction error IMMEDIATELY after the per-element construct loop, before ever building the AFTER_EVENT notify jlist or firing HOOK_DETAIL_JOB_NEW -- even for SIBLING array elements that constructed successfully and are already durably committed on a majority. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +handle_jobsubmit()'s post-commit construction loop set construct_rc<0 on any failed create_job() call but used continue (not break) to keep constructing remaining siblings correctly -- however, immediately AFTER the loop, "if (construct_rc < 0) { reply error; return; }" fired BEFORE the jlist-build + xce_run_hooks(HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW) block, so a single failed element suppressed notification for every successfully constructed sibling in the same array submission. Both the scheduler and apiserver are purely event-driven off this hook with no periodic reconciliation, so the stranded siblings (real, durable, present in c->jobmgr) were invisible until the next controller restart replayed the WAL. + +## Fix +Moved the construct_rc<0 early-return in handle_jobsubmit() (src/controller/job.c) to AFTER the jlist-build + xce_run_hooks(HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, ...) block instead of before it. The jlist-build loop already only finds jobs that key_find() locates in c->jobmgr, so a failed-to-construct element is naturally and silently absent from jlist with no extra bookkeeping -- the hook now fires unconditionally for every successfully constructed element regardless of a sibling's construct_rc, and the client-facing reply still reports the construct_rc error afterward. Added regression test test_jobsubmit_array_partial_construct_failure_notifies_survivors in src/controller/testsuite/test_commit_barrier_reply.c: drives handle_jobsubmit with a 2-element array on a worker pthread, ACKs element 0's WAL entry, then deterministically reproduces the TOCTOU outcome by inserting a stand-in job directly into c->jobmgr for element 1's jobid (via the existing cbrp_make_job helper) strictly after jobid_range_exists() has already run and strictly before the construction loop can run (element 1's commit is still pending on a withheld ACK) -- makeJob_()'s own key_find() then legitimately returns EEXIST for element 1. A new test hook capture (cbrp_jobnew_capture_fn, registered via xce_add_hook) records every jobid delivered through HOOK_AFTER_EVENT/HOOK_DETAIL_JOB_NEW so the test can assert element 0 was both constructed AND notified despite element 1 failing. RED-phase confirmed against the pre-fix code (job 9201 construction failed with errno=17/EEXIST as designed, and the notification assertion for job 9200 failed as predicted). CORRECTION (round-2 review): under the standard verification invocation (--run test_commit_barrier_reply, the full 25-test group), the failing TEST_ASSERT_TRUE_MESSAGE at test_commit_barrier_reply.c longjmps past this test's own inline cbrp_teardown() call (Unity TEST_ABORT semantics) -- the SAME longjmp-past-teardown pathology already documented for round 1's withheld-commit test, not a clean assertion FAIL. It leaks live persist/commit-engine background threads into the next test's setup, producing a deterministic HANG under the full-group command (reproduced 3/3 times, both threads blocked in futex_do_wait). A clean, fast FAIL was only observed when this test was isolated from the other 24 tests in the group -- that isolated form is what the original write-up described. GREEN after the fix (reordering removes the failure entirely, so this hang mode never triggers in the fixed code): both pre-existing single-element jobsubmit tests and the full 25-test test_commit_barrier_reply group plus the full controller_unit_test binary pass (0 failures) via the standard --run invocation. Hardened afterward: the test now captures its assertion-relevant booleans (cbrp_job_exists/cbrp_jobnew_notified) into locals and calls cbrp_teardown() unconditionally before asserting on the captured locals, so a future regression in this exact test can no longer skip cleanup and hang sibling tests. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1409.md b/.wolf/bugs/bug-1409.md new file mode 100644 index 00000000..5abd885b --- /dev/null +++ b/.wolf/bugs/bug-1409.md @@ -0,0 +1,38 @@ +--- +id: "bug-1409" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-07-04T17:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c"] +tags: ["raft-apply-decoupling", "task-6.1", "apply-pump", "busy-loop", "infinite-loop", "commit-resolve-pending", "regression-sweep"] +related: [] +legacy: true +--- + +# bug-1409: test_demotion_convergence hung/spun (18.5M+ log lines, timeout kill) once commit.c's… + +## Description +test_demotion_convergence hung/spun (18.5M+ log lines, timeout kill) once commit.c's commit_resolve_pending unconditionally called the new persist_apply_pump_kick + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +apply_pump_drain()'s recheck-then-clear retry loop ignored persist_apply_pump_run()'s return code: on a genuine read failure (e.g. committed_lsn advanced in a test fixture with no real WAL bytes on disk for that LSN, rc=-ENOENT), last_applied_lsn never advances, so the very next re-check inside the SAME drain loop sees identical unresolved work and immediately reclaims the scheduled flag and retries forever -- a tight busy loop with no backoff, hammering disk reads and the logger. + +## Fix +apply_pump_drain() now checks persist_apply_pump_run()'s rc immediately after clearing apply_pump_scheduled; a negative rc returns immediately instead of re-checking for more work and reclaiming the flag. The failure is already logged loudly inside persist_apply_pump_run itself. A future, genuinely NEW kick (a later real commit-advance event) retries then, at that event's natural cadence, never in a busy spin. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1410.md b/.wolf/bugs/bug-1410.md new file mode 100644 index 00000000..0acd268c --- /dev/null +++ b/.wolf/bugs/bug-1410.md @@ -0,0 +1,38 @@ +--- +id: "bug-1410" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-04T17:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_apply_pump.c"] +tags: ["raft-apply-decoupling", "task-6.1", "test-only", "va_arg", "hook-system"] +related: [] +legacy: true +--- + +# bug-1410: test_apply_pump's spy hook recorded jobid=0 for HOOK_DETAIL_JOB_UPDATE/HOOK_DETAIL_JOB_NEW instead… + +## Description +test_apply_pump's spy hook recorded jobid=0 for HOOK_DETAIL_JOB_UPDATE/HOOK_DETAIL_JOB_NEW instead of the real jobid + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The hook's per-subscriber `data`/cls pointer (bound at xce_add_hook registration time) is separate from the va_list's own variadic arguments. apply_committed_entry's notify calls pass (cluster_t*, payload) as the actual varargs -- the spy hook read the FIRST va_arg as the payload pointer directly, skipping the leading cluster_t* argument every real dispatch call (job.c, persist.c) always sends first. + +## Fix +Added `(void)va_arg(ap, cluster_t*);` before extracting the real payload (xlist_t*/job_t*) in both HOOK_DETAIL_JOB_NEW and HOOK_DETAIL_JOB_UPDATE cases, mirroring persist_job_hook_fun's own established va_arg consumption order. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1411.md b/.wolf/bugs/bug-1411.md new file mode 100644 index 00000000..8c63625f --- /dev/null +++ b/.wolf/bugs/bug-1411.md @@ -0,0 +1,38 @@ +--- +id: "bug-1411" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-04T17:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_apply_pump.c"] +tags: ["raft-apply-decoupling", "task-6.1", "test-only", "asan", "repl-broadcast", "fixture"] +related: [] +legacy: true +--- + +# bug-1411: AddressSanitizer global-buffer-overflow in dataq_enq when test_apply_pump.c called… + +## Description +AddressSanitizer global-buffer-overflow in dataq_enq when test_apply_pump.c called persist_submit_job_fact with a real 3-member repl setup + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl_broadcast tried to enqueue onto the dummy standby `int` stand-ins (g_tap_hep_a/b, opaque identity handles only) as if they were real endpoint_t* objects, overflowing past the 4-byte int into adjacent globals -- the fixture omitted the repl_broadcast_send_hook interception test_commit_barrier_reply.c's cbrp_setup already uses to keep repl_broadcast from ever dereferencing those dummy pointers. + +## Fix +Added the identical repl_broadcast_send_hook = tap_drop_send_hook (a no-op stub returning success) in tap_setup/tap_teardown, mirroring test_commit_barrier_reply.c's cbrp_drop_send_hook exactly. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1412.md b/.wolf/bugs/bug-1412.md new file mode 100644 index 00000000..f694adb6 --- /dev/null +++ b/.wolf/bugs/bug-1412.md @@ -0,0 +1,38 @@ +--- +id: "bug-1412" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-04T17:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_apply_pump.c"] +tags: ["raft-apply-decoupling", "task-6.1", "test-only", "comment-terminator", "build-break"] +related: [] +legacy: true +--- + +# bug-1412: test_apply_pump.c failed to compile: 'unknown type name RETAIN', stray bytes, __gnuc_va_list… + +## Description +test_apply_pump.c failed to compile: 'unknown type name RETAIN', stray bytes, __gnuc_va_list errors cascading from an unrelated location + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +A doc comment contained the literal substring 'WAL_JOB_*/RETAIN' -- the '*' immediately followed by '/' forms a real C block-comment terminator, closing the file's top doc comment many lines early and turning the rest of the comment text into stray code tokens. + +## Fix +Reworded to 'a WAL_JOB_SUBMIT WAIT-mode entry mixed with WAL_JOB_{START,FINISH} RETAIN-mode facts', removing the accidental */ sequence. Lesson: never write '*/' inside a C block comment, even when '*' is meant as a wildcard/glob character. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1413.md b/.wolf/bugs/bug-1413.md new file mode 100644 index 00000000..3e00d4a6 --- /dev/null +++ b/.wolf/bugs/bug-1413.md @@ -0,0 +1,38 @@ +--- +id: "bug-1413" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-07-04T17:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_apply_pump.c"] +tags: ["raft-apply-decoupling", "task-6.1", "test-only", "longjmp-past-teardown", "off-by-one", "test-hygiene"] +related: ["bug-1408"] +legacy: true +--- + +# bug-1413: test_apply_pump's burst-ordering test hung the NEXT test (longjmp-past-teardown) after a wrong… + +## Description +test_apply_pump's burst-ordering test hung the NEXT test (longjmp-past-teardown) after a wrong off-by-one assertion (expected lsn_before+3, actual watermark is lsn_before+2 for 3 appended entries) failed + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +A TEST_ASSERT failure mid-test longjmps past the rest of the function body, including the explicit tap_teardown() call at the end -- leaving the fsync thread/tmpdir/hooksys from the failed test alive when the next test's tap_setup() ran, hanging it. Root trigger was a genuine off-by-one in the test's own math (current_lsn before N appends is the FIRST assigned lsn, not one less than it; the watermark after N appends is first_lsn + N - 1, not first_lsn + N). + +## Fix +Fixed the off-by-one (first_lsn + 2 == watermark for 3 appends starting at first_lsn). Restructured all three tests in the file to capture every assertion-relevant observable into local variables BEFORE any TEST_ASSERT call, and to call tap_teardown() unconditionally before running any assertion -- the exact pattern already established in this codebase for task 3.1's identical hazard (tasks.md Implementation Notes, task 3.1 round 2/3). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1408 diff --git a/.wolf/bugs/bug-1414.md b/.wolf/bugs/bug-1414.md new file mode 100644 index 00000000..972050e7 --- /dev/null +++ b/.wolf/bugs/bug-1414.md @@ -0,0 +1,38 @@ +--- +id: "bug-1414" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-04T18:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c", "src/controller/persist.c", "src/include/wal_types.h", "src/include/persist.h", "src/controller/include/cluster.h"] +tags: ["raft-apply-decoupling", "task-6.1", "round-2", "job-state-corruption", "merge-staleness", "deferred-apply", "wal-wire-format", "wal_jobstate_t", "_pad-discriminator"] +related: ["bug-1413"] +legacy: true +--- + +# bug-1414: raft-apply-decoupling task 6.1 round 1 REJECTED: FINISH (LSN N) then a POST_EXEC-stage FAIL (LSN… + +## Description +raft-apply-decoupling task 6.1 round 1 REJECTED: FINISH (LSN N) then a POST_EXEC-stage FAIL (LSN N+1) for the same job, with FAIL received before FINISH applies, corrupts the job to RUN|EXIT|PERR (0x106) instead of the correct DONE|PERR (0x108) once both apply in LSN order -- DONE is silently lost and a stray RUN bit appears. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +record_job_fail_fact (job.c) and record_job_update_fact (postdone) precomputed a FINAL absolute state at RECEIPT time (scratch-copy merge for FAIL, inline backstop+PDONE merge for postdone) and baked it into wal_jobstate_t.state -- correct only when receipt and apply are the same event. Once task 6.1 deferred apply past the commit barrier, a second in-flight fact for the same job (e.g. FINISH) could apply BETWEEN this fact's receipt and its own apply, making the receipt-time snapshot stale by the time apply_committed_entry's generic apply_job_update_transition() did its blind absolute overwrite -- silently clobbering whatever the other fact had just applied. + +## Fix +Repurposed wal_jobstate_t._pad (previously always-zero, unused) as a discriminator: _pad==0 still means legacy absolute state (job_ctrl's mark_job_fail/mark_job_update live paths, which mutate+capture under one lock atomically -- no staleness risk, unchanged); a nonzero _pad (WAL_JOB_FAIL_STAGE_FLAG|stage for FAIL, WAL_JOB_UPDATE_POSTDONE_MARK for postdone) marks the entry for apply-time recomputation. apply_committed_entry (persist.c) now dispatches WAL_JOB_FAIL/WAL_JOB_UPDATE on this bit, calling the REAL apply_job_fail_transition() / new apply_job_postdone_transition() (job.c) against the job's ACTUAL current state at apply time instead of a stale baked value. New persist_submit_job_fact_ex(p,type,jobid,state,pad) primitive added; persist_submit_job_fact becomes a pad=0 thin wrapper so zero existing call sites needed changes. Also fixed 2 pre-existing test_commit_barrier_reply.c tests that asserted the old wire format directly (cbrp_count_jobstate_entries reading state bits) -- extended the helper to a _ex variant exposing _pad, updated assertions to check the new marker plus a direct call to the guard core on a scratch job_t. New regression test test_apply_pump_fail_racing_finish_merges_against_current_state in test_apply_pump.c drives FINISH then FAIL through the real receipt handlers with both entries durably appended but unapplied, then a single covering ACK; RED-confirmed byte-for-byte (Expected 264 Was 262) against the reverted round-1 code, GREEN after the fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1413 diff --git a/.wolf/bugs/bug-1415.md b/.wolf/bugs/bug-1415.md new file mode 100644 index 00000000..27264889 --- /dev/null +++ b/.wolf/bugs/bug-1415.md @@ -0,0 +1,38 @@ +--- +id: "bug-1415" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-04T19:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/commit.c (commit_resolve_pending", "commit_resolve/dispatch_continuation)", "src/controller/persist.c (persist_apply_pump_kick", "persist_apply_pump_run)"] +tags: ["raft-apply-decoupling", "task-6.2", "vtaskmgr", "race-condition", "orphaned-registration", "lost-reply", "commit_resolve_pending", "persist_apply_pump_kick", "self-healing"] +related: ["bug-1409"] +legacy: true +--- + +# bug-1415: raft-apply-decoupling task 6.2 review finding (NOT reproduced under the test suite's pool-less… + +## Description +raft-apply-decoupling task 6.2 review finding (NOT reproduced under the test suite's pool-less fixtures, NOT fixed): a real vtaskmgr pool (n_vcpus > 1) can run the commit continuation's dispatch (job_commit_resolve_adapter -> jobsubmit_commit_continuation -> persist_register_pending_apply) and the apply pump's kick (persist_apply_pump_kick -> apply_pump_drain -> persist_apply_pump_run) on two DIFFERENT worker threads with no completion-order guarantee, even though commit_resolve_pending() enqueues them in that order. If the apply-pump task happens to finish first, persist_take_pending_apply() finds nothing registered yet for that lsn (0, correct no-op per its own contract), applies the entry, and advances last_applied_lsn past it -- then when the continuation task finally runs and calls persist_register_pending_apply() for that now-already-applied lsn, the registration is permanently orphaned: the walk only ever revisits [last_applied_lsn+1..committed_lsn], so nothing will ever call persist_take_pending_apply() for that lsn again. Net effect: the job is correctly constructed and visible in c->jobmgr, but the client that submitted it never receives a reply -- silently, forever. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +commit_resolve_pending() (commit.c ~793-835, landed by task 1.2/6.1) resolves every ready continuation via commit_resolve() -> dispatch_continuation(), which for a real pool (ce->pool != NULL) enqueues the continuation onto vtaskmgr via vtask_enq_classified(VTASK_LIVE_BLOCK) and returns WITHOUT waiting for it to run; commit_resolve_pending() then calls persist_apply_pump_kick(ce->p, ce->pool), which enqueues a SECOND, independent VTASK_LIVE_BLOCK task onto the SAME pool. vtaskmgr.c's dispatcher (vdispatcher_entry) hands queued tasks to whichever vcpu worker thread is next idle (pick_idle_vcpu/pick_eligible_task) -- with n_vcpus > 1 (the only realistic production configuration; every test fixture in this spec deliberately never calls commit_set_pool, so ce->pool stays NULL and both dispatches run synchronously inline instead, which is why no test in this suite reproduces the race), two tasks enqueued back-to-back can be picked up by two different idle vcpus at nearly the same time, and their COMPLETION order depends on each task's own runtime (a disk-reading WAL-scan-and-apply walk vs. a calloc+mutex-guarded list insert) -- not their enqueue order. This is a pre-existing structural gap from task 1.2 (async continuation dispatch) + task 6.1 (apply-pump-kick wired into the same choke point, same pool) that had no observable consequence until task 6.2 added a real consumer (persist_take_pending_apply) whose registration can now race the very entry it targets. + +## Fix +FIXED round 2 (task 6.2): persist_register_pending_apply() (persist.c) now self-heals option (b) from this entry's own proposed fixes -- before linking a new node onto pending_applies, it checks `lsn <= persist_last_applied_lsn(p)`; if the apply pump's walk has already applied `lsn` (last_applied_lsn already at or past it), `on_applied(ctx, lsn)` fires immediately and synchronously on the CALLING thread instead of being linked for a walk pass that will never revisit that lsn. Race-safe in both interleavings: when the walk has not reached `lsn` yet, the check is false and registration proceeds exactly as before (unchanged behavior, verified against the full existing regression suite); when the walk already passed `lsn`, its own persist_take_pending_apply() call for that lsn already ran and found nothing (that is precisely how we know registration is late), so the immediate inline fire can never double-fire alongside it. New deterministic regression test `test_apply_pump_late_pending_apply_registration_self_heals` (test_apply_pump.c) wires a REAL multi-vcpu vtaskmgr_t (n_vcpus=4) via commit_set_pool -- the one test in this spec to do so -- and forces the exact bad interleaving via a barrier (the test's stand-in commit continuation polls persist_last_applied_lsn() until the apply pump has already applied the lsn, THEN registers), rather than relying on natural timing luck. Verified genuine RED (fails with 'Expected 1 Was 0' against the unfixed check, confirming the hang is real and reproducible) -> GREEN (fires exactly once, resource released exactly once, no orphaned registration) after the fix. Full regression sweep (test_apply_pump, test_commit_barrier_reply, test_apply_reapply_idempotency, test_commit_resolve, test_commit_admission_pool, test_demotion_convergence) shows zero new failures -- the only 2 failures present (test_jobsubmit_returns_before_commit_and_registers_pending_apply, test_jobsubmit_array_mixed_outcome_no_phantom_job_for_failed_elements, both in test_commit_barrier_reply.c) are pre-existing/expected per task 6.1's own Implementation Notes (their assertion text predates task 6's apply pump landing; task 7.3's job to update), confirmed identical before and after this fix by reverting to the pre-round-2 baseline and re-running. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1409 diff --git a/.wolf/bugs/bug-1416.md b/.wolf/bugs/bug-1416.md new file mode 100644 index 00000000..e965fa21 --- /dev/null +++ b/.wolf/bugs/bug-1416.md @@ -0,0 +1,38 @@ +--- +id: "bug-1416" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-05T06:15:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 11 +files: ["src/controller/persist.c (persist_verify_apply_correctness / persist_recover", "cluster.c cluster_init_phase2)"] +tags: ["raft-apply-decoupling", "task-7.2", "xce-test-ha", "factstream", "persist_recover", "APPLY_DIVERGENT", "false-positive-candidate", "boot-race", "concurrency", "live-regression-found", "recovery-complete-gate", "myapp-boot-sequencing"] +related: ["bug-1409", "bug-1415"] +legacy: true +--- + +# bug-1416: [xce-test-ha] FAIL: factstream round 1: cluster did not reconverge to 3-of-3 after restarting… + +## Description +[xce-test-ha] FAIL: factstream round 1: cluster did not reconverge to 3-of-3 after restarting ctrl-1 -- root cause: persist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip (expected last_applied_lsn=72 new_job_count=23); persist_recover_loud_fence_if_unrecoverable: UNRECOVERABLE, staying alive/non-promotable + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Found live via the new raft-apply-decoupling task 7.2 kill-mid-fact-stream soak scenario (--factstream), reproduced 4/4 runs (ASAN debug build, paced-wave ASAN build, release build, and a cleanup-suppressed debug postmortem run) in this sandboxed environment. cluster_init_phase2() calls cluster_open_listener() (which starts accepting network I/O, including replicated/catch-up WAL traffic from an already-serving primary) BEFORE persist_recover() runs its boot-time apply-correctness verification (task 5.1, Req 8.1/8.2). Under a slow/contended boot (observed: persist_recover firing ~36s after process start instead of the normal ~5s, plausibly due to heavy concurrent nix-develop/test-process CPU contention in this sandbox), a standby's listener can already be live and processing real traffic for tens of seconds before its own persist_recover() call finally executes. persist_verify_apply_correctness() (persist.c ~3597) reads p->last_applied_lsn and live-queries count_jobmgr_entries(p->cluster) AFTER replay_wal completes -- both of these are also mutated by the LIVE apply pump (raft-apply-decoupling task 6.1, wired through commit_resolve_pending) reacting to replicated commit-advance events. If live traffic is already flowing and being applied through the live pump DURING the extended pre-persist_recover window, the live-read 'actual' values used by persist_verify_apply_correctness can reflect state beyond what persist_recover's own single WAL-scan pass computed as 'expected', producing a DIVERGENT verdict even though no real apply defect exists -- confirmed via ctrlstatus probes showing committedLsn identical (79) and no actual data loss across all 3 nodes; the fence engaged correctly per its own fail-loud design (stayed alive, non-promotable, kept retrying) but two of three nodes (including the node that went on to win the next election, term=5, first_boot=1) were fenced at the exact moment persist_recover ran. This matches task 5.1's own documented residual-risk note (tasks.md Implementation Notes, task 5.1 entry): 'the position check is diagnostically tautological for THIS SPECIFIC call site today... this will change once task 6's live apply pump gives expectation and actual state a real chance to diverge across threads/time' -- that predicted future divergence has now materialized under a slow-boot + concurrent-live-traffic combination that only a scenario driving traffic this early/aggressively after cluster boot (the factstream scenario) exercises; existing soak scenarios (chaotic/sequential/force-snapshot) submit far fewer jobs, more slowly, and do not reliably land traffic inside a slow node's pre-recovery window. REFINED after further reproduction (10/10 runs total, across ASAN/release builds and job counts from 8 to 20): the trigger is specifically the RESTART/catch-up path, not the initial multi-node cold boot. A standalone control test (3 nodes, no kill, 20s settle before any traffic) submitted jobs and drove jstart/jfinish/jfail cleanly with zero APPLY_DIVERGENT -- confirming the race needs a node that was DOWN while the cluster kept moving (i.e. an actual failover), not merely a fast cold boot: the returning node's persist_recover()/cluster_init_phase2 vtask must complete while it is ALSO receiving a compressed catch-up replication burst covering everything it missed while down, and that burst's volume is inherent to 'kill mid-fact-stream' (the whole point of the scenario) -- a run-ha-soak.sh-side pacing fix cannot eliminate the trigger the way it did for the (separately real but distinct) initial-boot instance the settle-delay mitigation addresses. Reproduced identically down to 8 fact-stream jobs with 2-job/1s-gap waves, so the trigger threshold is low. This plausibly explains why no prior soak scenario in this spec's history (chaotic/sequential/force-snapshot, all using NJOBS=3 by default, un-paced but small) ever surfaced it: none combine a real kill-during-traffic failover with this much job+fact volume until raft-apply-decoupling task 7.2's factstream scenario. Also independently added test.c's print_jobinfo state_raw hex field (task 7.2) and confirmed via a live probe that apply_job_fail_transition correctly applies EXIT|PERR bits even when job_state_str's RUN-checked-first priority ordering hides them in the pretty-printed state string (job showed 'state: RUN' / 'state_raw: 0x106' after a jstart+jfail-without-jfinish sequence) -- a separate, minor, likely pre-existing (not introduced by raft-apply-decoupling) display-only quirk in job_state_str, also not fixed here (out of boundary), but which the factstream phantom-check was updated to work around by reading state_raw instead of the ambiguous pretty string. + +## Fix +FIXED (round 2, raft-apply-decoupling task 7.2 review remediation). Root cause confirmed exactly as hypothesized in the prior entry's candidate #2: persist_verify_apply_correctness() compared a WAL-scan-derived 'expected' snapshot against a LIVE re-read of last_applied_lsn/count_jobmgr_entries(), while TWO live mutation paths could write those same values concurrently -- the live apply pump (persist_apply_pump_run, driven by persist_apply_pump_kick off any of commit.c's 6 committed_lsn-advance sites) and the standby live-replication apply path (ha_apply_single_entry / ha_apply_term_entry_via_library in ha.c). Fixed via a new _Atomic bool persist_t.recovery_complete gate (persist.h/persist.c): persist_apply_pump_kick() now no-ops while the gate is closed; ha_apply_single_entry/ha_apply_term_entry_via_library now skip ONLY the in-memory apply_wal_entry() call while closed (durability write + replication ACK + applied_lsn advance are never gated, preserving the existing persist-before-apply-before-ACK contract and bug-1043's durability/materialization decoupling precedent). persist_recover() closes the gate at entry and reopens it (plus issues one deliberate persist_apply_pump_kick catch-up pass) right before returning, regardless of verdict -- covering any direct unit-test caller. ROUND 2 FINDING during live verification: this alone was insufficient -- a node stuck retrying its own election as a losing/timing-out candidate can receive and durably apply live-replicated entries via the ungated (persist_init default=true) path for tens of seconds BEFORE cluster_init_phase2() ever calls persist_recover() for the first time, reproducing the identical false-DIVERGENT symptom via an earlier entry point that persist_recover()'s own bracket cannot see. Closed by adding persist_mark_recovery_pending(&c.persist) in myapp_run() (myapp.c), called immediately after persist_init()/persist_start() succeed and BEFORE ha_start_election_async()/cluster_run() can process any network traffic -- closing the gate from the earliest safe moment in the real boot sequence. persist_init()'s default stays TRUE (ungated) so none of this spec's ~20 existing unit-test fixtures that construct a persist_t directly (bypassing myapp_run entirely) are affected; persist_mark_recovery_pending is never called outside myapp.c. Verified: live 3-node factstream soak (2 rounds, kill+restart both rounds) now PASSes clean with zero APPLY_DIVERGENT/UNRECOVERABLE across 5 persist_recover() calls (2 initial-boot delayed by the SAME pre-existing election-instability condition that originally exposed the round-2 gap, 2 post-kill restarts, all APPLY_CLEAN); broader controller_unit_test sweep (23 groups incl. test_apply_correctness_verification's 3 real-divergence tests, test_apply_pump/_integration, test_ha_core_safety, test_demotion_convergence) shows zero new failures (only the 2 pre-existing, documented task-6.1/6.2 test_commit_barrier_reply.c failures remain, task 7.3's job). Real-divergence detection independently re-confirmed intact (test_apply_divergent_decision_forces_unconditional_fence, test_apply_divergent_loud_fence_sets_sticky_flag_and_stays_alive, test_real_replay_divergence_sets_unrecoverable_fence all still PASS). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1409, bug-1415 diff --git a/.wolf/bugs/bug-1417.md b/.wolf/bugs/bug-1417.md new file mode 100644 index 00000000..e3e26c6b --- /dev/null +++ b/.wolf/bugs/bug-1417.md @@ -0,0 +1,38 @@ +--- +id: "bug-1417" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-07T09:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["vote", "raft", "self_idx", "index-mapping", "chaotic-failover", "off-by-one", "ha", "ha_repl_ack_abs_peer_idx", "election"] +related: ["bug-828"] +legacy: true +--- + +# bug-1417: [xce-test-ha --chaotic] FAIL round 4/10: election stall after killing ctrl-1 — ctrl-2 PreVote wins… + +## Description +[xce-test-ha --chaotic] FAIL round 4/10: election stall after killing ctrl-1 — ctrl-2 PreVote wins but real election times out despite ctrl-3 granting RequestVote + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +vote_reply_dispatch passes local votes_idx (compressed, excludes self) directly to raft_on_vote_reply which expects ABSOLUTE peer index (includes self). For self_idx=1 (ctrl-2), ctrl-3's vote (absolute=2, local=1) lands in vote_replies[1]=ctrl-2's own self-slot → treated as duplicate → vote silently dropped. With ctrl-1 dead in chaotic mode, ctrl-2 only has self-vote (1 < quorum 2) → eternal timeout. Self_idx=0 and self_idx=2 are partially/fully immune due to index alignment luck. ha_repl_ack_abs_peer_idx() already exists (ha.c:3779) with correct mapping and passing tests — vote_reply_dispatch just doesn't call it. + +## Fix +FIXED (src/controller/ha.c vote_reply_dispatch, ~line 5233): raft_on_vote_reply now receives ha_repl_ack_abs_peer_idx(ha, 1, idx) instead of the raw compressed local idx. ha_repl_ack_abs_peer_idx (ha.c:3779, already existed + unit-tested for the heartbeat-ACK path) performs the exact inverse of raft_cb_send_vote_request send-side remap: votes_idx < self_idx ? votes_idx : votes_idx + 1. ha_record_vote_reply (immediately preceding line) is UNCHANGED -- it correctly wants the compressed idx for ctx->votes[]. Confirmed pre_vote_reply_dispatch needs NO equivalent fix: raft_cb_send_pre_vote_request (ha.c:301) already stores the ABSOLUTE peer_idx in vctx->peer_idx for the PreVote path (explicit code comment there), matching raft_on_pre_vote_reply/pre_vote_replies[] absolute-index expectation -- only the real-vote send path (raft_cb_send_vote_request, ha.c:175) stores compressed. RED->GREEN: new unit test test_bug1417_middle_index_real_vote_from_peer2_not_dropped (test_ha_vote_send.c) drives a REAL 3-node election (self_idx=1) through raft_start_election then a genuine peer-2 grant via vote_reply_msg_handler_for_test (the actual decode/dispatch path, not the xce_vote_reply_hook bypass every other real-vote test in this file uses) -- reproduced Expected 2 Was 1 pre-fix, GREEN post-fix (votes_granted=2, role=RAFT_LEADER). Verified: full 79-group controller_unit_test sweep 0 new failures; 10-round --chaotic xce-test-ha soak PASS (committed_divergence=0, committed_job_loss=0, 18 jobs). Soak surfaced an UNRELATED, pre-existing --chaotic harness bug (run-ha-soak.sh run_chaotic_rounds: DOWN variable is not cleared on a skip-this-round no-primary-to-kill path, so a subsequent round calls start_node on the same already-running node without killing it first -- 6 stacked ctrl-2 processes observed piling up over rounds 5-10) -- confirmed via git diff this script was untouched by this fix, logged separately, NOT fixed here (out of bug-1417 boundary). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-828 diff --git a/.wolf/bugs/bug-1418.md b/.wolf/bugs/bug-1418.md new file mode 100644 index 00000000..de4cc26b --- /dev/null +++ b/.wolf/bugs/bug-1418.md @@ -0,0 +1,41 @@ +--- +id: "bug-1418" +status: "fixed" +status_updated: "2026-09-24" +status_note: "9b173c9a + bug-1421 + raft-apply-decoupling" +component: "controller" +severity: "hang" +timestamp: "2026-07-09T10:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 5 +files: ["src/controller/ha.c"] +tags: ["ha", "raft", "apply-divergence", "promotion-gate", "chaotic-failover", "liveness", "term-desync", "resolved-via-upstream"] +related: ["bug-1417", "bug-1419", "bug-1421", "bug-1416"] +fix_commits: ["9b173c9a"] +legacy: true +--- + +# bug-1418: Final resolution pass over bug-1418 (all 4 prior occurrences): the two contributing factors from… + +## Description +Final resolution pass over bug-1418 (all 4 prior occurrences): the two contributing factors from the 2026-07-07T18:20 analysis (term desync + apply-divergence promotion-gate liveness deadlock) are both now resolved, not by the originally-proposed gate relaxation. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Re-audited both factors against the CURRENT code (post raft-apply-decoupling 32/32, post bug-1421): (1) TERM DESYNC — ha_start_prevote_async (ha.c:6715) already computes its prospective term from ha->raft.term + 1, not the stale ha->term the original analysis described; ha.c:1009-1010 explicitly syncs ha->term from the raft term on every relevant transition. This half was already fixed independently of bug-1418's own tracked fix. (2) APPLY-DIVERGENCE LIVENESS DEADLOCK — the promotion gate (ha.c raft_cb_become_leader ~line 1130, and promotion_fence_timeout_fn ~line 876) unconditionally refuses promotion while persist_apply_verdict()==APPLY_DIVERGENT, by deliberate design (raft-apply-decoupling task 5.3, Req 8.4: a divergent local materialization must never be trusted to self-clear via any live/structural condition, including durable_floor>=cluster_hwm — only a fresh persist_recover() replay may clear it). The buglog's own previously-proposed fix (relax the gate to self-apply when durable_floor>=cluster_hwm) would have violated this intentional invariant and risked serving genuinely divergent state to clients. Instead, raft-apply-decoupling's tasks 5.1 (checkpoint_lsn mislabeled from WAL tip instead of the logical applied cursor, causing a false APPLY_DIVERGENT on an ordinary crash-during-checkpoint) and 7.2/bug-1416 (a boot-time TOCTOU where a returning node's persist_recover() replay-verify raced the live apply pump / live-replication apply path, producing a false APPLY_DIVERGENT on the restart/catch-up path specifically — the exact scenario bug-1418's residual occurrence hit: 'apply pump lagged behind WAL tip during kill') closed the false-positive detection paths that were producing spurious APPLY_DIVERGENT verdicts in the first place. A genuine (non-false-positive) divergence still correctly fences forever by design. + +## Fix +[Consolidated 2026-08-15: this ID previously had 4 duplicate entries in buglog.json (a pgrep-guard test-harness fix, a partial term-desync analysis, and a stale mid-investigation 'NOT FIXED' snapshot -- all superseded by this final resolution pass, which already accounted for all 3 in its own analysis). Collapsed to this single entry; the other 3 removed.] + +No code change to the promotion gate itself — it is intentionally correct and must stay unconditional. The actual fix is the combination of already-shipped upstream work: bug-1421 (AE-reject term-discovery blindspot, this session) + raft-apply-decoupling tasks 5.1/5.3/7.2 (false-APPLY_DIVERGENT elimination, closed prior to this session). Validated: two 40-round soaks today (sequential 123 jobs zero-loss all-3-nodes-led; chaotic 123 jobs zero-committed-divergence zero-job-loss) — an order of magnitude more election churn than the original 10-round soak that first surfaced bug-1418 — produced zero apply-divergent promotion refusals and zero liveness deadlocks. Further corroborated by the 30-round sequential+chaotic regression run immediately after this entry. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1417, bug-1419, bug-1421, bug-1416 diff --git a/.wolf/bugs/bug-1419.md b/.wolf/bugs/bug-1419.md new file mode 100644 index 00000000..fcce8103 --- /dev/null +++ b/.wolf/bugs/bug-1419.md @@ -0,0 +1,47 @@ +--- +id: "bug-1419" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-07-07T14:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/persist.c (apply-correctness verification) + src/controller/ha.c (PreVote freshness gate) + WAL duplicate-submit mechanism (unlocated)"] +tags: ["ha", "election", "raft", "apply-divergence", "promotion-gate", "chaotic-failover", "liveness", "wal-duplicate-submit", "checkpoint-lsn", "kill-mid-apply"] +related: ["bug-1417", "bug-1060", "bug-845", "bug-1418", "bug-1420"] +fix_commits: ["fe4352f"] +legacy: true +--- + +# bug-1419: [xce-test-ha --chaotic] genuine, reproducible election-liveness stall (not the bug-1418 harness… + +## Description +[xce-test-ha --chaotic] genuine, reproducible election-liveness stall (not the bug-1418 harness process-stacking artifact, which was a separate, now-fixed issue): after 2+ overlapping kills, the surviving 2-node set can permanently deadlock -- neither node can ever become primary again for the rest of the run. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Direct log/WAL analysis (waldump on the affected node's data dir) during a live repro superseded two prior, less-precise theories filed as bug-1418 (a shell-script process-stacking artifact, unrelated and separately fixed; and an unverified ha->term/ha->raft.term "term desync" claim that does not match the code -- raft_begin_pre_vote/ha_start_prevote_async both read s->term == ha->raft.term directly, there is no separate stale-field read on the PreVote path). + +ACTUAL mechanism observed: waldump on ctrl-3's WAL (the node with the longest/freshest log after the chaotic kill sequence) showed "duplicate submits: 3 (byte-identical: 0)" -- the SAME jobid appended to the WAL twice with DIFFERING payloads (different subtime, different CRC), e.g. lsn=13 and lsn=14 both jobid=0-0-1783418222-0 but distinct subtime/PCRC. This is NOT bug-845's mechanism (that fix, landed 2026-06-23, was for byte-IDENTICAL double-hook-registration dupes and is confirmed still working -- these are non-identical). count_jobmgr_entries() legitimately sees these as 2 distinct job-table mutations (or the entity count otherwise fails to match the WAL-replay-derived expectation), so persist_recover()'s post-replay apply-correctness check (task 5.1, Req 8.1/8.2) correctly fires APPLY_DIVERGENT -> the loud, sticky, non-promotable fence (raft-apply-decoupling Req 8.3/8.4) -- this fencing behavior is WORKING AS DESIGNED, not itself a bug. + +The resulting LIVENESS bug: with the fenced node holding the only sufficiently-fresh (longest) log, the other surviving node can never win the PreVote freshness gate against it (correctly, per Raft/PreVote semantics -- it really is behind), while the fenced node itself can win Raft elections (quorum_reached/became_leader observed at terms 27/30/32 in logs) but its own raft_cb_become_leader promotion gate refuses CTRL_PRIMARY every time (apply_divergent=1). With the third node dead (killed by the chaotic driver) and never restarted long enough to catch up, NO node can ever become primary again -- a genuine, reproducible liveness deadlock, distinct from (and downstream of) the WAL duplicate-submit root cause. Confirmed via direct log grep across ctrl-1.log/ctrl-3.log at the exact stall timestamps (09:57:16 divergence detected, repeating "PreVote DENIED ... reason=freshness" and "promotion REFUSED ... apply_divergent=1" through 10:05+); NOT root-caused via speculation. + +## Fix +FIXED (2026-07-07). Layer (1), the WAL duplicate-non-identical-submit mechanism: raft-apply-decoupling task 3.1 deferred job/entity construction in c->jobmgr until AFTER commit (apply_committed_entry, post-commit apply pump), but jobid_range_exists() -- the sole dup guard in handle_jobsubmit -- only checks c->jobmgr. A submission that is durably WAL-appended and awaiting its commit outcome does not yet exist in c->jobmgr, so a client-side retry for the SAME jobid (XCECLI_FAILOVER_CALL/xce_failover_classify's idempotent-replay contract resends on a transport error or failover-mid-flight) sailed straight past the guard and appended a second, non-identical WAL_JOB_SUBMIT entry (different subtime/CRC) for the same jobid -- exactly the 'duplicate submits: N (byte-identical: 0)' signature waldump showed. Fix: src/controller/job.c adds a process-lifetime in-flight-jobid-range registry (s_inflight_submits, guarded by s_inflight_submits_lock) that handle_jobsubmit atomically check-and-claims (inflight_range_try_add) immediately after the existing jobid_range_exists check, and releases (inflight_range_remove) from jobsubmit_async_ctx_release. Regression test test_jobsubmit_resubmit_before_resolution_rejected_no_dup_wal_entry added to test_commit_barrier_reply.c (RED->GREEN confirmed). Committed fe4352f. + +Layer (2), the PreVote-freshness/apply-divergence liveness deadlock: initially framed as needing a fundamental safety/liveness design tradeoff among 3 unpalatable options (force-promote a divergent node / truncate committed-but-unapplied entries / relax the freshness gate) -- this framing was WRONG, corrected after user pushback ("kill-mid-apply should be recoverable per Raft protocol", "I do not think break FSM invariant is a good idea"). Root-caused instead to a concrete, scoped bug: persist_checkpoint() (persist.c:2261-2263, now fixed) stamped p->checkpoint_lsn from p->current_lsn (the physical WAL append tip) instead of p->last_applied_lsn (the true 'jobmgr reflects up through here' cursor, persist.h:344-363 documents these as DISTINCT once apply is decoupled from commit). write_snapshot() serializes live c->jobmgr, so a checkpoint taken while the async apply pump lags behind the WAL tip mislabels the snapshot: on restart, last_applied_lsn is re-seeded FROM checkpoint_lsn and replay resumes strictly above it, silently skipping committed-but-not-yet-applied entries that jobmgr never actually received -- the real mechanism behind kill-mid-apply producing a permanent APPLY_DIVERGENT fence. Confirmed apply_committed_entry() is fully deterministic/idempotent (rules out non-determinism) and write_snapshot() is crash-atomic at the file level (temp+fdatasync+rename, rules out torn-file corruption) -- the bug is purely a mislabeled watermark, not a structural divergence. Fix: persist_checkpoint() now reads checkpoint_lsn from last_applied_lsn and holds apply_pump_lock across the label-capture + write_snapshot() call so the label and the jobmgr state it describes are captured atomically w.r.t. the live apply pump (the pump only ever moves last_applied_lsn/jobmgr forward, so freezing it here rules out the label ever claiming more than jobmgr actually holds). test_checkpoint_sets_checkpoint_lsn updated (now simulates apply catch-up explicitly, since a cluster-less unit-test persist_t never runs the apply pump); new regression test test_checkpoint_lsn_tracks_last_applied_not_current_lsn added to test_persist_checkpoint.c (RED confirmed pre-fix: checkpoint_lsn took the WAL tip=7 instead of the simulated applied cursor=2; GREEN after). Full 78-group/782-test controller_unit_test sweep: 0 failures. + +ADDENDUM (2026-07-08, post-fix re-soak): a fresh chaotic soak on a rebuilt binary (both layers included) cleared rounds 1-4 (previously stalled at round 4) but still hit a fresh APPLY_DIVERGENT fence at round 5. Deep dive (waldump + log correlation + code trace) proved BOTH layer 1 and layer 2 are working correctly and are NOT the cause: added test_jobsubmit_resubmit_after_full_resolution_rejected_no_second_job (test_commit_barrier_reply.c) proving the post-resolution dedup window is also airtight (-EEXIST, no second WAL entry) -- both known dedup windows are provably race-free. The round-5 duplicate jobid was traced instead to a TEST-HARNESS-ONLY defect: xce_test (test.c:1868) hardcoded idgen(&idctx, 0, 0, ...) (clusterId=apiserverId=0) and run-ha-soak.sh's submit_jobs() spawns a fresh xce_test process per retry attempt, resetting idgen's static idctx.sequence to 0 each time -- two unrelated job submissions from two fresh processes landing in the same wall-clock second collide on an identical jobid (ts+tag+seq all equal). Confirmed NOT a production risk: apiserver's GenerateJobID (clib.go) holds one process-lifetime idgen_ctx_t via sync.Once and each deployed apiserver instance has its own distinct apiserverId, so config.h's jobid bit-layout (cluster/apiserver bits) already partitions production's namespace. Fixed by folding getpid() into the harness's apiserverId (test.c:1868) so cross-process collisions become structurally impossible there too; filed as bug-1420. bug-1419 itself (both production layers) stands as correctly fixed -- this addendum exists so a future session doesn't misread the round-5 re-soak stall as evidence layer 1/2 are insufficient. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1417, bug-1060, bug-845, bug-1418, bug-1420 diff --git a/.wolf/bugs/bug-1420.md b/.wolf/bugs/bug-1420.md new file mode 100644 index 00000000..351c3b6a --- /dev/null +++ b/.wolf/bugs/bug-1420.md @@ -0,0 +1,38 @@ +--- +id: "bug-1420" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-08T04:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c"] +tags: ["test-harness", "jobid", "idgen", "chaotic-soak", "false-positive"] +related: ["bug-1419"] +legacy: true +--- + +# bug-1420: xce_test CLI harness generates colliding jobids across separate process invocations within the… + +## Description +xce_test CLI harness generates colliding jobids across separate process invocations within the same wall-clock second, surfacing as a spurious APPLY_DIVERGENT fence in chaotic HA soak (misattributed at first to bug-1419's controller-side dedup logic). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +idgen() (src/lib/idgen.c) derives jobid from a per-context {timestamp (1s resolution), sequence (reset to 0 whenever the second rolls over or the context is freshly initialized)} pair, plus caller-supplied clusterId/apiserverId bits (src/include/config.h: 6+4 bits respectively) that are meant to partition the jobid namespace across independent generators. xce_test's cmd_jsub called idgen(&idctx, 0, 0, arsize, ...) -- both partition fields hardcoded to 0 -- so every xce_test process shares the SAME (degenerate, unpartitioned) jobid space. run-ha-soak.sh's submit_jobs() (run-ha-soak.sh:446) spawns a fresh xce_test process per submit/retry attempt (`printf script | xtest`, xtest() = `timeout 60 "$XT"`), and idgen_ctx_init() resets sequence=0 on every fresh process start. Two unrelated job submissions from two different xce_test process launches landing in the same wall-clock second therefore compute an IDENTICAL jobid (ts, tag, seq all equal) -- a genuine, reproducible collision between logically distinct jobs, not a retry of the same one. This is NOT a production defect: apiserver's GenerateJobID (src/frontend/apiserver/backend/clib.go) initializes its idgen_ctx_t exactly once per process via sync.Once (sequence only resets on an actual apiserver restart, far rarer than per-request), and each deployed apiserver instance is assigned its own distinct apiserverId, so config.h's bit layout already partitions the production jobid namespace across instances even in the rare restart-collision case. + +## Fix +Folded getpid() into the harness's apiserverId at the idgen() call site (test.c:1868): idgen(&idctx, 0, (size_t)((uint64_t)getpid() & MAX_APISERVER_ID), arsize, &(jobsubmit.jobid)) -- gives every xce_test process invocation its own slice of the same namespace partitioning production already relies on, making a same-second cross-process collision structurally impossible. Mechanical, test-tool-only change; no production code touched. Verified via clean rebuild of both xce_test and controller_unit_test targets (no coupling, no regressions). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1419 diff --git a/.wolf/bugs/bug-1421.md b/.wolf/bugs/bug-1421.md new file mode 100644 index 00000000..e24b16dd --- /dev/null +++ b/.wolf/bugs/bug-1421.md @@ -0,0 +1,38 @@ +--- +id: "bug-1421" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-08T06:55:15.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c"] +tags: ["ha", "raft", "term-desync", "liveness", "election-regression", "ae-reject", "replication"] +related: ["bug-1417", "bug-1418", "bug-851"] +legacy: true +--- + +# bug-1421: Sequential HA soak regression: no primary elected at term>8 by round 6 (initially suspected to… + +## Description +Sequential HA soak regression: no primary elected at term>8 by round 6 (initially suspected to stem from the bug-1417 fix); root-caused 2026-07-09 as the primary never discovering a follower's higher term via AppendEntries-reject responses + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_raft_on_repl_ack only routes AE-reply term-adoption for success=1 ACKs; AE rejects (success=0) never reach raft_on_append_entries_reply. That function additionally drops any reply with reply_term != s->term (raft.c:832-833, stale-term drop) before it can adopt a higher term. Net effect: once a follower is ahead in term (e.g. it steps down from PreVote to CTRL_STANDBY at a higher term while an existing primary keeps campaigning/replicating at its old term), every AppendEntries the stale primary sends is rejected with ae_reject_stale, the primary never learns the higher term, and it can never regain write-quorum (needs 2/3, only self-vote counts) — a permanent livelock. First surfaced as an election regression (sequential soak: no primary elected at term>8 by round 6, chaotic stalled at round 5) suspected to be caused by the bug-1417 vote-index-mapping fix; re-reproduced 2026-07-09 as a round-4 sequential stall (primary ctrl-2 term=6, follower ctrl-3 term=7, 490 ae_reject_stale events, primary writable_lsn stuck at 17), which pinned the exact mechanism above. + +## Fix +Added two fallback term-discovery vectors that fire regardless of AE-reject outcome. (1) StateReport heartbeat-reply now carries the standby's current term (controller.capnp +term field @2, typemap.h state_report_t +term, ha_nop_reply_state_report populates it from ha->term) so the primary learns a higher term on every convergence cycle. (2) New library entry point raft_on_higher_term_observed(s, observed_term) mirrors UpdateTerm (strictly-greater-than only; touches term/voted_for/role; on_persist before on_become_follower for durability ordering) and is called from repl_ack_decode_and_dispatch on both the heartbeat state-report branch and the WAL-NACK branch (ha.c). Primary now steps down instead of retrying AppendEntries against a follower that rejects forever. Validated: 265 controller + 287 lib unit tests pass (zero regressions); 10-round sequential soak PASS (33 jobs, zero loss, all 3 nodes led ctrl-1/ctrl-2/ctrl-3); 10-round chaotic soak PASS (33 jobs, zero committed divergence, zero committed job loss). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1417, bug-1418, bug-851 diff --git a/.wolf/bugs/bug-1422.md b/.wolf/bugs/bug-1422.md new file mode 100644 index 00000000..9b5b7e0d --- /dev/null +++ b/.wolf/bugs/bug-1422.md @@ -0,0 +1,38 @@ +--- +id: "bug-1422" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-07-09T10:14:14.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".claude/skills/xce-test-ha/run-ha-soak.sh"] +tags: ["xce-test-ha", "false-positive", "client-failover", "retry-budget-exhaustion", "E_XCE_STANDBY", "test-harness"] +related: ["bug-854", "bug-1421"] +legacy: true +--- + +# bug-1422: 40-round sequential HA soak FAIL at round 36: verify_no_loss reported '0/111 jobs confirmed… + +## Description +40-round sequential HA soak FAIL at round 36: verify_no_loss reported '0/111 jobs confirmed present' right after a fresh election (new primary ctrl-1, term=40) — xce_test's expect-job got E_XCE_STANDBY (10014) from a fresh client session whose own wait-primary had JUST confirmed ctrl-1 as serving 45ms earlier + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +False-positive test-harness failure, NOT real committed-job loss: ctrl-1's own controller log confirms it stayed CTRL_PRIMARY throughout the window and successfully committed the 3 new post-failover jobs (log_append/ae_reply_success/commit_advanced at 10:14:22-23). ha_dispatch_gate's CTRL_PRIMARY branch (ha.c:3451-3531) unconditionally permits OPCode_get (joblist) even while fenced, so ctrl-1 itself could not have returned E_XCE_STANDBY for this query — the reply must have come from a standby (ctrl-2, just killed, or ctrl-3) that the client's failover retry loop landed on. Root cause traced to client_failover.c's xcecli_retry_next/xcecli_retry_decide (client_failover.c:305-319): E_XCE_STANDBY classifies as XCE_FO_RETRY_NOW (client_failover.c:59-60, prompt rotate, no backoff), and the 20-attempt retry budget (3 masters) should reliably reach the real primary — but when the WHOLE budget is exhausted while every recent classification was RETRY_NOW (as opposed to RETRY_BACKOFF), xcecli_retry_decide's budget-exhaustion branch (line 312-318) does NOT override *rc the way it does for RETRY_BACKOFF exhaustion (which maps to the distinct -E_XCE_NOT_READY, line 316-317) — it just returns false with *rc left at whatever the last attempt's raw status was (here, a stray E_XCE_STANDBY from a standby hit during the rapid post-election retry storm). verify_no_loss() (run-ha-soak.sh) then did a single one-shot check with zero retry at the driver level, unlike the job-submit loop just above it in the same file which already retries on this exact 'primary likely just promoted, regaining write-quorum' condition (run-ha-soak.sh:483-486). Only surfaced at round 36 of a 40-round run — narrow enough timing that shorter 6-10 round soaks across many prior sessions never hit it. + +## Fix +Added a bounded retry (3 total attempts, 2s settle sleep between) around the WHOLE verify_no_loss() check in run-ha-soak.sh, mirroring the existing submit-retry loop's rationale: a job still missing after a real settle window + 2 retries is a genuine loss, not this client-side retry-storm race. Did not touch client_failover.c (the battle-tested libxce-failover retry/rotation library with many other production consumers) — fixing at the test-driver level is safer and fully local. Re-ran both 40-round soaks after the fix: sequential PASS (123 jobs, zero loss, all 3 nodes led) and chaotic PASS (123 jobs, zero committed divergence, zero committed job loss). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-854, bug-1421 diff --git a/.wolf/bugs/bug-1423.md b/.wolf/bugs/bug-1423.md new file mode 100644 index 00000000..df149e1f --- /dev/null +++ b/.wolf/bugs/bug-1423.md @@ -0,0 +1,38 @@ +--- +id: "bug-1423" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-07-09T13:58:16.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["ha", "raft", "election", "liveness-deadlock", "vote-request", "dead-peer", "quorum", "permanent-stall"] +related: ["bug-1417", "bug-1418", "bug-1421"] +legacy: true +--- + +# bug-1423: 30-round sequential HA soak FAIL at round 5: post-failover submit did not get 3 acks after 5 retry… + +## Description +30-round sequential HA soak FAIL at round 5: post-failover submit did not get 3 acks after 5 retry attempts (15s+). Root-caused as a genuine permanent election deadlock, not a client-side false positive: ctrl-3 won PreVote and entered CTRL_ELECTING (real election, term=10) with ctrl-1 dead and ctrl-2 alive, then never showed another HA state-machine event for 2+ minutes (until the harness killed the cluster on FAIL) — stuck in CTRL_ELECTING forever. Meanwhile ctrl-2 endlessly retried its own PreVote every ~15-17s, correctly denied every time by ctrl-3 on freshness grounds, but that denial never unstuck ctrl-3's already-in-flight real election. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +raft_cb_send_vote_request (ha.c, sends the real RequestVote RPC to each peer) has two failure branches — endpoint_connect() failing and iomgr_monitor() failing — that are exactly what happens when the RPC target is a dead/unreachable peer (here, ctrl-1, killed at the start of round 5). The endpoint_connect failure branch only called the legacy handle_request_vote_reply() when `c == NULL` (unit-test mode) — in PRODUCTION (c always non-NULL) a connect failure to a dead peer was silently dropped with NO notification to the raft library at all. The iomgr_monitor failure branch called handle_request_vote_reply() unconditionally, including in production — but per vote_reply_dispatch's own design comment, handle_request_vote_reply is the LEGACY/ctx-done-signaling path; 'raft_on_vote_reply remains the SOLE production promotion driver' (vote_reply_dispatch calls ha_record_vote_reply + raft_on_vote_reply, NOT handle_request_vote_reply). Net effect: a candidate's vote request to a dead peer never registers as a 'reply' with the production raft library's raft_state_t.replies_received counter. raft.c's Step 6 fast-path ('all peers have replied, no quorum → become follower and retry', gated on replies_received == n_peers-1) can therefore NEVER trigger whenever any configured peer is down — with quorum=2 in a 3-node cluster and one peer dead, replies_received caps at 1 (only the live peer's real reply, if any) and can never reach n_peers-1=2. Recovery then depends ENTIRELY on the raw per-campaign election_timeout (5-10s, ha_jittered_election_delay) firing via delaymgr — in the observed failure, no election_timeout_fn trace appeared in the log at all for the full 2+ minute window, leaving the candidate permanently wedged. + +## Fix +Routed both raft_cb_send_vote_request failure branches (connect failure and monitor failure) through vote_reply_dispatch(ha, ectx, votes_idx, term, vote_granted=0, 0, 0) in production (c != NULL) — the exact same call a genuine vote reply makes (ha_record_vote_reply + raft_on_vote_reply), so a dead/unreachable peer now correctly counts as 'replied, not granted' immediately on connect/monitor failure instead of only via the timeout. Test mode (c == NULL) keeps the legacy handle_request_vote_reply call unit tests depend on. Added a forward declaration of the previously-file-local-only vote_reply_dispatch near the top of ha.c so raft_cb_send_vote_request (defined much earlier in the file) can call it. Verified: full controller_unit_test suite clean (755 PASS / 0 Fail, exit 0) both before and after removing temporary diagnostic logging used during root-cause; re-ran the exact failing scenario — 30-round sequential HA soak PASS (93 jobs, zero loss, all 3 nodes led) and 30-round chaotic HA soak PASS (93 jobs, zero committed divergence, zero committed job loss) — both previously hit this stall reliably (round 5 of the sequential run, 100% reproduction before the fix). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1417, bug-1418, bug-1421 diff --git a/.wolf/bugs/bug-1424.md b/.wolf/bugs/bug-1424.md new file mode 100644 index 00000000..fc4e73a3 --- /dev/null +++ b/.wolf/bugs/bug-1424.md @@ -0,0 +1,38 @@ +--- +id: "bug-1424" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-09T17:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["ha", "raft", "prevote", "dead-peer", "latency"] +related: ["bug-1423"] +legacy: true +--- + +# bug-1424: PreVote twin of bug-1423: raft_cb_send_pre_vote_request's connect/monitor failure branches (a… + +## Description +PreVote twin of bug-1423: raft_cb_send_pre_vote_request's connect/monitor failure branches (a dead/unreachable peer) never registered a reply with the raft library's PreVote tally, in EITHER test or production mode (worse than the real-vote path, which at least handled test mode). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same missing-registration pattern as bug-1423, in raft_cb_send_pre_vote_request instead of raft_cb_send_vote_request. Found while auditing for the analogous gap after fixing bug-1423. Lower severity than bug-1423: PreVote has an unconditional round-timeout fallback (ha_prevote_round_timeout_fn, confirmed firing reliably every round across dozens of observed rounds in the bug-1423 investigation logs) that always resolves the round regardless of tally completeness, so this was never a permanent-deadlock class bug like bug-1423 was — just a latency waste: every PreVote round against a down peer burns the full round timeout instead of concluding as soon as the live peer(s) reply. + +## Fix +Routed both failure branches through pre_vote_reply_dispatch(ha, c, peer_idx, granted=0) — the same production PreVote-reply path a real reply uses (raft_on_pre_vote_reply) — mirroring bug-1423's fix exactly. Added a forward declaration near the top of ha.c (pre_vote_reply_dispatch is defined much later in the file). Verified: full controller_unit_test suite clean (0 Fail, exit 0) after the change. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1423 diff --git a/.wolf/bugs/bug-1425.md b/.wolf/bugs/bug-1425.md new file mode 100644 index 00000000..1e9c505c --- /dev/null +++ b/.wolf/bugs/bug-1425.md @@ -0,0 +1,60 @@ +--- +id: "bug-1425" +status: "fixed" +status_updated: "2026-09-24" +status_note: "8a6c339a" +component: "controller" +severity: "hang" +timestamp: "2026-07-09T21:15:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/persist.c or src/controller/commit.c (unrooted)"] +tags: ["ha", "commit-engine", "election", "intermittent", "raft-apply-decoupling-followup", "live-repro-found", "concurrent-connect-trigger", "root-caused", "tdd-red-green"] +related: ["bug-1416"] +fix_commits: ["8a6c339a"] +legacy: true +--- + +# bug-1425: Intermittent (~2 of 9 attempts) client hang in the synchronous jobsub (COMMIT_MODE_WAIT) path… + +## Description +Intermittent (~2 of 9 attempts) client hang in the synchronous jobsub (COMMIT_MODE_WAIT) path shortly after a fresh single-node election. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT ROOT-CAUSED. Flagged during raft-apply-decoupling task 7.4's throughput benchmark work. Reproduced identically on both the pre-fix and post-fix (task 6.1 apply-pump) binaries — confirmed unrelated to that fix. Resembles but is not confirmed identical to the closed bug-1416 boot-race class (persist_recover() replay-verify racing the live apply pump / live-replication apply path). + +## Fix +[UPDATE 2026-08-16] Live investigation (single-node deploy via /xce-deploy-ha, controller_unit_test's own fix build): found a RELIABLE reproduction recipe the original ~2/9-sequential-attempts description never had, and narrowed the root cause to a specific mechanism, but did NOT land a fix -- reporting findings honestly per the explicit scope of this investigation pass. + +RELIABLE REPRO (not sequential -- CONCURRENT): deploy a fresh single-node cluster, poll the log for CTRL_PRIMARY, then fire 9 xce_test client processes (each: connect + one jobsub) SIMULTANEOUSLY (backgrounded, not sequential) immediately after election. Reproduced in 3 of 4 rounds (1-3 of 9 concurrent clients failing per round, closely matching the original ~2/9 ratio) -- 0/45 reproductions when the SAME 9 attempts were fired sequentially instead (one xce_test process at a time). Concurrency at connect time, not raw timing after election, is the actual trigger. + +SYMPTOM, precisely: the failing client's xcecli_jobsubmit() (client.c, wrapped in XCECLI_FAILOVER_CALL) returns rc>=0 (success-shaped, NOT a negative error) after exactly ~10.0-10.1s, with the output jobid=0 (never written) instead of the submitted JobId -- test.c's cmd_jobsub (bug-786's own guard) correctly catches this as 'no clean ack' rather than misreporting success, so no data is actually lost/corrupted -- but the client hangs ~10s before getting a clear answer, matching the bug's own description exactly. + +CONFIRMED CLIENT-SIDE, not server-side: cross-referenced the controller's own log (fresh --logfile per round) against the 9 concurrent attempts -- server-side WAL only ever shows 8 job-submit entries committed (lsn sequence has no gaps, no admission-cap rejection WARN, no error of any kind) when 9 concurrent clients attempted -- the failing 9th request never reaches handle_jobsubmit at all. Ruled out the obvious admission-cap suspect: commit.c's COMMIT_MODE_WAIT admission control (cap=n_vcpus-VTASK_LIVENESS_RESERVED=3 on this 4-vcpu deploy) never logged its 'WAIT rejected, admission cap exhausted' WARN in any reproduction -- 8 concurrent WAITs completing well under the cap is plausible given each takes <100ms server-side once received. + +THE ~10s NUMBER, precisely sourced: xceapi.c's xce_jobsubmit() (the innermost call XCECLI_FAILOVER_CALL wraps) has a HARDCODED cbuf_deadline_10s() wait on cbuf_timedget waiting for a reply to land in the single-slot ctx->wbuf. Read cbuf_timedget itself (mylock.c) -- confirmed CORRECT, it returns -ETIMEDOUT (negative) on a genuine timeout, which xce_jobsubmit correctly propagates as a negative rc. So a bare cbuf_timedget timeout cannot by itself explain the observed rc>=0-with-jobid=0 shape; something in the retry/rotate/recover layer ABOVE it must be involved. + +NARROWED SUSPECT (not confirmed): client_internal.h's own documented CI1 hazard -- 'handle_disconnect may have injected a stale status into the single-slot ctx->wbuf ... xcecli_ctx_recover drains any pending entry with a non-blocking cbuf_tryget so the next attempt's cbuf_get cannot consume a prior-attempt reply' -- but this drain only runs when xce_failover_rotate() succeeds (client_failover.c:130-153, mirrored by xcecli_retry_next per its own doc comment: 'only recover when the rotate succeeded'). On this single-candidate cluster, rotate reconnects to the SAME (only) node via a real xce_ctx_reconnect() -- plausible this usually succeeds (ruling out the simplest 'rotate never succeeds on single-node, drain never runs' theory) but NOT independently verified under the exact concurrent-connect contention this repro needs; a transient reconnect failure specifically during the connection-establishment storm (9 simultaneous TLS handshakes against a listener seconds old) could leave a stale wbuf entry undrained, which a LATER retry's cbuf_timedget could then either (a) consume immediately (fast, not 10s -- doesn't match) or (b) never resolve at all if the drain path interacts badly with an in-flight send, leaving the single wbuf slot permanently empty until the full 10s deadline. Did not instrument (fprintf/gdb) the actual attempt count or wbuf state during a live failing run to confirm this mechanism specifically -- xce_test's own -vvv flag produced no additional client-side retry/rotate trace output (goes through a different, unwired logger in this CLI context), and adding real instrumentation would need a fresh lib rebuild + re-repro cycle not completed in this pass. + +NOT YET DONE: instrument xcecli_retry_next/xce_jobsubmit directly (attempt count, wbuf drain outcome, rotate rc, per-attempt elapsed) and re-run the now-reliable concurrent-burst repro to catch the exact mechanism in the act -- this is the recommended next step, using the repro recipe above (no gdb needed; the reproduction is now reliable enough for print-based debugging). Confirmed unrelated to raft-apply-decoupling task 6.1's apply-pump fix (reproduces identically on that spec's pre-fix and post-fix binaries per the original entry) and to this session's bug-1428/1426/1474/1472-remainder fixes (none touch xceapi.c/client.c/client_failover.c's client-side retry path). + +[Original entry] +NOT FIXED. Worked around via retry for task 7.4's benchmarking purposes only. Needs a dedicated root-cause investigation with the same live-repro rigor bug-1416 got (gdb-attach via GDB_WRAP seam, or targeted debug logging around the commit-wait barrier right after a fresh single-node election). + +[UPDATE 2026-08-16, part 2] FIXED. Root-caused via static code trace (no gdb/instrumentation needed after all): XCECLI_FAILOVER_CALL's idempotent-replay rule (client_failover.c, xce_failover_classify -- 'a duplicate on a re-sent write means the replay guard already applied our write, treat as success') can normalize a retried resend that hits the server's EEXIST/inflight-collision rejection into overall success (rc==0) -- but that rejection reply carries NO payload (handle_jobsubmit's EEXIST/inflight-collision branches reply with RequestReply_data_nodata, job.c:1441-1467), so xce_jobsubmit() (xceapi.c) never populates its *jobid output on the winning (dup) attempt, and never did on the earlier failing attempt either (its raw status was already negative). Net: xcecli_jobsubmit() can return rc>=0 with *jobid left at the caller's initial value (0) -- matching the confirmed symptom exactly (rc>=0, jobid=0, ~10.0-10.1s: one full 10s cbuf_timedget deadline on the failing first attempt, then a fast dup-rejection reply on the resend). + +FIX (TDD): added jobsubmit_finalize_jobid(uint64_t* jobid, uint64_t submitted_jobid, int overall_rc) (src/lib/client_internal.h declaration, src/lib/client.c definition), called from xcecli_jobsubmit() right after XCECLI_FAILOVER_CALL returns. Sets *jobid = submit->jobid (the CLIENT-generated id already on the wire -- job ids are client-generated via idgen(); the controller commits/echoes this exact value verbatim on every genuine success reply, bug-786/job.c:1439) whenever overall_rc >= 0, otherwise leaves *jobid untouched. Deliberately placed at the jobsubmit-specific wrapper layer, not the generic classify/decide layer (which is correctly type-erased and has no concept of jobid) -- test.c's cmd_jobsub (bug-786's own gate, comparing returned jobid == locally-generated jobsubmit.jobid) was ALREADY the right verification signal and needed no changes. 3 new pure unit tests in test_libxce_retry.c (test_jobsubmit_finalize_populates_jobid_on_success/leaves_jobid_untouched_on_failure/null_jobid_safe) -- socket-free, following this codebase's established direct-state-simulation test style for I/O-adjacent logic (test_netio_on_connected.c's precedent) rather than mocking a live connection. + +LIVE VALIDATION: full ASAN rebuild (0 compiler errors; controller_unit_test 1101 tests, 2 pre-existing unrelated test_config_repl.c failures; libxce test suite 1/1 OK, all 3 new tests PASS). Deployed a fresh single-node cluster and ran the same concurrent-burst repro as the original investigation (fresh election -> 9 simultaneous xce_test jobsub clients) for 5 rounds / 45 total clients: 0 'no clean ack' failures, 45/45 clean acks with jobid matching the submitted id (vs. the original repro's 1-3/9 failures in 3 of 4 rounds pre-fix). Honest caveat: this is probabilistic/timing-dependent (like the original repro), so 45/45 clean is strong but not airtight proof the exact race window is unreachable -- the fix is correct by construction regardless (it closes the *jobid gap for EVERY path that can produce rc>=0, not just the one race that was reproduced), so residual risk is limited to a still-undiscovered SEPARATE mechanism producing the same symptom, not this one going unfixed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1416 diff --git a/.wolf/bugs/bug-1426.md b/.wolf/bugs/bug-1426.md new file mode 100644 index 00000000..82a886fe --- /dev/null +++ b/.wolf/bugs/bug-1426.md @@ -0,0 +1,41 @@ +--- +id: "bug-1426" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-09T21:15:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c (trim_wal_segments", "apply_pump_read_entry cache)"] +tags: ["ha", "persistence", "wal-trim", "apply-pump", "low-priority", "raft-apply-decoupling-followup"] +related: [] +legacy: true +--- + +# bug-1426: trim_wal_segments's retention floor has no awareness of the local apply pump's own lag — a… + +## Description +trim_wal_segments's retention floor has no awareness of the local apply pump's own lag — a reachable scenario since apply-lag-behind-commit is raft-apply-decoupling's entire premise. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Identified during raft-apply-decoupling task 7.4's apply-pump segment-read cache fix (round-2 review finding): the new fast-path cache can (POSIX unlink-keeps-fd-alive) silently advance last_applied_lsn past already-deleted data if the pump is lagging when trim_wal_segments fires. Currently bounded by the pre-existing APPLY_DIVERGENT boot fence (deferred loud fencing on next restart, not silent corruption) — a real behavior change from the pre-cache code, not literally risk-free, but not an active defect either. Same family as an already-tracked standby-cursor variant (per task 7.4's own notes). + +## Fix +FIXED (TDD RED->GREEN). trim_wal_segments() now caps its trim floor at the local apply pump's own progress (last_applied_lsn), not just remote standby-acked positions: floor = min(repl_trim_lsn's standby/disconnect-guard floor, last_applied_lsn+1). The +1 matters -- repl_trim_lsn's floor treats equality as 'still needed, preserve' (a standby acked through LSN N might still need that segment for retry/resync), but the LOCAL apply cursor's equality case means the opposite: last_applied_lsn==N means this process is DONE with entry N and will never re-read it, so a segment starting exactly at N is safe to delete. Feeding last_applied_lsn+1 into the same shared 'seg_start_lsn < floor' comparison gets correct delete-on-equal semantics for this term without a second, differently-shaped check. This closes the real gap: apply_pump_read_entry()'s fast-path cache holds an open fd into whatever segment it's currently reading; POSIX unlink-keeps-fd-alive meant a lagging pump could keep reading via that stale fd even after trim_wal_segments deleted the segment out from under it (no immediate corruption, since the fd's data was still valid) -- but a FRESH read of the same range (a restart's persist_recover() replay, or the pump's own cache falling back to its slow path after an unrelated miss) would find the segment already gone, producing a genuine APPLY_DIVERGENT-class gap between what the live process quietly caught up to and what a fresh read of disk state can see. Regression test (test_e2e_checkpoint.c's test_trim_guard_preserves_unapplied_local_segments): no standbys registered (so the pre-fix floor alone would allow trimming everything below the active segment) but last_applied_lsn deliberately lags behind one segment -- proves the already-applied segment IS trimmed (no unnecessary retention) while the not-yet-applied one is preserved (the actual guard). Also had to advance last_applied_lsn in the 3 pre-existing trim tests in the same file (test_checkpoint_wal_trim_after_checkpoint, test_trim_guard_preserves_standby_segments, test_trim_without_standbys_proceeds_normally), which call persist_wal_async directly (bypassing the whole apply path) and so never advance last_applied_lsn on their own -- without the added stamp, this fix would have correctly-but-unexpectedly blocked ALL trimming in those fixtures, since last_applied_lsn defaults to 0. Full controller_unit_test suite: 1099 tests, 2 pre-existing unrelated failures (bug-1561), no regressions. + +[Original entry] +NOT FIXED — deliberately deferred. task 7.4's own scope explicitly excluded redesigning trim_wal_segments's retention-floor awareness of apply-pump lag; documented via an accurate code comment instead. A future fix would give trim_wal_segments a floor input derived from the local apply cursor, not just standby-acked LSNs. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1427.md b/.wolf/bugs/bug-1427.md new file mode 100644 index 00000000..892552e5 --- /dev/null +++ b/.wolf/bugs/bug-1427.md @@ -0,0 +1,41 @@ +--- +id: "bug-1427" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-09T21:15:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c (persist_verify_apply_correctness", "count_jobmgr_entries)"] +tags: ["ha", "persistence", "apply-divergent", "worker", "low-priority", "raft-apply-decoupling-followup"] +related: [] +legacy: true +--- + +# bug-1427: APPLY_DIVERGENT's entity-count correctness check (persist_verify_apply_correctness) covers job… + +## Description +APPLY_DIVERGENT's entity-count correctness check (persist_verify_apply_correctness) covers job entities only — worker entities can diverge (worker_from_capnp()/restore_worker() genuinely creates a new worker on a WAL_WORKER_CTRL/HOOK_DETAIL_WORKER_JOIN entry when absent) but this is currently invisible to the check. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Disclosed, accepted scope gap from raft-apply-decoupling task 5.1's own design: the entity-count check was scoped to jobs only (queues correctly excluded — config-derived, WAL_QUEUE_CTRL never creates a new one, verified by two review rounds). Workers were never brought in scope. Not a regression — a known, deliberate boundary, just not yet extended. + +## Fix +FIXED (TDD RED->GREEN). Extended persist_verify_apply_correctness() with a worker-count check mirroring the existing job-count check exactly: added count_workermgr_entries() (persist.c, mirrors count_jobmgr_entries()), a new expected_new_worker_count field on apply_expectation_t (persist.h), and a new workermgr_count_before_replay parameter threaded through persist_verify_apply_correctness()/persist_recover(). The harder part: apply_committed_entry()'s WAL_WORKER_CTRL case previously always returned 0 regardless of outcome (JOIN-creates-new, UPDATE-applies-to-existing, and idempotent-noop were all indistinguishable to callers), so replay_wal() had no signal to tally new worker joins from. Fixed by making worker_from_capnp() (worker.c) return 1 specifically for a genuine new-worker creation (discarding restore_worker()'s raw idx, which can legitimately be 0 and would have collided with the update-applied case) and propagating that rc through apply_committed_entry() instead of the hardcoded 0 -- verified no caller of either function depended on the exact prior return value (only sign and -EEXIST were ever checked). replay_wal() tallies apply_rc==1 for WAL_WORKER_CTRL exactly like it already does for WAL_JOB_SUBMIT. New regression tests: test_verify_apply_correctness_detects_worker_count_mismatch (test_apply_correctness_verification.c, mirrors the existing job-count-mismatch test) proves a worker-count divergence is now caught as APPLY_DIVERGENT -- RED before this fix (no worker check existed at all), GREEN after. Also updated test_apply_committed_entry.c's test_ace_worker_ctrl_join_restores_and_notifies, which asserted rc==0 for a genuine JOIN creation under the old (now-corrected) contract -- now asserts rc==1 with an explanatory message. Full controller_unit_test suite: 1097 tests, 2 pre-existing unrelated failures (bug-1561), no regressions. + +[Original entry] +NOT FIXED. Extend persist_verify_apply_correctness() with a worker-count check mirroring count_jobmgr_entries() if/when a worker-creation divergence needs to be closed (task 5.1's own forward note). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1428.md b/.wolf/bugs/bug-1428.md new file mode 100644 index 00000000..391c159e --- /dev/null +++ b/.wolf/bugs/bug-1428.md @@ -0,0 +1,41 @@ +--- +id: "bug-1428" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "security" +timestamp: "2026-07-09T21:15:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c (persist_register_pending_apply)"] +tags: ["ha", "persistence", "apply-pump", "toctou", "unconfirmed", "raft-apply-decoupling-followup"] +related: ["bug-1415"] +legacy: true +--- + +# bug-1428: Narrow TOCTOU residual in persist_register_pending_apply() (task 6.2, part of bug-1415's fix)… + +## Description +Narrow TOCTOU residual in persist_register_pending_apply() (task 6.2, part of bug-1415's fix) — not reliably exercised by any current test. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Flagged during raft-apply-decoupling task 6.2 as a non-blocking fast-follow while closing bug-1415. Specific race window not fully documented in tasks.md's own notes; needs a fresh look at the function to characterize exactly what window remains and whether it's reachable in production timing. + +## Fix +FIXED (characterized then TDD RED->GREEN, per the entry's own 3-step plan). Characterized the exact window: persist_register_pending_apply()'s self-heal check (an UNLOCKED read of last_applied_lsn) ran BEFORE acquiring pending_applies_lock, separate from the locked insert that followed. Window: this thread reads last_applied_lsn (sees 'not yet applied') -> BEFORE it locks and inserts, the apply pump's thread advances last_applied_lsn past this lsn AND completes its own persist_take_pending_apply(lsn) call (finding nothing, since this thread hasn't inserted yet) -> this thread then inserts a node the pump will never revisit -- permanently orphaned, identical failure mode to bug-1415 (a hang, on_applied never fires) via a different, narrower trigger window. Production reachability: yes -- persist_register_pending_apply() is called from a different thread/vtask than the apply pump's walk (commit.c's continuation dispatch vs persist_apply_pump_run, per the function's own pre-existing doc comment), with no synchronization between them before this fix beyond the already-locked insert. Fix: moved the self-heal check inside pending_applies_lock, so the whole check-then-(insert-or-self-heal) sequence is one atomic critical section against persist_take_pending_apply's own locked scan-and-remove -- either this thread's check now runs before the pump's take (sees 'not yet applied', inserts, and the pump's later take, now forced to wait on the same lock, finds the freshly-inserted node) or after (sees 'already applied', self-heals instead of inserting, and the pump's take correctly finds nothing) -- never the interleaved gap that orphaned it. Regression test (test_apply_pump.c's test_register_pending_apply_race_against_concurrent_apply_self_heals): deterministic reproduction via controlled state (a main thread holding pending_applies_lock forces the register call to observe last_applied_lsn only after the lock's own update, removing dependence on real-time race timing) -- RED under the pre-fix code (on_applied never fires, node orphaned on pending_applies), GREEN under the fix (self-heals correctly, no orphan). Full controller_unit_test suite: 1098 tests, 2 pre-existing unrelated failures (bug-1561), no regressions. + +[Original entry] +NOT FIXED — explicitly deferred as a fast-follow, not a blocker for task 6.2 or the spec's closure. Needs: (1) characterize the exact TOCTOU window, (2) determine production reachability, (3) add a regression test that exercises it before fixing. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1415 diff --git a/.wolf/bugs/bug-1429.md b/.wolf/bugs/bug-1429.md new file mode 100644 index 00000000..b93ad7a6 --- /dev/null +++ b/.wolf/bugs/bug-1429.md @@ -0,0 +1,54 @@ +--- +id: "bug-1429" +status: "fixed" +status_updated: "2026-09-25" +status_note: "b5d59253" +component: "controller" +severity: "crash" +timestamp: "2026-07-10" +last_seen: "2026-09-24T10:51:31Z" +occurrences: 5 +files: ["src/controller/testsuite/test_commit_barrier_rollout.c", "test_demotion_convergence.c", "test_convergence_verification.c", "test_egress_filter.c", "test_apply_pump.c", "test_apply_pump_integration.c", "test_job_array_fence.c", "test_ha_role_lifecycle.c", "test_job_array_replay_fence.c", "test_commit_durability.c", "test_commit_engine.c", "test_p2_integration.c (test-fixture teardown ordering", "21 sites across 12 files); src/controller/commit.c (commit_engine_fini", "unchanged -- confirms the documented contract); src/controller/persist.c (fsync_worker", "unchanged -- confirms the race window)"] +tags: ["FIXED", "VTASK_LIVE_BLOCK", "commit_engine_fini", "config_repl.c", "controller-tests", "delaymgr", "fsync_worker", "gdb-confirmed", "ha_convergence.c", "hang", "live-repro", "multi-location", "persist.c", "persist_stop", "root-caused", "teardown-ordering", "test-fixture-bug", "test_commit_barrier_reply", "timeout", "toctou-race", "vtask"] +related: ["bug-1452", "bug-1687", "bug-924"] +fix_commits: ["b5d59253"] +legacy: true +gitea_issue: 264 +gitea_synced_status: "fixed" +--- + +# bug-1429: controller_unit_test suite TIMEOUT (300s, killed by SIGTERM) during /xce-build controller run… + +## Description +controller_unit_test suite TIMEOUT (300s, killed by SIGTERM) during /xce-build controller run immediately after uncommitted edits to persist.c, ha_convergence.c, config_repl.c (delaymgr-inline blocking I/O moved to VTASK_LIVE_BLOCK vtasks with inline-fallback-on-failure). Compile succeeded with zero errors (only pre-existing warnings). meson test summary: Ok:0 Fail:0 Timeout:1. Last test output before the hang was test_demotion_replay_yields_committed_only (test_commit_barrier_reply.c) printing overlayfs-SKIP, followed by a burst of HA FSM / persist_apply_pump_run log lines ending in "HA quorum lost: connected=0 required=1", then total silence for the remaining ~294s until SIGTERM. No ASAN/crash report -- looks like a genuine hang/deadlock, not a crash. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED 2026-08-30, LIVE-REPRODUCED with a full gdb thread backtrace (the first time in this bug's history any session captured a live hang instead of just reporting the SIGTERM/timeout). The original delaymgr/VTASK_LIVE_BLOCK hypothesis carried by this bug since 2026-07-10 was WRONG -- a static read of every vtask_enq_classified(..., VTASK_LIVE_BLOCK) call site in ha.c/persist.c/config_repl.c/commit.c found every one of them NULL-checks its pool pointer (or the trampoline is gated behind `c->delaymgr.running`, which test fixtures never set) and falls back inline on failure -- there is no path where a controller_unit_test fixture leaves a non-NULL-but-never-started vtaskmgr_t that a VTASK_LIVE_BLOCK task could be silently swallowed into. That entire subsystem is a red herring for this bug. + +The REAL root cause is the same TOCTOU test-teardown race already root-caused once before, for a DIFFERENT symptom, in bug-1452 (2026-07-19: an ASAN SEGV in test_persist_truncate.c) and partially re-applied once more in bug-1687 (2026-08-25: cbrp_teardown in test_commit_barrier_reply.c). Every affected test fixture calls `commit_engine_fini(&ce)` BEFORE `persist_stop(&persist)` in its teardown helper -- the exact opposite of the contract documented in commit_engine_fini's own comment (commit.c: 'Callers are expected to have stopped the flusher (persist_stop) first'). Because persist_stop had not yet run, the persist_t's live fsync_worker background thread (persist.c:800, started by persist_start) can still be mid-flight calling `p->on_durable_advance(p->durable_cb_ctx)` -- i.e. commit_on_durable_advance(ce) -- with NO lock held (persist.c:866-868), concurrently with commit_engine_fini destroying ce->lock/ce->cond (commit.c:476-477) and freeing ce->continuations. commit_engine_fini's own persist_set_durable_cb(ce->p, NULL, NULL) unhook is 'defensive and cheap' per its own comment but is NOT sufficient on its own: the flusher thread can have already read the OLD non-NULL callback pointer a few instructions before the unhook takes effect. Depending on exact thread interleaving this TOCTOU race manifests as either bug-1452's SEGV (a NULL/freed dereference inside commit_advance_locked) OR -- newly confirmed this session -- a hang: pthread_cond_destroy(&ce->cond) blocking forever because the racing fsync_worker thread is concurrently inside pthread_cond_broadcast(&ce->cond) / holding ce->lock from commit_on_durable_advance, which glibc's cond-destroy implementation cannot safely proceed past. This explains every symptom previously logged against this bug: the varying hang location across sessions (different tests hit their OWN teardown's copy of this same race, at whatever moment the fsync_worker thread happened to be flushing), the total silence after the hang (both racing threads are genuinely blocked in futex/pthread_cond primitives, not spinning), zero CPU time accruing, and "both threads parked in futex_do_wait" from the 2026-08-23 sighting. + +LIVE REPRO (this session): `meson test -C src/xce.build.bug1429 --suite controller --verbose` hung 40s into the run with 3 threads (main + 2x fsync_worker) all in futex_do_wait, 0 CPU time advancing over a 3s recheck. gdb -p -batch -ex 'thread apply all bt full' (attached BEFORE the 300s SIGTERM, while the process was still alive) showed: Thread 1 (main) stuck in pthread_cond_destroy() called from commit_engine_fini (commit.c:476) <- jap_teardown() (test_commit_barrier_rollout.c:1877) <- test_job_array_submit_payload_size_bounded_at_max_array_idx (test_commit_barrier_rollout.c:2294) <- UnityDefaultTestRun <- test_commit_barrier_rollout() <- main(). Threads 2 and 3 both stuck in pthread_cond_timedwait at fsync_worker (persist.c:827), BOTH with `p = &g_jap` (the same global persist_t) -- confirming jap_teardown's exact commit_engine_fini-before-persist_stop ordering bug (test_commit_barrier_rollout.c:1877-1879, at the time of capture) as the live mechanism. This is the 4th distinct hang location logged against this bug (previously: test_demotion_replay_yields_committed_only, the test_configack_* group, test_jobsubmit_repeated_truncation_forced_failures_no_phantom_jobs) -- all four are different call sites of the identical teardown-ordering bug, which is why the location never repeated and why every prior static-inference pass aimed at the vtaskmgr/delaymgr subsystem instead never found it. + +## Fix +FIXED 2026-08-30. Test-fixture-only fix -- no production code changed (commit.c and persist.c are exactly as documented; the bug is every affected fixture violating commit_engine_fini's own documented calling contract). Inserted an explicit `persist_stop(&persist);` call immediately before every confirmed-vulnerable `commit_engine_fini(&ce);` call (mirroring bug-1452/bug-1687's established fix pattern exactly), leaving each function's pre-existing later persist_stop()/teardown-helper call in place as a safe, idempotent no-op (persist_stop's own fsync_thread_running and wal_fd>=0 guards make the second call a clean skip). Where a fixture also tears down a real multi-vcpu vtaskmgr_t pool (test_apply_pump.c's tpr_teardown, test_apply_pump_integration.c's tpi_teardown), vtaskmgr_fini() was left in its original position AFTER commit_engine_fini() (a separate, unrelated invariant: no new vtask may be dispatched onto the pool mid-teardown) -- only persist_stop()'s position moved earlier. + +21 sites fixed across 12 files, all confirmed vulnerable by verifying (a) the fixture's persist_t is genuinely persist_start()'d (has a live fsync_worker thread) and (b) the teardown called commit_engine_fini() before persist_stop(): test_commit_barrier_rollout.c (3 sites -- g_fce/g_fp inline teardown, cb5_teardown, jap_teardown -- the last being the exact live-reproduced hang site), test_demotion_convergence.c (dc_teardown -- this EXACT site was flagged but explicitly left unfixed by bug-1452's own remediation note as a known follow-up, never actioned until now), test_convergence_verification.c (cv_teardown), test_egress_filter.c (ef_teardown), test_apply_pump.c (2 sites -- tap_teardown, tpr_teardown), test_apply_pump_integration.c (tpi_teardown), test_job_array_fence.c (tf_teardown), test_ha_role_lifecycle.c (lifecycle_cluster_teardown), test_job_array_replay_fence.c (1 site of its 4 persist_start'd fixtures -- the other 3 never wire a commit engine), test_commit_durability.c (3 sites, including one with a comment that incorrectly claimed persist_set_durable_cb's unhook alone was sufficient), test_commit_engine.c (3 sites, all sharing sync_persist_teardown), test_p2_integration.c (3 sites, 2 sharing var `p` + 1 using `g_p9`). Confirmed SAFE and left unchanged: barrier_teardown (test_commit_barrier_rollout.c), engine_teardown (test_commit_engine.c), and the fixtures in test_repl_majority.c/test_commit_admission_pool.c/test_coldstart_lsn0.c/test_commit_resolve.c -- all use a 'stub' persist_t that is never actually persist_start()'d (no live fsync_worker thread), confirmed by grep for persist_start against each fixture's global before touching it. + +KNOWN REMAINING EXPOSURE (not fixed this session, explicit follow-up for a future pass, same as bug-1452 left one for this session): test_config_repl.c has ~23 call sites following the identical inline `commit_engine_fini(&c->commit); t42_teardown(c);` pattern (t42_teardown, line 629, confirmed calls persist_stop THEN persist_fini -- so every one of these is the same bug) -- not fixed here because the file is large (~7000 lines) with some sites (e.g. the c_control/c_raced pair around line 2911-2912) not yet individually verified for uniformity, and a bulk edit without that verification was judged too risky for this session's budget. test_term_safety_adapter.c has persist_start'd fixtures (lines 59, 245, 2382) and 5 commit_engine_fini call sites (676, 930, 1776, 1950, 2338) not individually checked. Neither file's exposure has been observed to hang or crash yet (this bug's own history shows the race is intermittent and timing-dependent -- absence of a prior sighting there is not evidence of safety). Recommended next step: a dedicated /xce-audit-invariants or grep-driven sweep of these two files using the exact same verification method used here (confirm persist_start presence, confirm fini-before-stop order, insert persist_stop() immediately before commit_engine_fini()). + +VERIFICATION: full `meson test -C xce.build.bug1429 --suite controller` run 3 times back-to-back after the settled fix (no edits during any of these three runs) -- 39.01s / 38.86s / 38.91s, Ok:6 Fail:0 every time, zero hangs, zero timeouts (previously: TIMEOUT 300.08s-300.30s on effectively every attempt this session before the fix, reproduced live on the very first try). No regression test added (per the task's own documented exception for a suite-level timing-dependent teardown race): the race requires a live fsync_worker thread mid-flush at the exact moment of teardown, which is not reliably triggerable in a bounded, fast unit test without reintroducing the same flakiness this fix removes; the 3x full-suite re-run is the verification method this project's own bug-1452 precedent already established for this exact bug class. + +UPDATE (2026-09-24): 2026-09-24 ROOT CAUSE of the residual 300s TIMEOUT (live-captured with gdb under a 6-way parallel stress harness, unshare -rn per instance): any failing TEST_ASSERT longjmps out and SKIPS the fixture teardown; the fixture's commit engine sweep thread (and persist fsync_worker) stay asleep on the static engine's ce->cond / p->fsync_cond; the next test re-inits the same static struct underneath them (pthread_cond_init on a condvar with a live waiter), corrupting the waiter count; a later teardown then blocks forever in pthread_cond_destroy (commit_engine_fini / persist_fini) or pthread_join (cb5_teardown) -- or, for a stack-local engine, the sweep thread reads a dead frame (ASAN stack-use-after-return in commit_sweep_worker, test_persist_truncate.c). The failing assertion itself was never seen because Unity's stdout was fully buffered and lost on SIGTERM/abort. Triggering flaky assertions found once visible: test_job_array_submit_one_entry_one_continuation_across_sizes (the test in flight when the 11:34 meson run hung; continuation resolved on the fsync_worker thread, test read a plain int immediately), test_pump_mixed_facts_and_commands_catchup_original_order (reply count read once before trailing continuations), test_persist_flush_fires_durable_advance_callback (fsync_worker credited a concurrent persist_flush advance to itself -> double fire). FIX (uncommitted): (1) controller_test.c line-buffers stdout; (2) XCE_TESTING registries: tearDown() calls persist_test_reap_unstopped() then commit_test_reap_unfinalized_engines(), stopping/joining whatever a failed test leaked (clean full run reaps 0; injected failure now = 1 FAIL in 2s instead of a hang); (3) persist.c flusher_flush_once reports only its own advance (fsync_worker no longer double-fires; regression test test_flusher_pass_does_not_claim_concurrent_flush_advance); (4) the two flaky tests wait (bounded) for cross-thread resolution. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1452, bug-1687, bug-924 +- Gitea: #264 diff --git a/.wolf/bugs/bug-1430.md b/.wolf/bugs/bug-1430.md new file mode 100644 index 00000000..553c4ad6 --- /dev/null +++ b/.wolf/bugs/bug-1430.md @@ -0,0 +1,38 @@ +--- +id: "bug-1430" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-07-10T07:12:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c"] +tags: ["asan", "use-after-free", "prevote", "delaymgr", "chaotic-soak", "raft"] +related: ["bug-1423", "bug-1424"] +legacy: true +--- + +# bug-1430: AddressSanitizer: heap-use-after-free on address ... WRITE of size 4 ... #0… + +## Description +AddressSanitizer: heap-use-after-free on address ... WRITE of size 4 ... #0 ha_prevote_cancel_round_timeout ../controller/ha.c:6712 #1 ha_start_prevote_async ../controller/ha.c:6889 #2 ha_failover_stagger_fn ../controller/ha.c:7259 -- crashed 2 of 3 nodes during a live chaotic HA soak (10-round overlapping-kill mode) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_prevote_round_timeout_fn (the delaymgr task body for a PreVote round's timeout) only cleared the static s_prevote_round_ctx pointer inside its 'round still in progress' branch (ha->raft.pre_vote_in_progress true). When a round instead got decided via a reply arriving through raft_on_pre_vote_reply (which clears pre_vote_in_progress without touching s_prevote_round_ctx) and the round-timeout task fired afterward anyway (a benign, expected race), the function fell straight through to free(ctx) with s_prevote_round_ctx still pointing at that about-to-be-freed memory. The next PreVote round's ha_start_prevote_async defensively calls ha_prevote_cancel_round_timeout() before arming its own timer, reading the dangling pointer and writing to old->cancelled -- UAF. Same call stack (single delaymgr thread T8) on both crashing nodes, all-100%-reproducible under chaotic-mode's rapid overlapping restarts. + +## Fix +ha_prevote_round_timeout_fn now does a CAS (atomic_compare_exchange_strong_explicit matching s_prevote_round_ctx against this specific ctx, clearing to NULL) UNCONDITIONALLY as the first action, before either branch -- so the static pointer is never left dangling regardless of which path is taken. Brings it in line with the already-correct pattern used by the sibling function raft_election_timeout_fn (same file), which unconditionally clears its own stored ctx pointer before checking cancelled. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1423, bug-1424 diff --git a/.wolf/bugs/bug-1431.md b/.wolf/bugs/bug-1431.md new file mode 100644 index 00000000..b3c71ee3 --- /dev/null +++ b/.wolf/bugs/bug-1431.md @@ -0,0 +1,38 @@ +--- +id: "bug-1431" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "data-loss" +timestamp: "2026-07-10T10:02:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/ha.c", "src/controller/persist.c", "src/include/persist.h"] +tags: ["raft", "promotion", "committed_lsn", "chaotic-soak", "apply-decoupling", "durability"] +related: ["bug-1432"] +legacy: true +--- + +# bug-1431: chaotic HA soak: committed_job_loss=5-8 across 10 rounds (VERDICT result=FAIL) after… + +## Description +chaotic HA soak: committed_job_loss=5-8 across 10 rounds (VERDICT result=FAIL) after raft-apply-decoupling task 16 gated standby apply strictly on commit; committed_divergence stayed 0 and all jobs eventually reappeared at final settlement -- a visibility-latency gap, not true data loss + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Two stacked gaps in raft_cb_become_leader promotion, both masked pre-task-16 by eager apply-on-WAL-receipt: (1) persist_truncate_wal_above only sets p->committed_lsn on branches of truncate_wal_above_locked that perform a REAL truncation; its common no-op early-return (leader own log already covers safe_lsn) left committed_lsn stuck at its pre-election value, so newly-computed safe_committed_lsn was never applied to the actual commit watermark. (2) Even after fixing (1) with a durable_lsn-capped store, a newly-promoted leader whose own fsync had not yet caught up to its own WAL tip (group-commit batches fsync on a timer) stayed capped below safe_lsn indefinitely -- nothing re-drove the promotion advance once the flusher eventually synced, since fsync_worker on_durable_advance only re-evaluates the commit engines PENDING per-entry continuations, and a standby has none for replicated entries. + +## Fix +New function persist_promotion_advance_committed(p, safe_lsn) (persist.c, declared in persist.h) called from raft_cb_become_leader right after persist_truncate_wal_above: unconditionally advances p->committed_lsn toward safe_lsn (monotonic), forcing a synchronous flusher_flush_once(p) first if durable_lsn < safe_lsn (safe/bounded -- by Raft Leader Completeness the new leaders own WAL tip is always >= safe_lsn, so the bytes are already on disk and only need fdatasync). Reduced but did not fully close the soak failures alone -- see bug-1432 for the second half of the fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1432 diff --git a/.wolf/bugs/bug-1432.md b/.wolf/bugs/bug-1432.md new file mode 100644 index 00000000..9de31a5d --- /dev/null +++ b/.wolf/bugs/bug-1432.md @@ -0,0 +1,38 @@ +--- +id: "bug-1432" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-10T10:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c"] +tags: ["raft", "heartbeat", "promotion", "delaymgr", "chaotic-soak", "latency"] +related: ["bug-1431"] +legacy: true +--- + +# bug-1432: chaotic HA soak still FAIL (committed_job_loss=8) after bug-1431 force-flush fix; jobs above the… + +## Description +chaotic HA soak still FAIL (committed_job_loss=8) after bug-1431 force-flush fix; jobs above the conservative 2-voter safe_committed_lsn floor stayed invisible via expect-job for ~3.1s post-promotion, matching the primary heartbeat interval (3000ms) almost exactly + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_start_primary_heartbeat armed the periodic heartbeat via delay_task_ms(interval_ms, -1, primary_heartbeat_fn, c), whose FIRST invocation fires only after interval_ms elapses (standard periodic-timer semantics) -- despite the functions own long-standing comment claiming to start the heartbeat immediately. A freshly-promoted leader therefore waited a full heartbeat interval before ever sending a NOP to a reconnecting/catching-up standby, so it could not learn that standbys true applied_lsn (via the heartbeat-reply stateReport, ha_process_standby_state) for up to 3s. That knowledge gates when catch-up WAL streaming starts and, in turn, the per-entry majority-ack that advances committed_lsn past the elections conservative safe_committed_lsn floor (jobs above that floor were legitimately majority-committed via the OLD primary before it died, per Leader Completeness, but the new leader cannot expose them without fresh quorum corroboration -- a correct Raft Figure-8 safety requirement, not a bug in itself). + +## Fix +ha_start_primary_heartbeat now calls primary_heartbeat_fn(c) synchronously once, immediately, before arming the periodic delay_task_ms timer -- matching the functions own documented intent. primary_heartbeat_fn only performs non-blocking async sends (xce_ha_heartbeat_req), so this is safe inline with no thread-role change. Single fix point covers all 3 promotion call sites (raft_cb_become_leader, ha_promote_to_primary, and the third raft-driven-decision-loop site) since all route through ha_start_primary_heartbeat. Closed the gap: 10-round chaotic soak went from committed_job_loss=8 to committed_job_loss=0, committed_divergence=0, result=PASS; 10-round sequential soak + leadership-coverage also PASS (36 jobs, zero loss). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1431 diff --git a/.wolf/bugs/bug-1433.md b/.wolf/bugs/bug-1433.md new file mode 100644 index 00000000..e19bfcb2 --- /dev/null +++ b/.wolf/bugs/bug-1433.md @@ -0,0 +1,39 @@ +--- +id: "bug-1433" +status: "fixed" +status_updated: "2026-09-27" +status_note: "255b0faa: peer-control traffic while CTRL_ELECTING classified VTASK_LIVE_BLOCK (become-leader tail preads)" +component: "controller" +severity: "unknown" +timestamp: "2026-07-10T10:59:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c", "src/controller/ha.c"] +tags: ["raft", "promotion", "vtask", "thread-role", "liveness", "audit-finding", "bug-1198-class"] +related: ["bug-1431", "bug-1432", "bug-1198"] +fix_commits: ["255b0faa"] +legacy: true +--- + +# bug-1433: controller-invariant-audit (2026-07-10, /xce-audit-invariants) async-I/O-thread-role dimension… + +## Description +controller-invariant-audit (2026-07-10, /xce-audit-invariants) async-I/O-thread-role dimension: bug-1431 round-2 force-flush fix (persist_promotion_advance_committed calling flusher_flush_once, a blocking fdatasync) is reachable via the real multi-node winning-vote-reply dispatch path on a VTASK_LIVENESS-classified vtask worker -- a task class the design documents as never admission-gated / never expected to block. This is the same bug class as bug-1198 (HA_REPL_SEND_SNAPSHOT persist_checkpoint on VTASK_LIVENESS), which was previously closed by offloading the blocking work off that worker class entirely. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +raft_cb_become_leader (the .on_become_leader raft-library callback) is invoked synchronously from within peer_ctrl_msg_handler -> peer_ctrl_dispatch_to_pool -> peer_ctrl_classify (ha.c ~5935-5940), which HARDCODES VTASK_LIVENESS for every inbound peer-control packet including vote replies -- the doc comment above it explicitly states "everything this handler dispatches is liveness work today" and was written BEFORE this session added a blocking fdatasync call reachable from that same path. ha_dispatch_election_start/raft_election_start_vtask_entry (the VTASK_LIVE_BLOCK dispatch used for the RDHUP-failover and single-node self-grant election-START triggers) does NOT cover the separate, later "win the election / become leader" event, which fires on vote-REPLY arrival via a completely different dispatch chain. + +## Fix +Removed the flusher_flush_once force-flush from persist_promotion_advance_committed entirely -- round-3 (durable-capped only, no blocking calls) plus the already-shipped bug-1432 immediate-first-heartbeat fix together close the original visibility-latency gap non-blockingly (confirmed by two live soaks: 6-round chaotic committed_job_loss=0/committed_divergence=0 PASS, 6-round sequential+leadership-coverage PASS). TRACKED RESIDUAL (not fixed this session): persist_apply_pump_run_sync (added earlier this session for the bug-1043-family apply-lag fix, also called from raft_cb_become_leader on the same VTASK_LIVENESS path) does bounded pread() of recently-written WAL segments -- lower practical risk than fdatasync (almost always page-cache-hot) but still, strictly, blocking I/O on a never-blocks task class. Removing it is NOT safe (reopens bug-1043-family job loss). Proper fix needs either reclassifying the winning-vote-reply case in peer_ctrl_classify or dispatching promotion tail onto a fresh VTASK_LIVE_BLOCK vtask -- both larger changes deferred to a dedicated follow-up with its own soak-validation pass. Comments at both call sites corrected to stop falsely claiming VTASK_LIVE_BLOCK safety. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1431, bug-1432, bug-1198 diff --git a/.wolf/bugs/bug-1434.md b/.wolf/bugs/bug-1434.md new file mode 100644 index 00000000..59c538a1 --- /dev/null +++ b/.wolf/bugs/bug-1434.md @@ -0,0 +1,38 @@ +--- +id: "bug-1434" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-10T10:56:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c", "src/controller/ha.c"] +tags: ["raft", "committed_lsn", "race-condition", "monotonicity", "audit-finding", "CAS"] +related: ["bug-1431"] +legacy: true +--- + +# bug-1434: controller-invariant-audit (2026-07-10, /xce-audit-invariants) Raft TLA+ checklist dimension… + +## Description +controller-invariant-audit (2026-07-10, /xce-audit-invariants) Raft TLA+ checklist dimension: persist_promotion_advance_committed and ha_standby_advance_committed_and_apply can race on persist_t.committed_lsn during the promotion window before role flips to CTRL_PRIMARY, via an unlocked load-then-conditional-store pattern in both functions -- a losing racer could clobber a larger value with a smaller one, regressing committed_lsn (violates the documented commitIndex-monotonic invariant). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +raft_cb_become_leader calls persist_promotion_advance_committed BEFORE setting ha->role = CTRL_PRIMARY (the role store sits ~170 lines later in the function). During that window this node is still nominally non-primary, so ha_apply_wal_catchup_entry (still processing an in-flight WAL catch-up entry on another thread, e.g. if this node was mid-catch-up when it won the election) can legitimately call ha_standby_advance_committed_and_apply concurrently with the promotion path own committed_lsn advance. Both did atomic_load(committed_lsn); if(target<=cur) return; atomic_store(committed_lsn, target) with no lock and no CAS, a classic lost-update race under concurrent writers. + +## Fix +Added persist_committed_lsn_advance_atomic(persist_t*, uint64_t target) (persist.c, declared in persist.h) -- a CAS retry loop (atomic_compare_exchange_weak_explicit) that is safe against any number of concurrent callers with no lock required, and can never regress committed_lsn regardless of interleaving. Both persist_promotion_advance_committed and ha_standby_advance_committed_and_apply now route through this shared helper. Validated: build clean (19/19 unit test suites pass), 6-round chaotic soak PASS (0 loss, 0 divergence), 6-round sequential+leadership-coverage soak PASS. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1431 diff --git a/.wolf/bugs/bug-1435.md b/.wolf/bugs/bug-1435.md new file mode 100644 index 00000000..89000403 --- /dev/null +++ b/.wolf/bugs/bug-1435.md @@ -0,0 +1,38 @@ +--- +id: "bug-1435" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-12T04:45:15Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (raft_state_t entry points)", "src/controller/ha.h (new raft_lock field)", "src/controller/cluster.c (raft_on_vote_request/raft_on_pre_vote_request call sites)"] +tags: ["ha", "raft", "raft-drive-async", "data-race", "raft_state_t", "vote-reply", "election", "verify_no_loss", "xce-test-ha", "E_XCE_STANDBY", "concurrency", "peer_ctrl_dispatch_to_pool"] +related: ["bug-608", "bug-1197", "bug-1198", "bug-1417", "bug-652", "bug-1282"] +legacy: true +--- + +# bug-1435: 40-round sequential /xce-test-ha FAIL at round 36/40: verify_no_loss reported 0/111 jobs confirmed… + +## Description +40-round sequential /xce-test-ha FAIL at round 36/40: verify_no_loss reported 0/111 jobs confirmed after 3 attempts; xce_joblist client call returned E_XCE_STANDBY (10014) against freshly-elected primary ctrl-1 (term=41) moments after wait-primary succeeded on the same session + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED (2026-07-12, static analysis + full call-graph trace, not live-reproduced under a debugger due to timing rarity -- see confidence note in fix). ha_dispatch_gate's CTRL_STANDBY branch is the ONLY production site that can return -XCE_ERR_STANDBY for a GET/joblist request (confirmed by exhaustive read of ha.c:3699-3865); under CTRL_PRIMARY, OPCode_get is unconditionally allowed regardless of quorum/fence state. So the answering node's ha->role was genuinely CTRL_STANDBY at the moment ha_dispatch_gate ran -- this was NOT a client-side discovery/retry-budget problem (ruled out: XCECLI_FAILOVER_CALL's retry loop -- src/lib/client_failover.c -- correctly retries E_XCE_STANDBY up to max_retries=20 with zero backoff via XCE_FO_RETRY_NOW, and its final surfaced rc staying exactly -E_XCE_STANDBY on budget exhaustion is the DOCUMENTED behavior for a RETRY_NOW-classified cause, not a symptom of a broken retry loop). The MISSING INFO 'xcecli failover: attempt N' log lines (zero matches across the whole 40-round run) are explained by the fact that libxce's wait-primary REPL command (cmd_wait_primary, testsuite/test.c) does its OWN silent primary-discovery/redirect-follow loop (xcecli_ctrlstatus_primary) that never calls the shared xcecli_retry_next/INFO-logging path -- once wait-primary hands off a live, correctly-connected-to-primary ctx, subsequent ops normally succeed on attempt 0 with nothing to log; this is a red herring, not a clue to a client bug. The REAL mechanism is a genuine, unsynchronized DATA RACE in raft_state_t (the embedded Raft consensus struct, ha->raft): raft_on_vote_reply/raft_on_vote_request/raft_on_pre_vote_reply/raft_on_pre_vote_request/raft_begin_pre_vote/raft_start_election/raft_check_and_start_election/raft_on_election_timeout/raft_on_higher_term_observed all read+mutate raft_state_t's PLAIN (non-atomic) fields (term, role, votes_granted, replies_received, election_decided, vote_replies[], pre_vote_* tally) with ZERO locking -- the raft library (src/lib/raft/raft.c) is written assuming a single-threaded/serialized caller (the classic Raft single-event-loop model). But the raft-drive-async feature (peer_ctrl_dispatch_to_pool in ha.c, default ON via g_drive_async_peerctrl_dispatch_enabled=1) dispatches each peer-control message (including RequestVoteReply from each individual voter) as an independent VTASK_LIVENESS vtask onto the shared vtaskmgr pool; peer_ctrl_dispatch_to_pool's own per-endpoint serialization (bind_vtask) only orders messages arriving on the SAME endpoint -- it does NOT serialize messages from DIFFERENT peers' endpoints against each other. So two vote replies from two different voters (e.g. ctrl-2 grants, ctrl-3 grants or denies) for the SAME election CAN be processed concurrently on two different pool worker threads, both calling raft_on_vote_reply(&ha->raft, ...) on the shared struct with no lock. Traced exact race for the observed symptom: worker A processes voter1's grant, reaches quorum (votes_granted=2>=2), sets election_decided=1, fires on_become_leader -> raft_cb_become_leader -> ha->role=CTRL_PRIMARY (term=41, matching ctrl-1's own logs exactly). Worker B, running concurrently (no synchronization), processes voter2's reply and -- racing on the unprotected votes_granted/replies_received/election_decided reads -- can independently satisfy raft_on_vote_reply's Step 6 'all replied, no quorum' branch (replies_received==n_peers-1 && votes_granted raft_cb_become_follower -> ha->role=CTRL_STANDBY for the SAME term, an instant after promotion, with no further election and no fresh log line calling out a 'demotion' that anyone was specifically looking for. Every subsequent client GET request then correctly, faithfully hits ha_dispatch_gate's CTRL_STANDBY branch and is rejected with -XCE_ERR_STANDBY -- explaining why the failure PERSISTED across all retries/attempts across multiple seconds (nothing else ever corrects role back to PRIMARY once flipped -- no bug, no external trigger, it just stays CTRL_STANDBY). This also explains why the bug needed ~36 rounds to surface once: it requires two vote replies to genuinely overlap in processing time on two different pool workers, a narrow, load/scheduling-dependent timing window. Verified raft-drive-async is on by default (g_drive_async_peerctrl_dispatch_enabled=1) and confirmed via full call-graph trace that raft_cb_become_leader/raft_cb_become_follower (the synchronously-fired callbacks) never themselves call back into another raft_* entry point (no re-entrancy via the callbacks), and that no existing ha->lock-held critical section ever calls into a raft_* function (so lock-order is always raft_lock-then-ha->lock, never reversed -- no ABBA risk versus the two pre-existing lock-order bugs in this exact area, bug-652 and bug-1282). CONFIDENCE: HIGH on mechanism (every piece of evidence in the original investigation -- ctrl-1's own healthy term=41 promotion log, zero ASAN errors, commit_advanced continuing for seconds after, the missing xcecli-failover logs, every retry returning the SAME E_XCE_STANDBY class -- is fully and specifically explained by this race, with no residual unexplained facts), but NOT live-reproduced under a debugger/instrumented build in this session (the race is timing-rare, needed ~36 rounds to surface once in the original soak) -- this is a well-evidenced, fully-traced static diagnosis, not a confirmed-via-repro one. + +## Fix +Added a new dedicated ha_state_t field `raft_lock` (pthread_mutex_t, PTHREAD_MUTEX_RECURSIVE -- src/controller/ha.h) serializing every mutating/reading entry point into raft_state_t on the election/vote path: raft_on_vote_reply (ha.c vote_reply_dispatch), raft_on_vote_request + raft_on_pre_vote_request (cluster.c, inbound vote RPCs), raft_on_pre_vote_reply + raft_begin_pre_vote (ha.c pre-vote dispatch), raft_start_election + raft_check_and_start_election x3 call sites (ha.c election-start paths, inline and vtask-dispatched), raft_on_election_timeout (ha.c delaymgr callback), and raft_on_higher_term_observed x2 (ha.c heartbeat-reply and WAL-NACK term-discovery paths). Initialized/destroyed in ha_init/ha_fini. Deliberately RECURSIVE (not a plain mutex): raft_on_pre_vote_reply's WON branch and raft_begin_pre_vote's single-node-win branch synchronously call ha_enter_real_election -> ha_start_election_async -> raft_start_election on the SAME thread within the same call chain (a legitimate same-thread nesting, not a race) -- a plain non-recursive mutex would self-deadlock on that path exactly like the precedent bug-652 (ha->lock self-deadlock from synchronous re-entry). Deliberately scoped to the ELECTION/VOTE path only, NOT the AE/commit-replication path (raft_log_append, raft_on_append_entries_request/reply, raft_advance_commit) -- ha_primary_log_append/raft_log_append is reachable from concurrent client-write vtasks with no visible lock either, which may be a SEPARATE, analogous concurrency gap, but touching the write/durability path carries much higher regression risk and wasn't the demonstrated mechanism for this specific symptom; flagging it as a follow-up investigation, not silently ignoring it. Verified no lock-order regression: no existing ha->lock-held critical section calls into any raft_* function (grepped and read all 9 ha->lock call sites), so raft_lock is always acquired before, never after, ha->lock -- consistent single order, no ABBA (cf. bug-1282, bug-652). Build: nix develop ./src -- bash src/tools/xce-build.sh controller -- clean compile, zero new warnings (3 pre-existing unrelated warnings in ha.c/cluster.c confirmed via git diff hunk-range check to be outside every edited region), controller_unit_test meson suite Ok:1 Fail:0 in 32.38s with no hang/timeout (a hang would have surfaced a recursive-mutex misuse or missed ha_init path, per the bug-1224/bug-1429 precedent of dispatch-related test hangs in this exact area -- none occurred). NOT live-reproduced/soak-validated in this session (explicitly deferred by the user to a separate follow-up 40-round /xce-test-ha run) -- this is a well-evidenced, carefully-scoped static fix for a data race, not a repro-confirmed one; flagged honestly as best-effort-but-high-confidence rather than 100%-certain. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-608, bug-1197, bug-1198, bug-1417, bug-652, bug-1282 diff --git a/.wolf/bugs/bug-1436.md b/.wolf/bugs/bug-1436.md new file mode 100644 index 00000000..01199167 --- /dev/null +++ b/.wolf/bugs/bug-1436.md @@ -0,0 +1,38 @@ +--- +id: "bug-1436" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-07-12T05:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/client_internal.h (new XCECLI_FAILOVER_CALL_COUNT macro)", "src/lib/client.c (xcecli_reslist/xcecli_workerlist/xcecli_joblist/xcecli_resval/xcecli_jobinfo switched to it). .claude/skills/xce-test-ha/run-ha-soak.sh's node_pattern()/run_pattern() workdir-scoping fix (prior pass) remains a legitimate", "independent improvement -- KEPT", "not reverted."] +tags: ["ha", "connection-reset", "econnreset", "xce-test-ha", "sequential-mode", "round-36-recurrence", "libxce-failover", "client_failover.c", "xce_failover_classify", "xcecli_joblist", "status-code-magnitude-collision", "ECONNREFUSED-111", "reconnect-storm", "ROOT-CAUSED", "FIXED", "live-repro"] +related: ["bug-1435", "bug-1418", "bug-854", "bug-646"] +legacy: true +--- + +# bug-1436: 40-round sequential /xce-test-ha FAIL at round 36/40: xce_joblist: Connection reset by peer… + +## Description +40-round sequential /xce-test-ha FAIL at round 36/40: xce_joblist: Connection reset by peer, immediately after wait-primary confirmed ctrl-2 serving in 46ms. RE-OPENED 2026-07-12 ~14:00-16:00Z after a live re-test (fresh isolated 40-round run, no concurrent soak) reproduced the IDENTICAL failure with ctrl-2 confirmed NEVER killed (same PID throughout, continuous RAFT/heartbeat log activity, still answering other clients after the failure) -- disproving the process-collision theory as the (sole) root cause. ROOT-CAUSED FOR REAL via live capture (tcpdump unavailable in sandbox -- no CAP_NET_RAW; used direct -vvv log correlation on a live reproduction instead) 2026-07-12 ~16:00-16:15Z. FIXED. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +TWO ROOT CAUSES, both now addressed. (1) SUPERSEDED/PARTIAL (kept as a real but non-exclusive fix): the original pass diagnosed this as run-ha-soak.sh's unscoped `pkill -9 -x xce_controller` letting a second concurrent soak invocation silently SIGKILL the first run's live primary. That fix (node_pattern()/run_pattern() workdir-scoped pgrep/pkill) is a genuine improvement and stays in place, but a FRESH, FULLY ISOLATED 40-round soak re-run (deploy dir xce-test-ha-40-v2, confirmed via `ps aux` before/during/after that no second soak was ever running) STILL FAILED at round 36/40 with the byte-identical signature, and ctrl-2's own log proved it was never killed (constant PID, unbroken RAFT event stream, still serving OTHER clients seconds after the failure) -- so process collision was real but was NOT the (sole) mechanism behind every occurrence. (2) THE ACTUAL ROOT CAUSE (found via live reproduction, not archaeology): `xcecli_joblist`/`xcecli_reslist`/`xcecli_workerlist`/`xcecli_resval`/`xcecli_jobinfo` (src/lib/client.c) are all built on `xce_get_with_condition` (src/lib/xceapi.c), which returns a NON-NEGATIVE ROW COUNT on success (`rc = getreply.nrows`) -- NOT the uniform '0 = success, negative = XCE/errno-style status' convention every OTHER xcecli_* wrapper follows. All of them were routed through the generic `XCECLI_FAILOVER_CALL` macro (client_internal.h), which unconditionally feeds the raw EXPR return value into `xce_failover_classify()` (client_failover.c) -- a function that recognizes retriable/terminal outcomes purely by MAGNITUDE (E_XCE_* codes >=10001, and POSIX errno values it deliberately accepts in EITHER sign via `fo_abs()`: ECONNRESET=104, ETIMEDOUT=110, ECONNREFUSED=111). A joblist reply whose row count happens to equal one of these magnitudes is silently misclassified as a transport failure on EVERY call with that exact count, even though the read fully succeeded -- forcing a spurious `xce_failover_rotate()` to the next configured master. Since get-class ops are primary-only, landing on a standby correctly (but pointlessly) fails with E_XCE_STANDBY and rotates again; the whole cycle repeats for every subsequent same-sized call in the same verify_no_loss batch, burning the finite `max_retries` budget (20, XCE_RETRY_DEFAULTS) on self-inflicted reconnect churn. run-ha-soak.sh's sequential mode submits NJOBS=3 per round, so the tracked job count follows the arithmetic progression 3+3*round -- of the three dangerous POSIX magnitudes (104, 110, 111), ONLY 111 is a multiple of 3, and it is hit for the FIRST time at EXACTLY round 36 (3+3*36=111=ECONNREFUSED) -- explaining both why this is genuinely rare (requires an exact numeric collision, not a generic race) and why it always surfaced at round 36 specifically for this test's job-count cadence. LIVE PROOF: a fresh, fully isolated 40-round soak (deploy dir xce-test-ha-40-bug1436-live) failed AGAIN at round 36/40 with 111 tracked jobs; the target primary's own -vvv log (ctrl-2, PID confirmed continuously alive via a parallel 1s-resolution forensic process/connection poller) showed a clear, damning pattern for the ENTIRE 'verify' client session: `cluster.c:786 server sock 4 accepted 10` (a BRAND NEW TCP connection) immediately before EVERY SINGLE `opcode get` request, followed by two `xce_reply` sends (the >100-row chunked-get path, xce_reply_get pktlimit=100) and then `cluster.c:64` (the TRACE line inside `handle_endpoint_error`, i.e. the connection being torn down) -- one full accept-serve-close cycle PER expect-job call, ~1799 accepts total on that one short-lived primary PID alone -- confirming the client was silently reconnecting on every single (mostly-successful) get call, exactly the reconnect storm this mechanism predicts, not a misreading of the log by a prior investigation pass. The run's own retry-storm messages ('verify_no_loss attempt 1/3: recheck failed... likely post-election client retry-storm race') were themselves an accurate but previously-misinterpreted symptom of this bug. Confirmed via codegraph_explore + direct reads of src/lib/client.c, src/lib/client_internal.h, src/lib/client_failover.c, src/lib/xceapi.c, src/lib/request.c (xce_reply_get chunking), src/controller/opget.c (do_reply_joblist/check_controller), src/lib/netio.c (endpoint_enq/handle_error/handle_output -- confirmed NOT independently implicated: zero `endpoint_update_epoll` WARN lines anywhere in the failing run's ctrl-2.log), and the existing `src/lib/testsuite/test_libxce_retry.c` StatusClassifier taxonomy tests (which confirm `xce_failover_classify` is INTENTIONALLY designed to accept genuine status codes in either sign -- the bug is that 5 call sites fed it a row count instead of a status code, not that the classifier itself is wrong). Two purpose-built synthetic repro harnesses (hammer2.sh: fixed 60-job list, ~5400+ calls with periodic real primary kills; hammer3.sh: fixed 495-job list, ~1500+ calls with periodic real primary kills) both ran clean with ZERO failures, because neither ever held the tracked job count at exactly 104/110/111 -- which in hindsight is the confirming, not disconfirming, result: the bug requires an EXACT count collision, not raw connection-churn volume, so a fixed-size synthetic repro that never lands on the magic number cannot hit it no matter how much volume it generates. That negative result is what motivated pivoting to the real /xce-test-ha harness (which naturally sweeps through every job count via its round-based growth) as the reproduction vehicle, per the live-capture mandate. + +## Fix +Added `XCECLI_FAILOVER_CALL_COUNT(ac, EXPR)` to src/lib/client_internal.h -- a sibling of `XCECLI_FAILOVER_CALL` for xce_get-family calls. It only enters the retry/classify loop when EXPR's raw result is NEGATIVE (`while ((_fc_rc < 0) && xcecli_retry_next(...))`); any non-negative result (0 or a genuine row count of ANY magnitude) is unconditionally treated as success and returned as-is, so it can never be misread as a retriable/terminal status code. A dead ctx (XCE_CTX_ERR) is still handled correctly: the per-op XCE_CTX_RUN guard in xceapi.c makes EXPR itself return a negative -EINVAL in that state, which enters the loop and is reclassified by xcecli_retry_decide's dead-connection branch exactly as before. Switched the 5 count-returning call sites in src/lib/client.c to the new macro: xcecli_reslist, xcecli_workerlist, xcecli_joblist (the confirmed live-reproduced offender), xcecli_resval (same xce_get_with_condition pattern, same latent risk), and xcecli_jobinfo (returns a fixed 1 on success -- lower risk since it never collides with a recognized magnitude and was previously misclassified as TERMINAL, which is harmless because TERMINAL does not rotate/rewrite rc, but fixed anyway for correctness/consistency rather than relying on that coincidence). Did NOT touch `xce_failover_classify` itself or any of the other 14 XCECLI_FAILOVER_CALL call sites (hello, join, jobdispatch, jobsubmit, clusterconf, ctrlstatus, update_resvals, update_jusage, jobctrl, queuectrl, hostctrl, runtask, set_pend_reason, clear_reason) -- all of those genuinely follow the 0/status-code convention the classifier is designed for, confirmed via test_libxce_retry.c's existing StatusClassifier taxonomy tests, which still pass unmodified (xcelib_test: 293 Tests 0 Failures 0 Ignored). controller_unit_test: 926 PASS / 0 FAIL / 0 Ignored (direct-binary run, ANSI-stripped grep census, exit 0). Did NOT touch run-ha-soak.sh further (its bug-1436-prior-pass workdir-scoping fix stays, unrelated to this mechanism) and did NOT add a client-side retry-more band-aid -- the actual defect was a classification-layer bug misinterpreting legitimate success data as a transport error, not a genuinely unavoidable transient condition, so the correct fix is at the classification boundary, not the retry budget. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1435, bug-1418, bug-854, bug-646 diff --git a/.wolf/bugs/bug-1437.md b/.wolf/bugs/bug-1437.md new file mode 100644 index 00000000..536475d3 --- /dev/null +++ b/.wolf/bugs/bug-1437.md @@ -0,0 +1,38 @@ +--- +id: "bug-1437" +status: "open" +status_updated: "2026-09-24" +status_note: "LSan leaks in test_jobj_hardening (makeInstance 240B + 7B strdup), no fix/suppression" +component: "lib" +severity: "crash" +timestamp: "2026-07-13T00:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/testsuite/test_jobj_hardening.c"] +tags: ["jobj", "test_jobj_hardening", "leak", "asan", "lsan", "datakiller", "dereference", "hard_build_accounts", "test_job_destroy_does_not_leak_accounts", "unverified-fix", "kvmgr", "job-array-single-wal-record", "mechanical-validation", "reconfirmed"] +related: [] +legacy: true +--- + +# bug-1437: Verification run of lib build + xcelib_test --run test_jobj_hardening… + +## Description +Verification run of lib build + xcelib_test --run test_jobj_hardening (ASAN_OPTIONS=detect_leaks=1) after user changed 4x .datakiller=destroy -> .datakiller=dereference in src/lib/testsuite/test_jobj_hardening.c (to fix a leak in the kvmgr-of-job_t test fixture, since job_t.destroy is NULL in RTTI typeinfo and teardown only runs via job_dereference): Unity reports 5/5 PASS, 0 Failures, 0 Ignored, but LeakSanitizer STILL reports two leaks totaling 247 bytes and the process exits 1 (no halt_on_error/abort_on_error set, so no SIGABRT with plain detect_leaks=1 -- differs from the meson full-suite run which sets abort_on_error=1 and gets killed by signal 6). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +TWO leaks detected, only one already known. (1) Direct leak of 240 bytes via calloc -> makeInstance -> makeJob_ (../lib/jobj.c:768) -> makeJob (../lib/jobj.c:789) -> test_job_init_propagates_makeJobsub_failure (test_jobj_hardening.c:133) -- this IS the already-diagnosed, out-of-repo makeInstance-does-not-free-on-negative-init-rc characteristic (vendored rtti library), unrelated to the datakiller fix. (2) Direct leak of 7 bytes via strdup -> hard_build_accounts (test_jobj_hardening.c:269) -> test_job_destroy_does_not_leak_accounts (test_jobj_hardening.c:328) -- this is a SEPARATE, NEW leak signature not previously logged, and it is attributed to the exact test (test_job_destroy_does_not_leak_accounts) that the datakiller=dereference fix was meant to make leak-free. NOT YET DIAGNOSED WHY: could be an unrelated small leak in the hard_build_accounts helper itself (e.g. a strdup()d account string never freed regardless of datakiller correctness), or could indicate the datakiller fix is incomplete. Not investigated further per explicit user instruction to build/run/report only, no fixing. + +## Fix +NOT FIXED IN THIS SESSION -- user explicitly instructed build+run+report only, no code changes. Logging the raw finding so the next session does not have to rediscover it: the accounts-leak test still leaks (7 bytes, strdup, hard_build_accounts) even after the .datakiller=dereference change, though it is a much smaller leak than the pre-existing 240-byte makeInstance one and does not cause any Unity test FAILURE (LSAN leak detection is a separate pass/fail axis from Unity PASS/FAIL in this harness). RECURRENCE 2026-07-16 (2nd occurrence, mechanical-validation pass for job-array-single-wal-record spec, dimension A/full-suite reconfirmation): full lib test binary run (ASan) again reports 638 PASS / 0 FAIL at the Unity level, but LeakSanitizer aborts the process at exit with the same 240-byte makeInstance->makeJob_ (jobj.c:768) leak signature, triggered via test_job_init_propagates_makeJobsub_failure (test_jobj_hardening.c:133). Confirmed this is the SAME pre-existing vendored-rtti-library characteristic already diagnosed in this entry's root_cause (1), not a new leak and not introduced by this feature's production code -- the feature's own new test is simply the first test in the suite to deliberately drive job_init() to fail after makeInstance() has already allocated, so it is the first to trip the latent characteristic. Still NOT FIXED (out of scope for a validation-only pass); flagging that under meson's full-suite invocation (abort_on_error=1) this leak turns into a SIGABRT / non-zero exit code for the lib test binary, so a naive exit-code-only CI gate would read this as a lib-suite failure even though zero Unity assertions fail and zero new markers/regressions exist in feature code. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1438.md b/.wolf/bugs/bug-1438.md new file mode 100644 index 00000000..7b218b7b --- /dev/null +++ b/.wolf/bugs/bug-1438.md @@ -0,0 +1,38 @@ +--- +id: "bug-1438" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-07-14T09:36:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_commit_barrier_rollout.c"] +tags: ["job-array-single-wal-record", "wal", "test-helper", "offset-arithmetic", "deadlock", "futex"] +related: [] +legacy: true +--- + +# bug-1438: test_job_array_submit_appends_exactly_one_wal_entry_and_one_continuation: Expected 11 Was 5 (WAL… + +## Description +test_job_array_submit_appends_exactly_one_wal_entry_and_one_continuation: Expected 11 Was 5 (WAL type mismatch); subsequent test hung all 3 threads on futex_do_wait + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +jap_read_last_wal_hdr()/jap_read_last_wal_payload() were copied from read_last_wal_hdr() in test_ha_role_lifecycle.c, which computes the header offset as seg_offset - sizeof(hdr). That arithmetic is only correct for zero-payload entries (its only prior caller was a WAL_NOOP test). Every WAL_JOB_ARRAY_SUBMIT entry carries a nonzero capnp payload, so seg_offset - sizeof(hdr) landed inside the payload bytes, not the header -- read garbage decoded as a bogus type (5 == WAL_JOB_STOP). The first array test's TEST_ASSERT_EQUAL_UINT32 failure then longjmp'd out of the test body BEFORE reaching jap_teardown(), leaking that test's persist_t (background WAL-sync thread still running) into the next test's jap_setup() memset -- corrupting a live mutex mid-flight and deadlocking all 3 threads on futex_do_wait in the very next test. + +## Fix +Replaced the back-computed-offset helpers with jap_read_wal_hdr_at()/jap_read_wal_entry_at(), which take the entry's start offset EXPLICITLY. Callers now snapshot g_jap.seg_offset immediately before each persist_submit_job_array_async() call and pass that snapshot in, instead of trying to infer the header position from the post-append offset. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1439.md b/.wolf/bugs/bug-1439.md new file mode 100644 index 00000000..3733c6fa --- /dev/null +++ b/.wolf/bugs/bug-1439.md @@ -0,0 +1,41 @@ +--- +id: "bug-1439" +status: "fixed" +status_updated: "2026-09-24" +status_note: "654c6e8c, c1c6c9a2" +component: "controller" +severity: "unknown" +timestamp: "2026-07-14T13:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 5 +files: ["src/xce.build/lib/controller.capnp.h (generated by [5/190] Generating lib/controllercapnp.[ch] custom command)"] +tags: ["ninja-race", "controller", "parallel-build-race", "build-failure", "unresolved", "retry-fixes-it", "ninja", "missing-dependency-edge", "reproduced", "capnp", "go-apiserver-link"] +related: [] +fix_commits: ["654c6e8c", "c1c6c9a2"] +legacy: true +gitea_issue: 30 +gitea_synced_status: "fixed" +--- + +# bug-1439: lib/controller.capnp.h:4170:43: error: unknown type name 'LimitCon'; lib/controller.capnp.h:1… + +## Description +lib/controller.capnp.h:4170:43: error: unknown type name 'LimitCon'; lib/controller.capnp.h:1: error: unterminated #ifndef — during ninja compile of lib/libxce.a.p/utils.c.o, cascading into ~440 bogus errors (glibc math.h storage-class errors, utils.h ectx_t redefinition) for the rest of that one translation unit + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +SUSPECTED (not confirmed): parallel-build race — gcc appears to have read lib/controller.capnp.h while the custom-command generation step was still writing it (non-atomic write, no write-to-tmp+rename). Evidence: re-inspecting the SAME on-disk file immediately after the failed build shows correct, well-formed content at line 4170 (get_LimitConfig(struct LimitConfig*, LimitConfig_list, int i);) and balanced #ifndef/#endif (9/9) — i.e. the finished file is NOT actually corrupt, only what gcc read mid-write was. Not yet reproduced a second time to confirm. + +## Fix +NONE APPLIED — this run was build+report only (nix develop ./src -- xce-build.sh controller), not a fix task. Build failed via set -euo pipefail before any tests ran; no controller/xce_controller, controller_unit_test, or xce_test binaries were produced. Next step if it recurs: check meson.build custom_target() for lib/controller.capnp.[ch] has correct output declarations / add explicit depends on all consumers, or file upstream against capnpc-c invocation for atomic output. RECURRENCE 2026-07-16 (2nd occurrence, CONFIRMS the race, different symptom): clean build (rm -rf xce.build) of `nix develop ./src --command bash src/tools/xce-build.sh controller asan` at HEAD 2a76940. This time gcc failed with `fatal error: controller.capnp.h: No such file or directory` (not corrupted content) for hook.c.o, threshold.c.o, xlist.c.o, utils.c.o, window.c.o, xceapi.c.o, wrap.c.o — all 7 consumers of utils.h#include controller.capnp.h. Critically, ninja's own completion log shows `[12/191] Generating lib/controllercapnp.[ch]` printed AFTER jobs [5..11] (the 7 failed compiles) had already completed/failed. This proves the custom_target that generates lib/controller.capnp.h has no dependency edge forcing it to run before these 7 specific compile units — confirms the suspected root cause in the original entry (missing depends()/output declaration in meson.build), not just a non-atomic-write timing fluke. Still NOT fixed (build+report only task, no source edits authorized). controller_unit_test and xce_test were never produced this run (clean build wiped the prior binaries and the new build failed before relinking). RECURRENCE 2026-07-17 (3rd occurrence, kiro-validate-impl smoke-boot build for job-array-single-wal-record): fresh (non-clean) build of `nix develop ./src --command bash src/tools/xce-build.sh controller` at HEAD ea8d9d3 failed on the FIRST invocation with yet another symptom of the same race: `fatal error: opening dependency file controller/controller_unit_test.p/ha.c.o.d: No such file or directory` AND simultaneously `fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory` for two unrelated targets at once ([187/192] and [188/192]) -- gcc could not open its OWN .o.d dependency-file output path, not an input header this time, reinforcing this is a general ninja/meson parallel-build filesystem race in this sandbox, not specific to controller.capnp.h generation. WORKAROUND CONFIRMED AGAIN: simply re-running the identical build command with no source changes succeeded cleanly (85/85 targets, controller_unit_test 3/3 OK), producing fresh xce_controller/xce_test binaries. Do-not-repeat: if xce-build.sh fails with any "fatal error: opening dependency file ... .o.d: No such file or directory" or "controller.capnp.h: No such file or directory", just retry the same command once before treating it as a real build break -- do not start debugging source code. RECURRENCE 2026-07-20 (4th occurrence, xce-build.sh all asan build+report task validating bug-1453/1454 C-side changes, HEAD after commit 619f41f): clean build failed on FIRST invocation with fatal error: controller.capnp.h: No such file or directory for xlist.c.o, htab.c.o, window.c.o, xstring.c.o, hook.c.o, wrap.c.o, threshold.c.o, xceapi.c.o, utils.c.o, zarray.c.o (10 consumers of utils.h#include controller.capnp.h) -- ninja log again shows [16/353] Generating lib/controllercapnp.[ch] printed AFTER the failed compiles. WORKAROUND CONFIRMED AGAIN (4th time): identical retry with zero source changes succeeded cleanly, 353/353 ninja targets, all 9 meson test suites OK (0 Fail), test_cluster (part of libxce test suite) and scheduler tests all passed. Still unfixed at the meson.build dependency-edge level -- do-not-repeat guidance holds: always retry once before treating this as a real regression. RECURRENCE 2026-07-26 (5th occurrence, build+report task validating bug-1474 repl.c fix via `nix develop ./src --command bash src/tools/xce-build.sh all asan`, HEAD c0b0c7d): TWO consecutive races in the same session, different symptoms each time. 1st invocation (clean build): classic capnp.h race -- hook.c.o etc. failed 'controller.capnp.h: No such file or directory' with the generator's status line ([6/361] Generating lib/controllercapnp.[ch]) printed AFTER the failed compiles -- identical to prior occurrences. 2nd invocation (retry, clean build): build progressed all the way to [361/361] (last job, frontend/apiserver's go_build) which FAILED with `/usr/bin/ld: cannot find -lxce: No such file or directory`; lib/libxce.a was confirmed absent from src/xce.build/lib/ immediately after the failure, and no 'Linking static target lib/libxce.a' line ever appeared in the ninja log for that invocation. NEW symptom variant: this is the first occurrence of the race hitting a Go custom_target (apiserver's CGO link against libxce.a) rather than a C compile/link step -- apiserver's CGO_LDFLAGS references @BUILD_ROOT@/lib/libxce.a only via an env-var string, which is suspected to carry no explicit ninja dependency edge on the lib archive target, letting the go_build job be eligible to run before the archive step completes. 3rd invocation (retry, clean build) succeeded cleanly: 361/361 ninja targets, meson test Ok:9 Fail:0, scheduler manual test 15/15 pass, frontend Go tests (apiserver + xcecli) all ok, xceadmin pytest 166 passed, producing fresh xce_controller/xce_test (confirmed containing the bug-1474 'reusing guard-holding slot' WARN string) and a fresh xceadmin/.venv/bin/xceadmin. WORKAROUND CONFIRMED AGAIN (do not debug source on first or second failure): simply retry the identical xce-build.sh command with no changes; this session needed up to 2 retries (3 total invocations) before a clean pass, still well within the established do-not-repeat guidance. Do-not-repeat addendum: the race is not limited to C compile/link edges -- it can also strand a Go custom_target waiting on a C static archive (apiserver <- lib/libxce.a) referenced only via env-var string interpolation, not a declared meson `depends:`/`link_with:` edge. Still unfixed at the meson.build dependency-edge level. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #30 diff --git a/.wolf/bugs/bug-1440.md b/.wolf/bugs/bug-1440.md new file mode 100644 index 00000000..d586776b --- /dev/null +++ b/.wolf/bugs/bug-1440.md @@ -0,0 +1,39 @@ +--- +id: "bug-1440" +status: "fixed" +status_updated: "2026-09-24" +status_note: "b7b90e9d" +component: "apiserver" +severity: "unknown" +timestamp: "2026-07-15T09:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/wire_dispatch_test.go"] +tags: ["cgo", "go-test", "toolchain-limitation", "wire_dispatch_test", "job-array-single-wal-record", "backend-package", "build-error", "golang-go-13938"] +related: [] +fix_commits: ["b7b90e9d"] +legacy: true +--- + +# bug-1440: use of cgo in test wire_dispatch_test.go not supported / FAIL apiserver/backend [setup failed] + +## Description +use of cgo in test wire_dispatch_test.go not supported / FAIL apiserver/backend [setup failed] + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Go toolchain hard restriction, not an environment/CGO_CFLAGS/nix issue: cmd/go's module indexer (GOROOT share/go/src/cmd/go/internal/modindex/read.go, ~line 589, inside the per-file import scan) explicitly checks `if imp.path == "C" { if isTest { badGoFile(name, fmt.Errorf("use of cgo in test %s not supported", name)) } }`. Any `_test.go` file that directly does `import "C"` is rejected outright the moment CGO is active for that file (CGO_ENABLED=1 and the file's build tag, here `//go:build cgo`, is satisfied) -- this is golang/go#13938, a long-standing toolchain limitation, confirmed by reading the actual installed Go 1.25.5 source in the nix shell. Because wire_dispatch_test.go and clib.go/convert.go/peers_test.go/handler_test.go/convert_test.go/heartbeat_test.go (the last four carry no build tag, so they build under BOTH cgo and !cgo) all live in the same package, the ENTIRE apiserver/backend package fails as `[setup failed]` under `make test-backend` (CGO_ENABLED=1) -- this also silently prevented the pre-existing cgo-buildable tests (handler_test.go, convert_test.go, peers_test.go, heartbeat_test.go) from running, even though none of them regressed on their own. The `!cgo`-tagged files (integration_test.go, control_host_test.go, stub_funcs_test.go) are unaffected since they're excluded from the CGO_ENABLED=1 file set entirely; `make test` (CGO_ENABLED=0) passed clean (apiserver/backend ok, 1.510s) confirming no regression on the non-cgo path. + +## Fix +NOT FIXED -- user explicitly instructed build/run/report only, no source edits. Reported fix path for the user: remove `import "C"` (and any direct `C.xxx` calls) from wire_dispatch_test.go itself; move the cgo-dependent helpers (buildEncodedJobArrayPush, deliverViaRealDispatchChain, and the `/* ... */ import "C"` preamble with the wire_test_* static C helpers) into a new non-test `.go` file in the same package (e.g. clib_wire_test_helpers.go, `//go:build cgo`), exposing plain-Go-signature functions the _test.go file can call without itself using cgo directly. This mirrors the established pattern already used for other CGO test-only helpers in this package. Because the file was rejected before type-checking, it is unknown whether wire_dispatch_test.go has any further errors (undefined C symbols, type mismatches) beyond this first, blocking one -- those can only be discovered after this structural fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1441.md b/.wolf/bugs/bug-1441.md new file mode 100644 index 00000000..c00ac397 --- /dev/null +++ b/.wolf/bugs/bug-1441.md @@ -0,0 +1,38 @@ +--- +id: "bug-1441" +status: "fixed" +status_updated: "2026-09-24" +status_note: "wire_dispatch_support.go direct Go call" +component: "apiserver" +severity: "unknown" +timestamp: "2026-07-15T09:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/wire_dispatch_support.go"] +tags: ["cgo", "wire_dispatch", "go_push_handler", "cross-file-declaration", "job-array-single-wal-record", "test-backend", "CGO_CFLAGS"] +related: ["bug-1440"] +legacy: true +--- + +# bug-1441: backend/wire_dispatch_support.go:154:8: could not determine what C.go_push_handler refers to /… + +## Description +backend/wire_dispatch_support.go:154:8: could not determine what C.go_push_handler refers to / FAIL apiserver/backend [build failed] + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +cgo declaration visibility is per-file, not per-package: go_push_handler is exported to C only via clib.go's own preamble comment ("extern int go_push_handler(xce_request_t*, void*);" immediately followed by "//export go_push_handler" on the Go func), and that extern declaration lives nowhere in any shared header -- it is a plain comment inside clib.go's /* ... */ block, not client.h/netio.h/request.h/controller.capnp.h. cgo processes each file's own preamble independently to build that file's C.xxx symbol table, so wire_dispatch_support.go (which #includes client.h/netio.h/request.h/controller.capnp.h but does NOT itself declare go_push_handler) has no visibility into that extern, even though both files are in the same Go package and both carry //go:build cgo. This surfaced only after fixing bug-1440's toolchain-level rejection (moving import "C" out of the _test.go file) and after separately discovering make test-backend also needs CGO_CFLAGS/CGO_LDFLAGS manually derived to match src/frontend/apiserver/meson.build (pkg-config --cflags/--libs for librtti, c-capnproto, openssl, tinylog, plus -I/src/include -I/src/build/lib -L/src/build/lib -lxce -pthread -lm) since the nix devshell itself does not export them and make test-backend invokes go test directly rather than through the meson/go_build path that normally injects them. + +## Fix +NOT FIXED -- user explicitly instructed build/run/report only, no source edits. Reported fix path for the user: add the same "extern int go_push_handler(xce_request_t*, void*);" declaration to wire_dispatch_support.go's own cgo preamble (matching clib.go's exact signature), OR move that extern declaration into a shared header both files already include. Because the toolchain stops at first error, it is unknown whether wire_dispatch_support.go/wire_dispatch_test.go have any FURTHER errors (e.g. C.free_JobSubmit_ptr signature, C.extract_data, C.xce_encode/xce_decode field access) beyond this first blocking one -- those can only be discovered after this fix. Separately (env-only, no source change): make test-backend also requires CGO_CFLAGS/CGO_LDFLAGS to be exported manually before invoking make, mirroring meson.build's pkg-config-derived flags -- the bare Makefile command alone (as documented) does not produce a working CGO_ENABLED=1 build in this worktree. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1440 diff --git a/.wolf/bugs/bug-1442.md b/.wolf/bugs/bug-1442.md new file mode 100644 index 00000000..507513ad --- /dev/null +++ b/.wolf/bugs/bug-1442.md @@ -0,0 +1,38 @@ +--- +id: "bug-1442" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "crash" +timestamp: "2026-07-15T09:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/wire_dispatch_support.go"] +tags: ["cgo", "wire_dispatch", "asan", "double-free", "use-after-free", "job-array-single-wal-record", "test-backend", "go_push_handler", "free_Dataset", "free_Dataset_ptr"] +related: ["bug-1441"] +legacy: true +--- + +# bug-1442: AddressSanitizer: attempting double-free on 0x...1b0 in thread T0 -- fires immediately on… + +## Description +AddressSanitizer: attempting double-free on 0x...1b0 in thread T0 -- fires immediately on TestWireDispatch_OnePushProducesNRecordsAndLogsOnce, aborts the whole backend.test binary before it can print PASS/FAIL, so TestWireDispatch_RedeliveryCreatesNoNewRecordsAndLogsNothing never runs at all + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT ROOT-CAUSED (build/run/report only, no fix attempted per user instruction). bug-1441 (C.go_push_handler cross-file cgo resolution) is now fixed, and a separate missing `import "apiserver/store"` in wire_dispatch_test.go was fixed as a trivial compile error, so the build now succeeds and the test actually runs -- and immediately hits a real ASan double-free at runtime, not a build failure. Leading hypothesis (unverified): wireDispatchDeliver() (lines 122-180) calls go_push_handler(&decoded, nil) at line 164, whose defer C.free_Request(req) (clib.go) frees *decoded including its Request_body_datamsg payload -- the comment at lines 166-172 asserts free_Dataset() is 'a documented no-op for every table' (citing task 5.3b's residual-gap note) and therefore safe for wireDispatchDeliver to ALSO free ds.rows via free_JobSubmit_ptr() and ds itself at lines 173-177. A 16-byte double-free immediately after go_push_handler returns is consistent with that no-op assumption being wrong for the Data_data_jobArray-derived dataset table specifically (i.e. free_Dataset() actually does free ds->rows or ds itself for this table, and wire_dispatch_support.go's explicit cleanup then frees the same memory again). NOT CONFIRMED -- needs a symbolized ASan backtrace (frames are bare hex addresses in this build) or manual trace of free_Dataset()/free_Request() to confirm which free is the duplicate. + +## Fix +FIXED and VERIFIED 2026-07-15T09:55:00Z. Root cause confirmed: go_push_handler's internal `defer C.free_Request(req)` (clib.go) calls free_Request(&decoded), whose Request_body_datamsg case calls free_Dataset_ptr(&decoded.body.datamsg) -- free_Dataset_ptr() (controller.capnp.c, generated) unconditionally free()s the dataset_t container pointer it is given, even though free_Dataset() itself is an empty no-op (never touches ->rows). So `ds` becomes a dangling pointer the instant go_push_handler returns -- the earlier NOT-ROOT-CAUSED hypothesis was correct in spirit but had the wrong culprit function (free_Dataset_ptr, not free_Dataset). wireDispatchDeliver() (wire_dispatch_support.go) had two bugs: (1) read ds.rows AFTER calling go_push_handler = use-after-free, (2) separately called C.free(ds) again = double-free on the same 16-byte region ASan caught. Fix: capture `rows := ds.rows` BEFORE calling go_push_handler (line 175), use that captured value afterward instead of re-reading ds.rows, and removed the redundant C.free(unsafe.Pointer(ds)) call entirely since go_push_handler's internal free_Request already frees it. Verified via full ASan CGO rebuild (make test-backend under nix develop ./src with CGO_CFLAGS/CGO_LDFLAGS = -fsanitize=address -g -DWITH_DEBUG=1 + pkg-config librtti/c-capnproto/openssl/tinylog): TestWireDispatch_OnePushProducesNRecordsAndLogsOnce and TestWireDispatch_RedeliveryCreatesNoNewRecordsAndLogsNothing both PASS with zero ASan errors; full ./backend/... suite (91 subtests, convert/handler/heartbeat/peers/wire_dispatch/auth) all PASS, zero regressions; make test (CGO_ENABLED=0) also fully green. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1441 diff --git a/.wolf/bugs/bug-1443.md b/.wolf/bugs/bug-1443.md new file mode 100644 index 00000000..fa92f685 --- /dev/null +++ b/.wolf/bugs/bug-1443.md @@ -0,0 +1,39 @@ +--- +id: "bug-1443" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-07-16T02:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c (restore_job_array()", "bulk-insert phase)"] +tags: ["job-array-single-wal-record", "restore_job_array", "bulk-insert", "partial-visibility", "requirement-2.3", "requirement-9.4", "requirement-9.5", "requirement-9.6", "key_del", "kv.c", "kvmgr.c", "open-gap", "kiro-validate-impl"] +related: [] +fix_commits: ["dcf9f39"] +legacy: true +--- + +# bug-1443: not a runtime failure report + +## Description +OPEN, NOT A CRASH -- discovered as a design-time/test-proven gap, not a runtime failure report. restore_job_array()'s final bulk-insert pass into c->jobmgr can itself fail partway through (e.g. realloc() exhaustion growing c->jobmgr's backing table). Because the underlying xkvmgr_type store has no key_del, an already-inserted prefix [0, k) cannot be rolled back and is left permanently, genuinely visible in c->jobmgr -- individually addressable (status/cancel/completion all work via direct jobid lookup) -- while the function honestly returns a negative errno (never silently claims success). Critically, that prefix is never included in out_new_jobs, so HOOK_DETAIL_JOB_NEW never fires for it, not on this attempt or on any later successful retry of the same LSN (the per-index idempotent-skip logic on retry treats the already-inserted prefix as 'already exists', excluding it from the retry's own out_new_jobs too). Practical consequence: these array elements exist and are queryable forever, but never reach the scheduler's or API-server's push and so are never actually scheduled to run -- a silent, permanent no-op job, not merely a bookkeeping gap. requirements-coverage validation (kiro-validate-impl, 2026-07-16) confirmed this is a literal violation of Requirement 2.3 ('a standby shall never observe a single array WAL record in a partially-applied form') and Requirement 9.4 ('shall not construct a partial/divergent set of elements'), and its shadow extends to 9.5 (self-heal, since the orphaned prefix never gets a fresh chance to hook-fire even once the retry succeeds) and 9.6 (the capstone fault-injection test deliberately targets the fully-atomic construction-phase seam instead of this bulk-insert seam, precisely because asserting 'zero partial elements' against this seam would be dishonest). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +No capacity-reservation or rollback primitive exists anywhere in this codebase's kv.c/kvmgr.c/zarray.c stack -- c->jobmgr (xkvmgr_type) grows its backing table via realloc() with no pre-check, and once key_add() has inserted an element there is no key_del to undo it. restore_job_array()'s design (scratch-build every element first, then one bulk-insert pass under a single write-lock) makes construction-phase failures fully atomic (nothing ever touches c->jobmgr until every element is ready), but the bulk-insert pass itself is NOT covered by that atomicity guarantee -- it is the one place a late-stage resource-exhaustion failure can leave real, permanent state behind that the function's own success/failure contract cannot fully describe. Discovered and documented during task 3.1's original implementation review (job-array-single-wal-record spec), re-confirmed live-testable via test_rja_bulk_insert_failure_reports_error_not_false_success (test_job_array_apply.c), and re-surfaced by name during the spec's final /kiro-validate-impl requirements-coverage pass. + +## Fix +FIXED and VERIFIED 2026-07-16 (commit dcf9f39, job-array-single-wal-record task 7.1, user-directed follow-up after the GO decision). Added kvmgr_reserve()/xzarray_reserve() -- new, purely-additive primitives in kvmgr.c/zarray.c following the existing kvmgr_capture_keys() template (bare extern function, no RTTI vtable slot, caller-holds-write-lock contract) -- that pre-grow c->jobmgr's backing xzarray for the whole array before the bulk-insert pass begins, closing the dominant N-scaling realloc-exhaustion trigger structurally. Discovered and fixed a genuine, previously-unknown off-by-one in the existing, unmodified xzarray_ensure_items(): its strict less-than sufficiency check (mincap < caps) means reserving exactly array_size and then performing array_size real stores still triggers one more growth cycle on the LAST store; reserving array_size + 1 is the exact minimal correct compensation, independently re-derived and confirmed with concrete arithmetic by adversarial review. For the now-much-narrower residual case (kvtab_add()'s own small per-entry hash-chain-node allocation, or a content-based -EEXIST collision -- both non-N-scaling, honestly still theoretically possible and documented as such, NOT claimed to be eliminated): restore_job_array() now populates out_new_jobs with whatever prefix was genuinely inserted before a mid-pass failure, and apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case fires HOOK_DETAIL_JOB_NEW for that non-empty partial prefix even when the overall return is negative, while still propagating the negative rc so the LSN correctly halts and the Live Apply Resource Fence still engages for the remaining un-constructed indices -- a genuinely-inserted element is never left un-pushed/un-scheduled again. Verified: retry-double-notify risk traced and confirmed absent (pre-existing per-index idempotent-skip logic excludes already-inserted indices from any subsequent attempt's out_new_jobs); apply_wal_entry()'s dead-code standby path confirmed genuinely untouched; full controller/lib/scheduler test suites clean, zero regressions in any other consumer of the shared kvmgr.c/zarray.c code. Full detail in tasks.md's Implementation Notes, task 7.1 entry. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1444.md b/.wolf/bugs/bug-1444.md new file mode 100644 index 00000000..72b20992 --- /dev/null +++ b/.wolf/bugs/bug-1444.md @@ -0,0 +1,39 @@ +--- +id: "bug-1444" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-16T02:48:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c (replay_wal()", "~line 3902-3904; consumed by persist_recover()/persist_verify_apply_correctness()", "~line 3960-3991)"] +tags: ["job-array-single-wal-record", "persist_recover", "replay_wal", "apply_verify_correctness", "APPLY_DIVERGENT", "boot-replay", "ha-availability", "non-promotable-fence", "xce-test-ha", "array-soak", "task-6.1", "raft-apply-decoupling", "ha-commit-durability"] +related: ["bug-1443"] +fix_commits: ["16747aa"] +legacy: true +--- + +# bug-1444: persist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip… + +## Description +persist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip (expected last_applied_lsn=5 new_job_count=3); a hidden apply defect is present (ha-commit-durability recover_verdict=RECOVER_CLEAN) / persist_recover: UNRECOVERABLE -- post-replay apply-correctness verification is DIVERGENT ... Staying ALIVE and PERMANENTLY non-promotable (refusing to serve divergent state) and RETRYING catch-up indefinitely -- reproduced live during job-array-single-wal-record task 6.1's HA soak scenario the FIRST time a node is killed and restarted after ANY WAL_JOB_ARRAY_SUBMIT record has ever been committed to its local WAL (e.g. after a 2-element array submission). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +replay_wal()'s post-replay apply-correctness expectation counter only tallies WAL_JOB_SUBMIT entries toward new_job_submits_out: `if (hdr.type == WAL_JOB_SUBMIT && new_job_submits_out && apply_rc == 1) (*new_job_submits_out)++;` (persist.c ~3902-3904). It has NO corresponding branch for WAL_JOB_ARRAY_SUBMIT (the new record type job-array-single-wal-record/task 1.1 introduced), even though apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case (persist.c ~2809-2851, restore_job_array()) constructs N new job_t entries per record, not 1. So expected_new_job_count silently undercounts by N for every array record replayed during boot -- while persist_verify_apply_correctness() (persist.c ~3982-3988) measures actual_new_jobs via a REAL jobmgr entry-count delta (count_jobmgr_entries before/after), which correctly reflects the N elements. The mismatch (actual_new_jobs != expected_new_job_count) trips APPLY_DIVERGENT on every single restart of every node whose local WAL contains a WAL_JOB_ARRAY_SUBMIT entry above its checkpoint -- deterministic and 100% reproducible, not a race or resource-exhaustion condition. persist_recover_loud_fence_if_unrecoverable then classifies this as UNRECOVERABLE, permanently non-promotable-fencing that node for the rest of its process life (own boot-time verification re-runs identically on every subsequent restart too, since the miscounting bug reproduces every time). This is a NEW, separate gap from this spec's three self-flagged residual items (task 3.1 bulk-insert partial-visibility / bug-1443, task 4.2 boot-replay-doesn't-respect-apply-failures gap, task 5.3b's small per-commit leak) -- it lives in a DIFFERENT, pre-existing mechanism (raft-apply-decoupling task 5.1's post-replay apply-correctness verification / ha-commit-durability's Req 8.1-8.4), which this spec's own task list never touched to account for the new WAL_JOB_ARRAY_SUBMIT type. Practical severity: in any cluster where a job array has EVER been submitted, every ordinary node restart (routine maintenance, crash-restart, rolling upgrade) permanently non-promotable-fences that node; over enough restarts across a 3-node cluster this can leave ZERO nodes eligible for promotion -- a serious HA-availability regression, not a cosmetic log line. Discovered live via job-array-single-wal-record task 6.1's HA soak harness (.claude/skills/xce-test-ha/run-ha-soak.sh, new --array-soak mode) -- specifically NOT reproducible by this spec's existing white-box unit tests (task 2.3/3.4), which never drive a real persist_recover()/replay_wal() boot cycle after committing an array record. + +## Fix +FIXED and VERIFIED (2026-07-15/16, commit 16747aa, task 6.0a of job-array-single-wal-record) -- independently root-caused by a separate implementation pass on the same branch, inserted as a blocking task ahead of 6.1/6.2/6.3 per explicit user sign-off after the exact same bug was live-reproduced during this task's own --array-soak run. Fix matches the shape predicted below almost exactly: replay_wal()'s tally gained a `WAL_JOB_ARRAY_SUBMIT` branch that adds `apply_rc` (the genuine per-record new-element count apply_committed_entry()'s array case already returns, not the flat/undercounted +1 the WAL_JOB_SUBMIT branch uses) to *new_job_submits_out when apply_rc > 0, mirroring the existing WAL_JOB_SUBMIT pattern. Verified via: (a) new unit tests in test_apply_correctness_verification.c with an independently-reproduced RED-before-fix/GREEN-after-fix cycle; (b) two from-scratch live 3-node HA soak re-runs (--array-soak), one by the fixing pass's implementer and one by its independent reviewer, both showing zero APPLY_DIVERGENT across all three of this bug's trigger paths (restart-mid-small-array, standby election-path recovery fallback, restart-mid-large-array) with one-WAL-record/one-commit-barrier and per-element status/cancel/completion all confirmed post-failover. Full detail in .kiro/specs/job-array-single-wal-record/tasks.md's Implementation Notes, task 6.0a entry. This buglog entry was originally filed (independently, before its author discovered the fix already existed on the branch) with fix=NOT FIXED; corrected here once the duplication was noticed, since the underlying finding is accurate and well-characterized and deserves its own tracked entry (task 6.0a's fix was documented in tasks.md but never separately logged here) -- kept as a distinct entry from bug-1443 (a different, still-open gap in the same feature) rather than merged or deleted. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1443 diff --git a/.wolf/bugs/bug-1445.md b/.wolf/bugs/bug-1445.md new file mode 100644 index 00000000..19b7e0ca --- /dev/null +++ b/.wolf/bugs/bug-1445.md @@ -0,0 +1,38 @@ +--- +id: "bug-1445" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-17T09:47:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["job-array-single-wal-record", "ha.c", "promotion-gate", "live-apply-resource-fence", "boot-replay", "first-boot", "persist_recover", "replay_wal", "ordering-bug"] +related: ["bug-1443"] +legacy: true +--- + +# bug-1445: A node's very first promotion-eligibility evaluation in a fresh process runs before that node's… + +## Description +A node's very first promotion-eligibility evaluation in a fresh process runs before that node's own first persist_recover()/replay_wal() call, so the Live Apply Resource Fence (job-array-single-wal-record tasks 4.1-4.3, extended to boot replay by task 7.2) is invisible to that one specific check. A node that would otherwise correctly fence itself non-promotable on a resource-exhaustion apply failure during its first-ever boot replay can instead pass its first promotion check looking fully eligible, because the fence flag hasn't been set yet at the point that first gate runs. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha.c's promotion-gate eligibility check (ha.c:1303-1408) runs before persist_recover()'s first-ever replay_wal() call (ha.c:1427-1439) on a freshly-started process. Every promotion evaluation AFTER a process's first one is correctly protected (persist_apply_resource_fenced() is threaded into both the periodic fence-release check and the election-time check per task 4.3) -- this is a narrow ordering gap specific to the single very-first evaluation of a brand-new process, discovered and documented (not fixed) during job-array-single-wal-record task 7.2's own review, out of that task's persist.c-only file boundary. + +## Fix +FIXED and VERIFIED (2026-07-17, src/controller/ha.c). raft_cb_become_leader()'s is_first_boot block now re-checks persist_apply_resource_fenced(&c->persist) immediately after cluster_init_phase2() completes (the call that triggers this process's own first persist_recover()/replay_wal()) -- if that first replay just engaged the fence, promotion is refused (same pattern as the pre-existing refusal branch: WARN log + ha_arm_promotion_fence_timer() + return), instead of falling through to CTRL_PRIMARY on a decision made before that first replay ever ran. Every later election is unaffected -- is_first_boot is false by then, so the earlier eligibility gate (ha.c:1337) already covers it correctly. Scope check: confirmed ha_elect_evaluate_and_signal (the OTHER election path, ha.c ~5030) is dead in production for the promotion outcome -- every live call site of handle_request_vote_reply passes vote_granted=0 (bug-828 already redirected the real grant-recording path to the record-only ha_record_vote_reply, which never signals it) -- so raft_cb_become_leader is confirmed the sole live promotion path and this fix's scope is complete, not partial. TDD: new test test_first_boot_promotion_evaluates_own_first_replay_fence (src/controller/testsuite/test_job_array_replay_fence.c) wires ha_phase2_hook to a stub that calls persist_recover() for real, reproducing production ordering exactly (a never-recovered persist_t entering the election path with ha->phase2_done==0) -- RED-confirmed failing pre-fix at the exact predicted assertion (role_after_election==CTRL_PRIMARY), GREEN-confirmed passing post-fix. Full controller_unit_test binary (bundles every testsuite/test_*.c including test_convergence_verification.c, test_ha_cluster.c, test_p1_integration.c) green: 16/16 tests in this file, 0 regressions cluster-wide. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1443 diff --git a/.wolf/bugs/bug-1446.md b/.wolf/bugs/bug-1446.md new file mode 100644 index 00000000..0859187f --- /dev/null +++ b/.wolf/bugs/bug-1446.md @@ -0,0 +1,40 @@ +--- +id: "bug-1446" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-07-18T00:47:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/test-commit-durability.sh (det_probe_once / date usage) -- test harness only", "NOT a product file"] +tags: ["test-harness", "false-positive", "busybox", "date", "kiro-validate-impl", "ha-commit-durability", "deterministic-mode"] +related: [] +legacy: true +gitea_issue: 31 +gitea_synced_status: "fixed" +--- + +# bug-1446: /kiro-validate-impl ha-commit-durability: test-commit-durability.sh --deterministic reported 3/3… + +## Description +/kiro-validate-impl ha-commit-durability: test-commit-durability.sh --deterministic reported 3/3 FAIL ('DURABILITY GAP -- primary acked with 0 fdatasync in-window'), i.e. a false RED that looked like a regression of the spec's core invariant on the current build. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +This sandbox's /sbin/date is BusyBox date, which does NOT support GNU date's %N (nanoseconds) format directive -- 'date +%H:%M:%S.%N' literally emits the unexpanded string 'HH:MM:SS.%N' (confirmed by direct invocation). det_probe_once() captures t0/t1 with this call to bound the submit-to-ack window, then _hms_to_sod() converts them to seconds-of-day for the in-window fdatasync count. With the literal '%N' suffix the sub-second component is unparseable/degenerate, so the computed window fails to contain the real fdatasync timestamp even though it lands in the same wall-clock second -- producing syncs=0 (RED) regardless of actual ordering. This is an environment tooling gap (BusyBox vs GNU date), not a code regression: the script assumes GNU date is on PATH. + +## Fix +NOT a product bug, no source fix applied (out of this validation's scope; script is owned by xce-test-e2e-primitives per this spec's task 1.3 boundary note). Diagnosed by cross-checking strace's own microsecond timestamps (which DO work correctly, e.g. 'fdatasync(10) = 0' at 00:47:01.128244) against the controller's committed_lsn advance log lines (commit_advance_locked '1 -> 2' at 00:47:01,144) for the same three probes: fdatasync consistently preceded the commit-advance (that gates the client ack) by 16-43ms in all 3 probes -- i.e. the real durable-before-ack ordering the spec requires DOES hold on the current build; the script's verdict was the false signal, not the product. Recommend the script detect/require GNU date (or use 1784335901.164264 / a C helper) rather than assuming %N support, so it fails loud on an incompatible date instead of silently producing a false RED. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #31 diff --git a/.wolf/bugs/bug-1447.md b/.wolf/bugs/bug-1447.md new file mode 100644 index 00000000..d0bd72bc --- /dev/null +++ b/.wolf/bugs/bug-1447.md @@ -0,0 +1,47 @@ +--- +id: "bug-1447" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-07-18T01:05:02Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/netio.c (handle_ssl_handshake / io_handler / io_worker -- crash site", "shared iomgr epoll dispatch); src/controller/ha.c (ha_connect_prep_peer/ha_connect_finish_peer/reconnect_lane_fn", "raft-drive-async Task 2.1/2.2 reconnect-offload -- suspected trigger); src/controller/cluster.c (handle_connection / endpoint_accept -- inbound accept path", "alternate suspect)"] +tags: ["ASAN", "FIXED", "SEGV", "endpoint_ref", "handle_ssl_handshake", "io_worker", "kiro-validate-impl", "netio.c", "on_standby_connected", "openssl", "raft-drive-async", "reconnect-offload", "regression-test", "round-9", "same-batch-uaf", "sequential-soak", "ssl3_read_bytes"] +related: ["bug-1282", "bug-1286", "bug-1298"] +legacy: true +gitea_issue: 32 +gitea_synced_status: "fixed" +--- + +# bug-1447: /kiro-validate-impl raft-drive-async: 10-round SEQUENTIAL HA soak (the fully-supported… + +## Description +/kiro-validate-impl raft-drive-async: 10-round SEQUENTIAL HA soak (the fully-supported single-failover lifecycle, hard-fail-on-any-loss) failed at round 9/10 -- 'FAIL: round 9: post-failover submit did not get 3 acks' (0/3 acked across 5 retry attempts). Root traced to ctrl-3 (a standby) crashing: AddressSanitizer DEADLYSIGNAL SEGV on address 0x40 (near-null, READ) in openssl's ssl3_read_bytes <- tls_get_message_header <- state_machine <- handle_ssl_handshake (src/lib/netio.c:548) <- io_handler (netio.c:588) <- io_worker (netio.c:633), thread T2 (an io_worker thread). With ctrl-1 already killed (the round's own kill target) and ctrl-3 now also dead, no 2-of-3 quorum remained for ctrl-2 (freshly elected PRIMARY, term=11) to commit writes -- explaining the observed 0/3-acks symptom as a downstream consequence, not the primary defect. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED (deterministic regression test + live soak, not speculative). The earlier 'unsynchronized concurrent SSL_do_handshake from two io_worker threads' hypothesis is WRONG and is retracted: iomgr_start() creates exactly ONE io_worker thread per iomgr_t (src/lib/netio.c:iomgr_start), and a single controller process has exactly one such iomgr/thread for all peer connections -- there is no second io_worker thread to race against. + +REAL mechanism: io_worker's epoll dispatch loop processes every ready fd from ONE epoll_wait() batch in a plain for-loop calling io_handler() per event (src/lib/netio.c, old io_worker ~line 612-640). A handler for an EARLIER event in that SAME batch can free a DIFFERENT endpoint that also has a ready event LATER in the SAME batch. In production this is exactly src/controller/ha.c:8257-8331 on_standby_connected(), the handshake-complete callback for a freshly-reconnected outbound standby channel (raft-drive-async Task 2.1/2.2's reconnect-offload path): when it detects a stale duplicate outbound channel to the same peer (ha.c:8286-8303, 'Different ep, same peer -> stale duplicate to evict'), it calls iomgr_unmonitor() (ha.c:8320-8321) then endpoint_unref() (ha.c:8327) on the STALE endpoint. iomgr_unmonitor()'s epoll_ctl(EPOLL_CTL_DEL) only stops a FUTURE epoll_wait() from returning events for that fd -- it does NOT invalidate an event for that SAME stale endpoint that is already sitting in the CURRENT batch's stack-local events[] array (very plausible during a reconnect burst, where several peer connections' fds become ready close together right after a new primary election). endpoint_unref() reaching refcnt 0 immediately SSL_free()s ep->ssl and free()s the endpoint_t. The loop's next iteration then calls io_handler() for that now-freed endpoint -- a heap-use-after-free that reads freed memory as 'ep', and (if it still looks like a plausible pointer chain) hands a freed/reused ep->ssl to SSL_do_handshake(), producing exactly the observed ASAN DEADLYSIGNAL SEGV deep inside OpenSSL's ssl3_read_bytes rather than an immediate top-of-function null-ssl crash. + +This is a genuine gap in the bug-1282/1286/1298 hardening family: INVOKE_MSG_HANDLER (netio.h) already refs/unrefs an endpoint around its OWN msg_handler call (bug-1298 note in that macro's comment) so a handler cannot free the SAME ep it is currently dispatched on -- but that bracket never covered a DIFFERENT, already-batched endpoint torn down as a side effect (on_standby_connected's cross-endpoint eviction). io_worker's own epoll dispatch loop held no reference of any kind on the endpoints in its batch. Reproduced deterministically (no live soak needed to hit it) via a new unit test that drives io_worker's real batch-dispatch loop with a synthetic 2-event epoll batch where event 0's handler endpoint_unref()s event 1's endpoint -- confirmed RED pre-fix: ASAN heap-use-after-free at src/lib/netio.c:587 (io_handler reading ep->ssl_handshake_done), freed by src/lib/netio.c:843 (endpoint_unref) called from the injected handler, exactly mirroring the production on_standby_connected -> stale endpoint eviction -> later-batched io_handler() read chain. + +## Fix +FIXED (2026-07-18). src/lib/netio.c: extracted io_worker's per-batch dispatch into a new static io_worker_dispatch_batch(struct epoll_event*, int) (~line 639-654), called from io_worker's while(1) loop in place of the old inline for-loop. It now takes an endpoint_ref() on every endpoint in the batch BEFORE dispatching any of them, dispatches all events via io_handler() as before, then endpoint_unref()s every endpoint only AFTER the whole batch has been processed -- mirroring the SAME endpoint_ref/endpoint_unref bracket idiom this codebase already uses for the vtask dispatch path (bug-1298) and INVOKE_MSG_HANDLER (netio.h), applied here to io_worker's own epoll dispatch loop, the one dispatcher that didn't yet have it. With this bracket, on_standby_connected's endpoint_unref() on a stale duplicate can only drop that endpoint's refcount from 2 (owner + io_worker's batch ref) to 1 during the batch -- never to 0 -- so the free is deferred until io_worker's own trailing unref runs after the ENTIRE batch (including that endpoint's own later event) has been safely dispatched. ref/unref are the existing lock-free _Atomic-int operations (netio.c endpoint_ref/endpoint_unref) with no I/O and no lock acquisition, so this adds zero blocking risk to the io_worker hot path (satisfies the async-io steering rule against blocking in vtask/event-loop handlers). Also added a small always-compiled test seam, io_worker_dispatch_batch_for_test() (netio.c + netio.h) -- not XCE_TESTING-gated because netio.c compiles once into libxce.a with no per-target defines (unlike controller/ha.c, which the controller test binary compiles directly with -DXCE_TESTING), so it follows this file's existing convention of unconditionally exporting its internals (endpoint_ref, endpoint_unref, endpoint_mark_tearing_down, etc. are all plain public functions already). + +Regression test: src/lib/testsuite/test_netio_batch_dispatch_uaf.c (test_batch_dispatch_survives_cross_endpoint_eviction), registered in src/lib/meson.build and src/lib/testsuite/test.c. Drives io_worker_dispatch_batch_for_test() with a synthetic 2-event batch: event 0 -> an endpoint whose conn_handler calls endpoint_unref() on event 1's endpoint (simulating on_standby_connected's stale-duplicate eviction); event 1 -> an endpoint with refcnt==1 (its only ref is the one event 0's handler drops) whose conn_handler just records that it ran. TDD verified: RED confirmed pre-fix (ASAN heap-use-after-free at netio.c:587 in io_handler, object freed at netio.c:843 endpoint_unref from the injected handler -- captured verbatim before the fix was restored); GREEN confirmed post-fix (both handlers run safely, exit 0, re-confirmed again standalone after both live soaks completed). Full regression: libxce lib suite 294/0/0 (xcelib_test, ASAN, ~/xce.build), controller_unit_test 972/0/0 with zero AddressSanitizer/heap-use-after-free/heap-buffer-overflow/double-free/SEGV hits in the full test log. Live verification beyond the unit test: two independent fresh sequential 10-round /xce-test-ha HA soaks on real ext4 (/home/claude/tmp/xce-bug1447-verify-1: 10/10 rounds, 33 jobs, zero job loss, all 3 nodes led; /home/claude/tmp/xce-bug1447-verify-2: 10/10 rounds, 36 jobs, zero job loss, all 3 nodes led including a forced-election leadership-coverage check on ctrl-3) -- both fully clean, zero crashes, and an explicit grep of every node's logs in both runs for the bug-1447 signature (AddressSanitizer/DEADLYSIGNAL/SEGV/ssl3_read_bytes/handle_ssl_handshake) found zero matches. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1282, bug-1286, bug-1298 +- Gitea: #32 diff --git a/.wolf/bugs/bug-1448.md b/.wolf/bugs/bug-1448.md new file mode 100644 index 00000000..cb71956d --- /dev/null +++ b/.wolf/bugs/bug-1448.md @@ -0,0 +1,40 @@ +--- +id: "bug-1448" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-07-18T01:54:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/xce-build.sh (skill invocation pattern", "not the script itself); .claude/skills/xce-build/SKILL.md's 'Always dispatch the build via the Agent tool' instruction"] +tags: ["xce-build", "Agent-tool", "run_in_background", "nohup", "setsid", "disown", "subagent-teardown", "kiro-validate-impl", "ha-heartbeat-committed-propagation", "process-detach"] +related: [] +legacy: true +gitea_issue: 33 +gitea_synced_status: "fixed" +--- + +# bug-1448: /kiro-validate-impl ha-heartbeat-committed-propagation: the xce-build skill's own documented… + +## Description +/kiro-validate-impl ha-heartbeat-committed-propagation: the xce-build skill's own documented pattern ("dispatch via a plain Agent-tool call, run_in_background: false, wait for it") produced an INCOMPLETE build -- ninja log ends with 'ninja: build stopped: interrupted by user.' at 85/192 targets, roughly 14 minutes after launch with zero nix/meson/ninja processes still running and no error in the log. The synchronous Agent call returned to the caller almost immediately (~38s) with text claiming 'The build is running in the background... I'll wait for it to complete' -- i.e. the subagent itself spawned an internal Bash run_in_background job and then ended its own turn, rather than blocking until the command finished. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same defect family as the already-documented xce-test-ha/run-ha-soak.sh 'context: fork' SIGHUP-teardown bug (fixed 2026-07-08), but manifesting through a DIFFERENT mechanism: xce-build's skill doc explicitly avoids 'context: fork' and claims a plain Agent-tool dispatch is safe -- but that safety claim implicitly assumes the subagent's own Bash tool call blocks synchronously for the whole build. In practice the subagent chose to run the long nix/meson/ninja command via its OWN internal run_in_background Bash call and then finished its turn, un-detached (no setsid/nohup/disown) -- so when that subagent's session/turn ended, the still-attached background job's process group got interrupted (ninja saw SIGINT/SIGHUP-equivalent, logged literally as 'interrupted by user'). A second SendMessage asking the same agent to 'wait for it now' resumed a session with nothing left alive to wait for. + +## Fix +Relaunched the build via a fresh Agent dispatch whose ONLY instruction was to run `setsid nohup nix develop ./src --command bash src/tools/xce-build.sh controller asan LOG 2>&1 & disown` and return immediately (fire-and-forget), then polled LOG/binary-mtimes independently from the main session via the Monitor tool. That build completed cleanly in ~10 minutes (85/85 ninja targets, clean link, meson test suite started normally) with no interruption. General fix for any long shell command dispatched through the Agent tool that must outlive the dispatching subagent's own turn: always explicit setsid+nohup+disown, never rely on the subagent's internal run_in_background alone or on run_in_background:false at the Agent-tool-call level to guarantee synchronous blocking all the way through. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #33 diff --git a/.wolf/bugs/bug-1449.md b/.wolf/bugs/bug-1449.md new file mode 100644 index 00000000..981c704f --- /dev/null +++ b/.wolf/bugs/bug-1449.md @@ -0,0 +1,40 @@ +--- +id: "bug-1449" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-07-18T09:56:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".claude/skills/xce-test-ha/run-ha-soak.sh (sequential mode round-1 kill timing) / src/controller/ha.c ha_on_primary_ctrl_fd_rdhup (ha-config-bootstrap Req 2.8 voter-only guard", "working as designed)"] +tags: ["ha-elect-ctx-lifecycle", "xce-test-ha", "run-ha-soak.sh", "ha-config-bootstrap", "test-harness", "flaky", "timing-race", "config-push", "round-1", "sequential-mode"] +related: [] +legacy: true +gitea_issue: 34 +gitea_synced_status: "fixed" +--- + +# bug-1449: [xce-test-ha] FAIL: round 1: no new primary elected after kill (no serving primary at term>2)… + +## Description +[xce-test-ha] FAIL: round 1: no new primary elected after kill (no serving primary at term>2) -- hit once during ha-elect-ctx-lifecycle kiro-validate-impl soak, did NOT reproduce on immediate retry (10/10 clean) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Test-harness timing race, not a code defect. Root-caused from the controller logs (not assumed): cold-start election was unusually slow this run -- ctrl-1 only reached CTRL_PRIMARY (term=2) ~10.1s after process start (raft_cb_become_leader). Immediately on becoming primary it began its config-push handshake to both standbys (on_standby_connected: 'outbound standby connection to peer[N] established, ctrlhello sent', same ~10.1s timestamp). The driver's round-1 kill fired only ~1.7s later (its budget is 'submit 3 jobs + verify + walcompare' after primary is detected, no floor on how fast that completes) -- before either standby had received and applied the pushed config. Both standbys were still at config tag=0 when ctrl-1 died, so ha_on_primary_ctrl_fd_rdhup correctly refused to start PreVote/RDHUP re-election per ha-config-bootstrap's own Req 2.8 guard ('no local config (tag=0, n_peers=3) -- voter only, not starting PreVote/RDHUP re-election; awaiting configured primary'), leaving the cluster with no primary and no candidate. This is the guard working exactly as designed against an unconfigured standby self-electing -- the actual gap is that run-ha-soak.sh's sequential mode has no wait for initial 3-of-3 config convergence before round 1's first kill (unlike its wait for FULL reconvergence between later rounds). Sequential mode's round-1 timing has apparently never lined up this unluckily in any of the many other Group-1 kiro-validate-impl soaks run earlier the same session (ha-wal-replication, ha-heartbeat-committed-propagation, ha-prevote, raft-sim-conformance, libxce-failover, ha-entity-snapshots, ha-commit-durability, ha-term-safety, controller-status, job-array, job-array-single-wal-record) -- all of which use the same driver script unmodified and ran clean. + +## Fix +FIXED (.claude/skills/xce-test-ha/run-ha-soak.sh). Added a round-1-only config-convergence gate that mirrors wait_for_full_convergence's exact polling style (same alive-count guard via pgrep -c -f "$(run_pattern)", same STABLE_TICKS=2-consecutive-converged-probes, same sleep 2 backoff, same ~90s/30-iteration bound, same log()/fail() convention) but gates on a NEW signal, cluster_config_converged (all 3 nodes report a non-zero configVersion via a new node_config_tag() single-node ctrlstatus client probe, mirroring node_probe's technique), instead of the existing cluster_committed_converged (committed-prefix catch-up) -- the two convergence signals are independent, and only the config-tag one covers the ha-config-bootstrap Req 2.8 voter-only guard. Added functions node_config_tag() / cluster_config_converged() / wait_for_initial_config_convergence() right after wait_for_full_convergence (~line 556). Call site inserted right after the chaotic-mode branch's closing `fi` and before the 'Steps 7-10 looped' round loop (~line 1897), so it applies only to the sequential/default path (chaotic mode intentionally tolerates this race by design -- 'no new primary' is a COUNTED outcome there, never a hard fail, so it was correctly left untouched). Diff: +77 lines, 0 lines removed -- no C source touched. VERIFIED: 3 back-to-back --sequential 6-round soaks on a genuine ext4 path (/home/claude/tmp, confirmed via `mount`/`df -T`, not overlayfs -- bug-924), all 3 PASS (6 rounds/24 jobs/zero loss/all 3 nodes led, identical verdict each run), and the new 'waiting for 3-of-3 initial config convergence before round 1' / 'confirmed' log lines appeared correctly ahead of every round 1. Cross-checked the actual cold-start timing from controller logs across all 3 runs to size the margin: election (raft_cb_become_leader -> CTRL_PRIMARY) took 10.3-11.7s after process start each run, and the config-push (handle_repl_config_push on both standbys) landed only 40-46ms after election every time -- i.e. the actual condition resolves in well under 100ms, while the new gate's minimum dwell is ~4s (2 ticks x 2s) and its ceiling is ~90s, giving roughly 2 orders of magnitude of margin over the observed timing, not just enough for these 3 runs' luck. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #34 diff --git a/.wolf/bugs/bug-1450.md b/.wolf/bugs/bug-1450.md new file mode 100644 index 00000000..fd1da7c5 --- /dev/null +++ b/.wolf/bugs/bug-1450.md @@ -0,0 +1,42 @@ +--- +id: "bug-1450" +status: "fixed" +status_updated: "2026-09-27" +status_note: "7aac018c: demotion-convergence offload ON by default; inline fallback when slow lane not started" +component: "controller" +severity: "unknown" +timestamp: "2026-07-18" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha_convergence.c", "src/controller/ha.h"] +tags: ["async-io", "thread-role-violation", "delaymgr", "slowlane", "ha_convergence.c", "demotion", "raft-drive-async-pattern", "xce-audit-invariants", "controller-tests"] +related: ["bug-1429", "bug-1451"] +fix_commits: ["7aac018c"] +legacy: true +gitea_issue: 25 +gitea_synced_status: "fixed" +--- + +# bug-1450: xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_demotion_convergence_fn… + +## Description +xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_demotion_convergence_fn scheduled directly as a delaymgr task body with no vtask/slow-lane hop: it truncates the WAL (persist_truncate_wal_above, file I/O under p->lock) then calls ha_self_restart (tlog_exit synchronous log flush + execv) -- all inline on the delaymgr thread, the SAME single thread that drives heartbeats and election timeouts cluster-wide for the whole cluster. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_demotion_convergence_fn (ha_convergence.c) ran its full body -- truncate + self-restart -- synchronously on whatever thread invoked it (the delaymgr worker in production). A prior fix attempt (2026-07-10, see bug-1429) tried moving this to a VTASK_LIVE_BLOCK vtask via c->ha.pool and deadlocked controller_unit_test, because c->ha.pool is never initialized/started by this file's lightweight unit-test fixtures (dc_setup), so a vtask enqueued VTASK_LIVE_BLOCK blocked forever waiting for a worker that never existed; that attempt was reverted. + +## Fix +Split ha_demotion_convergence_fn into a dispatcher (fast admin work: single-flight CAS on new ha_state_t field demotion_convergence_inflight, then slowlane_submit) and a new demotion_lane_fn slow-lane worker body running the actual truncate+self-restart (extracted into ha_demotion_convergence_do_run) -- the SAME established slow-lane pattern already used for snapshot send / config-push / peer connect in ha.c (ha_snapshot_send_dispatch, ha_config_push_offloadable, ha_connect_to_peer), not the vtask pool that deadlocked before. New feature flag g_drive_async_demotion_offload (ha_set_demotion_offload_for_test / ha_demotion_offload_enabled, ha.h/ha_convergence.c) gates the offload. IMPORTANT: defaulting this flag to ON (1) was tried first and reproduced a NEW real hang -- raft_cb_become_follower (which schedules this task) is exercised by a very broad set of pre-existing controller_unit_test fixtures across many files that never call slowlane_init, so the offloaded work was silently queued on an uninitialized c->slowlane (queued, never run -- no live workers) and downstream code waiting on the now-never-happening completion hung. Reverted the default to OFF (0), matching the ORIGINAL rollout discipline of the three reference offloads (all shipped OFF-by-default, flipped ON only in a later dedicated compatibility-audit pass -- raft-drive-async task 3.4). The mechanism is correct and covered by dedicated tests (test_demotion_offload_task_returns_promptly / test_demotion_inline_task_blocks_RED in test_demotion_convergence.c) that explicitly flip the flag ON against a fixture with a real slowlane_init'd lane. Flipping the cluster-wide default to ON is a follow-up compatibility audit across the full test suite, explicitly out of scope for this fix. Verified: standalone controller_unit_test binary run directly (bypassing meson) reports 976 Tests 0 Failures 0 Ignored, no hang/timeout. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1429, bug-1451 +- Gitea: #25 diff --git a/.wolf/bugs/bug-1451.md b/.wolf/bugs/bug-1451.md new file mode 100644 index 00000000..01149934 --- /dev/null +++ b/.wolf/bugs/bug-1451.md @@ -0,0 +1,42 @@ +--- +id: "bug-1451" +status: "fixed" +status_updated: "2026-09-27" +status_note: "7aac018c: catch-up offload ON by default; inline fallback when slow lane not started" +component: "controller" +severity: "unknown" +timestamp: "2026-07-18" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c", "src/controller/ha.h"] +tags: ["async-io", "thread-role-violation", "delaymgr", "slowlane", "config_repl.c", "catchup-continuation", "raft-drive-async-pattern", "xce-audit-invariants", "controller-tests", "spurious-election-risk"] +related: ["bug-1429", "bug-1450"] +fix_commits: ["7aac018c"] +legacy: true +gitea_issue: 35 +gitea_synced_status: "fixed" +--- + +# bug-1451: xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_catchup_continuation_fn… + +## Description +xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_catchup_continuation_fn scheduled directly as a delaymgr task body with no vtask/slow-lane hop: catchup_stream_range -> ha_stream_wal_to_standby does blocking WAL file reads + socket writes to a standby directly on the delaymgr thread. Flagged as the MORE URGENT of the two violations -- catch-up can stream a variable, potentially large amount of WAL data, so it can block delaymgr (and therefore heartbeats to every OTHER peer) for an unbounded, unpredictable time, a real spurious-election risk. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_catchup_continuation_fn (config_repl.c) ran its full body -- re-derive target, then catchup_stream_range's blocking WAL reads/socket writes -- synchronously on whatever thread invoked it (the delaymgr worker in production). Source comment at config_repl.c confirmed a prior VTASK_LIVE_BLOCK vtask-pool attempt here (2026-07-10) also deadlocked controller_unit_test and was reverted (bug-1429), tracked as a known open violation. + +## Fix +Split ha_catchup_continuation_fn into a dispatcher (fast admin work: slowlane_submit keyed by peer_idx+1) and a new catchup_lane_fn slow-lane worker body running the original streaming logic unchanged -- the SAME established slow-lane pattern used for snapshot/config-push/reconnect in ha.c, not the vtask pool. c->ha.catchup_inflight[peer_idx] (set by ha_walack_on_ack before scheduling) remains the authoritative per-standby single-flight guard; the lane's own per-key dedup is a redundant backstop, mirroring ha_connect_to_peer's reconnect offload. New feature flag g_drive_async_catchup_offload (ha_set_catchup_offload_for_test / ha_catchup_offload_enabled, ha.h/config_repl.c) gates the offload. Defaulting this flag to ON (1) was tried first and reproduced a real hang: test_convergence_verification.c's pre-existing ACK-driven catch-up fixtures (which directly exercise this exact code path via ha_walack_continuation_for_test) never call slowlane_init, so offloaded work was silently queued and never ran, hanging controller_unit_test at ~90s+ (confirmed reproducible both via meson and a direct standalone binary run with SIGQUIT timeout). Reverted the default to OFF (0), matching the same rollout discipline as bug-1450's demotion-convergence fix and the three original raft-drive-async reference offloads. Mechanism covered by dedicated tests (test_catchup_offload_probe_runs_promptly / test_catchup_inline_probe_blocks_RED in test_config_repl.c) using a REAL, started, single-worker delaymgr (delaymgr_init+delaymgr_start, not just an un-started fixture) plus a probe task queued behind the continuation to directly prove the delaymgr thread is/isn't blocked. Flipping the cluster-wide default to ON is a follow-up compatibility audit, out of scope for this fix. Verified: standalone controller_unit_test binary run directly reports 976 Tests 0 Failures 0 Ignored, no hang/timeout. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1429, bug-1450 +- Gitea: #35 diff --git a/.wolf/bugs/bug-1452.md b/.wolf/bugs/bug-1452.md new file mode 100644 index 00000000..1c3fd34e --- /dev/null +++ b/.wolf/bugs/bug-1452.md @@ -0,0 +1,41 @@ +--- +id: "bug-1452" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-07-18T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/commit.c (crash site", "commit_advance_locked); src/controller/persist.c (fsync_worker background thread", "apply_committed_entry/restore_job/makeJob chain also appears in a related non-crashing stack from the same investigation); src/controller/ha_convergence.c + config_repl.c (g_drive_async_demotion_offload / g_drive_async_catchup_offload -- the flags whose flip triggers this)"] +tags: ["ha_convergence.c", "config_repl.c", "commit.c", "persist.c", "test_persist_truncate.c", "teardown-ordering", "toctou-race", "commit_engine_fini", "persist_stop", "raft-drive-async-pattern", "xce-audit-invariants", "controller-tests", "ASAN", "SEGV", "FIXED", "root-caused"] +related: ["bug-1429", "bug-1450", "bug-1451"] +legacy: true +gitea_issue: 36 +gitea_synced_status: "fixed" +--- + +# bug-1452: AddressSanitizer: SEGV on unknown address 0x000000000038 in commit_advance_locked… + +## Description +AddressSanitizer: SEGV on unknown address 0x000000000038 in commit_advance_locked (controller/commit.c:208), called from commit_on_durable_advance (commit.c:341) <- fsync_worker (persist.c:399), a background ASAN thread started by persist_start (persist.c:723) inside test_persist_truncate.c's make_fresh_persist / test_truncate_above_resolves_pending_continuations_as_failed. Reproduced by temporarily flipping BOTH g_drive_async_demotion_offload and g_drive_async_catchup_offload (bug-1450/1451) from their default 0 to 1 and running the standalone controller_unit_test binary directly (976-test suite) -- crashed immediately, before completing the suite. Confirmed absent with both flags at their committed default of 0: the same binary/build (just reverted) completed cleanly, 976 Tests 0 Failures 0 Ignored, twice. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED 2026-07-19: the g_drive_async_demotion_offload/g_drive_async_catchup_offload flags were a RED HERRING -- this exact SEGV signature (NULL deref in the fsync_worker -> commit_on_durable_advance -> commit_advance_locked chain) was already independently observed on 2026-07-07, long before those flags existed (see memory: 'Pre-existing ASAN SEGV in test_persist_truncate Unrelated to Bug-1417 Work'). The flags only perturbed thread-scheduling timing enough to make a PRE-EXISTING, always-latent test-teardown race fire more reliably; they do not participate in the crash's causal chain at all. Actual root cause: in src/controller/testsuite/test_persist_truncate.c, the three continuation-related tests (test_truncate_above_resolves_pending_continuations_as_failed, test_truncate_noop_does_not_resolve_pending_continuations, test_truncate_concurrent_burst_resolves_above_cutoff_exactly_once) called `commit_engine_fini(&ce)` BEFORE `teardown_persist(p)` (which calls `persist_stop(p)`) -- the exact opposite of the contract documented in commit_engine_fini's own comment (commit.c:464-468: 'Callers are expected to have stopped the flusher (persist_stop) first'). Because persist_stop had not yet run, the fsync_worker background thread was still alive and could be mid-flight invoking the durable-advance callback (fsync_worker, persist.c:398-399, calls p->on_durable_advance(p->durable_cb_ctx) with NO lock held) concurrently with commit_engine_fini destroying ce->lock/ce->cond and, as its final step, setting ce->p = NULL. This produces a genuine TOCTOU data race with no synchronization between the two threads: commit_on_durable_advance's own guard (`if (ce == NULL || ce->p == NULL) return;`) can read ce->p as still non-NULL (racing ahead of or concurrent with commit_engine_fini's teardown), pass the guard, and then commit_advance_locked's SEPARATE, later re-read of ce->p (commit.c:208, `atomic_load(&ce->p->committed_lsn)`) observes the NOW-NULLED value written by the concurrently-running commit_engine_fini on the main thread -- producing a NULL-pointer dereference at exactly offset 0x38 (offsetof(persist_t, committed_lsn)), matching the crash address bit-for-bit. persist_set_durable_cb's unhook (called from commit_engine_fini) is 'defensive and cheap' per its own comment but is NOT sufficient on its own without persist_stop's pthread_join guaranteeing the flusher thread cannot be in-flight at all -- confirmed by the fact that persist_set_durable_cb existed and ran in this path the whole time and the crash still occurred. + +## Fix +FIXED 2026-07-19 (commit pending): added an explicit `persist_stop(p);` call immediately before `commit_engine_fini(&ce);` in all three affected test functions in test_persist_truncate.c (previously commit_engine_fini ran first, then teardown_persist -> persist_stop ran second -- backwards from the documented contract). persist_stop joins the fsync_worker thread, so by the time commit_engine_fini tears down ce->lock/ce->cond/ce->p, no thread can be mid-callback referencing them. teardown_persist's own internal persist_stop(p) call becomes a safe idempotent no-op on the second invocation (fsync_thread_running is already 0 and wal_fd is already -1, so both of its internal branches skip). No production code changed -- this was purely a test-fixture ordering bug, not a bug in commit.c/persist.c itself. Verified: full ASAN controller_unit_test suite (976 tests) run 6 times total (1 via meson + 5 standalone direct binary runs per project convention of not trusting meson's aggregate alone) -- 976 Tests 0 Failures 0 Ignored on every run, zero ASAN reports (SEGV/UAF/etc.), all 8 test_persist_truncate.c tests PASS every run. NOTE for a future session: the identical commit_engine_fini-before-persist_stop ordering also exists in test_demotion_convergence.c (line 233 commit_engine_fini(&g_dc.commit) before line 235 persist_stop(&g_dc.persist), against a real persist_start'd fixture) -- same latent race, not yet observed to crash there (likely a narrower window against a static/global fixture rather than a per-test stack fixture), out of scope for this fix but worth a follow-up sweep of all commit_engine_fini call sites. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1429, bug-1450, bug-1451 +- Gitea: #36 diff --git a/.wolf/bugs/bug-1453.md b/.wolf/bugs/bug-1453.md new file mode 100644 index 00000000..ca42c71e --- /dev/null +++ b/.wolf/bugs/bug-1453.md @@ -0,0 +1,40 @@ +--- +id: "bug-1453" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-20T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["multiple — see root_cause"] +tags: ["dead-code", "tls", "ca-cert", "cleanup", "controller", "worker", "scheduler", "xcecli", "apiserver"] +related: [] +legacy: true +gitea_issue: 26 +gitea_synced_status: "fixed" +--- + +# bug-1453: Unnecessary CA certificate plumbing across C daemons and Go CLI — ca_cert_path fields, --ca-cert… + +## Description +Unnecessary CA certificate plumbing across C daemons and Go CLI — ca_cert_path fields, --ca-cert flags, and CACert config are dead or misleading; TLS verification is SSL_VERIFY_NONE everywhere + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +XCE design uses server-side TLS only (encryption without peer certificate verification). The CA cert plumbing was intended for future P3 mTLS hardening but was never implemented: (1) C-side: zero SSL_CTX_load_verify_locations calls in entire codebase — ca_cert_path in xce/cluster.h:35 is set by controller/worker/scheduler but never loaded into any SSL_CTX; do_connect() production path in cluster.c:73 is a NULL-returning stub; (2) Go-side: HTTPS is disabled by default; xcecli uses trust-on-first-use pinning of the server leaf cert (not a CA) stored via a misleadingly-named CACert field; falls back to InsecureSkipVerify:true. Design docs (persistence-and-ha-design.md:1700) explicitly tag peer verification as optional P3 hardening, confirming current encryption-only posture is intentional. + +## Fix +DONE. C-side: removed ca_cert_path field from xce_cluster_config_t (src/include/xce/cluster.h); do_connect()/xce_connect_fn_t dropped the ca_cert param (src/lib/cluster.c); removed ca_cert_path non-empty check from xce_cluster_config_validate(); removed --ca-cert flag from worker (src/worker/src/main.c) and scheduler (src/scheduler/src/main.c); removed ca_cert_path population from src/controller/main.c:279-281; updated src/lib/testsuite/test_cluster.c mock_connect signature + fixture. Go-side: deleted xcecli/cmd/auth/cert/{cert.go,parser.go} (self-signed cert TOFU-pinning helper); removed CACert field from config.go APIServerInfo and models/logon.go LogonResponse; removed SetRootCAs/loadRootCAs from common/request/request.go and certPath param from Get/Post/DeleteRequestFromAPIServer; updated all ~18 xcecli cmd/* call sites; updated cmd/auth/logon.go to drop the https cert-fetch block; removed TestSetRootCAs_*/TestBuildTLSLogic from request_test.go. Verified: full C build (all components, ASAN) 353/353 targets, meson test suite 9/9 OK; xcecli go build/vet/test all PASS; grep -rn 'CACert|SetRootCAs|loadRootCAs|ca_cert' src/ clean (only historical design-doc mentions remain in src/docs/*.md, left as-is). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #26 diff --git a/.wolf/bugs/bug-1454.md b/.wolf/bugs/bug-1454.md new file mode 100644 index 00000000..254ecef8 --- /dev/null +++ b/.wolf/bugs/bug-1454.md @@ -0,0 +1,41 @@ +--- +id: "bug-1454" +status: "fixed" +status_updated: "2026-09-24" +status_note: "94f50ce6" +component: "scheduler" +severity: "leak" +timestamp: "2026-07-20T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/scheduler.c"] +tags: ["scheduler", "peers", "verified-working", "minor-ctx-leak", "nmasters"] +related: [] +fix_commits: ["94f50ce6"] +legacy: true +gitea_issue: 37 +gitea_synced_status: "fixed" +--- + +# bug-1454: Scheduler --peers correctly wired for initial controller discovery; minor issues: nmasters=1 (no… + +## Description +Scheduler --peers correctly wired for initial controller discovery; minor issues: nmasters=1 (no EndpointRotator) and leaked ctx on HA path + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +VERIFICATION: --peers IS correctly parsed via xce_parse_peers() (main.c:129), stored via sched_set_cluster_cfg() (main.c:156), and iterated in sched_init() HA loop (scheduler.c:462-493) BEFORE sched_init — init ordering is correct. Narrowed findings: (1) nmasters=1 per attempt (line 468) — manual candidate loop instead of filling config->masters[] from all candidates so libxce EndpointRotator never engages. Functional but suboptimal. (2) Leaked apictx on HA path — xcecli_ctx_init at line 454 allocates sched->apictx, then HA loop at line 472 re-inits without xcecli_ctx_fini on the first allocation. On standby-failure branches (lines 480, 492) cleanup is correct. OUT OF SCOPE: runtime reconnect/rejoin after controller death belongs to .kiro/specs/component-failover-adoption/ (requirements + design approved, tasks 3.1/3.2 not yet approved). + +## Fix +DONE (leak fix only). Fixed the confirmed defect: sched_init() called xcecli_ctx_init(&sched->apictx, config) unconditionally, then on the HA path re-called xcecli_ctx_init() inside the candidate loop without freeing the first ctx -> leaked one xcecli_ctx_t per sched_init() call. Fix: xcecli_ctx_fini(sched->apictx) added immediately before the HA loop starts (src/scheduler/src/scheduler.c), since the loop's first iteration always replaces it. Verified via full ASAN build: all 5 scheduler meson test suites + libxce suite pass, no regression. REJECTED the nmasters>1+EndpointRotator suggestion: memory obs 11825 (this session) confirms scheduler/worker deliberately use the candidates[]-loop path instead of filling config->masters[], so changing that would contradict the established design, not fix a bug. Runtime reconnect/rejoin gap remains out of scope, deferred to component-failover-adoption spec (unchanged from original triage). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #37 diff --git a/.wolf/bugs/bug-1455.md b/.wolf/bugs/bug-1455.md new file mode 100644 index 00000000..2ab56ff6 --- /dev/null +++ b/.wolf/bugs/bug-1455.md @@ -0,0 +1,41 @@ +--- +id: "bug-1455" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-07-20T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/scheduler.c", "src/scheduler/src/main.c", "src/scheduler/include/scheduler.h"] +tags: ["scheduler", "peers", "ha", "config", "api-design"] +related: ["bug-1454"] +legacy: true +gitea_issue: 38 +gitea_synced_status: "fixed" +--- + +# bug-1455: Scheduler --peers silently ignored: sched_init() memset(sched,0,...) wipes sched->cluster_cfg that… + +## Description +Scheduler --peers silently ignored: sched_init() memset(sched,0,...) wipes sched->cluster_cfg that main.c populated via sched_set_cluster_cfg() just before calling sched_init(), so the HA candidate loop's candidates_count check always reads 0 and falls back to the hardcoded localhost:9990 single-host path regardless of --peers. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_set_cluster_cfg(&sched, &cluster_cfg) at main.c was called before sched_init(&sched, &config, handler); sched_init's first action is memset(sched, 0, sizeof(*sched)), which zeroes the cluster_cfg field just written. sched_init's own internal HA-vs-single-host branch reads sched->cluster_cfg.candidates_count immediately after, so it always sees 0. User-reported regression found via code review after bug-1454's leak fix landed in the same function. + +## Fix +Restructured instead of patched around: sched_init() signature changed to take `const xce_cluster_config_t*` directly (assigned to sched->cluster_cfg AFTER the memset, inside sched_init, not before via a separate setter). Deleted sched_set_cluster_cfg() (no other callers). main.c hoists cluster_cfg/cluster_cfg_ptr to function scope and passes cluster_cfg_ptr (NULL or &cluster_cfg) straight into sched_init(). Also fixed an adjacent smell in the same HA loop: it used to mutate the caller's xce_client_config_t* config in place per candidate, leaving it in a surprising mutated state after sched_init returned — changed to a per-iteration local copy. Verified: full ASAN build (all components) 353/353 targets, meson 9/9 OK, manual scheduler testsuite 15/15, Go tests all pass. No test exercises sched_init's HA path with real --peers input (pre-existing gap, not introduced by this fix). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1454 +- Gitea: #38 diff --git a/.wolf/bugs/bug-1456.md b/.wolf/bugs/bug-1456.md new file mode 100644 index 00000000..2bab979b --- /dev/null +++ b/.wolf/bugs/bug-1456.md @@ -0,0 +1,41 @@ +--- +id: "bug-1456" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-07-20T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/scheduler.c"] +tags: ["scheduler", "ha", "dangling-pointer", "lifetime", "client-config", "regression-of-bug-1454"] +related: ["bug-1454", "bug-1455"] +legacy: true +gitea_issue: 39 +gitea_synced_status: "fixed" +--- + +# bug-1456: Dangling pointer risk in sched_init() HA candidate loop: a later refactor (following bug-1454)… + +## Description +Dangling pointer risk in sched_init() HA candidate loop: a later refactor (following bug-1454) changed the loop to build a per-iteration stack-local xce_client_config_t candidate_config and pass &candidate_config into xcecli_ctx_init(). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xcecli_ctx_init() (src/lib/client.c) stores the raw pointer it is given as ctx->conf — it does not copy the struct. That pointer is dereferenced later, long after sched_init() returns, by client_failover.c runtime reconnect/rotation logic (e.g. ac->conf->masters[next]). When the candidate loop used a stack-local xce_client_config_t per iteration, ctx->conf pointed at a stack frame that no longer existed once sched_init() returned — a classic dangling pointer that only manifests later during HA reconnect/rotation, making it easy to miss in a build+unit-test pass. + +## Fix +Reverted to mutating the caller-supplied xce_client_config_t* config parameter in place (config->nmasters, config->masters[0], config->master_ports[0]) instead of a stack-local copy. config is owned by main()'s stack frame, which lives for the whole process, so storing &config-derived state in ctx->conf is lifetime-safe. Added a comment in scheduler.c explaining why the stack-local-copy pattern is unsafe here, to prevent it from being reintroduced. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1454, bug-1455 +- Gitea: #39 diff --git a/.wolf/bugs/bug-1457.md b/.wolf/bugs/bug-1457.md new file mode 100644 index 00000000..321b5bd1 --- /dev/null +++ b/.wolf/bugs/bug-1457.md @@ -0,0 +1,40 @@ +--- +id: "bug-1457" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-20T22:56:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/myapp.c"] +tags: ["controller", "startup", "single-node", "pre-existing", "myapp_run", "cluster_open_listener", "discovered-during-review", "incidentally-fixed"] +related: [] +legacy: true +gitea_issue: 27 +gitea_synced_status: "fixed" +--- + +# bug-1457: xce_controller with no --peers flag exits with -EINVAL from cluster_open_listener instead of… + +## Description +xce_controller with no --peers flag exits with -EINVAL from cluster_open_listener instead of starting as single-node primary + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Discovered while independently reviewing clusterconfig-schema-split task 5.3 by building and running the actual pre-spec baseline (commit 94f50ce) for comparison. Single-node startup (no --peers) was ALREADY broken before this spec began: myapp_run() calls cluster_open_listener() before the primary config (cluster_config) is loaded from disk, so the pre-spec fallback branch (cluster_config->port, only reachable when cluster_config != NULL) always hit its else branch and returned -EINVAL, unconditionally, for every zero-flag single-node start. This invalidates part of clusterconfig-schema-split's premise (requirements.md's Introduction claimed cluster_config->port was 'the sole live port-sourcing mechanism for single-node deployments' and 'single-node controller startup SHALL continue to work exactly as it does today' — in reality it did not work today). Not caused by, or a regression from, clusterconfig-schema-split's changes — task 2.1's port-resolution rewrite (commit 1229f9a) incidentally fixed this as a side effect, since its hardcoded-default fallback no longer depends on cluster_config being loaded. + +## Fix +RESOLVED -- incidentally repaired by clusterconfig-schema-split task 2.1 (cluster_open_listener now falls back to a hardcoded XCE_CONTROLLER_DEFAULT_PORT constant instead of an unloaded cluster_config->port), not as a dedicated fix for this entry. Live-verified post-fix: single-node startup with no --peers now correctly binds port 9990 and reaches CTRL_PRIMARY. No further action needed unless a future audit wants to correct the historical claim in clusterconfig-schema-split's requirements.md/design.md that pre-spec single-node startup was previously working — it was not, for an unrelated reason (myapp_run's listener-open/config-load ordering). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #27 diff --git a/.wolf/bugs/bug-1458.md b/.wolf/bugs/bug-1458.md new file mode 100644 index 00000000..62b1caf9 --- /dev/null +++ b/.wolf/bugs/bug-1458.md @@ -0,0 +1,38 @@ +--- +id: "bug-1458" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-07-22T03:22:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/nixSupport/xce.nix"] +tags: ["nix", "xceadmin", "python", "pip", "xce-build", "admin-config-tool", "task-1.3"] +related: [] +legacy: true +--- + +# bug-1458: tools/xce-build.sh: line 134: pip: command not found (inside nix develop ./src, running the new… + +## Description +tools/xce-build.sh: line 134: pip: command not found (inside nix develop ./src, running the new `xceadmin` build case, exit 127) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +admin-config-tool task 1.3 added python3Packages.{pycapnp,pyyaml,jsonschema,click,pytest,ruff} to xce.nix nativeBuildInputs per brief.md's Nix Environment section, but that list (copied verbatim from brief.md) omits python3Packages.pip. None of those packages transitively pull in a `pip` executable on PATH in a nix devShell built this way, so `pip install -e .` in the new xce-build.sh `xceadmin)` case failed immediately with command-not-found the first time it was actually exercised end-to-end inside `nix develop ./src`. + +## Fix +Added python3Packages.pip to xce.nix nativeBuildInputs alongside the other python3Packages.* additions. Verified via `nix develop ./src --command bash -c 'which pip; pip --version'`. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1459.md b/.wolf/bugs/bug-1459.md new file mode 100644 index 00000000..ad5b54c7 --- /dev/null +++ b/.wolf/bugs/bug-1459.md @@ -0,0 +1,38 @@ +--- +id: "bug-1459" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-07-22T03:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/xce-build.sh"] +tags: ["nix", "xceadmin", "python", "pip", "pep668", "externally-managed-environment", "xce-build", "admin-config-tool", "task-1.3"] +related: ["bug-1458"] +legacy: true +--- + +# bug-1459: pip install -e + +## Description +pip install -e . inside `nix develop ./src` fails: "error: externally-managed-environment ... This command has been disabled as it tries to modify the immutable /nix/store filesystem" (PEP 668) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The `xceadmin)` case (admin-config-tool task 1.3) ran bare `pip install -e .`/`pip install .` against the nix-provided python3 directly, matching brief.md's harness sketch. But nixpkgs' python3 site-packages lives under the read-only /nix/store, and nixpkgs patches pip to refuse writes there (PEP 668 externally-managed-environment guard) even with --break-system-packages in some configurations. brief.md's sketch was never actually exercised end-to-end against a real nix devShell before this task, so the guard was never hit. + +## Fix +Changed the `xceadmin)` case to create a throwaway venv (`$XCEADMIN_DIR/.venv`, recreated fresh every invocation) via `python3 -m venv --system-site-packages`, which inherits the nix-provided pycapnp/pyyaml/jsonschema/click/pytest (so pip never needs to rebuild those heavy binary deps from PyPI) while giving pip a normal writable site-packages to install xceadmin itself into. All subsequent commands (pip install, pytest) now run via $VENV_DIR/bin/pip and $VENV_DIR/bin/python. Verified end-to-end for both the default (editable) and release (pinned) profiles. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1458 diff --git a/.wolf/bugs/bug-1460.md b/.wolf/bugs/bug-1460.md new file mode 100644 index 00000000..67896863 --- /dev/null +++ b/.wolf/bugs/bug-1460.md @@ -0,0 +1,38 @@ +--- +id: "bug-1460" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-07-22T03:52:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/pyproject.toml"] +tags: ["xceadmin", "python", "packaging", "setuptools", "package-data", "wheel", "admin-config-tool", "task-1.3", "requirement-5.5"] +related: ["bug-1458", "bug-1459"] +legacy: true +--- + +# bug-1460: Under `xce-build.sh xceadmin release` (pinned, non-editable install), the installed xcecommon… + +## Description +Under `xce-build.sh xceadmin release` (pinned, non-editable install), the installed xcecommon package in site-packages has no schema_files/ directory at all — controller.capnp/c.capnp/JSON schemas are silently dropped from the wheel + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +pyproject.toml had no [tool.setuptools.package-data] entry for xcecommon, so setuptools' default wheel build only includes *.py files. The xce-build.sh sync step correctly copies controller.capnp/c.capnp/*.schema.json into src/xcecommon/schema_files/ in the SOURCE tree, but `pip install .` (non-editable) builds a wheel from that source tree and, without package-data config, drops every non-.py file. An editable install (`pip install -e .`) never surfaces this because it resolves imports straight into the checked-out source tree, bypassing wheel packaging entirely — exactly the gap design.md's Testing Strategy 'Unit — schema bundling' note warned only a release-profile test run could catch (Requirement 5.5). + +## Fix +Added [tool.setuptools.package-data] xcecommon = ["schema_files/*.capnp", "schema_files/json/*.schema.json"] to src/xceadmin/pyproject.toml. Verified: after `xce-build.sh xceadmin release`, site-packages/xcecommon/schema_files/ now contains controller.capnp, c.capnp, and json/*.schema.json, byte-identical to the synced source copies. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1458, bug-1459 diff --git a/.wolf/bugs/bug-1461.md b/.wolf/bugs/bug-1461.md new file mode 100644 index 00000000..1fc877ae --- /dev/null +++ b/.wolf/bugs/bug-1461.md @@ -0,0 +1,38 @@ +--- +id: "bug-1461" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-07-22T02:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/schema/controller.capnp"] +tags: ["capnp", "schema", "controller.capnp", "shadowing", "type-collision", "wal-replication", "snapshot-transfer", "admin-config-tool", "task-3.3", "xceadmin", "cross-component"] +related: [] +legacy: true +--- + +# bug-1461: pycapnp / capn-c: setting UpgradeConfigRequest.sha256 or .bytes raises… + +## Description +pycapnp / capn-c: setting UpgradeConfigRequest.sha256 or .bytes raises capnp.lib.capnp.KjException: 'expected reader.type == STRUCT [6 == 10]; Value type mismatch' -- and the generated C accessors (UpgradeConfigRequest_get_sha256 etc.) were typed Data_ptr instead of a real byte blob + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +controller.capnp declares `struct Data $C.mapname("xce_data_t")` at line 355 (a tagged union of OPCode_get query-result shapes: schedjobs/schedworkers/jobs/workers/resources/resvals/jobArray). Cap'n Proto resolves unqualified type names lexically within a file, so this local struct SHADOWS the Cap'n Proto builtin `Data` blob type for every later `: Data` field declaration in the same file -- confirmed against the real compiled schema via `capnp compile -o-` + `capnp decode`, and against the actual capnpc-c generated C output (fields typed `Data_ptr`/`XceData_ptr`, a struct-pointer wrapper, using `capn_getp`/`encode_Data_ptr`/`decode_Data_ptr`/`free_Data_ptr`, instead of the correct `capn_data` blob type + `capn_get_data`). Affected 7 fields across 6 structs, several HA-critical: UpgradeConfigRequest.sha256/.bytes, ReplConfigReply.sha256/.bytes, ReplConfigPushRequest.sha256/.bytes, ReplEnvelope.payload (WAL replication envelope), SnapshotTransfer's chunk payload, WalCatchupEntry.payload/.batchPayloads (WAL catchup), CtrlHelloRequest.key. Production C code had already worked around this for years via hand-rolled raw capn_ptr manipulation (see request.c's own comment: "Pull sha256 and bytes via raw capn_list8 access (NOT decode_Data which would corrupt via xce_data_t)") -- the bug was known-and-routed-around, never fixed at the root, because nothing before xceadmin's pycapnp-based dynamic reflection needed the generated codec/accessors to work correctly for these fields (the C raw-workaround functions never called them). Two fields, `getdata` (RequestReply's OPCode_get reply payload) and `pushmsg` (Request's generic OPCode_push payload), correctly and intentionally referenced the LOCAL struct Data all along -- their surrounding dispatch code (Data_data_jobArray/jobs/workers/etc. switches in apiserver.c/opget.c/sched.c/handler.c/xceapi.c/helper.c) depends on it. + +## Fix +Renamed `struct Data` to `struct XceData` in controller.capnp (kept `$C.mapname("xce_data_t")` so the generated C typedef name is unchanged), and explicitly repointed `getdata`/`pushmsg` at `XceData` by name (since removing the shadow means bare `Data` now correctly means the builtin blob, which those two fields do NOT want). Updated every derived-identifier reference (`Data_ptr`->`XceData_ptr`, `Data_data_`->`XceData_data_`, `struct Data`->`struct XceData`, `read_/write_/new_/encode_/decode_/free_Data(_ptr)?`->the `XceData` equivalents) across 19 hand-written files spanning controller, lib, scheduler, and the Go apiserver/xcecli CGo bindings (generated controller.capnp.c/.h regenerate automatically, never hand-edited). Fixed 5 call sites in src/lib/request.c (extract_data_bytes's parameter + 4 local variables) that had been declared XceData_ptr but now legitimately receive capn_data from the now-correct UpgradeConfigRequest_get_sha256/_get_bytes/ReplConfigPushRequest_get_sha256/_get_bytes accessors -- a pure type-name swap with zero logic change, since Data_ptr/XceData_ptr and capn_data share the identical {capn_ptr p} memory layout (confirmed: request.c already had a comment noting this exact equivalence). Verified via full rebuild+test of lib (1/1), controller (3/3, including the job-array push regression tests), scheduler (15/15), and the Go apiserver+xcecli backends (all packages green), plus a direct pycapnp round-trip proving UpgradeConfigRequest.sha256/.bytes now encode/decode correctly. Unblocks admin-config-tool task 3.3 (xceadmin.push's push_config()). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1462.md b/.wolf/bugs/bug-1462.md new file mode 100644 index 00000000..c01a55e4 --- /dev/null +++ b/.wolf/bugs/bug-1462.md @@ -0,0 +1,42 @@ +--- +id: "bug-1462" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-23T03:04:50.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["ha", "config-replication", "late-joiner", "committed_config_tag", "raft-promotion", "auto-detected"] +related: ["bug-1060"] +legacy: true +--- + +# bug-1462: ctrl-3 conf/ empty after HA failover deploy at /tmp/xce-ha-deploy -- config never replicated, but… + +## Description +ctrl-3 conf/ empty after HA failover deploy at /tmp/xce-ha-deploy -- config never replicated, but WAL/jobs replicated fine + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Post-election baseline for ha->committed_config_tag (raft_cb_become_leader, ha.c:1279, set to safe_tag from ha_compute_safe_committed_lsn, ha.c:744-802) is computed as the MAJORITY-SAFE FLOOR across granted voters' self-reported tags (median of sorted tags, same conservative math used for WAL safe_committed_lsn). On a cluster where no admin config push has ever happened live (deploy-ha.sh places cluster.bin on disk pre-boot, not via a live push), every peer voter reports tag=0 except the newly-elected primary itself (self_tag=nonzero, e.g. 860cfd4f59ef9381) -- so the majority-floor computation picks 0, even though the primary demonstrably has a real, valid, currently-loaded config. This 0-floor is CORRECT for WAL LSN safety (a leader must not assume unacknowledged entries are durable) but WRONG for config_tag: a node's own actively-loaded config is not speculative the way an unacknowledged WAL entry is. + +Effect: the ONLY path that successfully pushes config after promotion is the one-shot, promotion-instant fan-out (ha.c:1583-1602) to vote endpoints still connected AT THAT EXACT MOMENT (ectx->votes[i].ep != NULL) -- it correctly bypasses committed_config_tag and reads myconf->config_version_tag directly. Any standby that is NOT connected at that instant (a late joiner reconnecting seconds later via on_standby_connected, or any future reconnect/ restart for the rest of the term) relies entirely on the ONGOING heartbeat-driven decision loop (ha_dispatch_repl_action_for_standby -> ha_evaluate_repl_action, ha.c:3633-3663 / 3194-3199: 'if standby_tag < committed_tag -> HA_REPL_PUSH_CONFIG'). Since committed_config_tag is permanently floored at 0 for the rest of the term (nothing advances it except a live admin xceadmin config push via config_commit_change -> push_config_to_standbys, config_repl.c:1524/1556-1558, which never happens in a normal deploy-ha.sh-deployed cluster), standby_tag(0) < committed_tag(0) is false forever -> HA_REPL_PUSH_CONFIG never fires for that standby. Meanwhile WAL/job replication is unaffected since it correctly tracks committed_lsn (a genuinely-advancing value, set from real job commits), not the broken config_tag floor -- exactly matching the observed symptom (jobs replicate, config does not). + +Reproduced live: /tmp/xce-ha-deploy, peers ctrl-1=9901/ctrl-2=9902/ctrl-3=9903 (src/build.dbg/controller/xce_controller). ctrl-1 promoted primary at term=2 via PreVote failover (ha_compute_safe_committed_lsn logged safe_committed_lsn=0 safe_config_tag=0 'from 2 granted voters'). ctrl-2 (already-connected vote ep[0]) received handle_repl_config_push 45ms after promotion (one-shot path). ctrl-3 connected 3s later as 'late-joiner standby peer[2]' (ha.c on_standby_connected), got WAL-caught-up correctly (ha_process_standby_state: standby[2] converged: config_tag=0 applied_lsn=1 committed_tag=0 committed_lsn=1) but NEVER received a config push -- ctrl-1's full log has zero further push_config/repl_config lines after promotion, and ctrl-3's log stays stuck at 'CTRL_STANDBY -- waiting for primary to connect and push config'. ctrl-3/conf/ is empty on disk (no cluster.bin.*, no current pointer) while ctrl-1/ctrl-2 both have cluster.bin.860cfd4f59ef + current. + +## Fix +FIXED 2026-07-23. Implemented candidate direction (a): added ha_promotion_config_tag_floor(safe_tag, self_tag) (ha.h/ha.c, declared beside ha_promotion_truncate_floor) returning safe_tag > self_tag ? safe_tag : self_tag -- mirrors bug-1060's WAL-truncation clamp pattern but for config_tag instead of LSN. raft_cb_become_leader (ha.c ~1279) now clamps committed_config_tag to this floor instead of the raw majority-safe tag, so a newly-elected primary's own actively-loaded config tag is never dropped back to a stale majority-of-zero baseline. RED test added: test_promotion_config_tag_never_drops_below_own_loaded_config (testsuite/test_ha_election.c), covers the exact bug-1462 repro (safe_tag=0, self_tag=0x860cfd4f59ef9381 -> floor=self_tag), the equal case, safe_tag-above-self_tag case, and fresh-boot (0,0)->0. Unit suite green 977/977, zero regressions. Live regression: 20-round sequential soak (xce-test-ha, ~/tmp/xce-ha-soak-seq) PASS -- 63 jobs, zero job loss, all 3 nodes (ctrl-1/ctrl-2/ctrl-3) cycled through leadership across 20 promotions, 9m00.8s wall-clock. 20-round chaotic (overlapping-kill) soak (~/tmp/xce-ha-soak-chaos) also PASS -- VERDICT mode=chaotic rounds=20 jobs=57 committed_divergence=0 committed_job_loss=0 excluded_bug854_discovery=8 excluded_bug924_fdatasync=0 result=PASS (the 8 excluded bug-854 hits are client primary-discovery misses during overlapping-kill windows, not server-side loss). Confirms the fix holds under both normal and double-fault-within-reconvergence-window failover. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1060 diff --git a/.wolf/bugs/bug-1463.md b/.wolf/bugs/bug-1463.md new file mode 100644 index 00000000..f45ae959 --- /dev/null +++ b/.wolf/bugs/bug-1463.md @@ -0,0 +1,38 @@ +--- +id: "bug-1463" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xcecli" +severity: "unknown" +timestamp: "2026-07-23T06:37:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/xcecli/common/request/request.go", "src/frontend/xcecli/cmd/jobs/peek.go"] +tags: ["xcecli", "tls", "https", "dead-code", "build-break", "ca-cert-removal", "auto-detected"] +related: ["bug-1453"] +legacy: true +--- + +# bug-1463: xcecli build fails: ../common/request/request.go:143:32: undefined: tls / :144:13: undefined… + +## Description +xcecli build fails: ../common/request/request.go:143:32: undefined: tls / :144:13: undefined: loadRootCAs -- FAILED: [code=1] frontend/xcecli/xcecli + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Commit 94f50ce6 (bug-1453, CA-cert-plumbing removal) deleted the crypto/tls import, loadRootCAs, and SetRootCAs from request.go and stripped certPath from GetRequestFromAPIServer/PostRequestToAPIServer/DeleteRequestFromAPIServer plus their ~18 call sites, but missed GetJobOutput() (still referenced tls.Config{RootCAs: loadRootCAs(certPath)} with no import) and its sole caller cmd/jobs/peek.go, which still called apiClient.SetRootCAs(serverInfo.CACert) (SetRootCAs no longer exists anywhere) and passed serverInfo.CACert (the CACert field was already removed from APIServerInfo in config/config.go). Go compiles per-package, so once common/request failed to compile, the compiler never got far enough to surface peek.go's own two dangling references in the same build run -- they were latent until request.go compiled again. + +## Fix +FIXED 2026-07-23. request.go: removed GetJobOutput's dead TLS block, dropped certPath from its signature (token, jobID, params). peek.go: removed the SetRootCAs call, updated the GetJobOutput call site to the new 3-arg signature. Also enabled NewAPIClient's previously-commented-out `c.Client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})` -- per user request (xcecli needs to actually connect to an https API server) and per bug-1453's own documented design intent ("server-side TLS only, encryption without peer certificate verification"), since prior to this no xcecli method set any TLS config at all, meaning ANY https:// apiserver URL (self-signed/auto-generated cert, the common case) would have failed Go's default cert verification -- this was broader than just GetJobOutput. Added crypto/tls import accordingly. Documented both the api-server (http+https) and controller (server-side-cert-only, no client certs) TLS postures as a durable rule in .kiro/steering/security.md's Transport section, replacing a now-stale line claiming TLS is handled at the network layer. Verified: full `all debug` rebuild green (355/355 ninja targets), frontend/xcecli/xcecli binary produced with fresh mtime, no new warnings, no test fallout (grep for GetJobOutput/SetRootCAs/loadRootCAs across *_test.go in xcecli returns nothing). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1453 diff --git a/.wolf/bugs/bug-1464.md b/.wolf/bugs/bug-1464.md new file mode 100644 index 00000000..1ece5671 --- /dev/null +++ b/.wolf/bugs/bug-1464.md @@ -0,0 +1,38 @@ +--- +id: "bug-1464" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-07-23T09:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/tools/xce-build.sh"] +tags: ["build", "xce-build.sh", "meson", "xceadmin", "controller", "ha-soak", "workflow"] +related: ["bug-1486"] +legacy: true +--- + +# bug-1464: deploy-ha.sh FAIL: Error: xceadmin binary not found -- run 'xce-build.sh xceadmin' or 'nix build… + +## Description +deploy-ha.sh FAIL: Error: xceadmin binary not found -- run 'xce-build.sh xceadmin' or 'nix build .#xcecore' first (after controller had just been rebuilt and xceadmin had been built successfully earlier in the same session) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +setup_build() unconditionally does rm -rf "$BUILD_DIR" before every single-component build (controller, xceadmin, etc.), and all components share the same default BUILD_DIR=xce.build for the asan profile. Building controller after xceadmin (or vice versa) wipes the previous component entirely out of the shared build dir -- only the most-recently-built component survives. Only COMPONENT=all does one setup_build() + a single unfiltered meson compile, so it is the only invocation that leaves multiple components coexisting. + +## Fix +No code change; workflow fix -- when a downstream task (HA soak deploy-ha.sh) needs both xce_controller/xce_test AND the xceadmin venv present simultaneously in xce.build, run '/xce-build all' (or '/xce-build all asan') once instead of separate '/xce-build controller' + '/xce-build xceadmin' invocations, since the second call's setup_build() rm -rf wipes what the first call produced. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1486 diff --git a/.wolf/bugs/bug-1465.md b/.wolf/bugs/bug-1465.md new file mode 100644 index 00000000..bb2e3d5c --- /dev/null +++ b/.wolf/bugs/bug-1465.md @@ -0,0 +1,38 @@ +--- +id: "bug-1465" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-24T08:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_persist_snapshot_entities.c"] +tags: ["controller", "persist.c", "snapshot", "format-version", "test-regression", "config-upgrade-snapshot", "ha-entity-snapshots"] +related: [] +legacy: true +--- + +# bug-1465: test_snapent_checkpoint_job_content_roundtrip /… + +## Description +test_snapent_checkpoint_job_content_roundtrip / test_snapent_checkpoint_concurrent_mutation_not_torn / test_snapent_checkpoint_queue_status_roundtrip: FAIL: checkpoint file must parse as a well-formed entity-bearing format_version=1 payload + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +config-upgrade-snapshot task 1 bumped persist.c's SNAPSHOT_FORMAT_VERSION from 1u to 2u (reserving the version for the HostAdminState/QueueAdminState capnp migration). test_persist_snapshot_entities.c (ha-entity-snapshots spec) defines its OWN independent SNAPENT_TEST_FORMAT_VERSION=1u constant to structurally validate checkpoint bytes it reads directly, bypassing load_snapshot by design (testing the write-side/task 2.1-2.2 boundary, not the restore boundary). persist_checkpoint() now always writes format_version=2, so the test's stale hardcoded 1u check correctly (if inconveniently) rejected every checkpoint as unrecognized -- this is the same fail-loud format-mismatch behavior Requirement 5.7 mandates, just tripping up a test written before the bump existed. + +## Fix +Updated SNAPENT_TEST_FORMAT_VERSION from 1u to 2u in test_persist_snapshot_entities.c (one #define site; both use sites at the job-frame and queue-frame readers already referenced the constant, not a separate hardcoded literal). Mirrors the same fix already applied to test_config_repl.c's three fake_fmt sites for the same version bump. No other code in that file changed -- it belongs to a different, already-shipped spec. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1466.md b/.wolf/bugs/bug-1466.md new file mode 100644 index 00000000..4d7c234c --- /dev/null +++ b/.wolf/bugs/bug-1466.md @@ -0,0 +1,38 @@ +--- +id: "bug-1466" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-24T15:36:56.329888Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_config_repl.c"] +tags: ["checkpoint", "needs-owner", "severity-corrected", "persist.c", "vote-freshness", "config-upgrade-snapshot", "raft-log-index", "architectural-finding", "race-condition", "test-flake"] +related: [] +legacy: true +--- + +# bug-1466: test_config_repl.c: test_committed_marker_survives_checkpoint_trim and… + +## Description +test_config_repl.c: test_committed_marker_survives_checkpoint_trim and test_commit_advance_retry_clears_push_before_commit_fence flaked/failed (persist_checkpoint returned -EBUSY; new_mlsn <= committed_at_old) after wiring config-upgrade-snapshot task 5's WAL_CONFIG_MARKER apply case + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Two related findings from wiring apply_committed_entry's new WAL_CONFIG_MARKER case (persist.c, config-upgrade-snapshot task 5), which for the first time makes applying a config marker synchronously trigger a real checkpoint: (1) persist_apply_pump_kick's dispatch is a CAS race between the calling thread (inside commit_submit's synchronous commit_resolve_pending) and persist_start's independent fsync_worker thread (via commit_on_durable_advance) -- when the fsync thread wins, commit_submit's WAIT mode returns having waited only for COMMIT, not for the marker's now-real APPLY+CHECKPOINT side effect on that other thread, so a fixture's own subsequent persist_checkpoint()/LSN bookkeeping can race the still-in-flight background checkpoint. This race was always latent (applying a marker used to be a no-op) but was invisible until task 5 gave it a real, observable side effect. (2) DEEPER pre-existing architectural finding: persist_write_checkpoint_marker (called by any checkpoint, periodic or marker-triggered) appends its local-only, never-replicated WAL_CHECKPOINT entry by incrementing persist_t's current_lsn directly, WITHOUT going through ha_primary_log_append/raft_log_append -- this violates ha_primary_log_append's own documented co-tracking invariant 'p->current_lsn == s->log_len + 1 always holds' (ha.c:4057). Consequence: config_append_marker's returned lsn (sourced from raft's own s->log_len, which under-counts once a checkpoint has interleaved) becomes unreliable as an exact physical-position identifier for any marker appended after an earlier marker's checkpoint has run -- exactly the scenario Requirement 4.4 (overlapping markers) requires. Verified this does NOT corrupt on-disk data or commit/durability correctness: commit_advance_locked's majority_candidate_locked (commit.c) is driven entirely by persist_t's own durable_lsn/current_lsn, never by raft's s->log_len (that library value is documented as advisory-only for commit-ceiling purposes, ha-term-safety commit-ceiling-optA). The bug is confined to callers trusting the RETURNED lsn from config_append_marker/ha_primary_log_append for precise identity/ordering once a checkpoint has occurred earlier in the WAL. This is pre-existing (the periodic WAL-threshold checkpoint trigger has always had the same bypass) but was never observable before because periodic checkpoints require >=100 WAL records to fire, essentially never happening between two closely-spaced marker appends in a unit test; task 5's marker-triggered checkpoint fires on EVERY marker commit, making it the common case. SEVERITY CORRECTION (2026-07-24, kiro-review of task 5): the original "advisory-only... does NOT affect correctness" framing UNDERSTATES real risk. ha->raft.log_len (the same value as s->log_len) is a direct input to Raft RequestVote freshness comparisons (ha.c:258-261, my_last_log_index; ha.c:401-406, my_last_log_term via persist_entry_term(c, log_len)) -- NOT debug/advisory-only. ha_primary_log_append itself documents (ha.c:4054-4060) that its own LSN-assignment correctness RELIES on p->current_lsn == s->log_len + 1 holding; once a checkpoint marker breaks that invariant, the next primary append receives a library-assigned index that may no longer match its true physical WAL position, so persist_entry_term() lookups keyed by that index can resolve to the wrong WAL entry. Confirmed via kiro-review independent trace, not yet reproduced as a live failure. Pre-existing (not introduced by task 5 -- the periodic WAL-threshold checkpoint trigger already had this bypass; task 5 only made the interleaving common instead of rare). Still correctly out of config-upgrade-snapshot task 5's boundary to fix -- flagging with corrected severity for whoever next owns ha-term-safety/raft-log-replication so it is not deprioritized based on the original advisory-only framing. + +## Fix +(1) Added t42_wait_applied() (mirrors test_apply_pump.c's tap_wait_applied) and called it from t43_commit_marker and directly in test_committed_marker_survives_checkpoint_trim, polling persist_last_applied_lsn() before any LSN-sensitive step following a marker commit, closing the fsync-thread race. (2) Changed t43_append_marker_uncommitted to return the TRUE physical lsn read directly from persist_t (c->persist.current_lsn - 1) instead of trusting config_append_marker's returned (raft-library-derived, now-unreliable) lsn -- matching how production code that needs precise positioning (persist.c's apply seam, which receives lsn from the WAL entry's own on-disk header) already effectively does it. Did NOT attempt to fix the deeper co-tracking invariant gap in ha.c/raft.c (out of config-upgrade-snapshot task 5's scope -- would require raft-log-replication/ha-term-safety spec owners to add a log_len resync after any local-only checkpoint-marker append on a raft leader); flagged as a follow-up finding in the task 5 status report instead. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1467.md b/.wolf/bugs/bug-1467.md new file mode 100644 index 00000000..89eae21c --- /dev/null +++ b/.wolf/bugs/bug-1467.md @@ -0,0 +1,38 @@ +--- +id: "bug-1467" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-25T08:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c"] +tags: ["config-upgrade-snapshot", "task-8.4", "configcompare", "xce_test", "cli-dispatch", "reviewer-rejection", "testsuite"] +related: [] +legacy: true +--- + +# bug-1467: Reviewer rejected config-upgrade-snapshot task 8.4: configcompare_paths() (configcompare.c/.h) was… + +## Description +Reviewer rejected config-upgrade-snapshot task 8.4: configcompare_paths() (configcompare.c/.h) was a correct, fully-tested library function but was not reachable from anywhere outside the unit test binary -- unlike its siblings walcompare/snapverify/snapdiff, it was never registered as an xce_test CLI subcommand in test.c. `echo help | xce_test` confirmed configcompare was missing from the command list. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The prior implementer built and unit-tested configcompare_paths() as a library function but never wired a cmd_configcompare() wrapper into test.c's dispatcher (include, extern decl, top_commands[], client_commands[] entries, and the wrapper function itself) -- the step every other sibling tool in this family (walcompare, snapverify, snapdiff) already had. + +## Fix +Added #include "configcompare.h", extern int cmd_configcompare(int, char*[]) forward decl, a { "configcompare", cmd_configcompare, ... } entry in both top_commands[] and client_commands[], and a cmd_configcompare() wrapper (mirroring cmd_snapdiff's exact usage-message/return-value convention: argc<3 -> usage + -1, tool ran -> 0 on match / -1 on divergence) placed immediately after cmd_snapdiff's definition in test.c. Zero changes to configcompare.c/.h, snapverify.c/.h, test_configcompare.c, test_snapverify.c, controller_test.c, or meson.build (all previously reviewed and approved as-is). Verified: controller_unit_test build clean, 1022/1022 tests pass (pure regression check, no test files touched); `echo help | xce_test` lists configcompare; hand-built /tmp fixture member dirs (conf/current + conf/cluster.bin.) smoke-tested via piped stdin -- identical-content case printed 'IDENTICAL across 2 members' with exit 0, divergent-content case printed 'DIVERGENT -- member 1 differs from member 0' with exit 1. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1468.md b/.wolf/bugs/bug-1468.md new file mode 100644 index 00000000..3e7362e7 --- /dev/null +++ b/.wolf/bugs/bug-1468.md @@ -0,0 +1,38 @@ +--- +id: "bug-1468" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-25T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/cluster.c"] +tags: ["config-upgrade-snapshot", "task-8.6", "upgradeConfig", "cluster.c", "dispatch-gate", "regression", "admin-config-tool", "cross-spec"] +related: [] +legacy: true +--- + +# bug-1468: OP_upgradeConfig sent by any network client is permanently rejected -- handle_upgrade_config() was… + +## Description +OP_upgradeConfig sent by any network client is permanently rejected -- handle_upgrade_config() was unreachable via the wire dispatch path, exercised only by unit tests calling it directly. Independently diagnosed by the admin-config-tool spec's design.md (Boundary Commitments / Risks) as a blocking controller defect out of that spec's scope; re-discovered here while building config-upgrade-snapshot task 8.6's live OP_upgradeConfig client (no prior client existed to send it and hit the gate). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +commit fd93997f ("fix(election-iomgr-refactor): config replication and RDHUP failover chain", 2026-05-31) inverted the OPCode_upgradeConfig primary/standby guard in myreq_handler's switch: it changed `if (!ha_is_primary(&c->ha)) reject` (correct: only primary can commit a config change) to `if (ha_is_primary(&c->ha)) reject` (rejects the ONLY role -- primary -- that ha_dispatch_gate() ever lets reach this switch case at all, since OPCode_upgradeConfig is absent from the CTRL_STANDBY/CTRL_INIT/CTRL_ELECTING allow-lists in ha_dispatch_gate, ha.c). The commit's own message text ("standbys apply, not primaries") reflects a since-superseded design where OP_upgradeConfig was a primary-to-standby push; do_upgrade_config_with_bytes (config_repl.c) was later substantially rewritten (ha-term-safety R7 / config-upgrade-snapshot task 7) into a client-to-primary admin op that drives config_commit_change(), but this dispatch guard was never updated to match, leaving handle_upgrade_config() reachable only by unit tests calling it directly -- never via the network. + +## Fix +Restored the pre-regression polarity in controller/cluster.c's myreq_handler OPCode_upgradeConfig case: reject with -EAGAIN when NOT primary (`if (!ha_is_primary(&c->ha))`), fall through to handle_upgrade_config() when primary. Updated the stale comment to describe the current client-to-primary semantics and cite ha_dispatch_gate as the actual primary-only enforcement point (this switch-level check is now explicitly documented as defensive redundancy). No change to config_repl.c/ha.c/persist.c materialization logic -- pure dispatch-routing fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1469.md b/.wolf/bugs/bug-1469.md new file mode 100644 index 00000000..b1b323b3 --- /dev/null +++ b/.wolf/bugs/bug-1469.md @@ -0,0 +1,45 @@ +--- +id: "bug-1469" +status: "fixed" +status_updated: "2026-09-27" +status_note: "154a169f: reconcile_config_queues on activation creates added queues without restart" +component: "controller" +severity: "unknown" +timestamp: "2026-07-25T13:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c do_upgrade_config_with_bytes (~1611-1770)"] +tags: ["config-upgrade-snapshot", "task-8.6", "bug-857", "state-divergence", "config_repl.c", "do_upgrade_config_with_bytes", "config_activate_committed", "materialization", "live-soak-finding", "needs-owner", "persist.c", "re-investigated", "needs-design"] +related: ["bug-857", "bug-1466"] +fix_commits: ["154a169f"] +legacy: true +--- + +# bug-1469: config-upgrade-snapshot task 8.6's live soak (--config-upgrade, run-ha-soak.sh) FAILED on its… + +## Description +config-upgrade-snapshot task 8.6's live soak (--config-upgrade, run-ha-soak.sh) FAILED on its first real run: after a config upgrade committed successfully and genuinely interleaved with a 3-round chaotic kill/restart sequence (all 6 jobs survived, upgrade confirmed committed), the post-convergence materialized-state identity check failed -- snapdiff reported ctrl-1's latest checkpoint (snap-000000000000000b.capnp, LSN 11) as 51569 bytes vs ctrl-2/ctrl-3's identical 51621-byte file. This is a live, empirical reproduction of bug-857's originally-hypothesized failure mode (config upgrade leaves members with divergent materialized state), which bug-857's own buglog entry explicitly flagged as 'NOT empirically reproduced yet' and task 8.5 could only address at the unit level (test_bug857_regression.c, a DIFFERENT and already-fixed mechanism -- schedule_checkpoint's WAL-threshold trigger racing the marker boundary). This is a third, distinct mechanism, first observed here. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Byte-level diff of the two checkpoint files (both LSN 11) shows the string 'soakupgrade' (the deliberately-added 5th queue's name, part of task 8.6's upgrade-target config) appears ONCE in ctrl-1's file (inside the embedded raw active-config-bytes blob, persist.c's read_active_config_bytes -- always a fresh disk re-read) but TWICE in ctrl-2/ctrl-3's files -- the second occurrence, present only on ctrl-2/ctrl-3, sits in a queue-ENTITY list section (immediately followed by the next queue's name 'lost_and_found', a runtime enumeration, not raw config text). So ctrl-1's checkpoint correctly embeds the NEW config bytes but its runtime QUEUE-ENTITY table still reflects the OLD (pre-upgrade) queue set. Traced to the code: do_upgrade_config_with_bytes (the handler that runs on whichever node ORIGINATES/commits the upgrade -- ctrl-1 was primary in term 2 when this upgrade was submitted, confirmed via its own log line 'do_upgrade_config_with_bytes: Config upgraded to tag=... (acked=2 standbys)') saves the new bytes to disk (conf_save_config) and bumps c->myconf->config_generation, but NEVER calls config_activate_committed()/load_primary_config() to actually re-parse the new bytes INTO c->myconf -- so the very struct the checkpoint writer presumably walks to emit the queue-entity list keeps the OLD parsed queue set. apply_committed_entry's WAL_CONFIG_MARKER case (persist.c) only calls persist_checkpoint_marker_triggered() then upgrade_materialization_verify() (ha.c) -- and upgrade_materialization_verify does NOT reload/activate config either, it only (a) checks whether some PRIOR checkpoint already referenced this marker (cfgref_present/tag/sha, circularly self-referential to the checkpoint it is meant to gate) and (b) hash-verifies the LIVE ON-DISK bytes -- neither check touches c->myconf's in-memory queue table, so the Activation Fence reports 'VERIFIED' (confirmed in ctrl-1's log, 'upgrade materialization VERIFIED... materialization complete') even though c->myconf itself was never refreshed. config_activate_committed (which DOES call load_primary_config, the only place that actually re-parses queues into c->myconf) is invoked only from handle_repl_config_push (a STANDBY receiving an out-of-band config-push) and from a member's own boot/recovery path -- confirmed empirically: ctrl-1's log shows ZERO config_activate_committed calls in its original (primary, term-2) process instance, but TWO calls in its restarted instance (12:59:16 and 12:59:39, after persist_recover reported APPLY_CLEAN at last_applied_lsn=11, i.e. loading the already-frozen stale checkpoint rather than replaying). Consequence: the node that ORIGINATES a config upgrade never refreshes its own in-memory queue table as part of that commit -- its checkpoint stays consistent with the OLD queue set until some LATER, independent event (restart+recovery, or an incoming config-push) happens to call config_activate_committed. Task 8.6's scenario kills the originating primary shortly after it commits (chaotic multi-kill interleaved with the upgrade, by design), so that self-correction never gets a chance to run, and no FURTHER WAL activity occurred to trigger a follow-up checkpoint that could have picked up a later-corrected queue table -- the divergent checkpoint file is permanently frozen. + +## Fix +PARTIALLY FIXED, then RE-INVESTIGATED (2026-07-25) -- correcting this entry's own earlier premature 'FIXED' claim. Two real findings: + +(1) GENUINE FIX, LANDED: apply_committed_entry's WAL_CONFIG_MARKER case (persist.c) was calling config_activate_committed(c) AFTER persist_checkpoint_marker_triggered() (the checkpoint write), not before -- meaning even when activation ran, it ran too late to affect the checkpoint it was meant to fix. Reordered to run BEFORE the checkpoint call (still guarded on c->myconf != NULL, still positioned before upgrade_materialization_verify per task 6.2's own sequencing need). Verified: full controller_unit_test suite 1025/1025 PASS, 0 failures, no crash (including the previously-SEGV-prone test_marker_apply_via_live_pump_checkpoints_at_marker_own_lsn and the bug-1469 regression test_wal_config_marker_apply_activates_originating_member_config). + +(2) DEEPER GAP FOUND, NOT FIXED -- genuinely out of task 8.6's scope: re-running the live --config-upgrade soak AFTER the reordering fix landed produced the IDENTICAL divergence signature (same ~52-byte checkpoint delta, same duplicated-vs-single 'soakupgrade' queue name pattern) on a DIFFERENT node this time (still one node showing stale queue state). Root-caused via direct code trace: config_activate_committed() -> load_primary_config() only re-parses bytes into myconf->cluster_config (config_repl.c:614-714) -- it NEVER touches c->qmgr, the RUNTIME queue table collect_queue_frames() (persist.c:1895, the checkpoint's queue-entity-list writer) actually walks. The ONLY code that populates c->qmgr from cluster_config->queues[] is build_queues() (queue.c:28), called EXACTLY ONCE per process lifetime from xce_config_init's HOOK_AFTER_EVENT branch (cluster.c, fired during cluster_init_phase2 at process boot/restart) -- and build_queues() is UNCONDITIONAL/blind-create (no existence check before calling create_queue/makeQueue for every queue in the config), so it is UNSAFE to simply re-call on a live cluster with existing queue objects (would attempt to recreate already-existing queues, risking duplicate-key/corruption semantics that were never designed for a live re-provisioning call). Consequence: NO MEMBER -- neither the config-upgrade originator NOR a replicating standby NOR any node's own live marker-apply of any kind -- has a working mechanism to add a NEWLY-INTRODUCED queue to its runtime qmgr without a full process restart. The apparent 'originating member vs standby' asymmetry in this bug's original report was COINCIDENTAL, not causal: whichever node's restart (inevitable during a chaotic multi-kill sequence, unrelated to which node 'originated' the upgrade) happened to occur AFTER the new config bytes were already on disk got the new queue for free via its own full boot sequence (xce_config_init -> build_queues); a node whose restart happened to land BEFORE that point, or that never restarted during the scenario at all, does not -- regardless of the fix in (1). This directly contradicts design.md's own stated goal for this spec ('No controller restart is required for a configuration upgrade after this change') for the specific case of a config upgrade that ADDS a queue: today it demonstrably STILL requires a coincidental restart to take effect on any given already-running member. requirements.md's own Design Invariant #1 ('Queues are config-provisioned... update-only restore is safe') is only true for the SNAPSHOT-INSTALL restore path (a fresh/behind joiner materializing a full checkpoint, restore_queue_state, queue.c:208) -- it was never validated against the LIVE no-restart marker-apply path this spec's Requirement 7/Introduction promises, and does not hold there. + +NOT fixed as part of config-upgrade-snapshot task 8.6: doing so properly requires new, carefully-designed production logic (a live-safe, idempotent 'sync qmgr from the freshly committed config' function -- e.g. mirroring restore_host_state's exact find-or-create-only shape, worker.c:477-536, adapted for queue_t/c->qmgr) PLUS a deliberate design decision this task is not positioned to make safely under its own time/scope budget: what happens to a queue REMOVED by an upgrade (delete the queue_t? re-home its jobs to lost_and_found -- bug-857's own originally-hypothesized-but-never-implemented mechanism? refuse the upgrade while jobs remain queued?) -- a decision requiring its own requirements/design pass, not an implementation detail. Flagging squarely for whoever owns tasks 1-7 (persist.c/config_repl.c/queue.c materialization logic) as a genuine, previously-undiscovered gap in this spec's core 'no controller restart required' promise, confirmed via TWO independent live soak runs (before and after the ordering fix) reproducing the identical byte-level signature. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-857, bug-1466 diff --git a/.wolf/bugs/bug-1470.md b/.wolf/bugs/bug-1470.md new file mode 100644 index 00000000..1f43baa6 --- /dev/null +++ b/.wolf/bugs/bug-1470.md @@ -0,0 +1,38 @@ +--- +id: "bug-1470" +status: "fixed" +status_updated: "2026-09-24" +status_note: "later full controller suites pass" +component: "controller" +severity: "unknown" +timestamp: "2026-07-25T14:25:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: ["src/controller/testsuite/test_config_repl.c"] +tags: ["testsuite", "controller", "config-upgrade-snapshot", "bug-1469", "commit-engine", "test-config-repl", "needs-diagnosis", "possible-regression"] +related: ["bug-1407", "bug-1466", "bug-1469"] +legacy: true +--- + +# bug-1470: controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence… + +## Description +controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence (test_config_repl.c:1854) FAILS ("Expected TRUE Was FALSE") DETERMINISTICALLY (4/4 runs: 1x via full xce-build.sh controller clean build + meson test, 3x direct re-invocation of the built controller_unit_test binary with no rebuild) on the bug-1469 fix commit (persist.c +91/test_config_repl.c +144, uncommitted worktree diff at time of this build). controller_unit_test overall: 1025 Tests 1 Failures 0 Ignored / meson summary Ok:2 Fail:1. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT YET DIAGNOSED. Distinguishing context vs the two prior entries for this same test name: bug-1407 (closed 2026-07-03) root-caused an earlier failure of this exact test to a STALE INCREMENTAL ninja build artifact in this worktree, conclusively fixed by a fresh cycle -- but this run WAS already a fresh clean build (xce-build.sh always does rm -rf + meson setup, confirmed src/tools/xce-build.sh:59-60), so bug-1407s explanation does not apply here. bug-1466 (2026-07-24) found a genuine CAS timing race between the commit-thread and persist_start()s fsync_worker thread that could affect this same test family after task 5s WAL_CONFIG_MARKER-triggers-real-checkpoint change, and claimed to fix it via t42_wait_applied() polling -- but the failure seen here ("Expected TRUE Was FALSE", a plain boolean assertion) is 100% reproducible across reruns of the same binary, which argues AGAINST a timing race (races are not usually 4/4 deterministic within one build) and FOR either (a) a residual gap in bug-1466s fix, or (b) a genuine regression introduced by the just-made bug-1469 persist.c change (adding an unconditional config_activate_committed() call inside apply_committed_entrys WAL_CONFIG_MARKER case, immediately before upgrade_materialization_verify()) altering ordering/state that this specific test asserts on. bug-1469s own buglog entry claims a prior direct-ninja verification run saw 1024/1024 PASS 0 failures on the same source -- so either that verification predates a subsequent edit, or ran under different build state; this needs reconciling before task 8.6 soak work continues. + +## Fix +NOT FIXED -- reported to the requesting task/session for triage; out of scope for the build-verification task that discovered it (was asked only to get a clean component build + report status, not to debug). Repro is cheap: nix develop ./src --command bash src/tools/xce-build.sh controller (full clean rebuild) or, faster, just rerun the already-built binary: ASAN_OPTIONS=detect_leaks=0 ./src/xce.build/controller/controller_unit_test 2>&1 | grep -A2 test_commit_advance_retry_clears_push_before_commit_fence. Full logs preserved this session at src/xce.build/meson-logs/testlog.txt (search test_config_repl.c:1854). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1407, bug-1466, bug-1469 diff --git a/.wolf/bugs/bug-1471.md b/.wolf/bugs/bug-1471.md new file mode 100644 index 00000000..b20e4144 --- /dev/null +++ b/.wolf/bugs/bug-1471.md @@ -0,0 +1,41 @@ +--- +id: "bug-1471" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-25T14:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/persist.c collect_host_frames / capnp_encode_host_admin_state (~lines 141-162", "2028)"] +tags: ["capnp", "capnp_encode_host_admin_state", "collect_host_frames", "config-upgrade-snapshot", "distinguished", "host-frame", "hypothesis-1-confirmed", "hypothesis-2-refuted", "live-soak-finding", "materialization", "needs-owner", "out-of-scope", "root-caused", "routed-to-bug-1472", "single-byte-divergence", "task-8.6"] +related: ["bug-1469", "bug-1472"] +legacy: true +--- + +# bug-1471: config-upgrade-snapshot task 8.6 live --config-upgrade soak: after avoiding bug-1469's… + +## Description +config-upgrade-snapshot task 8.6 live --config-upgrade soak: after avoiding bug-1469's queue-creation gap (switched the upgrade-target config change from ADDING a queue to updating an existing queue's description text -- see bug-1469's updated entry), the materialized-state identity check (snapdiff of each member's latest checkpoint file, all at the same LSN) still fails, but now with a MUCH smaller divergence: exactly ONE byte differs (offset 42879/51161 bytes total) between ctrl-1 and ctrl-2/ctrl-3 (ctrl-2 and ctrl-3 are byte-IDENTICAL to each other). ctrl-1 has 0x50 at that offset; ctrl-2/ctrl-3 have 0x00. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED AND DISTINGUISHED (2026-07-25, follow-up investigation): the two prior hypotheses are now resolved -- Hypothesis 2 (uninitialized/non-zeroed capnp padding) is REFUTED, Hypothesis 1 (genuine content discrepancy) is CONFIRMED, and the true mechanism is a pre-existing, out-of-spec-boundary bug, NOT part of config-upgrade-snapshot. Evidence: (1) Wrote a byte-precise decoder in Python that walks the checkpoint's [host_count][{payload_len; capnp bytes}] host-frame section (same structural approach as snapverify.c task 8.4) and decodes each frame using the exact wire shape write_HostAdminState produces (HostAdminState_word_count=1, pointer_count=1: one 8-byte data word holding `status` via capn_write32 at data-section offset 0, one 8-byte pointer word for `name` Text). Decoding BOTH /tmp/xce-ha-configupgrade and /tmp/xce-ha-configupgrade-v2's preserved checkpoints for all 3 members shows: host[1] = 'cfgupgrade-host', ctrl-1 status=0x00500000, ctrl-2/ctrl-3 status=0x00000000 -- byte offset 42879 is byte index 2 of the little-endian `status` DATA FIELD itself (data-section file offset 42877-42880), NOT the 4-byte padding word that follows it (file offset 42881-42884), which is 0x00000000 IDENTICALLY across all 3 members in both reproductions. (2) 0x00500000 = XCE_WORKER_CLOSED (0x00100000) | XCE_WORKER_CLOSED_BY_ADMIN (0x00400000) -- exactly the bits worker.c:779 sets for HOST_CLOSE inside worker_ctrl(). This is semantically meaningful admin-closed status, not heap garbage. (3) Independently confirmed capn_init_malloc's segment allocator (create_local in the pinned c-capnproto build, /nix/store/.../capnpc-1.0.1/lib/libCapnC_Runtime.so, disassembled directly) calls calloc(1, size) -- zero-initializing -- so Hypothesis 2 was never physically possible for this library, consistent with the empirically-always-zero padding bytes. (4) Traced the actual mechanism: worker_ctrl() (worker.c:732-794) sets w->host.status on whichever member directly processes the hostctrl RPC (ctrl-1, primary at the time), then fires HOOK_DETAIL_WORKER_UPDATE, which persist.c:5722 maps to WAL_WORKER_CTRL and replicates. On EVERY member applying that WAL entry (standbys AND a member's own replay-from-WAL/restart path alike), persist.c:3724's case WAL_WORKER_CTRL calls worker_from_capnp() (worker.c:827-856) UNCONDITIONALLY, regardless of hook_detail (JOIN vs UPDATE vs DEAD all share this one apply path). worker_from_capnp's own idempotency guard (worker.c:847-853) does key_find() and returns -EEXIST if a worker with that name ALREADY EXISTS -- WITHOUT applying any of the just-decoded fields, including status. persist.c:3732-3733 then treats -EEXIST as 'idempotent re-apply, no hook' and silently returns 0, discarding the decoded status update entirely. Since cfgupgrade-host was JOINED (a prior, separate WAL_WORKER_CTRL/HOOK_DETAIL_WORKER_JOIN entry) BEFORE it was admin-closed, by the time the UPDATE entry replicates/replays, the worker already exists on every standby -- so the close never lands there. This is NOT specific to config upgrades, checkpoints, or any config-upgrade-snapshot code: it reproduces for ANY `hostctrl -c`/`hostctrl -o` issued against an already-joined host on ANY XCE HA cluster, with zero dependency on a config upgrade ever being submitted. `git log -L` on worker_from_capnp confirms it was introduced 2026-05-08 in commit 9b6a91a8 ('feat: implement restore_job/worker_from_capnp for WAL recovery'), ~2.5 months before config-upgrade-snapshot's spec work began (first spec commit visible ~2026-06-24); no config-upgrade-snapshot task touches this apply path (the spec's own worker.c addition, restore_host_state from commit 5b1600a8, is a wholly separate function used only by the SNAPSHOT-INSTALL restore path, task 4, never by the live WAL_WORKER_CTRL apply switch). collect_host_frames/capnp_encode_host_admin_state (this spec's task 2.1 code) are functioning exactly as designed -- they faithfully serialize whatever w->host.status currently holds; the bug is that w->host.status itself is wrong on ctrl-2/ctrl-3, entirely upstream of any checkpoint/materialization logic. Existing test coverage (test_apply_committed_entry.c:test_ace_worker_ctrl_join_restores_and_notifies) only exercises the fresh-JOIN case; no test exercises WAL_WORKER_CTRL UPDATE against an already-existing worker, confirming this is a genuine untested gap, not a previously-handled case. Filed separately as bug-1472 (the true root cause) since it is a distinct, real, pre-existing mechanism entirely outside config-upgrade-snapshot's boundary -- see that entry for the fix-owner routing. This entry (bug-1471) remains the observed SYMPTOM (checkpoint byte divergence) for task 8.6's soak; the underlying CAUSE is bug-1472. + +## Fix +FIXED -- this was the observed SYMPTOM (checkpoint host-frame byte divergence) of bug-1472's root cause (worker_from_capnp()'s WAL_WORKER_CTRL apply path silently discarding UPDATE payloads against an already-existing worker). bug-1472 was fixed 2026-07-25 (TDD RED->GREEN): worker_from_capnp() now decodes hook_detail before applying its existing-worker idempotency guard, and HOOK_DETAIL_WORKER_UPDATE replaces the existing worker's XCE_WORKER_ADMIN_MASK bits with the decoded payload instead of silently no-op'ing on -EEXIST. This directly closes bug-1471's exact repro (an admin-close status change failing to replicate to standbys, producing the byte-42879 checkpoint divergence). Note: HOOK_DETAIL_WORKER_DEAD was deliberately left unfixed in the same change -- tracked separately as bug-1472-remainder (needs a design decision, not yet resolved). + +[Original entry] +NOT FIXED, AND DELIBERATELY NOT ATTEMPTED IN THIS SPEC -- confirmed genuinely OUT OF config-upgrade-snapshot's scope (see root_cause above). The real fix belongs in worker.c's worker_from_capnp()/persist.c's WAL_WORKER_CTRL apply case (case WAL_WORKER_CTRL, persist.c:3724), which needs to distinguish JOIN semantics (create-if-missing, EEXIST is a legitimate no-op) from UPDATE/DEAD semantics (an already-existing worker is the EXPECTED case, and the decoded fields -- at minimum host.status -- must still be applied to it, not discarded) -- e.g. an update-in-place path mirroring how restore_host_state (task 3, worker.c, config-upgrade-snapshot's OWN precedent) ORs admin bits into an existing worker rather than treating existing-ness as terminal. That is a genuine, real, previously undiscovered defect (filed as bug-1472) but is squarely core HA worker-control replication, predating this spec by ~2.5 months and untouched by any of its tasks -- fixing it here would be scope creep into a different subsystem's correctness contract, not a config-upgrade-snapshot materialization fix. Task 8.6's soak will keep observing this exact symptom (host-frame byte divergence whenever its scenario admin-closes a host before the final checkpoint) until bug-1472 is fixed by whoever owns worker.c/persist.c's general worker-control replication path. Recommend task 8.6 either (a) accept this as a known, separately-tracked limitation and adjust the soak's own PASS criteria to tolerate it (e.g. skip the admin-close-a-host step, or snapdiff-exclude the host-admin-status byte range) until bug-1472 lands, or (b) block final PASS on bug-1472's fix -- an orchestrator/owner decision, not this investigation's to make. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1469, bug-1472 diff --git a/.wolf/bugs/bug-1472-remainder.md b/.wolf/bugs/bug-1472-remainder.md new file mode 100644 index 00000000..47457e04 --- /dev/null +++ b/.wolf/bugs/bug-1472-remainder.md @@ -0,0 +1,41 @@ +--- +id: "bug-1472-remainder" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-25T20:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/worker.c worker_from_capnp (~827-895", "post-bug-1472-fix); mark_worker_unavail (~661-679); src/schema/controller.capnp struct Worker (~136-144)"] +tags: ["worker.c", "worker_from_capnp", "mark_worker_unavail", "WAL_WORKER_CTRL", "HOOK_DETAIL_WORKER_DEAD", "capnp-schema-gap", "resource-value-replication", "standby-divergence", "post-promotion-scheduling-risk", "needs-owner", "needs-design-decision", "deferred"] +related: ["bug-1472"] +legacy: true +--- + +# bug-1472-remainder: Follow-up split out of bug-1472: HOOK_DETAIL_WORKER_DEAD entries applied via WAL_WORKER_CTRL… + +## Description +Follow-up split out of bug-1472: HOOK_DETAIL_WORKER_DEAD entries applied via WAL_WORKER_CTRL against an already-existing worker still return -EEXIST (no-op) -- a dead-worker transition detected by one member (mark_worker_unavail(), worker.c:661-679) still never replicates its state to other members (standbys, or this member's own WAL replay), unconditionally. NOT a regression: this was already true before bug-1472's investigation; bug-1472's fix deliberately scoped itself to the byte-precise-confirmed HOOK_DETAIL_WORKER_UPDATE case only and left DEAD as -EEXIST/no-op, unchanged. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +mark_worker_unavail()'s mutation is NOT confined to XCE_WORKER_ADMIN_MASK the way worker_ctrl()'s is -- it sets XCE_WORKER_UNAVAIL (a non-admin bit), dead_since, resets hostattrs_t to 'unknown' sentinel strings, AND zeroes the live per-host resource-value slots via set_host_slots()/update_resvals(), which mutates xce_worker_t.resvals (a zarray_t of live xce_val_t entries). Critically, w->host.resvals is NOT part of the Worker capnp wire schema at all (confirmed by reading src/schema/controller.capnp's struct Worker and the generated decode_Worker/encode_Worker in controller.capnp.c) -- only the separate resvals_to_sched/nresvals_to_sched fields (a different, scheduler-push-facing concept) are encoded. So even applying a simple 'replace admin-masked bits' fix analogous to bug-1472's UPDATE fix would NOT fully replicate a DEAD transition's effects: the resource-value slot zeroing would still be silently lost on every other member, which matters materially at PROMOTION time (a newly-promoted primary would read stale, non-zeroed slots for a worker that died before promotion, potentially over-scheduling a dead host). A correct fix needs either (a) a schema change adding a wire-encodable resource-value-slots field so the DEAD payload can carry it, or (b) a different replication strategy for DEAD (e.g. re-deriving 'this worker's connection is gone locally' independently on each member rather than replicating a mutation), which is a design decision beyond a same-task patch. + +## Fix +FIXED (TDD RED->GREEN), design direction chosen by user: wire-encode the resource-value-slot zeroing rather than a non-replicated per-member re-derivation. Investigation revised the original design framing: NO new capnp schema field was actually needed. capnp_encode_worker (persist_worker_hook_fun) already serializes the FULL xce_worker_t.host -- including `attrs` (HostAttrs) -- on every WAL_WORKER_CTRL entry, and mark_worker_unavail() calls reset_hostattrs_unknown() BEFORE firing the hook that triggers that encode, so a DEAD entry's wire payload already carries the post-reset 'unknown'/0 attrs state. The actual gap was narrower: worker_from_capnp() had no DEAD branch at all (fell through to the JOIN-duplicate -EEXIST no-op), so nothing ever APPLIED what was already on the wire. Fixed by adding a DEAD branch (worker.c) mirroring the existing UPDATE branch (bug-1472's own fix): sets XCE_WORKER_UNAVAIL, takes dead_since from the decoded payload's deadSince field (the PRIMARY's own death timestamp -- genuinely wire-driven, not locally re-derived, honoring the chosen design direction), and calls reset_hostattrs_unknown()+add_hostattrs_resvals()+set_host_slots(0) locally to re-derive the resvals from the now-reset attrs -- these three calls are deterministic/data-independent (same effect mark_worker_unavail() itself produces on the primary), so xce_worker_t.resvals (the zarray_t the original investigation worried about) never needs to cross the wire at all. Regression test (test_apply_committed_entry.c's test_ace_worker_ctrl_dead_applies_to_existing_worker): pre-existing worker with live (non-'unknown') attrs receives a DEAD WAL entry with a specific non-zero deadSince -- RED under the pre-fix code (rc=-EEXIST, no mutation, no notification, the exact silent-discard this entry describes), GREEN under the fix (XCE_WORKER_UNAVAIL set, dead_since matches the payload exactly, hostattrs reset to 'unknown'/0, one notification fires). Full controller_unit_test suite: 1101 tests, 2 pre-existing unrelated failures (bug-1561), no regressions. + +[Original entry] +NOT FIXED -- deliberately deferred, investigated not skipped (see bug-1472's own fix note for the full investigation). Needs its own design decision on how (or whether) to wire-encode the resource-value-slot zeroing, or an alternative non-replicated detection strategy, before a fix can be written safely. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1472 diff --git a/.wolf/bugs/bug-1472.md b/.wolf/bugs/bug-1472.md new file mode 100644 index 00000000..efcf13ab --- /dev/null +++ b/.wolf/bugs/bug-1472.md @@ -0,0 +1,41 @@ +--- +id: "bug-1472" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-25T16:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/worker.c worker_from_capnp (~827-856); src/controller/persist.c case WAL_WORKER_CTRL apply path (~3724-3786)"] +tags: ["worker.c", "persist.c", "worker_from_capnp", "WAL_WORKER_CTRL", "host-control-replication", "hostctrl", "admin-status", "standby-divergence", "pre-existing", "out-of-config-upgrade-snapshot-scope", "root-caused"] +related: ["bug-1471", "bug-1472-remainder"] +fix_commits: ["7547a8f1", "45400c3a"] +legacy: true +--- + +# bug-1472: Host admin-control status changes (hostctrl -c / hostctrl -o, i.e + +## Description +Host admin-control status changes (hostctrl -c / hostctrl -o, i.e. HOST_CLOSE/HOST_OPEN via worker_ctrl()) never propagate to standby members' in-memory worker state once the target host has already been joined -- discovered while root-causing bug-1471's single-byte checkpoint divergence, but the defect itself is general and has nothing to do with checkpoints, config upgrades, or the config-upgrade-snapshot spec. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +worker_from_capnp() (worker.c:827-856) is invoked UNCONDITIONALLY by persist.c:3731 for every WAL_WORKER_CTRL entry, regardless of its hook_detail discriminator (JOIN, UPDATE, or DEAD all share this one apply path -- see persist.c:3766-3783's switch, which only controls which post-apply NOTIFICATION hook fires, not whether the mutation itself is applied). worker_from_capnp's own idempotency guard (worker.c:847-853): if key_find() on c->workermgr finds a worker with the decoded name ALREADY PRESENT, it returns -EEXIST immediately, WITHOUT applying any decoded field (including host.status) to the existing worker object. persist.c:3732-3733 treats any -EEXIST return as 'idempotent re-apply (replay / resync) -- no hook' and silently returns 0. This EEXIST short-circuit is correct for its originally-intended purpose (a duplicate/replayed JOIN entry for a worker that's already been created should indeed be a no-op), but WAL_WORKER_CTRL is also the opcode for HOOK_DETAIL_WORKER_UPDATE (worker.c:732-794's worker_ctrl(), which handles HOST_OPEN/ HOST_CLOSE admin commands, sets w->host.status, then fires exactly this same HOOK_AFTER_EVENT/HOOK_DETAIL_WORKER_UPDATE/WAL_WORKER_CTRL path at worker.c:786-787) and HOOK_DETAIL_WORKER_DEAD. For UPDATE/DEAD events, the target worker EXISTING is the NORMAL, EXPECTED case (you can only update or kill an already-joined worker) -- yet the same 'already exists -> skip entirely' logic applies, so the decoded status change is silently discarded on every member OTHER than the one that directly processed the original RPC. Confirmed via git blame: worker_from_capnp was introduced 2026-05-08 (commit 9b6a91a8, 'feat: implement restore_job/worker_from_capnp for WAL recovery'), predating config-upgrade-snapshot's spec work by ~2.5 months; no config-upgrade-snapshot task touches this function or the WAL_WORKER_CTRL apply case. Empirically confirmed via byte-precise capnp decode of two independent live-soak checkpoint sets (/tmp/xce-ha-configupgrade{,-v2}): ctrl-1 (which processed a `hostctrl -c` against a previously-joined synthetic host) has host.status=0x00500000 (XCE_WORKER_CLOSED|XCE_WORKER_CLOSED_BY_ADMIN) in its checkpoint's HostAdminState frame; ctrl-2/ctrl-3 (standbys) both show host.status=0x00000000 for the identically-named host entry -- proving the close never reached their live c->workermgr entries. Because a standby's OWN restart+full-WAL-replay hits the identical code path (persist.c's replay call site also routes through apply_committed_entry's same WAL_WORKER_CTRL case), a member that fully replays from LSN 1 would ALSO reconstruct the JOIN first (creating the worker) and then skip the later UPDATE (already exists) -- so this is not even a restart-repairable staleness; the divergence appears to be permanent for that member until directly patched or until some other mechanism re-applies the admin action. General severity: this affects ANY admin host-control action (`hostctrl -c`/`hostctrl -o`) against an already-joined host, unconditionally, cluster-wide -- not just this soak's synthetic cfgupgrade-host scenario. + +## Fix +FIXED 2026-07-25 (TDD: RED test first, confirmed failing against the pre-fix code, then GREEN). worker_from_capnp() (src/controller/worker.c) now decodes the WAL_WORKER_CTRL entry's hook_detail discriminator (the same 4 leading bytes it previously skipped without inspecting) BEFORE applying its existing-worker idempotency guard. When the target worker already exists: HOOK_DETAIL_WORKER_JOIN keeps the original -EEXIST no-op (a duplicate JOIN replay is genuinely idempotent -- unchanged, regression-guarded). HOOK_DETAIL_WORKER_UPDATE now REPLACES (not ORs -- worker_ctrl()'s HOST_OPEN uses assignment-style &= ~(...) to CLEAR admin bits, so OR would leave a re-opened worker stuck closed) the existing worker's XCE_WORKER_ADMIN_MASK bits with the decoded payload's admin-masked bits, under the worker's own write lock (mirrors restore_host_state's masking discipline, worker.c:487, with replace instead of OR semantics), then returns 0 so persist.c's WAL_WORKER_CTRL apply case (unchanged -- its existing '0 on any successful apply, -EEXIST only for genuine JOIN no-op' contract already handled this generically) falls through to fire the correct HOOK_DETAIL_WORKER_UPDATE notification instead of silently discarding the status change. HOOK_DETAIL_WORKER_DEAD is DELIBERATELY LEFT UNFIXED (still -EEXIST/no-op) -- investigated, not skipped: mark_worker_unavail()'s mutation (worker.c:661-679) is NOT confined to XCE_WORKER_ADMIN_MASK (sets XCE_WORKER_UNAVAIL + dead_since + resets hostattrs_t to 'unknown' + zeroes resource-value slots via set_host_slots), and critically, the live per-host resource-value array xce_worker_t.resvals (a zarray_t, what set_host_slots actually mutates and what scheduling reads) is NOT part of the Worker capnp wire schema at all -- only the separate resvals_to_sched/nresvals_to_sched fields are (confirmed by reading the generated decode_Worker/encode_Worker in controller.capnp.c and the schema in controller.capnp). A naive status-replace for DEAD would be an incomplete fix (would not zero standbys'/replayers' resource slots on a dead-worker event, a real correctness gap for POST-PROMOTION scheduling since the newly-promoted primary would read stale, non-zeroed slots) and needs its own investigation into whether/how to extend the wire schema -- filed separately as bug-1472-remainder below rather than rushed. Fix verified via 2 new + regression tests in test_apply_committed_entry.c (test_ace_worker_ctrl_update_applies_to_existing_worker, test_ace_worker_ctrl_update_replaces_not_ors_admin_bits, test_ace_worker_ctrl_join_duplicate_existing_is_noop -- the last one a regression guard proving JOIN's existing no-op behavior is unchanged) plus the full controller_unit_test suite: baseline (this worktree, immediately pre-fix) 1030 Tests 0 Failures, post-fix 1033 Tests 0 Failures (net +3, exactly the new tests, zero regressions). persist.c was NOT touched -- confirmed its WAL_WORKER_CTRL apply case (persist.c:3724-3786) already treats any non-negative worker_from_capnp() return as a successful apply generically (re-looks-up the worker by name and fires the hook_detail-keyed notification unconditionally), so returning 0 from the new UPDATE branch required no caller-side change. + +[UPDATE 2026-08-16] Tag hygiene: this entry's own UPDATE-case fix landed 2026-07-25 (commit 7547a8f1) but its tags were never updated off 'not-fixed'/'needs-owner'. The DEAD-case gap this entry deferred is bug-1472-remainder, separately FIXED and landed today in commit 45400c3a. Both halves of bug-1472 are now fixed and committed; closing out. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1471, bug-1472-remainder diff --git a/.wolf/bugs/bug-1473.md b/.wolf/bugs/bug-1473.md new file mode 100644 index 00000000..23c3ff9e --- /dev/null +++ b/.wolf/bugs/bug-1473.md @@ -0,0 +1,38 @@ +--- +id: "bug-1473" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-25T19:12:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_config_repl.c (test-only; no production code involved)"] +tags: ["config-upgrade-snapshot", "task-8.8", "test-only", "upgrade_materialization_verify", "activation-fence", "delaymgr", "test-construction", "not-a-production-bug"] +related: [] +legacy: true +--- + +# bug-1473: test_second_config_marker_supersedes_first_still_installing_on_member (task 8.8) FAILED on first… + +## Description +test_second_config_marker_supersedes_first_still_installing_on_member (task 8.8) FAILED on first run: Expected 0 Was 1 -- the fence must clear once materialization for the latest marker (B) completes. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Test-construction gap, not a production bug. After releasing the checkpoint_in_progress CAS in the raced fixture, the test called the bare persist_checkpoint() to resolve the in-flight-checkpoint scenario, then read upgrade_materialization_fenced() directly -- but plain persist_checkpoint() never calls upgrade_materialization_verify() itself. That call only happens from two production call sites: apply_committed_entry's WAL_CONFIG_MARKER case (fires only on a NEW marker apply, not a bare checkpoint), and the delaymgr-driven retry trampoline (schedule_upgrade_materialization_retry), which explicitly no-ops when delaymgr is not running -- its own doc comment says so verbatim: "No-op when... the delaymgr is not running (the unit-test fixture path...) -- caller drives directly." The bare t42/t43 test fixture (used throughout test_config_repl.c) never inits a delaymgr, so nothing in the raced fixture would ever re-evaluate the fence after the resolving checkpoint landed -- the fence stayed fenced=1 even though materialization for marker B had genuinely completed. + +## Fix +Added an explicit upgrade_materialization_verify(c_raced) call immediately after the resolving persist_checkpoint() call, mirroring the exact pattern already used by the precedent 6.1-c fence tests earlier in this same file (which call upgrade_materialization_verify(c) directly after t43_commit_marker for the same delaymgr-less-fixture reason). Also asserted verify_rc == 0. Confirmed via log trace that upgrade_materialization_verify only logs on a fence 0->1 transition (ha.c:254-256, "never spams on repeated failed retries") -- explaining why no second ERROR line appeared for markerB in the raw log even before the fix, which was the first clue this was a missing-caller issue rather than a fence-logic issue. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1474.md b/.wolf/bugs/bug-1474.md new file mode 100644 index 00000000..1f3fd0f1 --- /dev/null +++ b/.wolf/bugs/bug-1474.md @@ -0,0 +1,53 @@ +--- +id: "bug-1474" +status: "fixed" +status_updated: "2026-09-27" +status_note: "c3a17b19: root cause of angle (a) -- standby_endpoints loops bounded by count skipped entries past zarray_del holes (lost heartbeats, spurious re-dial); now scan caps" +component: "controller" +severity: "unknown" +timestamp: "2026-07-25T20:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/ha.c (primary_heartbeat_fn's 'reconnect to any peer missing from standby_endpoints' loop", "~9219-9253; on_standby_connected", "~8658-8708; raft_cb_become_leader's vote-ep promotion", "~1888-1921); src/controller/repl.c (repl_add_standby's first-empty-slot allocation", "~85-133", "and its memset(s", "0", "...) which wipes last_disconnected_lsn on reuse)"] +tags: ["ha.c", "repl.c", "trim_wal_segments", "repl_trim_lsn", "repl_add_standby", "bug-1147", "standby_endpoints", "late-joiner", "slot-reuse", "duplicate-connection", "promoted-vote-channel", "config-upgrade-snapshot", "task-8.9", "live-soak-finding", "root-caused", "reproducible-2-of-2", "needs-owner", "out-of-scope", "wal-trim", "mid-catch-up", "defense-in-depth", "log-hygiene", "review-remediation", "angle-a-fixed"] +related: ["bug-1147", "bug-1471", "bug-1472", "bug-1819"] +fix_commits: ["c3a17b19"] +legacy: true +--- + +# bug-1474: config-upgrade-snapshot task 8.9's new live --config-upgrade-wal-trim soak (run-ha-soak.sh… + +## Description +config-upgrade-snapshot task 8.9's new live --config-upgrade-wal-trim soak (run-ha-soak.sh, run_config_upgrade_wal_trim_round): the marker-triggered checkpoint's WAL-segment trim (persist_checkpoint_marker_triggered -> trim_wal_segments -> repl_trim_lsn, bug-1147's guard) strands a genuinely mid-catch-up standby's still-needed WAL segment -- but NOT because bug-1147's guard logic itself is wrong. Reproduced identically on 2/2 independent live 3-node runs: a killed, still-down standby's disconnect-guard (last_disconnected_lsn) is silently DESTROYED before the trim ever runs, because an UNRELATED, already-connected peer's redundant reconnect attempt reuses the dead standby's just-freed replicator slot. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Confirmed via direct log evidence on 2 independent live-soak runs (deploy dirs /home/claude/tmp/xce-ha-waltrim and /home/claude/tmp/xce-ha-waltrim2, both --config-upgrade-wal-trim). Sequence, identical both times: (1) An election promotes surviving VOTE endpoints to standby replication channels (raft_cb_become_leader: 'promoted vote ep[i] -> standby channel', repl_add_standby registers them at replicator slots 0/1 -- e.g. run 1: ep[0]=ctrl-1 -> slot 0, ep[1]=ctrl-3 -> slot 1). (2) The test kills a non-primary TARGET standby (ctrl-1) while it is caught up to a known WAL floor. The primary's err_handler (vote_ep_err_handler, correctly wired) detects the RDHUP and calls repl_remove_standby, freeing slot 0 and setting last_disconnected_lsn = ctrl-1's real acked_lsn (confirmed matching the test's own captured floor segment, e.g. seg-5 in run 1, seg-c in run 2) -- bug-1147's guard is set CORRECTLY at this point. (3) Seconds later (matching the ~9s default heartbeat interval), primary_heartbeat_fn's periodic 'reconnect to any peer missing from standby_endpoints' sweep (ha.c ~9219) runs. It correctly identifies ctrl-1 (peer[0]) as missing (reconnect attempt fails with ECONNREFUSED, confirmed in both runs' logs: 'HA: connect to peer[0] (localhost:9991) failed: -111') -- but it ALSO, incorrectly, identifies ctrl-3 (peer[2]) as missing, even though ctrl-3 has been continuously connected the whole time via its slot-1 promoted-vote-channel connection (confirmed acking WAL entries without any gap throughout, via ctrl-2.log's 'commit: ack standby_idx=1' sequence). This triggers a redundant outbound reconnect to ctrl-3 (on_standby_connected fires: 'HA: outbound standby connection to peer[2] established' + 'HA: repl registered late-joiner standby peer[2] (slot 0, localhost:9993)'). (4) repl_add_standby's slot allocation picks the FIRST EMPTY slot by lowest index (repl.c ~85-110) with no regard for whether that slot still holds an unexpired bug-1147 disconnect guard for a DIFFERENT peer -- it picks slot 0 (ctrl-1's just-freed, guard-holding slot, since it is lower-indexed than slot 1) and unconditionally memsets it to zero, WIPING OUT ctrl-1's last_disconnected_lsn guard entirely. (5) When the marker-triggered checkpoint's trim runs shortly after, repl_trim_lsn (repl.c ~650-683) sees slot 0 now occupied by a live, fully-caught-up ctrl-3 duplicate connection (contributing no meaningful floor-lowering) and slot 1 also fully caught up -- with NO surviving trace that ctrl-1 (still down, still needing its old floor) was ever disconnected. floor collapses to the active segment itself, and every segment below it -- including the one ctrl-1 genuinely still needs -- is deleted. This is NOT a bug in bug-1147's guard logic (repl_trim_lsn/trim_wal_segments are working exactly as designed against the replicator state they are handed); it is an upstream defect in how a duplicate/late-joiner reconnect for an ALREADY-connected peer gets allocated a replicator slot. Root cause of WHY the 'peer already has a standby endpoint' check (ha.c ~9225-9249) fails to recognize ctrl-3's existing promoted-vote-channel connection is not fully pinned to a single line -- standby_ep_ctx_t is deliberately layout-compatible with vote_ep_ctx_t (confirmed by direct struct read, ha.c:8590-8603, both start with ha/c/ctx/peer_idx in the same order, so the blind cast the check performs is NOT a type-confusion bug) -- but vote_ep_ctx_t.peer_idx appears to be populated under inconsistent conventions across its several creation call sites in ha.c (e.g. line 542 sets peer_idx=votes_idx [relative, votes-array index] while a comment at line ~6120 asserts 'vctx->peer_idx is the ABSOLUTE cluster index' for a different call site) -- a mismatch between the relative votes_idx_for_i the check computes and whichever convention the SPECIFIC promoted vote_ep_ctx_t for ectx->votes[i].ep was actually built under would make the sctx->peer_idx == votes_idx_for_i comparison fail deterministically for promoted-vote-channel entries, matching the 100% reproduction rate observed. This needs further tracing by whoever owns ha.c's HA connection-management code to identify the exact mismatched call site. + +## Fix +[UPDATE 2026-08-16] Angle-a FIXED (TDD RED->GREEN, root-caused precisely -- not the original peer_idx-convention hypothesis, a different, concrete bug in an adjacent function). Traced every vote_ep_ctx_t.peer_idx assignment site as recommended: raft_cb_send_vote_request (real vote, ha.c) correctly sets peer_idx=votes_idx (RELATIVE) and is the ONLY function that stores its endpoint into ectx->votes[] (the array raft_cb_become_leader's promotion loop reads) -- confirmed by full read, no ectx->votes[].ep assignment exists anywhere else. raft_cb_send_pre_vote_request deliberately sets peer_idx=ABSOLUTE (correct, documented, needed by the PreVote tally) but NEVER stores into ectx->votes[] either -- so PreVote endpoints structurally can never reach promotion, ruling out the original hypothesis (PreVote-vs-real-vote convention mismatch reaching the reconnect-sweep check). ha->my_priority is set once at init and never reassigned, ruling out a self-idx-drift hypothesis too. The reconnect-sweep check itself (primary_heartbeat_fn, ~line 9787) already correctly discriminates promoted-vote-ep (relative) vs standby-ep (absolute) via sctx->ctx != NULL, unchanged since 2026-06-01 (predates this bug's 2026-07-25 discovery, confirmed via git log -L) -- it was never the bug. The REAL bug: on_standby_connected's own same-peer duplicate-eviction check (the function's OWN doc comment already named this exact hazard -- 'two live outbound state-push channels to one peer') compared cur_sctx->peer_idx == sctx->peer_idx DIRECTLY, with no relative/absolute discrimination at all -- unlike the adjacent, correct reconnect-sweep check ~500 lines away. sctx (the newly-connecting standby) always carries the ABSOLUTE index; if the existing entry for the same physical peer is a promoted vote ep (RELATIVE), the raw comparison can never match, so the stale promoted-vote channel is never evicted -- both channels survive, reproducing the buglog's exact live evidence (heartbeats/WAL fan out to a stale duplicate; the genuinely-connected promoted channel's own bug-1147 disconnect-guard slot gets silently reused by the SECOND, spurious connect attempt for the SAME live peer). Fix: apply the identical votes_idx_for_i conversion the reconnect-sweep check already uses, in on_standby_connected's eviction loop, discriminating on cur_sctx->ctx != NULL exactly as that check does. Regression test (test_ha_liveness.c's test_reconnect_resolves_dual_channel_promoted_vote_ep): stale entry is a genuine promoted-vote-ep test double (vote_ep_ctx_build_for_test, relative peer_idx) for the same physical peer a fresh standby reconnect targets (absolute peer_idx) -- RED under the pre-fix code (both channels survive, zero evictions -- the existing sibling test, test_reconnect_resolves_dual_channel_same_peer, only ever exercised the standby-vs-standby case, both absolute, which is why this mixed case slipped through originally), GREEN under the fix (exactly one channel survives, one eviction). Full controller_unit_test suite: 1100 tests, 2 pre-existing unrelated failures (bug-1561), no regressions. NOT independently re-verified via a live multi-node chaotic soak (the class of test that originally discovered this bug) -- the unit test directly constructs and proves the exact mechanism the live soak's log evidence showed, but a live re-run of config-upgrade-snapshot task 8.9's --config-upgrade-wal-trim scenario would be the strongest additional confirmation if this is revisited. + +[Original entry -- angle-b, already fixed 2026-07-26] + +NOT FIXED -- explicitly out of config-upgrade-snapshot task 8.9's boundary (task 8.9 is a regression test for the pre-existing, unmodified bug-1147 guard under the NEW marker-triggered trigger path, not a fix for this separately-discovered HA connection-management defect). Two independent, complementary fix angles for whoever owns ha.c/repl.c's HA connection layer: (a) fix the 'peer already has a standby endpoint' check (ha.c ~9225-9249) so it correctly recognizes an existing promoted-vote-channel connection for a peer, eliminating the spurious duplicate/late-joiner reconnect entirely (the more targeted fix, addresses the root cause) -- likely requires auditing every vote_ep_ctx_t.peer_idx assignment site (ha.c lines 542, 697, 6956, and whichever one specifically constructs ectx->votes[i].ep's ep->data) against the relative-votes_idx convention votes_idx_for_i expects; (b) independently harden repl_add_standby's slot allocation (repl.c) to never silently clobber a slot whose last_disconnected_lsn guard has not yet expired per snap_interval_s -- e.g. skip a guard-holding slot in the first-empty-slot scan (falling through to a genuinely empty slot, or failing loud if none exists) -- this is a defense-in-depth hardening of bug-1147's own retention mechanism against ANY future caller that might reuse a slot prematurely, not just this specific duplicate-connection path. Recommend (b) regardless of whether (a) is fixed, since it directly protects the invariant Requirement 7.4 cares about. The run_config_upgrade_wal_trim_round test infrastructure (run-ha-soak.sh, config-upgrade-snapshot task 8.9) is CORRECT and reproducibly catches this defect via direct on-disk WAL segment inspection -- re-run `run-ha-soak.sh --config-upgrade-wal-trim` after either fix lands to confirm a clean PASS; task 8.9 itself should stay incomplete/blocked on this bug per its own scope boundary (mirrors task 8.6's bug-1471/1472 precedent). + +[UPDATE 2026-07-26] FIXED (fix angle (b), the repl.c defense-in-depth hardening): repl_add_standby's slot-allocation scan (repl.c) now does a two-pass search instead of first-empty-slot-by-index. Pass 1 prefers a genuinely virgin slot (ep==NULL && last_disconnected_lsn==0). Pass 2 (fallback, only when no virgin slot exists) reuses the guard-holding slot (ep==NULL && last_disconnected_lsn!=0) with the OLDEST last_disconnect_time -- a deliberate policy: that guard is closest to its own natural repl_expire_disconnect_slots expiry, so reusing it discards the LEAST remaining WAL-retention protection of any guard-holding slot. With MAX_STANDBYS==3, the exhausted case (both non-primary slots simultaneously guard-holding, no virgin slot free) is handled by this same fallback rather than -ENOSPC -- refusing a legitimate new/reconnecting peer would be worse for cluster availability than accepting an early guard loss in an already-degraded state; the pre-existing -ENOSPC path (more distinct live peers than MAX_STANDBYS slots) is untouched, reserved for that genuinely-impossible case only. +TDD: two new unit tests added to src/controller/testsuite/test_repl_unit.c -- test_repl_add_standby_prefers_virgin_slot_over_guard (RED under the old code: asserted a new unrelated peer must not land in a lower-indexed guard-holding slot when a virgin slot exists, and that the guard's last_disconnected_lsn must survive untouched) and test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect (RED under the old code: asserted the exhausted-fallback deliberately picks the OLDER-disconnected guard slot, not merely the lowest-indexed one -- the test deliberately makes 'oldest-disconnected' and 'lowest-indexed' point at DIFFERENT slots so an accidental first-guard-by-index fallthrough would still fail it). Both new tests failed pre-fix, pass post-fix. +GREEN: full controller_unit_test suite = 1030 tests (1028 known-good baseline + 2 new), 0 failures, no regressions (ASAN_OPTIONS=detect_leaks=0, nix develop build via `ninja -C src/xce.build controller/controller_unit_test`). +repl_remove_standby, repl_expire_disconnect_slots, repl_trim_lsn, and trim_wal_segments were NOT modified -- all confirmed correct and untouched, per this fix's scope. +This fix does NOT address fix angle (a) -- the separate ha.c heartbeat-reconnect-sweep defect (vote_ep_ctx_t.peer_idx convention mismatch hypothesis, ha.c ~9219-9254 / ~8687-8708) that triggers the spurious duplicate reconnect in the first place -- that remains a separate, not-fully-root-caused issue, out of this fix's scope. Recommend re-running `run-ha-soak.sh --config-upgrade-wal-trim` (config-upgrade-snapshot task 8.9) to confirm this defense-in-depth closes the observed WAL-segment-loss symptom even without (a) landing. + +[UPDATE 2026-07-26, post-review remediation] Independent review REJECTED the initial fix on one finding: the Pass 2 exhausted-fallback branch silently destroyed the reused slot's last_disconnected_lsn/last_disconnect_time via memset with zero logging, undercutting the fix's own stated goal of making the previously-silent WAL-retention-guard destruction visible/bounded -- this file already has an established convention for exactly this class of event (repl_broadcast's WARN("repl_broadcast: no standbys registered...") and WARN("repl_broadcast: standby %s:%u evicted...")). Remediated: repl_add_standby now tracks whether Pass 1 found a virgin slot (used_fallback_slot) and, only on the genuine Pass 2 fallback path, logs WARN("repl_add_standby: reusing guard-holding slot %d (host=%s:%u, last_disconnected_lsn=%lu) -- no virgin slot available, WAL-retention protection discarded", ...) immediately before the memset, capturing the discarded slot's host/port/last_disconnected_lsn while they're still readable. No new test required (a log line has no observable return-value/state effect for a test to assert on) -- verified instead by confirming the WARN string is present in the built binary (`strings controller_unit_test | grep 'reusing guard-holding slot'`) and that both existing bug-1474 tests (test_repl_add_standby_prefers_virgin_slot_over_guard, test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect -- the latter specifically exercises the Pass 2 path this WARN sits on) still pass unchanged. GREEN: full controller_unit_test suite = 1033 tests (1030 baseline + bug-1472's 3 concurrently-landed tests), 0 failures. NOTE: a transient stale-object-file build artifact was observed once in this shared build directory during verification (a link picked up an out-of-date worker.c.o from a moment when a concurrent sibling agent's build was in flight, showing 2 spurious failures unrelated to this change) -- resolved by touching the affected sources to force recompilation; not a defect in this fix, noted here in case it recurs for a future concurrent multi-agent session in this worktree. + +[UPDATE 2026-09-16, live re-observation during controller-status task 8.6's live verification, NOT independently fixed here -- out of that task's boundary] Directly observed fix angle (a)'s symptom again, live, on a fresh 3-node cluster (/tmp/xce-deploy-ctrlstatus86): killed standby ctrl-2 (localhost:9992) only; ctrl-3 (localhost:9993) was never killed and its own log shows zero RDHUP/disconnect/reconnect lines for the entire session. ~2s after the ctrl-2 kill, ctrl-1's log shows `ha_connect_finish_peer: HA: peer[2] (localhost) not connected -- initiating outbound connect` for ctrl-3 (peer[2]) -- a peer that was continuously, genuinely connected the whole time -- followed by `on_standby_connected: HA: repl registered late-joiner standby peer[2] (slot 2, localhost:9993)`, consuming what would otherwise have been a virgin slot. Zero repl_remove_standby/evict/duplicate log lines appear anywhere in the entire run, so the angle-a on_standby_connected eviction-check fix (2026-08-16) either did not fire for this specific reconnect path or the original promoted-vote-channel slot for ctrl-3 was left live alongside the new one. Direct downstream effect: when the genuinely-killed ctrl-2 reconnected ~1 minute later, repl_add_standby's Pass 1 found 'no virgin slot available' (logged) and fell back to Pass 2, reusing ctrl-2's OWN prior guard-holding slot rather than landing in a fresh slot -- i.e. this spurious ctrl-3 duplicate-connect is what consumed the third MAX_STANDBYS slot in this run. Not investigated further (root-causing ha.c's reconnect-sweep peer-liveness check is explicitly out of controller-status task 8.6's boundary, which owns only ctrlstatus_fill's read-side status derivation, not connection-management) -- reported as a CONCERN alongside task 8.6's own fix (bug-1819) rather than silently worked around. task 8.6's ctrlstatus_fill fix tolerates this correctly either way (single-slot reuse or a genuine duplicate-live-slot pair both resolve to the right status), so it did not block that task, but fix angle (a) itself remains open. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1147, bug-1471, bug-1472, bug-1819 diff --git a/.wolf/bugs/bug-1475.md b/.wolf/bugs/bug-1475.md new file mode 100644 index 00000000..2f5c4c03 --- /dev/null +++ b/.wolf/bugs/bug-1475.md @@ -0,0 +1,48 @@ +--- +id: "bug-1475" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-26T04:36:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (ha_dispatch_repl_action_for_standby", "ha_consume_append_entries", "ha_evaluate_repl_action — HA_REPL_STREAM_WAL decision/dispatch loop); src/controller/config_repl.c (ha_stream_wal_to_standby catch-up streamer)"] +tags: ["ha.c", "config_repl.c", "ha_dispatch_repl_action_for_standby", "ha_stream_wal_to_standby", "ha_consume_append_entries", "ae_reject_logmismatch", "catch-up-stall", "write-quorum-loss", "standby-liveness", "config-upgrade-snapshot-task-8.6", "force_elect_node", "not-bug-1472", "bug-1472-fix-verified-good", "same-family-as-bug-1028", "needs-owner", "root-caused-partially"] +related: ["bug-1472", "bug-1028"] +legacy: true +--- + +# bug-1475: Live config-upgrade-snapshot task 8.6 soak (`run-ha-soak.sh --config-upgrade`), run to verify… + +## Description +Live config-upgrade-snapshot task 8.6 soak (`run-ha-soak.sh --config-upgrade`), run to verify bug-1472's fix, FAILED — but NOT at bug-1472's own assertions (materialized-state checkpoint byte-identity and active-configuration identity both PASSED cleanly across all 3 members, confirming bug-1472's fix holds). Failure occurred one step later, inside the Requirement 6.4 host-admin-status-rotation check's own `force_elect_node("ctrl-1")` maneuver: after 3 rounds of overlapping chaotic kills + a live config upgrade all completed and the cluster reconverged 3-of-3, `force_elect_node` killed the spoiler (ctrl-3) and tried to submit a job batch on the surviving {ctrl-2 primary, ctrl-1 standby} pair — 5 consecutive attempts all got 0/3 acked ("waiting for write-quorum"), and the round hard-failed: `[xce-test-ha] FAIL: force-elect ctrl-1: submit on {ctrl-2,ctrl-1} failed`. The Req 6.4 workerlist/admin-status assertion itself was never reached. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT data loss, NOT bug-1472 (worker/host admin-status replication is unaffected — unrelated subsystem, and its own checks passed). This is a standby catch-up LIVENESS stall: ctrl-1 (which had itself served as primary in term=2 before being killed in round 1 of the chaotic sequence, then restarted) got permanently stuck at applied_lsn=12 and never advanced again for the rest of the run. ctrl-1's own log (`ha_consume_append_entries`) shows a revealing pattern: a zero-entry PROBE AppendEntries at prevIdx=12,n=0 correctly succeeds with match=12 (confirming index 12 IS ctrl-1's true, correct log tip), but the very next entry-carrying AppendEntries from the SAME dispatch cycle uses prevIdx=13 (not 12) for n=1, which ctrl-1 correctly rejects with event=ae_reject_logmismatch (success=0 match=0) since it has no entry at index 13 yet to check a predecessor against. This 1-off prevIdx computation (using prevIdx=standby_applied_lsn+1 instead of prevIdx=standby_applied_lsn for the FIRST entry of a catch-up batch) means every retry of the SAME heartbeat-driven HA_REPL_STREAM_WAL decision (ha_evaluate_repl_action: standby_lsn(12) < committed_lsn(31) -> stream) recomputes and resends the IDENTICAL doomed range, forever, with zero backtrack/self-correction -- confirmed via ctrl-2's log: 'HA repl: standby[0] lsn=12 < committed_lsn=31 -> stream WAL batches' repeats verbatim every ~3s from 04:31:26 through 04:32:26 (60+ seconds, 20+ identical retries, zero progress) after ctrl-2 had already logged 'repl_broadcast: standby localhost:9991 evicted (pending_ack_count=17 > 16) -- triggering catch-up reconnect' and 'HA quorum lost: connected=0 required=1'. Once the soak's own force_elect_node killed the OTHER standby (ctrl-3) as its spoiler maneuver, ctrl-1 became the sole remaining standby, and its permanent inability to catch up meant the primary could never regain 2-of-3 write quorum again -- every subsequent job submission timed out (`commit: async continuation deadline expired`, `jobsubmit_commit_continuation: lsn=N commit failed/timed out (rc=-110)`), exactly matching the soak's observed '0/3 acked; waiting for write-quorum' x5 before hard-failing. Superficially similar symptom signature to already-FIXED bug-1028 (same 'ha_dispatch_repl_action_for_standby: standby[0] lsn=X < committed_lsn=Y -> stream WAL batches' log line looping indefinitely post-failover) but a DIFFERENT specific AE-reject reason -- bug-1028 was a logOk TERM-mapping mismatch (raw hdr.term vs persist_entry_term legacy mapping), fixed 2026-06-23; this is `ae_reject_logmismatch`, which reads like a prevLogIndex/off-by-one issue in how the catch-up streamer picks the FIRST entry's predecessor index, not a term problem -- so this is either a narrower unfixed variant of the same family, or a new regression in a related code path. NOT root-caused down to the exact defective line (this investigation was a live-soak VERIFICATION run for a DIFFERENT bug, not a fix task) -- flagging for whoever picks this up next to read ha_dispatch_repl_action_for_standby/ha_stream_wal_to_standby's prevIdx computation directly and compare against ctrl-1's actual persisted log at lsn=12/13 (waldump). Also notable: this specific soak step (force_elect_node's spoiler-kill-then-submit maneuver, previously only exercised by --sequential mode's post-loop 'leadership coverage' step against an otherwise-calm, fully-settled cluster) had never previously been run immediately after --config-upgrade's THREE rounds of aggressive overlapping chaotic kills plus a live config upgrade -- this specific combination/history may be what's needed to reproduce the stall, rather than it being config-upgrade-specific. + +## Fix +FIXED (2026-07-27/28, via a regression test + real-instrumentation debugging session that continued this entry's original live-soak finding). Landed as THREE coordinated changes, all required together -- the first two alone left the new regression test (test_bug1475_catchup_survives_midstream_checkpoint, test_config_repl.c) RED with applied_lsn stuck at 0 for the whole run: + +1. SEND side (config_repl.c, ha_stream_wal_to_standby): when scanning the WAL for entries to stream, a WAL_CHECKPOINT entry is still never built/sent as its own catchup entry, but the prev_log_index/prev_log_term tracker now advances across it like every other scanned entry, instead of freezing one entry behind. A standby that independently applied the same committed WAL_CONFIG_MARKER writes its OWN local checkpoint at the identical LSN (deterministic per-member trigger), so the checkpoint's LSN is a real, held position on both sides, not a gap to skip past chaining-wise. + +2. RECEIVE side (ha.c, ha_apply_term_entry_via_library): a JIT resync reads persist_durable_floor(p) and monotonically raises ha->raft.log_len to match before consuming each AppendEntries -- mirroring ha_sync_raft_log_from_recovery's boot-time computation, but applied live on the AE-receive path. This corrects for persist_write_checkpoint_marker (bug-1466, still not fixed at ITS root) advancing persist_t's current_lsn without going through raft_log_append, which otherwise leaves ha->raft.log_len stale and fails logOk's prev_log_index<=log_len bound forever. + +3. TERM-INDEX side (persist.c, record_entry_term/entry_term_locked) -- a THIRD, previously-undiscovered gap, found only via live fprintf instrumentation of the new regression test after fixes 1-2 above still left it RED. Fixes 1-2 both depend on persist_entry_term(checkpoint_lsn) reading back 'absent' (0) FOREVER on every member (the explicit, twice-documented design intent in both fixes' own comments and in persist_write_checkpoint_marker's comment). That invariant was actually FALSE: record_entry_term's geometric-growth realloc zero-filled any never-explicitly-recorded gap slot (like a checkpoint's own LSN, which is deliberately never passed to record_entry_term), and entry_term_locked's `lsn <= term_index_count` presence check then read that zero-fill back as WAL_LEGACY_TERM (1), not absent (0), the INSTANT any LATER lsn was recorded and grew term_index_count past the gap -- which happens immediately on a live leader (it appends real entries right after the checkpoint) but not yet on a receiver that has not itself appended past its own checkpoint. So the checkpoint LSN's 'term' was TIME-DEPENDENT: the sender (which streams AFTER its own later entries exist) computed prev_log_term=1 for the entry right after the checkpoint, while the receiver's honest, not-yet-grown term index still (correctly, per design) read 0 for the same LSN -- logOk's equality check spuriously mismatched (0 != 1) and rejected forever, exactly reproducing this entry's original ae_reject_logmismatch signature. Confirmed with real fprintf instrumentation at the streamer, the JIT resync, and raft.c's logOk check (added and removed during this investigation, not left in the tree): prev_log_term flipped from 0 to 1 in the SAME test run, immediately after the leader appended its 2nd post-checkpoint entry. Fixed with a TERM_INDEX_UNSET sentinel (UINT64_MAX) that record_entry_term's growth path now fills gap slots with (instead of 0), and entry_term_locked treats as absent -- so a never-explicitly-written slot reads absent PERMANENTLY, independent of later array growth. Confined to persist.c's two functions plus the term_index doc comment in persist.h; no other call site reads p->term_index[] directly. + +Also completed as part of this fix: (a) audited whether WAL entry types OTHER than WAL_CHECKPOINT bypass record_entry_term the same way -- confirmed NO: WAL_CHECKPOINT (value 100) is the numeric max of wal_entry_type, persist_wal_async and persist_wal_write_replicated both explicitly reject type==WAL_CHECKPOINT up front (it cannot reach record_entry_term via either live path), and replay_wal's `hdr.type < WAL_CHECKPOINT` recording guard already covers every other currently-defined type (WAL_JOB_UPDATE=7, WAL_QUEUE_CTRL=20, WAL_WORKER_CTRL=30, WAL_NOOP=40, WAL_CONFIG_MARKER=50); (b) confirmed no interaction with the RequestVote/PreVote freshness gates (raft.c raft_log_ok, shared by raft_on_vote_request/raft_on_pre_vote_request): that gate also reads on_get_entry_term(s->log_len), which can legitimately land on a checkpoint LSN after the JIT resync in fix 2 -- the TERM_INDEX_UNSET fix makes that read deterministic (always absent) instead of time-dependent, which is a consistency IMPROVEMENT for vote freshness too, not a behavior change in the unsafe direction; full test_ha_prevote.c/test_ha_failover_prevote.c suites pass unchanged. Full regression suite: 1034 Tests, 0 Failures (was 1033/1034 with the new test failing before this fix). Files changed: src/controller/config_repl.c (REAL logic change, landed earlier in this same bug-1475 investigation: ha_stream_wal_to_standby's WAL_CHECKPOINT branch now falls through to the unconditional prev-tracker advance via `hdr.lsn > standby_lsn && hdr.type != WAL_CHECKPOINT`, instead of a hard `continue` that froze the tracker one entry behind -- plus cross-reference comments added in this later session); src/controller/ha.c (REAL logic change, also landed earlier in this investigation: a ~30-line JIT resync block added to the top of ha_apply_term_entry_via_library that raises ha->raft.log_len from persist_durable_floor(p) before consuming each AppendEntries -- plus cross-reference comments added in this later session); src/controller/persist.c (the TERM_INDEX_UNSET sentinel fix + doc comments, the new logic landed in THIS session); src/controller/testsuite/test_config_repl.c (test_bug1475_catchup_survives_midstream_checkpoint + updated doc comments describing all three fix parts). Correction (coordinator review 2026-07-27/28, Finding 1): an earlier version of this fix summary mischaracterized config_repl.c's and ha.c's changes as 'comment-only, no logic change' -- both carry genuine logic changes; they were simply written in an earlier part of this same bug-1475 investigation than the TERM_INDEX_UNSET sentinel (session 3 of the fix), not left unmodified by it. + +Finding 2 (coordinator review, 2026-07-27/28, CONFIRMED REAL and FIXED): after the TERM_INDEX_UNSET fix, the reviewer traced a plausible NEW instance of the identical permanent-wedge bug class, reachable via the SNAPSHOT-INSTALL catch-up path (HA_REPL_SEND_SNAPSHOT, dispatched whenever a standby has fallen too far behind for incremental WAL streaming -- a live, common path) instead of WAL-streaming catch-up. ha_snapshot_do_send (ha.c) computes checkpoint_term = persist_entry_term(leader, c->persist.checkpoint_lsn) and ships it on the wire; the receiver's ha_handle_snapshot_chunk calls persist_reconcile_checkpoint_term(p, checkpoint_lsn, checkpoint_term), which called record_entry_term(p, checkpoint_lsn, checkpoint_term) directly -- an EXPLICIT write, not a growth-fill, so it never got the TERM_INDEX_UNSET sentinel even when checkpoint_term==0. entry_term_locked's pre-existing legacy-0 rule then read that raw 0 back as WAL_LEGACY_TERM(1), forever -- permanently diverging from the leader's own permanently-absent(0) value for the SAME lsn whenever c->persist.checkpoint_lsn (== last_applied_lsn at general persist_checkpoint() time, NOT the WAL_CHECKPOINT entry's own physical lsn) happens to coincide with a checkpoint's own un-recorded lsn -- a real, reachable quiescent-state timing (the apply pump has walked, one lsn at a time including a no-op pass over checkpoint slots, all the way to exactly the checkpoint's own position before the next persist_checkpoint()+snapshot-send cycle runs, e.g. right after a config upgrade with no immediate follow-up activity) -- not the common case, but real and worth closing. Proven with a dedicated regression test (test_bug1475_finding2_snapshot_install_checkpoint_term_absent, test_config_repl.c) that drives the REAL ha_send_snapshot_to_standby -> ha_handle_snapshot_chunk -> persist_reconcile_checkpoint_term path (via the xce_snapshot_chunk_hook test seam) with a leader whose checkpoint_lsn is deliberately seeded (via last_applied_lsn) to coincide with its checkpoint's own physical lsn -- confirmed RED by temporarily reverting only persist_reconcile_checkpoint_term's fix (receiver read back term=1, not the expected absent=0) and GREEN after restoring it. Fixed with the SAME TERM_INDEX_UNSET sentinel: persist_reconcile_checkpoint_term now substitutes TERM_INDEX_UNSET for a wire checkpoint_term of exactly 0 before calling record_entry_term, instead of recording the raw 0 -- a wire value of 0 is unambiguous today (ha_snapshot_do_send is the ONLY sender and always computes it via persist_entry_term, whose public contract already returns a DIFFERENT value, WAL_LEGACY_TERM, for a present-but-legacy-0 entry -- so a public 0 can only mean genuinely, permanently absent). A genuinely non-zero checkpoint_term (any real historical term) is still recorded verbatim and still correctly overwrites a stale local term at that LSN, unchanged -- the pre-existing STALE-TERM reconcile test (test_ha_core_safety.c's test_3_1_snapshot_install_does_not_reconcile_boundary_term, CHECKPOINT_TERM=7, a synthetic non-zero value) is unaffected and still passes. Also fixed the stale 'zeroed-on-grow' comment at persist.c's truncate_term_index_locked (grown slots are TERM_INDEX_UNSET-filled since the original TERM_INDEX_UNSET fix, not zero-filled -- comment had not been updated). Re-verified RequestVote/PreVote freshness non-interference for this second fix too: persist_reconcile_checkpoint_term's caller (ha_handle_snapshot_chunk) never touches ha->raft.log_len or any vote-path state; the fix only changes what value entry_term_locked returns for a checkpoint-boundary LSN after a snapshot install (now consistently absent instead of inconsistently WAL_LEGACY_TERM), which is the SAME direction of improvement (deterministic absence vs time/path-dependent) the original TERM_INDEX_UNSET fix already established as safe for the freshness gate. Full suite: 1035 Tests, 0 Failures (was 1034 before this new test). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1472, bug-1028 diff --git a/.wolf/bugs/bug-1476.md b/.wolf/bugs/bug-1476.md new file mode 100644 index 00000000..270be5e1 --- /dev/null +++ b/.wolf/bugs/bug-1476.md @@ -0,0 +1,44 @@ +--- +id: "bug-1476" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-07-26T04:44:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/config_repl.c (load_primary_config ~line 656-666 free_ClusterConfig/free(myconf->cluster_config) call site; config_activate_committed ~line 360; handle_repl_config_push ~line 1280); lib/controller.capnp.c (decode_ClusterConfig_ptr ~line 17074 calloc site", "decode_Resource_list ~line 4057 crash site)"] +tags: ["config_repl.c", "load_primary_config", "config_activate_committed", "handle_repl_config_push", "heap-use-after-free", "decode_Resource_list", "controller.capnp.c", "concurrency-race", "config-upgrade-snapshot", "task-8.9", "live-soak-finding", "bug-1474-verification-run", "new-crash", "needs-owner", "possibly-related-to-bug-1474-fix-angle-a-duplicate-reconnects", "mylock"] +related: ["bug-976", "bug-981", "bug-1474", "bug-1475"] +legacy: true +--- + +# bug-1476: Live config-upgrade-snapshot task 8.9 re-verification soak (run-ha-soak.sh… + +## Description +Live config-upgrade-snapshot task 8.9 re-verification soak (run-ha-soak.sh --config-upgrade-wal-trim, run to confirm bug-1474's repl.c fix), deploy /home/claude/tmp/xce-ha-waltrim3: ctrl-3 ABORTed via AddressSanitizer heap-use-after-free during the round's live config-upgrade submission, well before TARGET(ctrl-1) was ever restarted. The round's own bug-1474 WAL-trim assertions (floor segment preserved, below-floor segments trimmed, active segment present) all PASSED cleanly before this crash was noticed -- ctrl-3's death was only discovered because it, combined with a second unrelated issue (bug-1475), prevented the round's final '3-of-3 reconvergence' assertion from ever succeeding. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +AddressSanitizer heap-use-after-free, WRITE of size 8, in decode_Resource_list (lib/controller.capnp.c:4057), thread T5. Full ASAN trace (ctrl-3.stderr): +WRITE by T5: decode_Resource_list <- decode_ClusterConfig <- decode_ClusterConfig_ptr <- load_cluster_config_from_file (config_repl.c:606) <- load_primary_config (config_repl.c:664) <- config_activate_committed (config_repl.c:360) <- handle_repl_config_push (config_repl.c:1280) <- myreq_handler (cluster.c:654) <- vtask_entry (cluster.c:786) +FREED by T6: load_primary_config (config_repl.c:659, the 'free any prior decoded config before overwriting the pointer' step: free_ClusterConfig(myconf->cluster_config); free(myconf->cluster_config)) <- config_activate_committed (config_repl.c:360) <- handle_repl_config_push (config_repl.c:1280) <- myreq_handler <- vtask_entry +ALLOCATED by T5: calloc in decode_ClusterConfig_ptr (lib/controller.capnp.c:17074) <- load_cluster_config_from_file (config_repl.c:606) <- load_primary_config (config_repl.c:664) <- config_activate_committed <- handle_repl_config_push +This is a genuine, unguarded data race: load_primary_config's sequence is (a) if myconf->cluster_config is non-NULL, free it and NULL the pointer, then (b) decode a fresh ClusterConfig into myconf->cluster_config via load_cluster_config_from_file/decode_ClusterConfig_ptr. decode_ClusterConfig_ptr appears to publish the new struct pointer into *config (i.e. myconf->cluster_config) before it has finished populating nested fields (e.g. the Resource_list), since T5 is later observed still writing into the struct via decode_Resource_list AFTER T6 already saw a non-NULL myconf->cluster_config, freed it (T6's own step (a)), and proceeded to decode its own replacement. No mutex protects myconf->cluster_config across this read-free-decode sequence, so two overlapping invocations of handle_repl_config_push -> config_activate_committed -> load_primary_config for the same node race on the shared myconf->cluster_config pointer. This matches the class of concern already flagged (but not as a crash) in bug-976/bug-981 (handle_repl_config_push / config_activate_committed activation-ordering races) -- this is a NEW, more severe manifestation: an actual heap-corruption crash, not just an ordering/verification gap. Plausible trigger in this specific soak: heavy duplicate/late-joiner reconnect churn was independently observed on ctrl-2 (the primary) around the same window (bug-1474's WARN 'reusing guard-holding slot' fired twice, confirming the still-open, out-of-scope ha.c heartbeat-reconnect-sweep defect -- bug-1474's un-addressed 'fix angle (a)' -- was actively spawning duplicate/late-joiner standby connections during this round). If the primary sent the same config-push payload to ctrl-3 twice (once per duplicate connection), ctrl-3 could dispatch two concurrent handle_repl_config_push invocations on separate VTASK_LIVE_BLOCK workers, which is consistent with the observed T5/T6 race. This causal link (duplicate-connection churn -> duplicate config-push delivery -> concurrent load_primary_config race) is a plausible hypothesis from timeline correlation, not confirmed by direct evidence of two inbound config-push messages -- flagged as a hypothesis for whoever investigates, not a proven mechanism. + +## Fix +FIXED (2026-07-28). Two-part fix in load_primary_config() (config_repl.c), independently reviewed and verified via a real pthread-based deterministic race test: (1) added myconf_t.lock (mylock_t, mirrors replicator_t.lock's idiom in repl.c) serializing the entire free-decode-publish sequence against any concurrent load_primary_config() call for the same myconf_t -- initialized in main.c before first use, finalized in myconf_fini(), and in the t43_attach_myconf/t43_free_myconf test fixture. (2) load_cluster_config_from_file() now decodes into a LOCAL temporary (cluster_config_t *tmp), never directly into the shared myconf->cluster_config field -- decode_ClusterConfig_ptr (lib/controller.capnp.c) calloc's and publishes its pointer via the out-parameter BEFORE decode_ClusterConfig populates nested fields (e.g. resources), so decoding into a local closes the early-publish exposure at its source; myconf->cluster_config is only swapped in after the new config is fully decoded and validated, still under myconf->lock. Part (2) is independently necessary because several other readers (cluster.c, queue.c, sched.c, worker.c, apiserver.c, ctrlstatus.c, reply.c) read c->myconf->cluster_config without ever taking myconf->lock, so the field itself must never be observably half-decoded. New regression test test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free (test_config_repl.c) drives two REAL concurrent pthreads through load_primary_config() for the same myconf_t via a deterministic test-only hook (load_primary_config_race_hook, XCE_TESTING-only, NULL/no-op in production) that parks the first thread exactly at the retire-old-config step the live crash raced -- pre-fix this reliably reproduces a genuine ASan-catchable double-free from the same missing-lock root cause; post-fix myconf->lock keeps the second thread blocked for the entire duration, verified both by a captured never-raced-through flag and by the final published config being fully valid, never torn. Full suite: 1035 -> 1036 tests, 0 failures. + +[UPDATE 2026-07-28, kiro-impl config-upgrade-snapshot resume] Prior fix landed with a comment block (config_repl.c:639-646) admitting it had been reverted to a RED-phase reproduction (no lock, decode-directly-into-shared-field) for test-construction purposes and NEVER restored to GREEN — the mylock_init/fini wiring (myconf.h, main.c, myconf.c) landed, but load_primary_config() itself still ran fully unguarded. Full suite run confirmed: test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free FAILED ("thread B... Expected 0 Was 1"), i.e. B raced through while A was parked -- the lock was never acquired. Fixed for real: mylock_lock(&myconf->lock) now wraps the ENTIRE load_primary_config() body (acquired first, released at every return including success), and Cap-n-Proto decode now targets a local cluster_config_t *new_config first -- myconf->cluster_config is only freed+swapped after new_config is fully decoded and validated (reserved-name check, generation default), never decoded directly into the shared field. Verified 4/4 clean full-suite runs (1036 Tests 0 Failures 0 Ignored, ASAN_OPTIONS=detect_leaks=0) after the fix, including the specific race test. Committed as part of config-upgrade-snapshot task 8.6/8.9 unblock resume. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-976, bug-981, bug-1474, bug-1475 diff --git a/.wolf/bugs/bug-1477.md b/.wolf/bugs/bug-1477.md new file mode 100644 index 00000000..2e1c9525 --- /dev/null +++ b/.wolf/bugs/bug-1477.md @@ -0,0 +1,44 @@ +--- +id: "bug-1477" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-28T08:25:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/myapp.c (~line 388-425", "the \"leader already known -- skip own election\" fast path and its single guarded cluster_init_phase2 retry", "bug-849 lineage); src/controller/ha.c (raft_cb_become_follower ~line 2000-2012", "raft_cb_become_leader ~line 1772-1783", "ha_elect_evaluate_and_signal ~line 5609-5629/5699-5710 -- all three unconditionally set ha->phase2_done=1 after calling cluster_init_phase2(c) without checking its return value); src/controller/cluster.c (cluster_init_phase2 ~line 1342-1598", "the sole call site of commit_engine_init/persist_recover/ha_sync_raft_log_from_recovery); src/controller/persist.c (~line 796-799", "on_durable_advance documented as registered ONLY by commit_engine_init)"] +tags: ["apply_pump_never_wired", "not-bug-1475", "commit_engine_init", "cluster.c", "config-upgrade-snapshot-task-8.6", "marker-triggered-checkpoint", "out-of-scope-upstream", "skip_election", "retry-mechanism", "phase2_done", "ae_reject_logmismatch", "bug-849-lineage", "permanent-catchup-wedge", "myapp.c", "bug-1475-fix-verified-good", "ha.c", "needs-owner", "cluster_init_phase2"] +related: ["bug-1475", "bug-1466", "bug-849"] +legacy: true +--- + +# bug-1477: Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade… + +## Description +Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade), run on top of bug-1475's already-fixed and already-built-in commit ce8df990 (confirmed via binary mtime 08:16:27 postdating ce8df990 04:48:53Z and b08e392e 08:13:54Z, plus `strings` matching TERM_INDEX_UNSET/persist_durable_floor), FAILED: "ctrl-3 has no on-disk checkpoint after the upgrade". ctrl-1/ctrl-2 both have data/snap/snap-000000000000000b.capnp (51161 bytes, LSN 11 hex = the WAL_CONFIG_MARKER's own lsn); ctrl-3's data/snap/ is completely empty. ctrl-3's log shows a PERMANENT loop (repeats every ~3s/heartbeat for the entire ~3-minute run, never progressing): a bare heartbeat probe at prevIdx=11 succeeds (ae_already_have, match=11), but every walCatchupEntry at prevIdx=12/13/14 is rejected forever with ae_reject_logmismatch. Unlike bug-1475's own signature, this is NOT a transient race that self-heals once durable_lsn catches up -- it never recovers for the rest of the run. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED with byte/log-precise evidence, NOT config-upgrade-snapshot's own defect. ctrl-3 (my_priority=2, the highest-priority-number/most-deferred candidate in the 3-node cluster, per myapp_run's startup-stagger nanosleep) never ran its own raft candidacy for the ENTIRE test: zero occurrences of raft_cb_become_follower/raft_cb_become_leader/"-> CTRL_STANDBY" anywhere in ctrl-3.log. Instead, by the time its stagger delay elapsed, ctrl-1 (priority 0) had already voted/established itself, so myapp_run's own "leader already known" check (myapp.c ~343-390: voted_for_name set, or raft.term>0, or primary_ctrl_fd>=0) was true, and ctrl-3 took the `goto skip_election` fast path, entirely bypassing ha_start_election_async and therefore every raft-library election callback for the rest of its process lifetime (it only ever GRANTS votes to others afterward -- confirmed via repeated vote_granted/pre_vote_granted RAFT[p2] events with no corresponding become_follower/become_leader). The skip_election fast path (myapp.c ~415-425, bug-849, commit 20e59d0b, 2026-06-21 -- three-plus weeks before config-upgrade-snapshot's first commit 9e68335a) makes exactly ONE, correctly-guarded attempt at cluster_init_phase2(&c) (checks prc, only sets c.ha.phase2_done=1 on success) -- and at 08:18:26,930, that ONE attempt fired before c->myconf->cluster_config was populated (ctrl-3 had no local config file and had not yet received/activated a push), so cluster_init_phase2 returned -EINVAL immediately (cluster.c:1349-1358, before doing ANY of its real work) and the WARN 'cluster_init_phase2 on standby rejoin failed (rc=-22) -- WAL not recovered; falling back to election-path recovery' fired. phase2_done correctly stayed 0 -- but there is NO OTHER retry mechanism anywhere in the codebase for a node that (a) took the skip_election fast path and (b) never subsequently runs its own contested election: the only other three call sites that could retry (raft_cb_become_follower ha.c:2000-2012, raft_cb_become_leader ha.c:1772-1783, ha_elect_evaluate_and_signal's CTRL_PRIMARY/CTRL_STANDBY branches ha.c:5609-5629/5699-5710) never fire for ctrl-3 because it never engages the raft callback machinery at all. (Separately confirmed those three call sites ALSO unconditionally set ha->phase2_done=1 after calling cluster_init_phase2(c) with NO check of its return value -- unlike myapp.c's own careful guard -- a second, adjacent defect of the same class that did not trigger THIS failure only because ctrl-3 never reached them, but would identically strand any node whose FIRST raft-driven role transition happens to race ahead of config load.) Consequence, confirmed via grep: ctrl-3.log has ZERO occurrences of "commit engine wired", "persist_recover", "xce_config_init", or "create_lost_and_found" for the entire run (vs. multiple occurrences on ctrl-1/ctrl-2, once per boot/restart/re-election) -- cluster_init_phase2 (and therefore commit_engine_init, cluster.c:1484-1496) NEVER completed on ctrl-3. persist.c:796-799 documents on_durable_advance (the sole wake hook driving persist_apply_pump_run) as registered ONLY inside commit_engine_init -- so without it, persist_apply_pump_run/apply_committed_entry NEVER runs on ctrl-3, for ANY committed LSN, for the whole process lifetime. This means persist_checkpoint_marker_triggered (config-upgrade-snapshot task 5's own WAL_CONFIG_MARKER-triggered checkpoint, apply_committed_entry's WAL_CONFIG_MARKER case) never fires on ctrl-3 -- explaining precisely why its data/snap/ is empty while ctrl-1/ctrl-2 (whose commit engines DID wire) both hold the identical snap-000000000000000b.capnp. ctrl-3's config nonetheless activates cosmetically at 08:18:48,572 via a SEPARATE, independent code path (handle_repl_config_push -> load_primary_config -> config_activate_committed, an RPC push handler that does not touch the commit engine or apply pump at all) -- masking the underlying defect and making the soak's materialized-state/checkpoint assertions the first thing ever to surface it. bug-1475's three-part fix (config_repl.c's tracker-advance-across-WAL_CHECKPOINT, ha.c's JIT persist_durable_floor resync of ha->raft.log_len, persist.c's TERM_INDEX_UNSET sentinel) is necessary and correctly targets its own class of bug (a receiving standby's log_len bookkeeping merely LAGGING behind a checkpoint it DID independently write) but structurally cannot cover this failure: bug-1475's own fix explicitly documents and depends on the invariant 'a standby that independently applied the SAME committed WAL_CONFIG_MARKER writes its OWN local checkpoint at the identical LSN (deterministic per-member trigger)' -- ctrl-3 violates that precondition outright (it never writes the checkpoint AT ALL, not merely late), so the primary's catch-up stream's prev_log_index=12 (the checkpoint's own lsn, per bug-1475's now-correct sender tracking) checks against an entry ctrl-3 genuinely, permanently does not have. ae_reject_logmismatch is therefore a CORRECT rejection of ctrl-3's true log state, not a stale/racy one -- and since a WAL_CHECKPOINT entry is local-only/non-replicated BY DESIGN (persist_write_checkpoint_marker's own doc comment; persist_wal_write_replicated/persist_wal_async both explicitly reject type==WAL_CHECKPOINT), ctrl-3 can NEVER receive lsn=12 over the wire, and can never independently write it either (cluster_init_phase2/commit_engine_init never retries) -- a genuinely permanent, not merely time-bounded, wedge, exactly matching the soak's observed behavior (unlike bug-1475's transient race, this repeated with zero progress for the entire ~3-minute run). git blame confirms both implicated pre-existing defects (myapp.c's skip_election fast path, commit 20e59d0b bug-849 2026-06-21; ha.c's raft-callback phase2_done latch structure, commit 8f0d2d55 task 6.2 raft-library) predate config-upgrade-snapshot's first commit (9e68335a) by roughly a month -- this is a pre-existing HA bootstrap/election gap, NOT something config-upgrade-snapshot's task 5 introduced. Task 5 merely exposed it by creating a new cross-member structural dependency (every member independently checkpointing at a synchronized marker-boundary LSN) that a node whose commit engine/apply pump never wired silently violates. Before config-upgrade-snapshot, the same underlying gap would have left such a node's materialized state (jobmgr/qmgr/workermgr) permanently stale/un-applied with a much less visible failure signature (no crash, no wedge -- just silently wrong query results if that node were ever promoted), plausibly unnoticed by prior soaks that do not assert per-member materialized-state identity. + +## Fix +FIXED (2026-07-28). Root cause: myapp_run's skip_election passive path, and 3 raft-callback call sites (raft_cb_become_leader, raft_cb_become_follower, ha_elect_evaluate_and_signal x2), all latched ha->phase2_done=1 UNCONDITIONALLY right after calling cluster_init_phase2 (or its ha_phase2_hook test seam), discarding its return value entirely -- so a single failed attempt (e.g. racing config availability, cluster_init_phase2 returning -EINVAL because cluster_config wasn't loaded yet) permanently stranded the member: commit_engine_init never ran, so apply_committed_entry never applied ANY committed WAL entry for the rest of that process's life, and -- for a node on the passive skip_election path specifically -- NOTHING else would ever retry it (the other 3 call sites only fire via this node's own raft-library state transitions, which never happen for a passive standby whose primary keeps heartbeating normally). + +Fix: extracted a shared, idempotent retry mechanism -- ha_phase2_retry_fn (the blocking retry itself, capturing cluster_init_phase2/ha_phase2_hook's real return value, latching phase2_done ONLY on rc==0) plus ha_schedule_phase2_retry (a delaymgr-scheduled, SELF-RESCHEDULING trigger at heartbeat_interval_ms cadence -- unlike upgrade_materialization_retry/config_activate_retry, which are re-triggered externally by recurring commit-advance events, nothing recurring would otherwise ever re-trigger this for a passive skip_election node, so this one self-arms). Wired into all 5 call sites: myapp.c's skip_election block now schedules a retry instead of just WARNing on failure; raft_cb_become_follower and ha_elect_evaluate_and_signal's STANDBY branch likewise (safe to stay passive while retrying); raft_cb_become_leader and ha_elect_evaluate_and_signal's PRIMARY branch additionally REFUSE promotion on failure in raft_cb_become_leader specifically (mirroring the sibling upgrade_materialization_fenced refusal in the same function -- a promoted primary with an unwired commit engine would be far worse than a delayed promotion); the legacy ha_elect_evaluate_and_signal PRIMARY branch does NOT refuse (its ctx->done semaphore is already posted to the caller before this point in that function, so refusing would leave an inconsistent signal -- it proceeds to CTRL_PRIMARY and lets the self-rescheduling retry wire the commit engine shortly after instead). + +TDD: 4 new tests in test_config_repl.c (test_ha_phase2_retry_fn_noop_when_already_done, test_ha_phase2_retry_fn_does_not_latch_on_failure_and_succeeds_on_retry -- the core regression, RED confirmed by temporarily reverting the rc-gating back to an unconditional latch and observing the exact predicted failure -- test_schedule_phase2_retry_noop_when_already_done, test_schedule_phase2_retry_noop_when_delaymgr_not_running). GREEN: full controller_unit_test suite = 1044 tests, 0 failures, clean across multiple runs (including the RED/GREEN round-trip). + +Live-verified: re-ran run-ha-soak.sh --config-upgrade (the exact scenario that originally found this bug) -- ctrl-2's log shows the retry mechanism firing for real (4 failed cluster_init_phase2 attempts at ~3s intervals, rc=-22, then success: "commit engine now wired (bug-1477)"), and the soak's own checkpoint-identity, config-identity, and host-admin-status-survival assertions (the exact things bug-1477 blocked) all PASS. The soak run subsequently hit a DIFFERENT, seemingly unrelated failure later in its own force-elect-rotation step (repeated commit timeouts, rc=-110, on ctrl-1-as-primary) -- flagged separately, NOT part of this fix's scope; this soak scenario has never previously reached that step (earlier runs always failed earlier, at bug-1471/1472's checkpoint divergence or this bug's own catch-up wedge), so it may be a pre-existing, newly-exposed issue rather than a regression from this fix -- needs its own dedicated investigation. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1475, bug-1466, bug-849 diff --git a/.wolf/bugs/bug-1478.md b/.wolf/bugs/bug-1478.md new file mode 100644 index 00000000..fdb7f44a --- /dev/null +++ b/.wolf/bugs/bug-1478.md @@ -0,0 +1,50 @@ +--- +id: "bug-1478" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-28T09:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c (repl_ack_decode_and_dispatch's heartbeat/stateReport reply branch", "~6379-6388 pre-fix -- the `target_lsn = max(committed_lsn", "last_written)` computation feeding ha_dispatch_repl_action_for_standby)"] +tags: ["ha.c", "repl_ack_decode_and_dispatch", "ha_dispatch_repl_action_for_standby", "ha_evaluate_repl_action", "target_lsn", "last_written", "committed_lsn", "WAL_CHECKPOINT", "persist_write_checkpoint_marker", "bug-1466", "bug-1474-fix-verified-good", "bug-1475-fix-verified-good", "not-bug-1477", "config-upgrade-snapshot-task-8.9", "permanent-catchup-wedge", "off-by-one", "waldump-confirmed", "root-caused", "narrowly-scoped-fix", "tdd", "red-green-verified"] +related: ["bug-1466", "bug-1474", "bug-1475", "bug-1477"] +legacy: true +--- + +# bug-1478: config-upgrade-snapshot task 8.9 live soak (run-ha-soak.sh --config-upgrade-wal-trim… + +## Description +config-upgrade-snapshot task 8.9 live soak (run-ha-soak.sh --config-upgrade-wal-trim ~/tmp/xce-ha-waltrim): bug-1474's floor-preservation fix confirmed working (floor segment seg-5 correctly preserved, all below-floor segments correctly trimmed) but the round still FAILED: "cluster did not reconverge to 3-of-3 after restarting ctrl-1 -- the preserved floor segment did not actually suffice for real incremental catch-up". ctrl-1 (TARGET, killed+restarted 3x) settles into a PERMANENT, zero-progress loop: its own raft AppendEntries exchanges cleanly reach match=22 with zero ae_reject events (heartbeat probe at prevIdx=22,n=0 -> success=1 match=22, repeating every ~3s forever), while ctrl-2 (primary) simultaneously and forever logs 'HA repl: standby[0] lsn=21 < committed_lsn=22 -> stream WAL batches' / 'ha_stream_wal_to_standby: streamed WAL [21+1..22] to standby' every heartbeat cycle with zero effect -- an off-by-one disagreement (21 vs 22) between the two sides that never reconciles. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED with byte/log-precise evidence; NOT a recurrence of bug-1474 (independently re-confirmed working in this same run -- floor preservation and below-floor trim both correct), bug-1475 (independently re-confirmed working -- ctrl-1's own raft log_len/term-index bookkeeping across the checkpoint boundary is clean, zero ae_reject_logmismatch anywhere in its log, unlike bug-1475's own signature), or bug-1477 (ctrl-1 clearly DID wire its commit engine every boot -- multiple 'commit engine wired'/persist_recover lines confirmed, unlike bug-1477's ctrl-3 which never ran its own election at all). This is a THIRD, previously-undiscovered gap in the same 'WAL_CHECKPOINT breaks LSN co-tracking invariants' family already flagged (not fixed at its root) as bug-1466. + +Ground truth via `xce_test waldump` on the preserved deploy dir (~/tmp/xce-ha-waltrim): ctrl-2's WAL shows lsn=21 as the WAL_CONFIG_MARKER (type UNKNOWN/40 bytes, matching ctrl-2's own log 'Config change committed (marker_lsn=21)') and lsn=22 as a WAL_CHECKPOINT (0-byte payload) -- confirming persist_write_checkpoint_marker (persist.c:1747, `hdr.lsn = p->current_lsn++`) fired immediately after the marker committed and consumed the very next LSN slot, exactly as apply_committed_entry's WAL_CONFIG_MARKER case (persist_checkpoint_marker_triggered) is designed to do (config-upgrade-snapshot task 5). Critically, ctrl-2's OWN commit_advance_locked (commit.c, the SOLE authoritative committed_lsn writer) never logs an advance past 21 anywhere in the run ('commit: committed_lsn 20 -> 21' is the LAST such line) -- the real, majority-gated persist.committed_lsn correctly and permanently stays at 21, since no standby can ever legitimately ack a local-only, non-replicated checkpoint entry. + +Yet ha_dispatch_repl_action_for_standby's own log line reports 'committed_lsn=22' from the very next heartbeat cycle onward. Traced to repl_ack_decode_and_dispatch's stateReportMsg branch (ha.c): `last_written = c->persist.current_lsn - 1` (the RAW physical WAL tip, which unconditionally counts EVERY locally-appended entry including a local-only WAL_CHECKPOINT) is combined via `target_lsn = max(committed_lsn, last_written)` and THIS value -- not the real committed_lsn -- is what gets passed into ha_dispatch_repl_action_for_standby's committed_lsn parameter and printed in its log line. Once ctrl-2 wrote its checkpoint, last_written jumped to 22 and target_lsn followed, permanently outrunning the real committed_lsn by exactly one. Since a checkpoint's own consumed LSN slot is, by design, NEVER streamed (ha_stream_wal_to_standby) or applied (persist_wal_write_replicated / persist_wal_async both explicitly reject type==WAL_CHECKPOINT) by any standby, no standby's applied_lsn can EVER legitimately reach 22 -- ha_evaluate_repl_action's `standby_lsn < committed_lsn` check can never become false, so HA_REPL_CONVERGED is permanently unreachable for any standby once a local checkpoint has interleaved ahead of the next real committed advance. ha_stream_wal_to_standby is correctly re-invoked every heartbeat (per its own logic) but finds nothing new in [22,22] to send every single time -- a deterministic, permanent, zero-progress busy-loop, exactly matching this run's observed symptom (and ctrl-1's own applied_lsn -- correctly capped at 21 by ha_advance_applied_lsn/ha_apply_term_entry_via_library's own last-contiguously-held-LSN semantic, mirrored by ha_handle_snapshot_chunk's snapshot-install path setting applied_lsn=checkpoint_lsn never checkpoint_lsn+1 -- can never satisfy it). + +The `target_lsn = max(committed_lsn, last_written)` fallback's OWN doc comment states its intended purpose narrowly: 'so standbys can catch up even when committed_lsn is stuck at 0 (no replicator-registered standbys to advance it)' -- a cluster-bootstrap convenience. But the code applies the max() UNCONDITIONALLY, not only when committed_lsn==0 -- broader than its own documented justification. This mismatch was latent and harmless until config-upgrade-snapshot task 5 introduced a LIVE, mid-operation local-only WAL entry (the marker-triggered checkpoint) that advances current_lsn without a corresponding real committed advance; bug-1466 already flagged the SAME persist_write_checkpoint_marker current_lsn bump as breaking raft's own log_len co-tracking invariant (affecting vote-freshness comparisons) -- this is a THIRD, distinct consequence of that same architectural gap, this time poisoning the replication-target computation instead. git blame confirms the `target_lsn` code (commit 06135954f, 2026-06-11) predates config-upgrade-snapshot's first commit by roughly two weeks -- a pre-existing HA replication-decision defect that config-upgrade-snapshot's task 5 merely exposed (same routing pattern as bug-1471/1472/1474/1477 before it), not something task 8.9 introduced. + +## Fix +FIXED (2026-07-28, narrowly-scoped TDD fix, in-scope to land here per explicit high-confidence + narrow-scope authorization -- unlike bug-1474's angle (a) and bug-1477, which were left for the HA-core spec owner): extracted the target_lsn computation out of repl_ack_decode_and_dispatch into a new small, pure, side-effect-free, directly-unit-testable function `ha_compute_repl_target_lsn(committed_lsn, last_written)` (declared in ha.h, defined in ha.c right after ha_process_standby_state). The fix restores the fallback to EXACTLY its own documented intent: `last_written` is used ONLY when `committed_lsn == 0` (genuine bootstrap, nothing ever committed) -- any other case returns `committed_lsn` unchanged, never leaking a trailing local-only checkpoint's consumed LSN into the convergence decision. repl_ack_decode_and_dispatch's call site now reads `target_lsn = ha_compute_repl_target_lsn(committed_lsn, last_written)`, replacing the old unconditional max(). No other function touched -- ha_evaluate_repl_action, ha_process_standby_state, ha_dispatch_repl_action_for_standby, ha_stream_wal_to_standby, persist_write_checkpoint_marker, and commit.c's majority_candidate_locked/commit_advance_locked are all confirmed correct and untouched (the bug was purely in what value got PASSED to them, not in their own logic). + +TDD: four new unit tests added to src/controller/testsuite/test_ha_role.c (all four call the new pure function directly, no capnp/network scaffolding needed): test_compute_repl_target_lsn_bootstrap_fallback (committed_lsn=0, last_written=42 -> 42, confirms the intended bootstrap behavior is preserved), test_compute_repl_target_lsn_zero_zero (0,0 -> 0, fresh-cluster edge case), test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn (this bug's EXACT reproduction shape: committed_lsn=21, last_written=22 -> must return 21, not 22 -- confirmed RED under the old unconditional-max code via a temporary revert, 'Expected 21 Was 22', then GREEN after restoring the fix), test_compute_repl_target_lsn_committed_already_ahead (100,50 -> 100, normal steady-state sanity check). GREEN: full controller_unit_test suite = 1040 Tests, 0 Failures, 0 Ignored, run twice clean (ASAN_OPTIONS=detect_leaks=0, nix devshell, `ninja -C src/xce.build controller/controller_unit_test`). RED confirmed independently: reverting only the new function's body to the old unconditional-max logic (leaving the new tests in place) reproduced exactly 1 failure at the predicted assertion ('Expected 21 Was 22'), then re-applying the fix restored 0 failures -- proving the test genuinely exercises this bug and the fix genuinely closes it, not a tautological test. + +NOT yet re-verified via a fresh live `run-ha-soak.sh --config-upgrade-wal-trim` run (recommended next step, mirroring bug-1474's own verification precedent, before considering config-upgrade-snapshot task 8.9 unblocked) -- this fix was validated at the unit level only within this investigation's scope. + +[UPDATE 2026-07-28, kiro-impl config-upgrade-snapshot task 8.9 verification] The initial fix landed only ONE of TWO call sites duplicating this formula: ha.c repl_ack_decode_and_dispatch (fixed, via the new ha_compute_repl_target_lsn) and config_repl.c catchup_derive_target (a SEPARATE, independently-duplicated inline copy of the exact same max(committed_lsn,last_written) formula, used by ha_walack_on_ack/catchup_lane_fn -- the ACK-driven catch-up continuation path, not the heartbeat/stateReport path the first fix covered). Rerunning the live --config-upgrade-wal-trim soak after the first fix STILL FAILED ("cluster did not reconverge to 3-of-3... preserved floor segment did not actually suffice for real incremental catch-up") -- confirmed via log correlation that ctrl-2 (primary) was stuck forever at ha_dispatch_repl_action_for_standby committed_lsn=22 while catchup_lane_fn's own INFO line independently reported target=22 (the OLD, unfixed formula) even though the heartbeat path's committed_lsn logging correctly showed 21. Root-caused to catchup_derive_target (config_repl.c:2617-2627, its own comment literally said "re-derive the replication target exactly as ha.c:2771-2776 / D6" -- a deliberate, now-stale duplication). Fixed by making catchup_derive_target call the SAME ha_compute_repl_target_lsn pure function instead of re-deriving the formula inline, so the two paths can never disagree again. Applying just this second-call-site fix then caused a NEW symptom: controller_unit_test hung indefinitely (0.8%% CPU, blocked not spinning) inside test_convergence_verification.c's test_walack_behind_target_schedules_one_continuation. Isolated via a controlled revert-and-rerun (confirmed clean 1040/1040 with the OLD formula restored, confirming the SECOND fix -- not something else -- was the trigger). Root cause of the hang: three pre-existing tests in that file ("behind", "single", "respawn") hardcoded committed_lsn=4 with appended_lsn=9, deliberately exercising the OLD buggy max(4,9)=9 behavior as their expected target -- with the fix, the correct target becomes 4, so acked_lsn=5/6 (used by the later tests) no longer legitimately reads as "still behind", and the FIRST test's own hardcoded assertion fails. Because this test file's harness (cv_setup/cv_teardown) does not run teardown from a signal-safe/finally-equivalent path, a failed TEST_ASSERT's Unity longjmp skips cv_teardown() entirely, leaking that test's live 8-thread commit_engine still bound to the soon-to-be-memset g_cv global -- the NEXT test's cv_setup() memset-ing g_cv out from under those still-running threads is what actually produced the observed permanent hang, not a defect in the production fix itself. Fixed by updating all three tests' committed_lsn from 4 to 9 (matching appended_lsn, so target=9 under EITHER formula) -- this decouples them from the bug-1478 edge case entirely (which already has dedicated, correct pure-function coverage in test_ha_role.c), preserving each test's TRUE original intent (continuation scheduling / single-flight / respawn-after-complete mechanics) with a minimal, surgical change. Stale docstrings referencing the old "max(committed_lsn, last_written)" formula were also corrected to reference ha_compute_repl_target_lsn. GREEN: full controller_unit_test suite = 1040 Tests, 0 Failures, run clean 2x after all fixes (plus the isolation-revert run confirming the pre-fix baseline was also clean). Live-verified: run-ha-soak.sh --config-upgrade-wal-trim now reports a clean PASS (18 jobs, zero job loss, ctrl-1 reconverged 3-of-3 using the preserved floor segment for real incremental catch-up) -- config-upgrade-snapshot task 8.9 unblocked. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1466, bug-1474, bug-1475, bug-1477 diff --git a/.wolf/bugs/bug-1479.md b/.wolf/bugs/bug-1479.md new file mode 100644 index 00000000..fad22cdf --- /dev/null +++ b/.wolf/bugs/bug-1479.md @@ -0,0 +1,48 @@ +--- +id: "bug-1479" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "leak" +timestamp: "2026-07-28T13:25:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/ha.c (repl_ack_decode_and_dispatch's WAL-ACK branch", "~6456-6644 -- the `!success` NACK path at ~6631-6642 only checks for a higher term and WARNs \"ignoring\"", "never calls repl_on_ack/repl_start_catchup/any retry); src/controller/repl.c (repl_broadcast ~295-380", "the live single-entry send path -- fire-and-forget", "no nextIndex backoff-and-retry on rejection; repl_on_ack ~250-271", "the ONLY site that decrements pending_ack_count", "never invoked on a NACK)"] +tags: ["ha.c", "repl.c", "repl_broadcast", "repl_ack_decode_and_dispatch", "repl_on_ack", "pending_ack_count", "MAX_PENDING_ACKS", "in_catch_up", "ae_reject_logmismatch", "raft-log-conflict", "nextIndex-backoff-missing", "nack-ignored", "catch-up-never-triggered", "permanent-quorum-loss", "config-upgrade-snapshot-task-8.6", "force_elect_node", "not-admission-slot-leak", "hypothesis-refuted", "same-family-as-bug-1466", "waldump-confirmed", "byte-precise", "root-caused", "needs-owner", "needs-design-decision", "live-soak-finding", "missing-nack-reply", "raft-adapter-conformance", "verified-live", "raft-tla-conformance", "stream-ceiling", "nack-now-sent"] +related: ["bug-1466", "bug-1471", "bug-1472", "bug-1474", "bug-1475", "bug-1477", "bug-1478", "bug-1421"] +legacy: true +--- + +# bug-1479: Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade… + +## Description +Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade2), run on top of bug-1477's fix (checkpoint-identity, config-identity, and host-admin-status-survival assertions ALL PASSED for the first time), FAILED later in the force-elect-rotation step: 5 consecutive job-submit attempts on {ctrl-1 (primary), ctrl-2 (standby)} all got 0/3 acked ("waiting for write-quorum"), then hard-failed. Investigated the ORIGINAL hypothesis (a leaked VTASK_LIVE_BLOCK/commit-admission slot from an earlier burst of 17 commit timeouts, lsn=15..31, rc=-110) and REFUTED it: commit_resolve()/wait_release_locked() correctly release the admission slot on every timeout (confirmed 17 'async continuation deadline expired' + 17 'commit failed/timed out' WARN pairs, 1:1, in ctrl-1.log) — no admission-pool leak exists. The REAL mechanism is unrelated to admission control entirely: a permanent Raft AppendEntries log-conflict wedge on standby ctrl-2, compounded by a NACK-handling gap that never triggers catch-up. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED with byte-precise waldump evidence (xce_test's built-in `waldump` REPL command against the preserved deploy dir ~/tmp/xce-ha-configupgrade2), cross-correlated with both ctrl-1's and ctrl-2's logs line-by-line. During the soak's chaotic round-3 kill/restart churn, THREE leadership changes happened in rapid succession (term 3 -> 4 -> 5) within about 20 seconds (13:18:57-13:19:19): ctrl-2 briefly won term=3 and appended its own arm-own-term-gate NOOP, was killed before that entry could be widely replicated/committed, a short-lived term=4 leader (never directly observed in either surviving node's own become-leader lines, but proven to have existed by its WAL fingerprint) won next and wrote ITS OWN own-term NOOP at the SAME log index (14), and finally ctrl-1 won term=5 and became the lasting primary. `xce_test waldump` on ctrl-1's canonical (surviving, majority) WAL shows: lsn=12 CHECKPOINT term=2 (the marker-triggered checkpoint from config-upgrade-snapshot task 5, matching bug-1475/1477/1478's exact family), lsn=13 NOOP term=4 (the short-lived term-4 leader's noop), lsn=14 NOOP term=5 (ctrl-1's own winning noop). `xce_test waldump` on ctrl-2's own local pre-restart WAL shows: lsn=12 CHECKPOINT term=2 (agrees with ctrl-1), lsn=13 NOOP **term=3** (ctrl-2's OWN aborted, never-widely-replicated noop from its deposed term-3 stint) -- a genuine, CONFLICTING entry at the SAME index (13) with a DIFFERENT term (3 vs the canonical 4). This is an entirely NORMAL, expected Raft outcome of rapid leader churn (Raft's own §5.3 log-matching property explicitly anticipates followers accumulating stale, uncommitted tail entries from deposed leaders) -- ctrl-2 correctly rejects ctrl-1's AppendEntries at prevLogIndex=13 with logOk's standard conflicting-term check (confirmed via ha_consume_append_entries's own trace: `prevIdx=13 n=1 -> success=0 match=0`, `RAFT[p1] term=5 event=ae_reject_logmismatch`, repeating for EVERY subsequent new entry ctrl-1 tries to send: prevIdx=14,15,16,...29, one per newly-committed lsn, all rejected, spanning 13:19:19 through 13:19:52). The BUG is what happens (or rather, does not happen) next, confirmed via two independent, compounding gaps: (1) ctrl-1 NEVER backs off and retries with a DECREMENTED prevLogIndex to find the point of log agreement (the standard Raft nextIndex-conflict-resolution protocol, which would let ctrl-2 truncate its conflicting lsn=13 tail and adopt the canonical term=4 entry) -- repl_broadcast (repl.c:295-380) is a pure fire-and-forget per-new-entry send with zero synchronous handling of the reply; it always advances prevIdx forward to (next-new-lsn - 1), never backward. (2) repl_ack_decode_and_dispatch's WAL-ACK reply-dispatch branch in ha.c (~6631-6642) handles a NACK (success=0) by checking ONLY for a higher observed term (raft_on_higher_term_observed, correctly added for the DIFFERENT bug-1421 stale-term scenario) and then explicitly `WARN("...success=0 — ignoring")` -- the comment says exactly what the code does: the rejection is discarded. Critically, `repl_on_ack` (repl.c:250-271) is the ONLY code path that decrements `standby_conn_t.pending_ack_count`, and it is called ONLY from the `success` branch -- so a standby that is stuck rejecting (for ANY reason) has pending_ack_count monotonically climb by 1 on every subsequent broadcast attempt with ZERO corresponding decrements, until repl_broadcast's own MAX_PENDING_ACKS=16 defensive eviction (repl.c:325-334, from ha-config-bootstrap task 7.2, commit 3969064e, 2026-05-28 -- a full month before config-upgrade-snapshot's first commit) fires and marks the standby in_catch_up=1 at 13:19:53 (confirmed: 'repl_broadcast: standby localhost:9992 evicted (pending_ack_count=17 > 16) — triggering catch-up reconnect'). This eviction correctly EXCLUDES the standby from repl_active_standby_count (hence from ha_has_write_quorum's majority calculation) -- but nothing in the codebase actually DRIVES a catch-up stream or snapshot-install for it afterward: ctrl-2's log shows ZERO entry-carrying AppendEntries for the remaining ~60 seconds of the run (13:19:53 through the log's end at 13:20:53) -- only bare heartbeat probes (prevIdx=13,n=0) that trivially succeed since a presence-only probe at the standby's own true log_len=13 doesn't hit the term-conflict check. With ctrl-2 permanently excluded (in_catch_up=1, never un-evicted) and ctrl-3 permanently failing to reconnect (separate, unrelated ECONNREFUSED loop, likely a real chaos-kill in this soak round), ctrl-1 has connected=0 standbys forever ('HA quorum lost: connected=0 required=1', logged once at the eviction and never restored) -- every subsequent job submission either times out waiting for write-quorum (0/3 acked, matching the force-elect step's failure verbatim) or gets an immediate -XCE_ERR_NOT_READY (-10013) fast-reject via ha_dispatch_gate's quorum check. This is a GENUINE, SERIOUS, and apparently PERMANENT availability defect: ANY standby whose log happens to diverge by even ONE entry at a single index during leader-election churn (an entirely normal, expected Raft occurrence, not itself a bug) can become irrecoverably stuck and permanently excluded from quorum, with no self-healing path -- in a 3-node cluster this is one bad-luck election sequence away from TOTAL, PERMANENT write-quorum loss if it hits 2 of 3 standbys. Same family as bug-1466 (open, 'WAL_CHECKPOINT breaks LSN/term co-tracking', NOT fixed at its root) and its now-3 fixed siblings (bug-1475, bug-1477, bug-1478) -- this is the family's 5th distinct member, but the trigger here (an ordinary post-election term-conflicted log tail, NOT specifically a checkpoint-boundary term-derivation issue) is narrower than bug-1466's checkpoint-specific framing and points instead at a genuinely MISSING piece of core Raft log-repair protocol (conflict-driven nextIndex backoff + NACK-triggered catch-up), which appears to have never been implemented for this replication path at all -- not a regression, a gap. git blame confirms both implicated mechanisms (repl_broadcast's pending_ack_count eviction, commit 3969064e, 2026-05-28; the NACK-handling 'ignoring' branch in repl_ack_decode_and_dispatch, present since at least bug-1421's fix, itself predating config-upgrade-snapshot) are pre-existing, core HA replication code untouched by any config-upgrade-snapshot task -- config-upgrade-snapshot's task 5 (marker-triggered per-member local checkpoint) merely adds one more entry into the WAL stream around the point where this pre-existing gap gets exercised; it does not create the gap itself. + +## Fix +FIXED AND VERIFIED LIVE (2026-07-29, see UPDATE 2 below for the final closing fix) -- two compounding gaps were closed after the initial investigation: (1) the standby's own reject path sent no NACK reply at all (ha_send_wal_ack_reply hardcoded success=1 unconditionally), fixed via ha_send_wal_ack_reply_ex with success=0 on reject; (2) even with NACKs flowing, ha_stream_wal_to_standby's catch-up range was hard-capped at committed_lsn, which could never advance in this exact scenario (self-referential deadlock) -- fixed via ha_compute_repl_stream_ceiling_lsn widening the ceiling to the leader's own log tail, matching raft.tla's real AppendEntries semantics. Verified via three successive live run-ha-soak.sh --config-upgrade re-runs tracking each iteration (v1/v2 still failed, v3 with both fixes together PASSED end-to-end: 3/3 jobs acked on both force-elect rounds, zero materialized-state or config divergence). 1053/1053 controller_unit_test PASS. Full investigation history preserved below for context (including the earlier partial-fix iterations and root-causing). + +[Original investigation, 2026-07-28] +NOT FIXED -- explicitly out of scope for a single safe pass, per this investigation's own conclusion (matches kiro-debug's guidance against forcing a fix when the true defect requires a protocol design decision, not a narrow patch). At least TWO separate, non-trivial design questions need to be resolved by whoever owns ha.c/repl.c's replication-catch-up protocol before a fix can be written safely: (a) should an ae_reject_logmismatch NACK immediately trigger repl_start_catchup (moving the standby to catch-up mode on the FIRST conflict, rather than silently accumulating up to 16 failed sends first)? This is the more targeted, root-cause-adjacent fix. (b) independently, once a standby IS marked in_catch_up (whether via this path or the existing MAX_PENDING_ACKS eviction), what mechanism is SUPPOSED to actually drive its catch-up (stream from join_applied_lsn, or snapshot-install if too far behind) and clear in_catch_up via repl_complete_catchup? Evidence here shows NOTHING does, for at least this eviction path -- config_repl.c's ACK-driven catchup_lane_fn/ha_walack_on_ack continuation (the only other repl_complete_catchup call site found) is itself ACK-driven, so it can never engage for a standby that never sends a genuine ACK in the first place; whether IT (or a periodic sweep independent of ACKs) is supposed to notice an in_catch_up standby and proactively start streaming it needs its own trace by the owning team. A correct fix likely needs both: NACK-triggered catch-up entry (closing gap (a)) AND a proactive (not purely ACK-reactive) catch-up driver that streams from join_applied_lsn or falls back to snapshot-install for any in_catch_up standby (closing gap (b)) -- implementing only one without the other would leave the defect only partially closed. Recommend routing to whoever owns ha.c/repl.c's HA replication/catch-up protocol (the same 'needs-owner' bucket as bug-1474's still-open fix angle (a) and bug-1472-remainder) for a proper design pass, THEN a TDD fix + live re-verification via `run-ha-soak.sh --config-upgrade` on a fresh deploy dir. Preserved evidence: ~/tmp/xce-ha-configupgrade2 (do not delete) -- `xce_test waldump ` (via the xce_test REPL: `printf 'waldump \nquit\n' | ASAN_OPTIONS=detect_leaks=0 timeout 60 /controller/xce_test`) reproduces the byte-precise lsn=13 term-3-vs-term-4 conflict evidence directly from the preserved WAL segments at any time. + +UPDATE (2026-07-29, this session): a fix WAS committed (uncommitted in the 'config' worktree at investigation time) implementing exactly gap (a) from this entry's own recommendation -- ha_raft_on_repl_nack() (ha.c/ha.h) routes a WAL-ACK-reply NACK into the raft library's nextIndex backoff and returns a corrected resume LSN; repl_ack_decode_and_dispatch's NACK branch calls it and immediately repl_start_catchup() with the corrected LSN; repl_get_catchup_join_lsn() (repl.c/repl.h) lets the heartbeat-driven ha_dispatch_repl_action_for_standby use the corrected position; repl_complete_catchup() now also resets pending_ack_count. 1048/1048 controller_unit_test PASS. HOWEVER: a fresh live-soak re-verification (run-ha-soak.sh --config-upgrade, deploy preserved at ~/tmp/xce_ha_bug1479_verify, NOT deleted) reproduced the ORIGINAL symptom verbatim -- force-elect ctrl-2 round: 5/5 submit_jobs attempts on {ctrl-1,ctrl-2} got 0/3 acked, hard FAIL. Root-caused: the fix's NACK-routing branch in repl_ack_decode_and_dispatch (ha.c ~6680-6720) is CORRECT but UNREACHABLE in production, because the standby's OWN reject path never sends a NACK (or any reply) in the first place. ha_apply_wal_catchup_entry has three reject paths (ha.c ~3230-3232 via ha_apply_term_entry_via_library for the term-carrying single/batch case -- the one actually hit live; ~3320-3321 for a batch sub-entry apply failure; ~3373-3374 for the legacy single-entry path) that all explicitly do 'no ACK' on a logOk reject/forward-gap. ha_send_wal_ack_reply (ha.c:3084-3106) is the ONLY function anywhere in the controller that ever constructs a WalAckReply, and it hardcodes success=1 unconditionally -- there is NO production code path that ever sends success=0. Live evidence: ctrl-2's own log shows repeating RAFT[p1] event=ae_reject_logmismatch / ha_consume_append_entries success=0 match=0 at growing prevIdx (40,41,...54) while stuck at match=20 forever; ctrl-1's log has ZERO 'WAL NACK from standby' or ha_raft_on_repl_nack lines the entire time -- the primary never learns of the rejection at all, so it never backs off nextIndex or re-arms catch-up, and committed_lsn stays frozen (the only alive standby can never re-sync, and the heartbeat-driven catch-up's own target is gated on the same frozen committed_lsn). STILL NOT FULLY FIXED. New follow-up spec .kiro/specs/raft-adapter-conformance/ created to (1) fix this concrete missing-NACK-reply defect at the three reject sites and (2) perform a broader raft.tla action-by-action conformance pass over the controller's Raft adapter (ha.c/repl.c/config_repl.c) -- distinct from raft-sim-conformance, which is scoped to src/lib/raft/raft_sim.c (library-only, in-process) and explicitly does not cover this adapter code. + +UPDATE 2 (2026-07-29, this session): fully FIXED and verified live end-to-end. Two more compounding gaps found and closed after the NACK-send fix (UPDATE 1) landed: +(1) ha_apply_term_entry_via_library's logOk-reject path (ha.c, the single live-reachable reject site) sent NO reply at all on rejection -- ha_send_wal_ack_reply hardcoded success=1 unconditionally, so the primary-side ha_raft_on_repl_nack (correct since UPDATE 1) was unreachable dead code. Fixed by adding ha_send_wal_ack_reply_ex(ha, ep, c, lsn, success) and calling it with success=0 on reject, plus a new ha_wal_ack_send_hook (ha.h, XCE_TESTING) so a unit test can observe the NACK without a live endpoint (test_bug1479_ae_reject_sends_nack_not_silence, test_config_repl.c). Confirmed live in a 2nd soak run: ctrl-1's log showed real 'WAL NACK from standby ... success=0 -- marking catch-up' lines for the first time. +(2) Even with (1) working, that 2nd soak run STILL failed at the identical force-elect ctrl-2 point: ha_stream_wal_to_standby's catch-up range was hard-bounded at committed_lsn, and committed_lsn can never advance in this exact scenario because advancing it requires an ACK from the very standby that's stuck catching up (2-of-3 alive, this standby is the only path to a new majority) -- a self-referential deadlock. Fixed by adding a new pure function ha_compute_repl_stream_ceiling_lsn(committed_lsn, last_written, checkpoint_lsn) (ha.c/ha.h), wired into ha_dispatch_repl_action_for_standby to widen the STREAM_WAL ceiling (and the ha_evaluate_repl_action convergence check that gates it) to the leader's own log tail instead of just committed_lsn -- matching raft.tla's real AppendEntries semantics (leaderCommit is advisory to the follower, never a cap on what gets sent) -- while still guarding against bug-1478's old trap (never widening to a trailing local-only checkpoint's own unstreamable LSN slot). 4 new pure-function tests in test_ha_role.c (test_compute_repl_stream_ceiling_lsn_*), RED-confirmed against a stubbed committed_lsn-only version before the real fix was restored. +Full verification: 1053/1053 controller_unit_test PASS twice clean. THREE successive live run-ha-soak.sh --config-upgrade re-runs tracked each fix: v1 (NACK-send only) still FAILED at force-elect ctrl-2 (0/3 acked); v2 confirmed NACKs now reaching the primary but still FAILED (frozen committed_lsn=14, ha_stream_wal_to_standby target never advanced); v3 (both fixes together) PASSED end-to-end: all 3 force-elect rounds succeeded (ctrl-2: 3/3 jobs acked, term=6; ctrl-3: 3/3 jobs acked, term=7), final verdict 'PASS (config-upgrade): 12 jobs, zero materialized-state divergence, zero configuration divergence, host admin-status confirmed across all members'. Evidence preserved at ~/tmp/xce_ha_bug1479_verify, ~/tmp/xce_ha_bug1479_verify_v2, ~/tmp/xce_ha_bug1479_verify_v3 (all not deleted). config-upgrade-snapshot task 8.6 unblocked. The raft-adapter-conformance spec created mid-investigation had its concrete seed defect (missing NACK reply) fixed here; the broader raft.tla action-by-action conformance audit it also scopes remains valid, separate future work. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1466, bug-1471, bug-1472, bug-1474, bug-1475, bug-1477, bug-1478, bug-1421 diff --git a/.wolf/bugs/bug-1480.md b/.wolf/bugs/bug-1480.md new file mode 100644 index 00000000..fdda8be2 --- /dev/null +++ b/.wolf/bugs/bug-1480.md @@ -0,0 +1,39 @@ +--- +id: "bug-1480" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-29T07:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c (handle_repl_config_push", "~1261-1337)"] +tags: ["config_repl.c", "handle_repl_config_push", "ReplConfigPushReply", "silent-drop-on-reject", "same-family-as-bug-1479", "raft-adapter-conformance", "found-not-yet-fixed", "config-push-channel", "term-discovery-gap"] +related: ["bug-1479"] +fix_commits: ["b7828eed"] +legacy: true +--- + +# bug-1480: raft-adapter-conformance systematic audit finding (not yet fixed): handle_repl_config_push… + +## Description +raft-adapter-conformance systematic audit finding (not yet fixed): handle_repl_config_push (config_repl.c:1261-1337) has 4 reject paths (malformed body, missing bytes/sha256, sha256 mismatch, conf_save_config failure) that all return without ever sending a ReplConfigPushReply. Only the success path (1326-1335) sends {accepted=1}; there is no {accepted=0} anywhere in this function, despite its own doc-comment implying a reject reply exists. cluster.c dispatch case (652-655) has no fallback error-reply either. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same defect class as bug-1479 (silent-drop-on-reject) but on the out-of-band config-push channel instead of the WAL catch-up channel: the reject branches never construct/send a reply message at all, so push_config_to_standbys / repl_wait_majority_ack cannot distinguish an active standby rejection from a lost packet -- both degrade identically to CONF_PUSH_TIMEOUT_MS. + +## Fix +FIXED (2026-07-29, ~09:43, commit b7828eed "feat(raft-adapter-conformance): send explicit reply on every config-push outcome"). handle_repl_config_push now constructs and sends ReplConfigPushReply{accepted=0, term=current_term} on all four reject early-returns (malformed request, missing bytes/sha256, sha256 mismatch, conf_save_config failure) instead of returning silently; term populated on the success path too. Landed as raft-adapter-conformance tasks 2.1-2.3 (see tasks.md), tested end-to-end in test_config_repl.c, confirmed live in task 7.3's soak. This buglog entry's original 'Not yet fixed' note (07:05 same day) predates the fix landing by ~2.5h and was never updated -- code is fixed, only this log entry was stale. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1479 diff --git a/.wolf/bugs/bug-1481.md b/.wolf/bugs/bug-1481.md new file mode 100644 index 00000000..181b7c50 --- /dev/null +++ b/.wolf/bugs/bug-1481.md @@ -0,0 +1,39 @@ +--- +id: "bug-1481" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-29T07:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (vote_ep_err_handler", "~7564-7608; correct sibling sites at ~693-726 and ~830-855)"] +tags: ["ha.c", "vote_ep_err_handler", "handle_request_vote_reply", "vote_reply_dispatch", "pre_vote_reply_dispatch", "silent-drop-on-reject", "same-family-as-bug-1423", "raft-adapter-conformance", "found-not-yet-fixed", "election-timeout-bounded", "not-a-deadlock"] +related: ["bug-1423"] +fix_commits: ["fce73b5b"] +legacy: true +--- + +# bug-1481: raft-adapter-conformance systematic audit finding (not yet fixed): vote_ep_err_handler… + +## Description +raft-adapter-conformance systematic audit finding (not yet fixed): vote_ep_err_handler (ha.c:7564-7608), the shared on_err handler wired onto both real-vote and PreVote endpoints (ha.c:688-689, 827-828), handles post-connect endpoint failures (RDHUP/SSL error/remote close arriving after connect+monitor succeeded but before a reply) by calling the legacy handle_request_vote_reply directly instead of routing through vote_reply_dispatch / pre_vote_reply_dispatch (the production tally path fixed by bug-1423/1424 for connect-time and monitor-time failures only). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Reopens the bug-1423/1424 defect class (vote-request send failure must reach the reply tally) for one failure window the original fix did not cover: connect/iomgr_monitor failures were fixed, but a failure arriving after a successful connect+monitor (before any reply) still bypasses the tally via the legacy path. Effect is bounded (election_timeout_ms / PreVote round timeout still fire), not a permanent deadlock, but the fast all-replied quorum-decision path is structurally unreachable for this failure mode. + +## Fix +FIXED (2026-07-29, ~13:53, commit fce73b5b "feat(raft-adapter-conformance): route post-connect vote/PreVote failures to production tally"). vote_ep_err_handler split into a real-vote handler and a new pre_vote_ep_err_handler; each now routes a post-connect election-phase failure through the same production tally path used for connect-time failures (vote_reply_dispatch / pre_vote_reply_dispatch) instead of the legacy handle_request_vote_reply. Landed as raft-adapter-conformance tasks 3.1-3.2 (see tasks.md), tested in test_ha_vote_send.c, confirmed live in task 7.4's soak. This buglog entry's original 'Not yet fixed' note (07:05 same day) predates the fix landing by ~6.8h and was never updated -- code is fixed, only this log entry was stale. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1423 diff --git a/.wolf/bugs/bug-1482.md b/.wolf/bugs/bug-1482.md new file mode 100644 index 00000000..8687c8f8 --- /dev/null +++ b/.wolf/bugs/bug-1482.md @@ -0,0 +1,38 @@ +--- +id: "bug-1482" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-07-29T10:43:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_config_repl.c; doc comment in src/controller/cluster.c (myreq_handler_for_test)"] +tags: ["cluster.c", "myreq_handler_for_test", "xce_decode", "free_packet", "use-after-free", "double-free", "misleading-doc-comment", "ownership-contract", "raft-adapter-conformance", "task-2.2", "review-finding-corrected"] +related: [] +legacy: true +--- + +# bug-1482: AddressSanitizer: heap-use-after-free in free_packet ../lib/netio.c:437, called from… + +## Description +AddressSanitizer: heap-use-after-free in free_packet ../lib/netio.c:437, called from test_config_push_reply_higher_term_adopts_and_steps_down (test_config_repl.c:3718) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +A code-review finding (task 2.2 re-review) reported that cpr22_build_config_push_reply_pkt's io_packet_t* pkt was never freed after myreq_handler_for_test(...), citing myreq_handler_for_test's own doc comment ('the caller still owns/frees pkt itself') as the ownership contract. That doc comment was WRONG: myreq_handler unconditionally calls xce_decode(pkt, &request) as its first step, and xce_decode() itself (src/lib/request.c:274-275) unconditionally calls free_packet(pkt) (frees pkt->data) then FREEUP(pkt) (frees the io_packet_t struct) before returning -- ownership of pkt is transferred into xce_decode, not retained by the caller. Applying the review finding literally (adding free_packet(pkt); free(pkt); after the call) was therefore a double-free/use-after-free, caught immediately by ASAN even under ASAN_OPTIONS=detect_leaks=0 (UAF detection is independent of leak detection). + +## Fix +Removed the erroneous free_packet(pkt)+free(pkt) call from the test (there is no leak -- pkt is already fully consumed). Corrected myreq_handler_for_test's doc comment in cluster.c to accurately state that pkt ownership is transferred into the call via xce_decode, and callers must not touch/free pkt afterward. Added an explanatory NOTE comment at the call site in the test to prevent recurrence. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1483.md b/.wolf/bugs/bug-1483.md new file mode 100644 index 00000000..a3c1cda7 --- /dev/null +++ b/.wolf/bugs/bug-1483.md @@ -0,0 +1,38 @@ +--- +id: "bug-1483" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "leak" +timestamp: "2026-07-29T10:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (ha_arm_heartbeat_timeout_vtask", "~8805-8848)"] +tags: ["ha.c", "ha_arm_heartbeat_timeout_vtask", "s_liveness_ctx", "test-mode-leak", "c-is-null-path", "raft-adapter-conformance", "task-3.2", "found-not-yet-fixed", "pre-existing", "prevote-lost-path"] +related: [] +legacy: true +--- + +# bug-1483: ha_arm_heartbeat_timeout_vtask (ha.c ~8805-8848), when called with c==NULL (unit-test mode driving… + +## Description +ha_arm_heartbeat_timeout_vtask (ha.c ~8805-8848), when called with c==NULL (unit-test mode driving a PreVote round to LOST without stubbing ha_prevote_rearm_liveness_hook), callocs a liveness_poll_ctx_t and stores it into the static s_liveness_ctx, but the guard skips scheduling anything to poll or free it -- the context leaks/dangles permanently in that code path. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Test-mode (c==NULL) entry into a production helper allocates state but has no corresponding free/cleanup path when the delaymgr-scheduling guard short-circuits -- a latent leak specific to the c==NULL unit-test invocation shape, not exercised by any live production path (a real cluster_t is never NULL there). + +## Fix +FIXED (2026-08-01). ha_arm_heartbeat_timeout_vtask (ha.c) published a calloc'd liveness_poll_ctx_t into the static s_liveness_ctx unconditionally, but only freed it via liveness_poll_fn's cancelled-check path -- which never runs if delay_task_ms was never called (c==NULL, delaymgr not running, or delay_task_ms itself failing). Fixed by clearing s_liveness_ctx and freeing ctx immediately in all three no-schedule cases (the pre-existing c==NULL/!running else-branch, plus the previously-unhandled delay_task_ms rc<0 branch, which had the identical defect shape and was found while fixing this). Establishes the invariant: s_liveness_ctx is non-NULL only when a delaymgr task actually holds a live reference to it. Regression test: test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx (test_ha_prevote.c) drives a real LOST PreVote round WITHOUT stubbing ha_prevote_rearm_liveness_hook (every other LOST-path test in that file stubs it specifically to avoid this code path) and asserts the new liveness_ctx_get_for_test() accessor returns NULL afterward. Full suite: 1071 Tests 0 Failures (verified via the direct-binary recipe, ASAN_OPTIONS=detect_leaks=0). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1484.md b/.wolf/bugs/bug-1484.md new file mode 100644 index 00000000..5212899a --- /dev/null +++ b/.wolf/bugs/bug-1484.md @@ -0,0 +1,38 @@ +--- +id: "bug-1484" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-07-29T21:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (~6690-6711 heartbeat-reply vector", "~6805-6814 WAL-NACK vector); no test file drives either"] +tags: ["ha.c", "raft_on_higher_term_observed", "repl_ack_decode_and_dispatch", "term-discovery", "bug-1421", "coverage-gap", "raft-adapter-conformance", "task-7.2", "found-not-yet-fixed", "pre-existing", "requirement-7.1", "requirement-7.2", "requirement-7.5"] +related: ["bug-1421"] +legacy: true +--- + +# bug-1484: No test anywhere in src/controller/testsuite/ drives the real production dispatch path for either… + +## Description +No test anywhere in src/controller/testsuite/ drives the real production dispatch path for either term-discovery vector bug-1421 introduced (heartbeat-reply state-report term adoption, ha.c:6690-6711; WAL-NACK reply_term adoption inside repl_ack_decode_and_dispatch, ha.c:6805-6814) and asserts raft_on_higher_term_observed actually fires. The look-alike test_intake_higher_term_heartbeat_adopts_term (test_term_safety_adapter.c:1232) drives a different code path (ha_consume_append_entries -> raft_on_append_entries_request library-level UpdateTerm, an AE-request vector, not the AE-reply/state-report fallback bug-1421 added). test_persist_recovery.c (cited by design.md line 146 as covering this) has zero term-adoption content -- pure recovery mechanics. test_config_repl.c only covers the NEW config-push term vector added by this feature (Req 7.4), not the pre-existing heartbeat/WAL-NACK vectors. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +bug-1421 (heartbeat-reply term-discovery fallback) shipped without a dedicated regression test asserting raft_on_higher_term_observed fires through the real repl_ack_decode_and_dispatch path for either vector -- found during raft-adapter-conformance task 7.2s regression-confirmation sweep, which searched the whole test suite (not just its own 8-file boundary) for `raft_on_higher_term_observed`/`higher_term` callers and found none exercising these two vectors end-to-end. + +## Fix +FIXED (2026-08-01) -- test-coverage gap closed, no production logic change (the underlying bug-1421 term-adoption code was already correct, just unverified end-to-end). Added test_term_adoption_vectors.c with two tests driving the real peer_ctrl_msg_handler -> repl_ack_decode_and_dispatch path (via peer_ctrl_msg_handler_for_test + standby_ep_ctx_build_for_test, c=NULL -- every c-dependent step downstream of the term check is already guarded): (a) test_heartbeat_state_report_higher_term_adopts_and_steps_down builds a wire-shape StateReport reply carrying a higher term and asserts raft_on_higher_term_observed adopts it and steps the node down from RAFT_LEADER (ha.c:6690-6711 vector); (b) test_wal_nack_higher_term_adopts_and_steps_down does the same for a WalAckReply{success=0} carrying a higher reply_term (ha.c:6805-6814 vector). Both went GREEN immediately, confirming Requirement 7.5's 'already verified' claim was correct in substance, just previously unverified by any test. Registered in meson.build controller_unit_test_src and controller_test.c. Full suite: 1071 -> 1073 Tests, 0 Failures. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1421 diff --git a/.wolf/bugs/bug-1485.md b/.wolf/bugs/bug-1485.md new file mode 100644 index 00000000..182ff63b --- /dev/null +++ b/.wolf/bugs/bug-1485.md @@ -0,0 +1,43 @@ +--- +id: "bug-1485" +status: "cannot-reproduce" +status_updated: "2026-09-24" +status_note: "stderr capture added a5560256, never re-seen" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-07-30T14:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 25 +files: ["src/xceadmin/tests/test_e2e_live_ha_push.py (live_ha_cluster fixture); root cause is in src/controller (exact site not yet isolated -- C-side investigation out of this task's boundary)"] +tags: ["flaky-test", "ha", "raft-election", "controller-crash", "admin-config-tool", "task-6.3", "e2e-live", "unresolved", "pytest-specific"] +related: ["bug-1477", "bug-1479", "bug-606"] +repro_attempts: "0/40 repro attempts (2026-08-01)" +legacy: true +--- + +# bug-1485: test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path flaky… + +## Description +test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path flaky (near-100% fail rate across ~25 live runs during admin-config-tool task 6.3 round-2 remediation): TimeoutError: controller never opened localhost:: [Errno 111] Connection refused, raised from a fresh wait_for_port re-probe of the elected primary's own address performed AFTER _wait_for_primary (CTRL_PRIMARY marker) + a new _wait_for_standbys (both other nodes show CTRL_STANDBY) + a 3s settle sleep all already succeeded. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +- Attempts: 0/40 repro attempts (2026-08-01) + +## Root cause +NOT a pure test-side readiness race as round-1 review assumed. Live forensic capture (ps aux mid-run) caught the freshly-elected ctrl-1 as a zombie process moments after its own log recorded the full raft_cb_become_leader promotion sequence ending at 'HA: repl registered standby ep[1] (slot 1, ...)' -- the log then goes silent (no crash trace, no further lines) and the process is gone. stderr was DEVNULL'd in the fixture so no ASAN/signal diagnostic was captured from a real failing pytest run. Confusingly, 3 separate standalone (non-pytest) repro scripts using byte-identical spawn code (same xce_controller build.dbg binary, same deploy-ha.sh cluster, same peers/certs/workdir args) ran the exact same primary-promotion sequence successfully every single time with the primary staying alive and the port reachable for 20-30s post-election -- while pytest-invoked runs of the identical fixture code failed roughly 24/25 times. Ruled out during investigation: pytest's default fd-capture (reproduced with -s / capturing disabled, still failed 3/3), ulimits (NPROC/NOFILE/AS/STACK identical between nix-develop plain-shell and pytest-invoked-via-python contexts), and orphaned-process port contention (ps aux clean before each batch). True root cause not isolated -- prime suspects are a genuine timing-sensitive controller-side crash/race in the CTRL_STANDBY->CTRL_PRIMARY raft promotion path (raft_cb_become_leader / repl registration) that pytest's process tree somehow makes far more likely to trigger, or a process-group/session/signal-mask difference inherited by subprocess.Popen children when the parent is pytest vs a bare python3 script. + +## Fix +UPDATE 2026-08-01 -- investigated per this entry's own "next step" (capture stderr instead of DEVNULL, re-attempt). Redirected each node's stderr to deploy_dir/logs/.stderr.log (permanent change to test_e2e_live_ha_push.py, replacing the DEVNULL redirect that left this entry's original investigation with no diagnostic) and added an XCE_KEEP_DEPLOY_DIR=1 debug env var to preserve deploy_dir across the fixture's own cleanup for inspection. Ran the live test 40 times total in this environment: 25 baseline (idle sandbox, load avg ~2/22 cores) + 15 more with a concurrent `xce-build.sh controller debug` compile racing for CPU (separate build.dbg dir, so it could not clobber the xce.build controller binary the test depends on -- unlike bug-1486's same-dir clobber shape). Result: 40/40 PASSED, 0/120 captured stderr logs (3 nodes x 40 runs) non-empty -- no crash, no ASAN report, in any run. Could NOT reproduce the ~24/25 failure rate from the original investigation despite matching the documented repro shape exactly (same binaries, same fixture code, same pytest invocation). + +NOT marking fixed -- absence of reproduction here is not proof of absence of the underlying bug, and this session's single concurrent compile is a much lighter contention scenario than the real multi-session sandbox reality bug-1486 documents (several concurrent AGENT SESSIONS, potentially each running their own live 3-node cluster). Leading hypothesis (unconfirmed): the original ~24/25 failure rate was itself a symptom of resource contention from OTHER concurrent sessions on the shared sandbox during that investigation (CPU starvation delaying the primary's post-election warmup past _POST_ELECTION_SETTLE_S, or an OOM-adjacent condition killing the freshly-forked primary outright) rather than a bug in the promotion path reachable in isolation -- consistent with the original investigation's own "pytest's process tree somehow makes far more likely" framing, since pytest's subprocess tree carries more memory/fd overhead than the bare python3 scripts that never reproduced it there either. + +Downgraded to low-priority/environment-flake, left OPEN. The stderr-capture infrastructure is now permanently in place (not reverted) so the next real live failure -- in pytest, in a genuinely contended multi-session sandbox, or in CI -- captures an actual ASAN report or signal instead of DEVNULL silence, which is what would actually close this out. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1477, bug-1479, bug-606 diff --git a/.wolf/bugs/bug-1486.md b/.wolf/bugs/bug-1486.md new file mode 100644 index 00000000..99c2f265 --- /dev/null +++ b/.wolf/bugs/bug-1486.md @@ -0,0 +1,38 @@ +--- +id: "bug-1486" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-07-31T03:48:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/xce-build.sh"] +tags: ["build", "xce-build.sh", "meson", "ninja", "race-condition", "concurrent-sessions", "multi-agent", "workflow"] +related: ["bug-1464"] +legacy: true +--- + +# bug-1486: Backgrounded `xce-build.sh all` run went silently stale mid-flight ([360/362], no ninja… + +## Description +Backgrounded `xce-build.sh all` run went silently stale mid-flight ([360/362], no ninja FAILED/error line, no completion marker, log just stopped growing at 03:38:35) with no matching build process left in `ps aux` + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Multiple DIFFERENT Claude agent sessions independently invoked xce-build.sh against the same shared worktree's src/xce.build directory at the same time (confirmed via `ps auxf`: sibling `bash src/tools/xce-build.sh controller` and a separate `... all` invocation running under different parent `claude --output-format stream-json` PIDs). xce-build.sh has no lock/mutex around its unconditional `rm -rf "$BUILD_DIR"` + `meson setup` step (same code path as bug-1464), so a peer session's rm -rf tore the build directory out from under this session's own in-progress ninja/meson compile with no error propagation -- the backgrounded process's output just stops, indistinguishable at first glance from a hang. + +## Fix +No code change; workflow fix. Before trusting a backgrounded xce-build.sh log that has gone stale, run `ps auxf | grep -E 'xce-build|meson|ninja|cc1'` to check for a sibling xce-build.sh invocation from a different session targeting the same worktree. If found, do not start a third concurrent build -- bounded-poll (`while kill -0 $PEER_PID; do sleep 5; done`, not a passive wait for a background-task notification) until all peer invocations exit, then run exactly one clean blocking `xce-build.sh all` and treat only that run's own output as ground truth. Real fix would be a flock/lockfile around setup_build()'s rm-rf+meson-setup step in xce-build.sh itself so concurrent sessions serialize instead of clobbering. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1464 diff --git a/.wolf/bugs/bug-1487.md b/.wolf/bugs/bug-1487.md new file mode 100644 index 00000000..ed198e8f --- /dev/null +++ b/.wolf/bugs/bug-1487.md @@ -0,0 +1,38 @@ +--- +id: "bug-1487" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-07-31T04:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".claude/skills/xce-test-ha/run-ha-soak.sh"] +tags: ["xce-test-ha", "chaotic-soak", "verify_classify", "raft-commit-barrier", "apply-decoupling", "false-positive", "test-harness"] +related: ["bug-1422", "bug-1431", "bug-1432"] +legacy: true +--- + +# bug-1487: 30-round chaotic HA soak (run-ha-soak.sh --chaotic 30): committed_job_loss=1 at round 25/30… + +## Description +30-round chaotic HA soak (run-ha-soak.sh --chaotic 30): committed_job_loss=1 at round 25/30 (VERDICT result=FAIL). expect-job reported job 0-6-1785473261-216067 not found on newly-elected primary ctrl-2 (term=27) immediately after the round-25 kill of ctrl-1 (term=26). committed_divergence stayed 0 for the entire run; the job (and all others) was confirmed present on every subsequent verify (rounds 26-30) and the final post-run sweep of all 93 tracked jobs. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT a controller bug and NOT genuinely-uncommitted-at-kill-time -- test-harness false-positive, root-caused with log/timestamp evidence from the preserved run (/home/claude/tmp/xce-ha-chaos30-run1/logs/ctrl-{1,2,3}.log). ctrl-1 (primary, term=26) majority-committed all 3 round-25 jobs (lsn 101,102,103; self + ctrl-2 ack) between 04:47:41.839 and 04:47:42.052 -- confirmed via commit_advance_locked log lines -- moments before being killed at 04:47:42+. ctrl-2 was elected primary at term=27 at 04:47:52.470 with ha_compute_safe_committed_lsn reporting safe_committed_lsn=99 "(from 2 granted voters)" -- the conservative Raft floor, since ctrl-3 (the only other reachable voter) was itself mid-restart/catch-up from round 25s own overlapping-kill design and had not yet caught up past lsn 99. Per the documented Raft Figure-8 safety mechanism (ha_arm_own_term_gate, ha.c:4997-5046: "the universal commit barrier still requires per-entry majority of a new-term entry"), a newly-elected primary cannot fold a PRIOR-term entry into its own committed_lsn watermark merely because it holds it durably -- it must first get a fresh majority ack, under its OWN term, for its own-term WAL_NOOP (armed at promotion). ctrl-2s committed_lsn stayed at 100 (excluding lsn 101-103) until ctrl-3s WAL catch-up stream let it ack the term-27 noop at lsn=104, at 04:47:58.702 -- 6.232s after promotion (commit_advance_locked: "committed_lsn 100 -> 104"). expect-job (joblist query) reads c->jobmgr, which the apply pump only populates for entries that have reached this committed_lsn watermark -- so the job was genuinely invisible, by design, for that 6.232s window. run-ha-soak.shs verify_classify() (chaotic mode) runs its expect-job check immediately once wait_for_new_primary detects the new primary is SERVING at a higher term -- unlike verify_no_loss (sequential mode; already carries a 3-attempt/2s-settle retry from bug-1422 for a related but distinct client-retry-storm race) and unlike wait_for_full_convergence (which chaotic mode intentionally skips between kills, by design, to land kills inside overlap windows) -- so its single, instantaneous query landed inside the 6.232s barrier window and counted a false LOSS. Comparison: round 24s election (ctrl-1, term=26) closed the identical barrier window in only 167ms because the other voting peer was not mid-catch-up at that moment -- the window duration is directly proportional to how far behind the second required peer is, which chaotic modes overlapping-restart design can occasionally push to several seconds. + +## Fix +Added a bounded retry (5 attempts, 2s settle between) around the expect-job/verdict-classify check in verify_classify() (.claude/skills/xce-test-ha/run-ha-soak.sh) -- mirrors verify_no_loss/bug-1422s "retry the whole check" precedent, only retrying on a LOSS classification (BUG854/OK still resolve on the first attempt, matching prior semantics). ~8s of total settle margin comfortably covers the measured 6.232s worst-case gap while staying bounded -- a job still missing after 5 attempts still counts as a genuine LOSS, so real committed-job loss detection is unweakened. Verified with a full, fresh 30-round chaotic re-soak (see run2 VERDICT). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1422, bug-1431, bug-1432 diff --git a/.wolf/bugs/bug-1489.md b/.wolf/bugs/bug-1489.md new file mode 100644 index 00000000..46e99232 --- /dev/null +++ b/.wolf/bugs/bug-1489.md @@ -0,0 +1,38 @@ +--- +id: "bug-1489" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-03T04:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/include/core.h"] +tags: ["scheduler-core", "task-4.1", "core-extraction", "typedef-forward-decl", "include-order"] +related: [] +legacy: true +--- + +# bug-1489: gcc: 'unknown type name xce_sched_core_t; did you mean xce_sched_ctx_t?' at core.h:92… + +## Description +gcc: 'unknown type name xce_sched_core_t; did you mean xce_sched_ctx_t?' at core.h:92 (xce_sched_core_init prototype) when compiling scheduler.h consumers (handler.c, main.c, etc). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +core.h defined `struct xce_sched_core_s { ... };` (the full body) but relied entirely on plugin.h's forward declaration (`typedef struct xce_sched_core_s xce_sched_core_t;`) for the typedef name -- core.h itself never declared it. scheduler.h includes core.h BEFORE plugin.h, so in that (the only real) include order, xce_sched_core_t was unresolved by the time core.h's own function prototypes referenced it. + +## Fix +Added `typedef struct xce_sched_core_s xce_sched_core_t;` directly in core.h, immediately before the struct body. An identical repeated typedef (same underlying type) is valid C11/gnu17, matching this codebase's existing cluster_t idiom (myconf.h/persist.h/ha.h/cluster_def.h all forward-declare independently) -- so the fix works regardless of which of core.h/plugin.h is included first. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1490.md b/.wolf/bugs/bug-1490.md new file mode 100644 index 00000000..10506902 --- /dev/null +++ b/.wolf/bugs/bug-1490.md @@ -0,0 +1,38 @@ +--- +id: "bug-1490" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-03T04:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/schedobj.c"] +tags: ["scheduler-core", "task-4.1", "core-extraction", "link-error", "whole-file-rewrite-risk"] +related: ["bug-1489"] +legacy: true +--- + +# bug-1490: ld: undefined reference to 'reset_sched_obj' when linking xce_scheduler (referenced from… + +## Description +ld: undefined reference to 'reset_sched_obj' when linking xce_scheduler (referenced from plugins/limit.c's limit_job_election()). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +While rewriting schedobj.c wholesale to relocate the CRUD accessors (sched_get_job/sched_add_job/etc, scheduler-core spec task 4.1) into core.c, the reset_sched_obj() function (unrelated to the relocation, a small xlist/zarray cleanup helper called by plugins/limit.c) was accidentally omitted from the rewritten file -- a transcription miss, not an intentional removal. + +## Fix +Re-added reset_sched_obj() verbatim (unchanged body) to schedobj.c, right after make_sched_obj() where it originally lived. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1489 diff --git a/.wolf/bugs/bug-1491.md b/.wolf/bugs/bug-1491.md new file mode 100644 index 00000000..49a277b8 --- /dev/null +++ b/.wolf/bugs/bug-1491.md @@ -0,0 +1,38 @@ +--- +id: "bug-1491" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-08-03T06:33:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_core_jbuf_capacity_pin.c"] +tags: ["scheduler-core", "task-4.2", "core-init-fini", "segv", "null-deref", "test-authoring-gotcha", "pend_updates"] +related: ["bug-1489", "bug-1490"] +legacy: true +--- + +# bug-1491: AddressSanitizer: SEGV on unknown address 0x000000000000 in dataq_deq_head (lib/dataq.c:15)… + +## Description +AddressSanitizer: SEGV on unknown address 0x000000000000 in dataq_deq_head (lib/dataq.c:15), called from dataq_fini <- xce_sched_core_fini (core.c:126), when a test called xce_sched_core_init() then xce_sched_core_fini() back-to-back on a fresh xce_sched_core_t. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce_sched_core_init() memset()s the whole xce_sched_core_t, leaving core->pend_updates[0]/[1] NULL -- those two dataq_t* are only allocated+dataq_init()d by scheduler.c sched_init() AFTER calling xce_sched_core_init() (see core.c file header comment: pend_updates storage lives in xce_sched_core_t but its dataq_init/killer wiring stays driver-owned in scheduler.c). xce_sched_core_fini() unconditionally calls dataq_fini(core->pend_updates[0]) and [1] with no NULL guard, so calling fini() right after init() without going through sched_init()'s extra setup step dereferences NULL inside dataq_deq_head. + +## Fix +Not a production bug (xce_sched_core_fini()'s real caller, scheduler.c sched_fini(), always runs after sched_init() has populated pend_updates) -- fixed the test instead: test_jbuf_capacity_is_runtime_one() (task 4.2) calls only xce_sched_core_init() and reads core.jbuf.ncap, deliberately never calling xce_sched_core_fini() on a fixture built by init() alone. Matches this testsuite directory's established "fixtures aren't torn down" convention (test_picker_core_loop_boundary.c's header comment). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1489, bug-1490 diff --git a/.wolf/bugs/bug-1492.md b/.wolf/bugs/bug-1492.md new file mode 100644 index 00000000..2a67ab8c --- /dev/null +++ b/.wolf/bugs/bug-1492.md @@ -0,0 +1,38 @@ +--- +id: "bug-1492" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-04T01:04:00Z" +last_seen: "2026-09-19T09:30:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugin.c"] +tags: ["scheduler-core", "task-5.2", "plugin-c", "guard-call-mismatch", "pre-existing", "legacy-fcfs-picker"] +related: ["bug-1682"] +legacy: true +--- + +# bug-1492: plugin_end_session() checks plugin->begin_session_fn != NULL but calls… + +## Description +plugin_end_session() checks plugin->begin_session_fn != NULL but calls plugin->end_session_fn(plugin) -- a guard/call field mismatch + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Pre-existing bug predating the scheduler-core spec (git blame: commit 638173dec, 2026-02-26). plugin_end_session()'s per-plugin loop guards on the wrong struct field before invoking the hook, so a plugin registering only end_session_fn (not begin_session_fn) never has it called; a plugin registering only begin_session_fn (not end_session_fn) would crash via a NULL function-pointer call if this branch were ever reached for it. Discovered incidentally during scheduler-core task 5.2's independent review while verifying the plugin_begin_session/plugin_end_session per-session bookkeeping fan-out is preserved in the new normal-mode driver loop -- not introduced by task 5.2, confirmed untouched by its diff. + +## Fix +FIXED (2026-09-19, /xce-audit-invariants scheduler dimension-1 audit, following scheduler-sim-live-allocation). Changed plugin_end_session()'s guard from plugin->begin_session_fn != NULL to plugin->end_session_fn != NULL (src/scheduler/src/plugin.c) -- exactly the one-line fix this entry originally proposed. Confirmed via TDD: added test_end_session_invokes_end_session_fn_when_only_it_is_set() (test_plugin_hook_dispatch.c) registering a plugin with ONLY end_session_fn set, proving it was silently never invoked before the fix (RED, 84/85) and is now correctly invoked (GREEN, 85/85). Duplicate of bug-1682 (same defect, independently re-discovered and re-logged during bug-1681's fix, 2026-08-24) -- this entry (bug-1492, filed 2026-08-04) is the original. Note: this entry's own tags array had been incorrectly stamped "fixed"/"closed-by-scheduler-sim-live-allocation" ahead of the actual fix landing (a bulk-tagging error found by this same audit) -- now genuinely accurate. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1682 diff --git a/.wolf/bugs/bug-1493.md b/.wolf/bugs/bug-1493.md new file mode 100644 index 00000000..61f8cac9 --- /dev/null +++ b/.wolf/bugs/bug-1493.md @@ -0,0 +1,38 @@ +--- +id: "bug-1493" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-08-04T12:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/include/typemap.h"] +tags: ["scheduler-core", "task-7.3", "capnp-schema", "typemap.h", "comment-syntax", "build-break"] +related: [] +legacy: true +--- + +# bug-1493: gcc: 'expected declaration specifiers before sched_snapshot_data_t' cascading into ~20 errors… + +## Description +gcc: 'expected declaration specifiers before sched_snapshot_data_t' cascading into ~20 errors across typemap.h/controller.capnp.h/controller.capnp.c when building lib/controller.capnp.c (scheduler-core task 7.3 SnapshotData C struct). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +A doc comment on sched_snapshot_data_t read '...raw uint8_t*/size_t pattern...' -- the literal substring '*/' inside the comment text (no space between the uint8_t* pointer-star and the following /size_t) prematurely closed the C block comment. Everything after that point (the rest of the comment prose plus the following struct's opening brace) was parsed as real C tokens, corrupting the sched_snapshot_data_t typedef and cascading into every generated function signature that references it. + +## Fix +Reworded the comment to avoid the accidental */ sequence ('uint8_t-pointer + size_t pattern' instead of 'uint8_t*/size_t pattern'). General gotcha: never write 'TypeA*/TypeB' (pointer-star immediately followed by slash) inside a /* */ C comment -- always put a space or word between the * and the /. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1494.md b/.wolf/bugs/bug-1494.md new file mode 100644 index 00000000..09e44ccb --- /dev/null +++ b/.wolf/bugs/bug-1494.md @@ -0,0 +1,38 @@ +--- +id: "bug-1494" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-04T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_snapshot_save_load.c"] +tags: ["scheduler-core", "task-8.1", "comment-syntax", "clang-format", "testsuite"] +related: ["bug-1493"] +legacy: true +--- + +# bug-1494: clang-format -i produced bizarre whole-file +1-space indentation drift (starting right after the… + +## Description +clang-format -i produced bizarre whole-file +1-space indentation drift (starting right after the top #include block, persisting to EOF) in a newly-written test_snapshot_save_load.c, with kill_sched_job() alone expanded to multiple lines while identical sibling one-liners stayed collapsed. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same class of bug as bug-1493 (task 7.3): the file's own top-of-file header comment contained the literal substring '(kill_sched_*/plugin_unregister/...)' -- the pointer-star-like '*' immediately followed by '/' prematurely closed the /* */ block comment. Everything after that point was parsed as real C code by clang-format's lexer, which is what produced the indentation drift (a strong diagnostic signal, not just a formatting quirk) -- gcc itself would also have failed to compile this. + +## Fix +Reworded 'kill_sched_*/plugin_unregister/sched_worker_release_host_ctx/sched_resolve_job_picker)' to 'kill_sched_* family, plugin_unregister, sched_worker_release_host_ctx, sched_resolve_job_picker)', removing the accidental */ sequence. General lesson (reinforcing bug-1493): a sudden, file-wide clang-format indentation shift starting at a specific line is a strong signal to grep that file for a stray '*/' inside a comment before assuming it's a clang-format config issue -- verified via a small python scanner that walks the file tracking in-comment state and flags any '*/' encountered while not already inside a comment. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1493 diff --git a/.wolf/bugs/bug-1495.md b/.wolf/bugs/bug-1495.md new file mode 100644 index 00000000..319e2cbb --- /dev/null +++ b/.wolf/bugs/bug-1495.md @@ -0,0 +1,38 @@ +--- +id: "bug-1495" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-04T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core.c"] +tags: ["scheduler-core", "task-8.1", "core.c", "get-or-create", "kvmgr", "key_find", "sched_get_user", "sched_get_project", "regression-fix"] +related: [] +legacy: true +--- + +# bug-1495: sched_get_user()/sched_get_project() (core.c) always returned NULL for any user/project name not… + +## Description +sched_get_user()/sched_get_project() (core.c) always returned NULL for any user/project name not already present in core->users/core->projects, even though both are documented and relied upon (core_events.c JOBS_PUSH, schedobj.c make_sched_job, and task 8.1's own snapshot_load.c) as get-or-create. Surfaced as job->user/job->project being silently NULL for every job referencing a brand-new user/project, discovered while writing test_snapshot_save_load.c's cross-core equivalence test (RED phase: TEST_ASSERT_NOT_NULL(sched_get_project(&core_a, "proj1")) failed immediately after a successful JOBS_PUSH). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Both functions call key_find() then do `if (rc < 0) { return NULL; }` before falling through to the make_sched_user()/make_sched_project() create path. key_find()'s actual not-found signal is a NEGATIVE errno (-ENOENT, confirmed via direct debug instrumentation: rc=-2 for an absent key, rc=1 for a present one) -- NOT rc==0 as the surrounding code assumed. Since -ENOENT < 0, the guard incorrectly treated the EXPECTED not-found outcome as a fatal error and returned NULL before ever reaching the create fallback, making get-or-create unreachable for any genuinely new name. This is a pre-existing bug from earlier tasks (4.1/4.3), not introduced by task 8.1, but it directly breaks task 8.1's own snapshot_load.c users/projects restoration onto a freshly-initialized core (which always starts empty, so every name is 'new'). + +## Fix +Changed the guard in both functions from `if (rc < 0)` to `if ((rc < 0) && (rc != -ENOENT))`, so only a genuine (non-ENOENT) error short-circuits before the create fallback. Two-line change per function, core.c lines ~341 and ~416. Verified: full scheduler suite 23/23 (up from 22/22 baseline, no regressions), plus the new test's own field-level assertions (users/projects created correctly, priority preserved) now pass. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1496.md b/.wolf/bugs/bug-1496.md new file mode 100644 index 00000000..c710d249 --- /dev/null +++ b/.wolf/bugs/bug-1496.md @@ -0,0 +1,38 @@ +--- +id: "bug-1496" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-04T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/snapshot_load.c"] +tags: ["scheduler-core", "task-8.1", "snapshot_load.c", "prng_seed", "round-trip-gap"] +related: [] +legacy: true +--- + +# bug-1496: xce_sched_core_load_snapshot() never restored core->prng_seed from the snapshot -- a freshly… + +## Description +xce_sched_core_load_snapshot() never restored core->prng_seed from the snapshot -- a freshly loaded core silently kept xce_sched_core_init()'s default (0) instead of the value xce_sched_core_save_snapshot() had captured. Caught by test_snapshot_save_load.c's field-level assertion: TEST_ASSERT_EQUAL_UINT64(0xDEADBEEFULL, core_b.prng_seed) failed with Expected 3735928559 Was 0 after a successful save+load round trip. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +save_snapshot() (snapshot.c) correctly writes `out->prng_seed = core->prng_seed;`, but the corresponding `core->prng_seed = in->prng_seed;` assignment was simply missing from load_snapshot()'s field-copy sequence -- an incomplete round trip for a field the function's own header comment (snapshot.c, 'PRNG seed resolution') explicitly documents as in scope (Req 5 AC 21's 'PRNG state' checklist row). + +## Fix +Added `core->prng_seed = in->prng_seed;` to xce_sched_core_load_snapshot() (snapshot_load.c), placed after the jobs-restoration loop and before the internal_state restoration block. Verified via the same field-level assertion, now passing. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1497.md b/.wolf/bugs/bug-1497.md new file mode 100644 index 00000000..d3573f9d --- /dev/null +++ b/.wolf/bugs/bug-1497.md @@ -0,0 +1,38 @@ +--- +id: "bug-1497" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-08-04T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_snapshot_save_load.c"] +tags: ["scheduler-core", "task-8.1", "test_snapshot_save_load.c", "xce_val_t", "zero-init-union-discriminant", "test-only", "asan-segv"] +related: [] +legacy: true +--- + +# bug-1497: ASan SEGV inside dup_val() -> makeString() -> hash_string() while building… + +## Description +ASan SEGV inside dup_val() -> makeString() -> hash_string() while building test_snapshot_save_load.c's cross-core equivalence test fixtures. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Test-side only, not production code: three test-owned xce_val_t resval fixtures were left with `.which` at its memset-zero default. XCE_VAL_STRING happens to be enum value 0 (xce/types.h), so the zeroed `.which` was silently interpreted as "this is a string" -- dup_val()/makeString() then reinterpreted the fixture's raw ivalue32 int as a char* pointer and hash_string() dereferenced it, crashing. + +## Fix +Explicitly set `.which = XCE_VAL_NUMERIC` on all three test-owned xce_val_t fixtures in test_snapshot_save_load.c before use -- never rely on struct-zero-init to select a union discriminant when a non-zero-valued enum member (XCE_VAL_NUMERIC) is the intended default. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1498.md b/.wolf/bugs/bug-1498.md new file mode 100644 index 00000000..035534e6 --- /dev/null +++ b/.wolf/bugs/bug-1498.md @@ -0,0 +1,38 @@ +--- +id: "bug-1498" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "security" +timestamp: "2026-08-05T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/recording.c"] +tags: ["scheduler-core", "task-9.2", "recording.c", "toctou", "hardlink", "O_TRUNC", "check-after-side-effect", "sched_record_start"] +related: [] +legacy: true +--- + +# bug-1498: test_toctou_hardlink_planted_after_validation_rejected FAILED: Expected Non-NULL -- fgets() on the… + +## Description +test_toctou_hardlink_planted_after_validation_rejected FAILED: Expected Non-NULL -- fgets() on the outside 'victim' file returned NULL (file was 0 bytes) after the TOCTOU regression test ran, even though sched_record_start()/the reproduced open-hardening sequence both correctly REJECTED the hard-link swap and returned an error. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_record_start()'s TOCTOU-hardened open used O_WRONLY|O_CREAT|O_TRUNC|O_NOFOLLOW. O_TRUNC truncates the target atomically as PART OF the open(2) syscall itself, before any userspace code -- including the fstat()+st_nlink==1 safety check right after open() -- ever runs. Against a hard-linked victim (same inode as the outside file, no symlink involved so O_NOFOLLOW never triggers), open() succeeds and O_TRUNC immediately zeroes the shared inode's content; the subsequent st_nlink>1 check then correctly rejects and closes the fd, but the outside victim file was already corrupted by that point. The check ran too late to matter -- classic 'check-after-side-effect' TOCTOU-adjacent bug, caught by the test's own outside-file-content assertion (added specifically to prove containment, not just that the return code was negative). + +## Fix +Removed O_TRUNC from the initial open() flags (now O_WRONLY|O_CREAT|O_NOFOLLOW). After fstat()+st_nlink==1 passes, explicitly call ftruncate(fd, 0) to get the same truncate-on-legitimate-overwrite behavior, but only once safety is verified. Same fix applied to the test's own toctou_hardened_open() helper (a verbatim reproduction of the production sequence, used because a literal multi-threaded race against sched_record_start()'s few-instruction internal window would be flaky). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1499.md b/.wolf/bugs/bug-1499.md new file mode 100644 index 00000000..997bb162 --- /dev/null +++ b/.wolf/bugs/bug-1499.md @@ -0,0 +1,38 @@ +--- +id: "bug-1499" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-05T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/driver.c"] +tags: ["scheduler-core", "task-9.2", "driver.c", "deadlock", "xce_core_signal_go", "accidental-deletion-in-edit", "xce_sched_driver_run_normal_mode_round"] +related: [] +legacy: true +--- + +# bug-1499: sched_driver_session_ordering_test TIMEOUT (30s, killed by SIGTERM) after adding task 9.2's… + +## Description +sched_driver_session_ordering_test TIMEOUT (30s, killed by SIGTERM) after adding task 9.2's recording-write hooks to xce_sched_driver_run_normal_mode_round() -- picker thread, allocator thread, and the main round all blocked forever. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +While inserting the new 'if (sched->recording) { rec_ctx = sched_record_session_begin(...); }' block between plugin_begin_session()/sched_account_jobs() and the decision-drain loop, the pre-existing `xce_core_signal_go(&(sched->core));` call was accidentally dropped from the Edit's old_string/new_string pair -- it was the last line of the block being replaced but was not included in the replacement text. Without xce_core_signal_go(), the picker thread's cbuf_get(&core->wbuf) rendezvous never unblocks, which cascades: the stand-in allocator thread never receives a jobid from cbuf_get(&core->jbuf), so it never pushes the end-of-session NULL decision, so the main thread's own cbuf_get(&core->decisions) drain loop blocks forever -- a 3-way deadlock across all three threads. + +## Fix +Restored `xce_core_signal_go(&(sched->core));` immediately after the new recording session-begin block (matching requirements.md's own annotated pseudocode order: build ReplaySession{beginRound} THEN sched_go()). Caught by re-running the full scheduler test suite after the recording hooks were added -- test_driver_session_ordering.c is the ONLY test in the suite that drives a real picker/allocator thread pair through this exact function, so it was the only test capable of catching a signal_go() regression. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1500.md b/.wolf/bugs/bug-1500.md new file mode 100644 index 00000000..195eed3e --- /dev/null +++ b/.wolf/bugs/bug-1500.md @@ -0,0 +1,38 @@ +--- +id: "bug-1500" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-06T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_sim_event_loop.c"] +tags: ["scheduler-core", "task-11.2", "sim_loop.c", "test-fixture", "sched_record_stop", "ReplayEvent_body_endSession", "premature-termination-marker"] +related: [] +legacy: true +--- + +# bug-1500: test_run_processes_multiple_sessions_in_order_and_no_longer_returns_enosys FAIL: Expected 5000 Was… + +## Description +test_run_processes_multiple_sessions_in_order_and_no_longer_returns_enosys FAIL: Expected 5000 Was 743938186628040 (a raw-looking CLOCK_MONOTONIC nanosecond value, not the fixture's own beginRound). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The test's fixture-building helper called the real sched_record_start() THEN sched_record_stop() to produce a valid header+snapshot prefix (matching test_sim_startup.c's own write_valid_fixture() precedent), then re-opened the same path in append mode ('a') to hand-write additional ReplayEvent/ReplaySession messages after it. But sched_record_stop() (recording.c) unconditionally writes its OWN ReplayEvent{endSession} marker immediately after the snapshot as part of every completed recording (Req 5 AC 20) -- so the fixture already contained a premature end-of-stream marker BEFORE any of the test's own appended events/sessions. sched_sim_driver_run()'s event loop (sim_loop.c, task 11.2) correctly treats ReplayEvent{endSession} as the whole-stream termination signal and stopped reading right there, leaving sim->core's clock at its xce_sched_core_init() default (virtual_clock_base=0, speed_num=1, speed_den=1 -- AC 3's normal-mode row), so xce_core_now() returned an unfrozen, elapsed-wall-clock-driven value instead of the intended frozen 5000. + +## Fix +Changed the fixture helper to call ONLY sched_record_start() (never sched_record_stop()), and return the STILL-OPEN FILE* (sched.record_file) so the test appends its own hand-built events/sessions directly onto the same open stream, ending with its own append_end_of_stream_marker() call, then fclose()s it directly -- bypassing sched_record_stop()'s real event/session-recording API entirely for this hand-built-fixture use case. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1501.md b/.wolf/bugs/bug-1501.md new file mode 100644 index 00000000..3963247f --- /dev/null +++ b/.wolf/bugs/bug-1501.md @@ -0,0 +1,38 @@ +--- +id: "bug-1501" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-07T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_verify.c"] +tags: ["scheduler-core", "task-11.4", "sim_verify.c", "block-comment", "accidental-comment-close", "bug-1493-recurrence", "build-failure"] +related: ["bug-1493"] +legacy: true +--- + +# bug-1501: gcc: 'expected =, ,, ;, asm or __attribute__ before is' / 'unknown type name is' / 'missing… + +## Description +gcc: 'expected =, ,, ;, asm or __attribute__ before is' / 'unknown type name is' / 'missing terminating character' cascade starting at sim_verify.c:59, plus 'implicit declaration of function sim_verify_str_eq' further down. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +File header doc comment's Comparison-strategy paragraph wrote the literal text 'xce_job_alloc_t*/xce_pendreason_t*' to mean 'an xce_job_alloc_t pointer or an xce_pendreason_t pointer' -- the '*/' substring is C's block-comment CLOSE token, so the /* ... */ doc comment terminated early, right in the middle of prose. Everything from that point until the next real */ (the following function's own closing delimiter) was parsed as C code, producing a cascade of nonsense syntax errors and, once the compiler resynced, an implicit-declaration warning/error for a function whose real prototype/definition had been swallowed into the truncated comment. Exact recurrence of the bug-1493 pattern already logged for typemap.h (same 'accidental */ in a C block comment' root cause). + +## Fix +Reworded to 'xce_job_alloc_t* or xce_pendreason_t*' (space before 'or' instead of a bare slash-star-slash-adjacent sequence), eliminating the literal '*/' substring. Ran a repo grep for the same '[A-Za-z_]*/[A-Za-z_]' pattern across both new/modified files (sim_verify.c, sim_loop.c) before rebuilding to rule out a second occurrence. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1493 diff --git a/.wolf/bugs/bug-1502.md b/.wolf/bugs/bug-1502.md new file mode 100644 index 00000000..6ed1f60f --- /dev/null +++ b/.wolf/bugs/bug-1502.md @@ -0,0 +1,38 @@ +--- +id: "bug-1502" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-07T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["scheduler-core", "task-11.5", "sim_loop.c", "red-phase-evidence", "review-rejection", "git-stash-pitfall", "methodology"] +related: [] +legacy: true +--- + +# bug-1502: Task 11.5 review round 1 REJECTED: previous implementer's self-reported RED-phase evidence (git… + +## Description +Task 11.5 review round 1 REJECTED: previous implementer's self-reported RED-phase evidence (git stash of 4 'behavioral production files' -> 3 tests failing under meson test) did not reproduce -- reviewer got a hard ninja build failure instead (test_sim_event_loop.c:467 calling sim_process_replay_event_message with 5 args against a stashed-back 4-arg sim_loop.c declaration). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +git stash scoped to only 4 production files silently desyncs test files (which already call the NEW post-task-11.5 signatures/behavior) against the stashed-back OLD production code -- a stash-based RED probe is not a valid methodology whenever tests and production code changed together in the same commit-to-be. The previous implementer's reported failure counts were therefore not reproducible as described. + +## Fix +Replaced the stash approach with a direct, targeted, in-place temporary edit of ONLY the behavioral logic inside sim_loop.c (never touching test files, sim_record.c, sim.c/sim.h, recording_session.c/recording.h, or either meson.build): hardcoded event_src to sim->replay_file (defeats AC16 what-if branching) and wrapped the 3 --sim-output write gates (header/snapshot, event pass-through, session-write via record_ctx) in 'if (0 && ...)' no-ops (defeats AC19). Rebuilt+ran `meson test -C xce.build --suite scheduler`: real RED = 30/32, sched_sim_alt_events_test and sched_sim_record_output_test FAIL with concrete assertion messages. Reverted all 4 edits using the exact original text; confirmed via `git diff src/scheduler/src/sim_loop.c` that only the original (pre-probe) task-11.5 diff remained, no probe residue (grep for 'TEMP RED-PHASE PROBE' and 'if (0 &&' both zero matches). Rebuilt+reran: 32/32 green again. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1503.md b/.wolf/bugs/bug-1503.md new file mode 100644 index 00000000..3b1c720d --- /dev/null +++ b/.wolf/bugs/bug-1503.md @@ -0,0 +1,38 @@ +--- +id: "bug-1503" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-08-07T03:11:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_sim_equivalence.c"] +tags: ["scheduler-core", "makeString", "strdup", "asan", "bad-free", "driver.c", "task-11.6"] +related: [] +legacy: true +--- + +# bug-1503: AddressSanitizer: attempting free on address which was not malloc()-ed, inside free_QueueConf() <… + +## Description +AddressSanitizer: attempting free on address which was not malloc()-ed, inside free_QueueConf() <- free_QueueConf_ptr() <- release_translated_payload() (driver.c) <- xce_sched_driver_translate_pending_events(). Crashing pointer was 56 bytes inside a 61-byte region allocated by makeString()->makeInstance(). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +makeString() (lib/xstring.c) is backed by the vendored rtti library makeInstance() allocator, which does NOT return a plain malloc()-compatible pointer at offset 0 of its allocation (it returns a pointer into a larger rtti-managed block). Elsewhere in the codebase makeString()-allocated strings are safe because they are only ever released by the SAME rtti-aware teardown path that created them (e.g. kill_sched_* killers, or never freed at all in white-box test fixtures). This new test constructed xce_queue_t/xce_job_t fixtures with q->queue/submit->queue/user/project_name/resreq set via makeString(), then drove them through the REAL production cleanup path release_translated_payload() (driver.c), whose free_QueueConf_ptr()/free_Job_ptr() (controller.capnp.c, capnp-generated) call plain free() on those fields directly -- exactly the mismatch: an rtti-backed allocation freed via plain free(). + +## Fix +Replaced every makeString() call used to populate xce_queue_t.queue / xce_job_t.submit->{queue,user,project_name,resreq} in test_sim_equivalence.c with strdup(), which returns a plain malloc()-compatible buffer safe for free_QueueConf_ptr()/free_Job_ptr() to free directly. Confirmed clean (no ASan bad-free) after the fix, full scheduler suite (33/33) green. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1504.md b/.wolf/bugs/bug-1504.md new file mode 100644 index 00000000..b9590a8e --- /dev/null +++ b/.wolf/bugs/bug-1504.md @@ -0,0 +1,38 @@ +--- +id: "bug-1504" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-08-07T08:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/schema/controller.capnp"] +tags: ["scheduler-core", "task-12.1", "capnp-schema", "codegen-limitation", "sigsegv", "uninitialized-memory", "controller.capnp", "snapshot"] +related: ["bug-1493", "bug-1505"] +legacy: true +--- + +# bug-1504: sched_sim_startup_test SIGSEGV in write_SnapshotData (lib/controller.capnp.c) -- deterministic… + +## Description +sched_sim_startup_test SIGSEGV in write_SnapshotData (lib/controller.capnp.c) -- deterministic crash inside libCapnC_Runtime's read_ptr/capn_setp, discovered during scheduler-core task 12.1 review remediation (full clean rebuild needed for a libedit dependency swap). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +SnapshotData.internalState was declared `:Data` in the capnp schema. This codebase's capnpc-c backend silently no-ops Data-field encoding inside generated encode_XXX() composite helpers (documented, deliberate limitation -- see controller.capnp's own prior comment citing ReplayHeader.configHash). But encode_SnapshotData_ptr()'s local `struct SnapshotData d;` is never zero-initialized before the (partially no-op) encode_SnapshotData() populates it, so the skipped Data field's capn_ptr is left as UNINITIALIZED STACK GARBAGE. write_SnapshotData() then unconditionally capn_setp()'s that garbage onto the wire. This is genuine UB: whether it crashes depends entirely on what garbage bytes land in that stack slot at each call site -- ReplayHeader.configHash has the identical gap but never crashed (lucky garbage); SnapshotData.internalState's stack layout in this one test's call chain was unlucky. A prior 'rm -rf xce.build' full rebuild (needed to pick up a new nix dependency) perturbed the whole binary's layout enough to flip this from dormant to a deterministic, reproducible SIGSEGV. + +## Fix +Changed SnapshotData.internalState from `Data` to `List(UInt8)` in controller.capnp, with a new companion internalStateLen (@11, UInt32) wire field for the maplistcount -- following the exact working precedent of FetchFileReply.content/bytesRead, which this same capnpc-c backend DOES correctly auto-generate real (non-no-op) encode/decode for (capn_new_list8/capn_set8 loop, not a placeholder comment). Updated typemap.h's sched_snapshot_data_t.internal_state_len from size_t to uint32_t to match the wire type, and updated test_e2e_capnp.c's test_snapshot_data_round_trip (which previously worked around the Data no-op with a manual low-level SnapshotData_get/set_internalState accessor) to rely on the now-working ordinary codec bridge instead. No hand-written low-level accessor code needed any more for this field. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1493, bug-1505 diff --git a/.wolf/bugs/bug-1505.md b/.wolf/bugs/bug-1505.md new file mode 100644 index 00000000..956204d9 --- /dev/null +++ b/.wolf/bugs/bug-1505.md @@ -0,0 +1,38 @@ +--- +id: "bug-1505" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-08-07T08:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/schema/controller.capnp"] +tags: ["controller", "ha", "wal-catchup", "capnp-schema", "codegen-limitation", "sigsegv", "uninitialized-memory", "controller.capnp", "deferred", "systemic"] +related: ["bug-1504", "bug-847"] +legacy: true +--- + +# bug-1505: controller_unit_test SIGSEGV in write_WalCatchupEntry (lib/controller.capnp.c)… + +## Description +controller_unit_test SIGSEGV in write_WalCatchupEntry (lib/controller.capnp.c), test_wal_catchup_append_term_fields_round_trip -- discovered incidentally while chasing bug-1504; NOT fixed, logged for its own follow-up. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same root cause as bug-1504 (Data-field codegen no-op leaves an uninitialized stack capn_ptr in the generated encode_XXX_ptr()'s local struct), a DIFFERENT field: WalCatchupEntry.payload (@2: Data), in the controller's HA/WAL-catchup subsystem (bug-847's own territory -- see controller.capnp's comment there), entirely unrelated to scheduler-core. Confirms this is a systemic, project-wide bug class affecting every `:Data` field in controller.capnp whose bytes are populated via the generic encode_XXX()/write_XXX() composite pipeline rather than fully hand-written low-level accessor code -- not a one-off. Known/suspected other Data fields with the same latent gap: ReplayHeader.configHash (dormant so far, no real consumer), CtrlHelloRequest.key, and possibly others not yet audited. + +## Fix +FIXED 2026-08-07. Investigated production exposure first: xce_wal_batch_encode_raw()/wal_entry_to_wire() (request.c) never call the generic encode_WalCatchupEntry() at all -- they memset() their own struct and call write_WalCatchupEntry() directly with every field hand-set, deliberately leaving payload unset (bug-847's own comment). So this bug was 100% test-only exposure, zero production risk, confirmed by grepping every caller of encode_WalCatchupEntry_ptr/decode_WalCatchupEntry (only test_e2e_capnp.c uses the generic path). Fix: same as bug-1504 -- changed WalCatchupEntry.payload (@2) and batchPayloads (@10) from Data to List(UInt8) in controller.capnp, with batchPayloadsLen (@11, new) as batchPayloads' maplistcount companion (payload reuses the existing wirePayloadLen @9). Verified batchPayloads' own top-level-single-blob shape means this does not reintroduce bug-847's per-element cross-segment far-pointer loss (that issue is inherent to PER-ELEMENT data regardless of Data vs List(UInt8) spelling -- payload, the per-element field, was already legacy/unpopulated in production before this fix and stays that way; batchPayloads is single top-level, never subject to that issue either way). Updated typemap.h's wal_catchup_entry_t field types (xce_data_t* -> uint8_t*, new batch_payloads_wire_len). Fixed request.c's extract_data_bytes() to accept a raw capn_ptr instead of capn_data (it already only ever used .p internally) so all 4 call sites (WalCatchupEntry.payload/batchPayloads, SnapshotTransfer.payload, per-sub-entry we.payload) work regardless of which Data-shaped wrapper type the accessor returns. Strengthened test_wal_catchup_append_term_fields_round_trip (test_e2e_capnp.c) with real payload/batchPayloads content assertions (previously never exercised at all, since the test's src.payload/batch_payloads were left NULL). Verified: controller_unit_test no longer crashes (3/3 controller suite), scheduler suite unaffected (37/37, unrelated schema area). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1504, bug-847 diff --git a/.wolf/bugs/bug-1506.md b/.wolf/bugs/bug-1506.md new file mode 100644 index 00000000..304cd7f9 --- /dev/null +++ b/.wolf/bugs/bug-1506.md @@ -0,0 +1,38 @@ +--- +id: "bug-1506" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-07T09:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_repl_breakpoints_inspection.c"] +tags: ["scheduler-core", "task-12.2", "repl", "xstring", "test-fixture", "makeString"] +related: ["bug-1507"] +legacy: true +--- + +# bug-1506: sched_repl_breakpoints_inspection_test: repl_dispatch(sim, "break JOB_STATE_CHANGE jobid=12345")… + +## Description +sched_repl_breakpoints_inspection_test: repl_dispatch(sim, "break JOB_STATE_CHANGE jobid=12345") stored NULL for the breakpoint's token/filter key/value -- TEST_ASSERT_EQUAL_STRING("jobid", ...) failed with 'Expected jobid Was NULL'. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +makeString() (src/lib/xstring.c) is an interned-xstring allocator backed by a process-global xstr_table that must be initialized via xstring_table_init() before first use -- xce_sched_core_init() (core.c) is the only call site in the scheduler tree and does this transitively. New task 12.2 unit tests built a bare xce_sched_sim_driver_t via memset(0) and called repl_state_ensure()/repl_dispatch() directly for pure command-dispatch tests, without ever calling xce_sched_core_init() first (the real driver flow always does, via sched_sim_driver_start() before xce_repl_start() runs, so this precondition is always satisfied in production -- only the test fixtures skipped it). With xstr_table uninitialized (NULL), makeString()'s internal key_find()/key_add() against a NULL kvmgr container fail gracefully and return NULL rather than crash, so the bug surfaced as silently-NULL breakpoint fields, not an immediate crash. + +## Fix +Added xce_sched_core_init(&(sim.core)) as the first statement (right after memset) in every Group A/B test in test_repl_breakpoints_inspection.c that calls repl_dispatch()'s break/delete commands or the repl_break.c matcher functions -- matching the real driver's own precondition ordering. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1507 diff --git a/.wolf/bugs/bug-1507.md b/.wolf/bugs/bug-1507.md new file mode 100644 index 00000000..907f4b6e --- /dev/null +++ b/.wolf/bugs/bug-1507.md @@ -0,0 +1,38 @@ +--- +id: "bug-1507" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-08-07T09:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_break.c"] +tags: ["scheduler-core", "task-12.2", "repl", "xstring", "heap-corruption", "free", "rtti"] +related: ["bug-1506"] +legacy: true +--- + +# bug-1507: sched_repl_breakpoints_inspection_test: 'free(): invalid pointer' SIGABRT inside… + +## Description +sched_repl_breakpoints_inspection_test: 'free(): invalid pointer' SIGABRT inside repl_breakpoint_free(), reached from repl_cmd_break()'s malformed-filter rejection path ("break JOB_PUSH nofiltermark"). gdb backtrace: malloc_printerr <- repl_breakpoint_free (repl_break.c:375) <- repl_cmd_break <- repl_dispatch <- test_break_malformed_filter_rejected. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl_breakpoint_free() called FREEUP() (a bare free()) on bp->token and each filter's key/value, all of which are makeString()-produced xstrings -- RTTI reference-counted objects (src/lib/xstring.c: xstring_type has custom .reference/.dereference interface functions; the returned pointer is the object's DATA region inside an RTTI-managed allocation, not a plain malloc() chunk). Releasing one requires going through the RTTI dereference path (xstring_dereference(), which only fires the real free when the refcount hits zero via the owning kvtab's key_del), never a bare free(). This codebase's own kill_sched_job()/kill_sched_queue() (schedobj.c) establish the identical precedent for every other makeString()'d field in the scheduler tree (xce_sched_job_t.submit fields, xce_sched_queue_t.name, etc.): none of them are ever explicitly released either -- they live for the process's lifetime, matching this whole test suite's 'detect_leaks=0 by design' convention. My new repl_break.c code was the first place in this file family to call free() directly on a makeString() result. + +## Fix +Removed the FREEUP() calls on bp->token/filters[i].key/filters[i].value from repl_breakpoint_free() (repl_break.c) -- it now only frees the plain calloc'd bp->filters array and bp itself, matching kill_sched_job()/kill_sched_queue()'s established never-release-xstrings precedent. Updated repl_internal.h's xce_repl_filter_t doc comment to state this explicitly so a future maintainer doesn't reintroduce the same free() on an RTTI-managed pointer. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1506 diff --git a/.wolf/bugs/bug-1508.md b/.wolf/bugs/bug-1508.md new file mode 100644 index 00000000..9500eb34 --- /dev/null +++ b/.wolf/bugs/bug-1508.md @@ -0,0 +1,41 @@ +--- +id: "bug-1508" +status: "fixed" +status_updated: "2026-09-26" +status_note: "7723a4f4: worker fetchfile accepts only /job__task.out for the request's jobid (exact match), O_NOFOLLOW|O_NONBLOCK, regular single-link file. Also closed the REST route (user out_file=/etc/shadow)." +component: "worker" +severity: "unknown" +timestamp: "2026-08-08T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/xcecd_worker.c (handle_fetchfile", "jobs-peek fetchfile feature)"] +tags: ["security", "path-traversal", "fetchfile", "worker", "jobs-peek", "needs-fix"] +related: [] +fix_commits: ["7723a4f4"] +legacy: true +gitea_issue: 287 +gitea_synced_status: "fixed" +--- + +# bug-1508: No user report / no test failure -- discovered incidentally during doc-cleanup fact-check audit of… + +## Description +No user report / no test failure -- discovered incidentally during doc-cleanup fact-check audit of src/docs/job_peek.md against current code. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +job_peek.md's design called for worker-side security validation on the fetchfile RPC (jobid ownership check, out/err-file path match against the job's actual output paths, path-traversal rejection) before Plan B (CGO/FetchFile) shipped. The validation was never implemented -- handle_fetchfile() opens whatever path it is given with no ownership or path-traversal check. + +## Fix +NOT FIXED -- out of scope for the doc-cleanup task that found it. Flagged to user as a follow-up; needs its own security-reviewer pass on handle_fetchfile() (job ownership check + path-traversal rejection, matching the validation job_peek.md originally specified). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #287 diff --git a/.wolf/bugs/bug-1509.md b/.wolf/bugs/bug-1509.md new file mode 100644 index 00000000..e7d1214c --- /dev/null +++ b/.wolf/bugs/bug-1509.md @@ -0,0 +1,38 @@ +--- +id: "bug-1509" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-10T07:34:55.093767+00:00" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/flake.nix", "src/nixSupport/xce.nix", "src/autoscaler/scaler/include/scaler.h"] +tags: ["tech-debt", "build", "nix", "low-risk"] +related: [] +legacy: true +--- + +# bug-1509: extralib flake dependency pulls external repo for trivial linked-list code used only by autoscaler… + +## Description +extralib flake dependency pulls external repo for trivial linked-list code used only by autoscaler scaler.h + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +basiclib/list.h (xcelist_t + list_insert_entry_back) is the only consumer; can be vendored locally in ~30 lines. + +## Fix +Port xcelist_t/xcelist_entry_t/list_insert_entry_back to local basic_list.h in autoscaler, remove extralib from flake.nix inputs and xce.nix nativeBuildInputs. ~40 min effort. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1514.md b/.wolf/bugs/bug-1514.md new file mode 100644 index 00000000..44c926b2 --- /dev/null +++ b/.wolf/bugs/bug-1514.md @@ -0,0 +1,38 @@ +--- +id: "bug-1514" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-10T18:46:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".kiro/specs/component-failover-adoption/design.md", "tasks.md", "research.md"] +tags: ["design-review", "thread-model", "kiro-validate-design", "scheduler", "critical-issue"] +related: [] +legacy: true +--- + +# bug-1514: kiro-validate-design Critical Issue 1: design.md claims sched_job_picker/sched_allocator are… + +## Description +kiro-validate-design Critical Issue 1: design.md claims sched_job_picker/sched_allocator are 'unaffected' by a blocked xcecli_jobdispatch call, violating Req 3 AC1 + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +design.md/tasks.md/research.md all wrongly claimed xcecli_jobdispatch executes on the sched_allocator thread. Verified against source: it actually runs on scheduler.c's main driver-loop thread (main.c while(1) -> xce_sched_driver_run_normal_mode_round -> driver.c on_decision -> sched_publish_dispatch, scheduler.c:1257), reached only after draining the decisions cbuf (capacity 5). sched_allocator only produces into decisions; it never calls xcecli_jobdispatch. jbuf (picker->allocator handoff) is a fixed capacity-1 scheduler-core invariant (core.c:161, never resizable). So a blocked dispatch call backpressures through decisions (5 slots) then jbuf (1 slot) -- picker/allocator get a bounded ~6-job grace window, then both stall. Not full isolation as originally claimed. + +## Fix +Corrected thread attribution and rejoin-call placement in design.md (SchedulerControllerClient section, sequence diagram note, traceability table) and tasks.md (task 3.1/3.2) to point at sched_publish_dispatch/main-loop thread instead of sched_allocator. Documented the real bounded-buffering model (jbuf cap 1 + decisions cap 5 = ~6-job grace window). Added a Req 3 AC1 scope-narrowing note as a Validation Hook -- flagged for human sign-off on requirements.md, not edited directly (requirements phase already approved). Corrected the same wrong claim at its source in research.md (lines 22, 30, 84). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1515.md b/.wolf/bugs/bug-1515.md new file mode 100644 index 00000000..3b251930 --- /dev/null +++ b/.wolf/bugs/bug-1515.md @@ -0,0 +1,38 @@ +--- +id: "bug-1515" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-10T20:20:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".kiro/specs/component-failover-adoption/requirements.md", "design.md", "tasks.md", "research.md"] +tags: ["design-review", "goal-loop", "requirements-math-error", "rejoin-retry", "shutdown-latency", "state-conflation", "scheduler", "apiserver", "worker"] +related: ["bug-1514"] +legacy: true +--- + +# bug-1515: /goal-driven deep re-review of component-failover-adoption requirements.md+design.md found 5… + +## Description +/goal-driven deep re-review of component-failover-adoption requirements.md+design.md found 5 additional distinct defects after bug-1514's fix, several introduced by bug-1514's own Req 7 fix + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +(1) Req 3 AC2's '~6-job grace window' was stated as 'at least 6' (a floor) when 6 is actually the best-case ceiling; the real guaranteed floor is 1-2 depending on decisions-cbuf occupancy at the moment of stall -- direction of the bound was backwards. (2) Citation scheduler.c:1253 for sched_publish_pend_reason was off by 2 (actual line 1251). (3) Cross-cutting: the shared 'check and clear needs_rejoin, then call xcecli_join' pattern (worker/scheduler/apiserver) cleared the flag unconditionally rather than only on xcecli_join's own success -- since xcecli_join is itself XCECLI_FAILOVER_CALL-wrapped and can hit its own retry-budget exhaustion, this could leave a component transport-connected but permanently un-joined with nothing left to retry (no requirement/design text previously covered this). (4) The Req 7 scheduler decision-publish retry loop only checked core.stop in the while-condition (after each attempt), so multiple decisions queued behind a stuck one would each burn a full ~100s budget cycle discovering shutdown was requested, compounding shutdown latency to minutes instead of the claimed ~1-2 min bound. (5) While fixing (3) for apiserver's heartbeat-idle-path, the first draft gated the retry on ApiserverBackendHealth.IsDegraded() -- but that state has its own independent TTL-based auto-clear for circuit-breaker purposes; if the TTL expired before an actual rejoin succeeded, the retry would silently stop, reintroducing the exact starvation bug (3) was fixing. Two unrelated state machines (rejoin-tracking vs request-path circuit-breaker) had been conflated onto one flag. + +## Fix +(1) Reworded Req 3 AC2 and all design.md/research.md/tasks.md echoes to state the guarantee qualitatively (bounded, occupancy-dependent, resumes with zero loss per Req 7) instead of a wrong fixed number. (2) Fixed the line citation. (3) Added Requirement 2 AC5 (rejoin retry-on-failure, applies to all three components) and updated the shared 'Rejoin callback constraint' section plus all three component sections in design.md and their tasks.md task bullets to clear the flag only on xcecli_join success. (4) Changed the Req 7 retry loop from do/while to while (check core.stop before each attempt, not only after) so only the actively in-flight decision can cost a full budget cycle; queued decisions behind it skip immediately once stop is set. (5) Introduced a dedicated needs_rejoin flag for apiserver's heartbeat path (set on transition, cleared only on join success) fully decoupled from ApiserverBackendHealth's TTL-governed degraded state; updated the sequence diagram, prose, and tasks.md task 4.4 accordingly, plus an explicit note in ApiserverBackendHealth's section warning against this conflation. Also added Req 7 AC6 documenting that the fix does not survive a scheduler process crash/restart (in-memory only, no persistence). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1514 diff --git a/.wolf/bugs/bug-1519.md b/.wolf/bugs/bug-1519.md new file mode 100644 index 00000000..0094e61e --- /dev/null +++ b/.wolf/bugs/bug-1519.md @@ -0,0 +1,39 @@ +--- +id: "bug-1519" +status: "fixed" +status_updated: "2026-09-24" +status_note: "b6a2f5ec (bug-1756)" +component: "worker" +severity: "unknown" +timestamp: "2026-08-11T12:49:17.815Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/worker/src/main.c (xcecd daemon startup)"] +tags: ["asan", "double-join", "worker", "xcecd", "unshare", "sandbox-environment", "unresolved", "component-failover-adoption"] +related: [] +fix_commits: ["b6a2f5ec"] +legacy: true +--- + +# bug-1519: ==ERROR: AddressSanitizer: Joining already joined thread, aborting + +## Description +==ERROR: AddressSanitizer: Joining already joined thread, aborting. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +UNRESOLVED / not root-caused. xcecd's root check (main.c:288 geteuid()!=0) was bypassed for sandbox testing via `unshare -r` (user-namespace fake root, euid=0 but not real root). Startup then immediately hit an ASAN double-pthread_join abort during/after the HA-cluster-config init log line (main.c:435), before any further logging. Never reproduced under real root -- unknown whether this is a genuine double-join bug in xcecd's daemonize/thread-spawn path (e.g. duplicate cleanup on an early-init error branch) or an artifact of unshare's user-namespace semantics (e.g. a syscall xcecd's threading depends on behaving differently under CLONE_NEWUSER, such as thread creation ordering or signal delivery). Blocked further component-failover-adoption Task 3.3 Item-3 E2E validation from getting a live worker registered, which in turn meant no real job ever produced a scheduler decision (picker/allocator never had a matching candidate to dispatch to or explicitly PEND against), so the scheduler's decision-publish/rejoin code path (task 3.2) could not be exercised via a fully live job in this sandbox. + +## Fix +Not fixed -- out of scope for Task 3.3 (which validates scheduler HA reconnect, not worker daemon startup). Worked around by validating scheduler join/failover/reconnect behavior via direct socket-state inspection (ss/proc fd) instead of a full live job pipeline; the decision-publish retry+rejoin logic itself already has white-box unit coverage from task 3.2 (test_scheduler_decision_publish_retry.c) and task 3.3 item 2 (test_scheduler_backpressure.c). If a future session needs a live worker in this sandbox, retry under real root (not unshare -r) first to isolate whether this is userns-specific. UPDATE (2026-08-20, build-time-versioning task 3.3 review): independently reproduced under REAL sudo root (not unshare -r) -- same signature, same timing (fires during later thread-join startup work, strictly AFTER the HA-cluster-config INFO log line, both with and without --peers). This refines/answers the prior 'never reproduced under real root' open question: it is NOT a userns/unshare-specific artifact -- the abort reproduces identically under genuine root. Still not root-caused/fixed; still does not block this session's version-display changes (main.c's new --version/log-line code runs entirely before the point where this abort fires). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1532.md b/.wolf/bugs/bug-1532.md new file mode 100644 index 00000000..fa8e86e0 --- /dev/null +++ b/.wolf/bugs/bug-1532.md @@ -0,0 +1,38 @@ +--- +id: "bug-1532" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "crash" +timestamp: "2026-08-12T00:20:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/clib.go"] +tags: ["use-after-free", "cgo", "apiserver", "failover", "component-failover-adoption", "task-4.2"] +related: [] +legacy: true +--- + +# bug-1532: apiserver clib.go tryConnect(): use-after-free on ctx->conf->masters[i] (host address string) on… + +## Description +apiserver clib.go tryConnect(): use-after-free on ctx->conf->masters[i] (host address string) on every post-initial-connect failover rotation attempt + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +tryConnect() allocated cAddr/cFrom/cKey via C.CString(), built a Go-function-local xce_client_config_t pointing at them, called xce_backend_init() once, then deferred C.free() on all three strings immediately after that single call returned. xcecli_ctx_init() (src/lib/client.c) stores ctx->conf = cfg as a RAW pointer (never copies the struct), and client_failover.c dereferences ctx->conf->masters[cur_master] on EVERY subsequent failover rotation attempt (client_failover.c:120,386), not just the initial connect -- so any reconnect/rotation after the first tryConnect() call returned read freed memory. A second, independent hazard: cConfig itself was a Go-local var; ctx->conf retained a raw pointer to it invisible to Go GC once tryConnect() returned. (Note: from/key were NOT actually part of this UAF -- xce_ctx_init() strdup()s them immediately at ctx-init time, so freeing cFrom/cKey was safe in isolation; only the masters[] host string and the client_config_t struct address itself were genuinely unsafe.) + +## Fix +component-failover-adoption task 4.2: replaced tryConnect() and the dual Go-level connect loops entirely with buildClusterConfig()+RegisterToBackend(), backed by two new package-level (process-lifetime, GC-rooted) vars gClusterCfg (C.xce_cluster_config_t) and gClientCfg (C.xce_client_config_t). Host strings now live in gClusterCfg.candidates[i].hostname (fixed char[256] arrays, no CString/free at all -- xce_cluster_config_to_client_config() aliases pointers directly into this storage). from/key point at gClusterCfg.identity[0]/key[0] (also fixed arrays) rather than separate CStrings. No C.CString/C.free is used anywhere in the connect path except the transient peers-string argument to xce_parse_peers(), which is safely freed right after that one call since xce_parse_peers() copies into gClusterCfg internally. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1542.md b/.wolf/bugs/bug-1542.md new file mode 100644 index 00000000..1e21cf2c --- /dev/null +++ b/.wolf/bugs/bug-1542.md @@ -0,0 +1,38 @@ +--- +id: "bug-1542" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-12T10:18:42.192Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/health_rejoin_test.go"] +tags: ["go", "cgo", "test-file-restriction", "import-C-in-test"] +related: [] +legacy: true +--- + +# bug-1542: health_rejoin_test.go referenced C.E_XCE_NOT_READY directly (via -int(C.E_XCE_NOT_READY)), which… + +## Description +health_rejoin_test.go referenced C.E_XCE_NOT_READY directly (via -int(C.E_XCE_NOT_READY)), which requires import "C" in a _test.go file -- illegal per golang/go#13938, would have failed to compile + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Wrote a test helper that needed the -E_XCE_NOT_READY errno value, and reached for C.E_XCE_NOT_READY directly without recalling that _test.go files can never import "C" themselves (established project constraint, see clib_test.go/reconnect_test.go doc comments) -- caught before ever running go test, by re-reading the file before the first test run. + +## Fix +Exposed -E_XCE_NOT_READY as a plain Go int via a new package-level var in clib.go (a cgo file): notReadyRC = -int(C.E_XCE_NOT_READY). health_rejoin_test.go references notReadyRC instead of C.E_XCE_NOT_READY -- same plain-Go-signature test-seam pattern as performRejoinFn (reconnect.go, task 4.3). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1548.md b/.wolf/bugs/bug-1548.md new file mode 100644 index 00000000..2efd600e --- /dev/null +++ b/.wolf/bugs/bug-1548.md @@ -0,0 +1,38 @@ +--- +id: "bug-1548" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-12T12:59:31Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/frontend/apiserver/backend/reconnect.go", "heartbeat.go", "clib.go (component-failover-adoption task 4.3/4.4's checkAndPerformRejoin/needsRejoin mechanism)"] +tags: ["apiserver", "rejoin", "push-notification", "failover", "component-failover-adoption", "task-5.1", "heartbeat", "live-e2e-finding", "task-5.2", "second-independent-repro"] +related: [] +legacy: true +--- + +# bug-1548: apiserver never re-registers (application-level xcecli_join with HostRole_apiserver) with the new… + +## Description +apiserver never re-registers (application-level xcecli_join with HostRole_apiserver) with the new primary after a live controller failover, even though its transport/write-call connectivity clearly recovers (job/host-control POSTs keep succeeding); observed for 10+ minutes and 20+ default-30s heartbeat ticks with zero occurrences of reconnect.go's own 'checkAndPerformRejoin: rejoin after transparent reconnect succeeded/failed' log lines in apiserver_out.log, across 3 separate write calls (2 job submits + 1 host control) issued after the primary was kill -9'd. TASK 5.2 UPDATE (2026-08-12, second independent live E2E reproduction, cleaner isolation): ran component-failover-adoption task 5.2's own scenario (apiserver-only, no worker/scheduler, 3-node controller cluster at localhost:19991-19993, apiserver -d /home/claude/tmp/xce-e2e-task52/apiserver) specifically to isolate the heartbeat-only idle-period path task 5.1's write-traffic-muddied run could not cleanly separate. Confirmed real election: ctrl-1 (primary, term=2) kill -9'd at 13:33:31Z; ctrl-2 promoted primary at term=3 within 9s (13:33:40Z, raft_cb_become_leader). Zero write requests issued for the full outage window: apiserver_out.log is BYTE-IDENTICAL (23 lines) from before the kill through 268s / ~9 heartbeat ticks after it (well past the task's 2-minute/3-4-tick minimum) -- zero occurrences of 'checkAndPerformRejoin'/'rejoin'/'needsRejoin'/'xcecli_join' anywhere. Diagnostic control issued only AFTER the observation window (13:39:52Z, ~6.5 min post-kill): POST /xce/v1/jobs/create succeeded with a real controller-assigned job ID (73930919093731328) and server-generated success message -- proving the transport had already reached the NEW primary (ctrl-2) -- yet this write call ALSO produced zero rejoin log lines, full apiserver_out.log grep for the same 4 terms still empty after it. This is strictly stronger evidence than task 5.1's: it directly demonstrates the write path's own XCECLI_FAILOVER_CALL/xcecli_invoke_rejoin also did not fire even when explicitly exercised, consistent with the transport having already been silently repaired by an earlier heartbeat tick's XCECLI_FF_CALL before the write call ever needed to retry -- exactly bug-1548's leading hypothesis, not merely 'consistent with' it as task 5.1 could only say. Corroborating bug-1549 downstream symptom also reproduced: GET /xce/v1/jobs immediately after the successful submit returned {"count":0,"rows":[]} -- the just-submitted job never reached apiserver's local store, consistent with apiserver never having actually re-issued xcecli_join(HostRole_apiserver) against ctrl-2. ctrl-2.log has no apiserver-join lines either direction (join.c logs only rejections, per task 5.1's prior finding -- reconfirmed, not independently informative here). Task 5.2's own observable-completion criterion ('the scenario passes without requiring any client-triggered write request during the outage window' -- i.e. apiserver rejoins and resumes push notifications via the heartbeat-triggered path alone) was NOT met: no rejoin evidence appeared at any point in the heartbeat-only window. Evidence preserved at /home/claude/tmp/xce-e2e-task52/ (apiserver/logs/apiserver_out.log, logs/ctrl-{1,2,3}.log, logon_response.json, diagnostic_job_submit_response.json, diagnostic_joblist_response.json, KILL_TIMESTAMP_UTC.txt, DIAGNOSTIC_CONTROL_START_UTC.txt, DIAGNOSTIC_CONTROL_SUBMIT_UTC.txt). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT root-caused (out of task 5.1's E2E-validation boundary to fix). Leading hypothesis from live log analysis: XCECLI_FF_CALL (the fire-and-forget macro backing xcecli_heartbeat) can silently self-heal the TRANSPORT (rotate to a live candidate) entirely within a single heartbeat tick without that tick ever returning a failure to Go -- so heartbeat.go's prevFailed-edge detection (onFailureToSuccess) never observes a failure->success transition and never sets needsRejoin. Separately, none of the 3 post-kill write calls (which DO route through XCECLI_FAILOVER_CALL, which per design.md should invoke the rejoin callback mid-call on a genuine failover) show any evidence of having hit a failed connection either -- consistent with the connection having already been silently fixed by a prior heartbeat tick before any write call needed to retry, so XCECLI_FAILOVER_CALL's own rejoin-invoking path is also never exercised. Net effect: needsRejoin is never set by any path, so checkAndPerformRejoin() stays a permanent silent no-op and apiserver's push-registration (join) with the new primary is never renewed. Could not independently confirm via controller-side logs either way -- discovered during this same investigation that the controller's join.c only logs REJECTED/failed joins (WARN/ERROR), never a successful one at any verbosity, so controller logs are not usable evidence for this question in either direction. REVIEW UPDATE (task 5.1 review, 2026-08-12): confirmed via source read that XCECLI_FF_CALL (client_internal.h:290-302) never calls xcecli_invoke_rejoin under any circumstance -- this half of the hypothesis is not speculative, it's structurally true by construction. Elevated confidence this is a REPRODUCIBLE gap, not scenario-specific timing luck: since real elections in this codebase's own logs complete in single-digit seconds while the default heartbeat interval is 30s, a single heartbeat tick landing after the election has already completed will both discover the dead connection AND repair it within that same XCECLI_FF_CALL invocation -- the failure never surfaces to Go's heartbeat.go loop as a separate tick, so the cross-tick prevFailed-edge detection (onFailureToSuccess) structurally cannot observe it. This is likely to recur on MOST real single-election failovers, not just this run's specific timing -- materially risks Req 2 AC4 (component-failover-adoption requirements.md) in production. Recommend escalating past ordinary buglog cadence to a dedicated follow-up spec/task, given it affects the already-approved task 4.3/4.4 mechanism. + +## Fix +FIXED (component-failover-adoption tasks.md Phase 6, task 6.1). Root cause confirmed as hypothesized: XCECLI_FF_CALL (client_internal.h:290-302) can rotate+reconnect the transport entirely inside a single call, holding ac->failover.lock, and still return success -- so a failure->success transition on the caller's own return code is not a reliable 'a reconnect happened' signal whenever an election (single-digit seconds) completes faster than the 30s default heartbeat interval, which is the common case. Fix: added a monotonic `_Atomic uint64_t reconnect_gen` field to `endpoint_t` (src/include/netio.h:137), incremented unconditionally on the success path of `xce_ctx_reconnect()` (src/lib/xcectx.c:402) -- confirmed via repo-wide grep as the sole reconnect call-site reachable from `xce_failover_rotate()`, covering both the write-path retry loop and the heartbeat fire-and-forget path. apiserver's heartbeat.go `runHeartbeat()` now compares this generation on every tick (via the existing `xcecli_get_my_endpoint()` accessor, atomic-loaded through clib.go) instead of waiting for a failure->success transition, replacing the old detector entirely. Regression test `TestRunHeartbeat_IdlePeriodRecovery_GenChangeAloneSetsNeedsRejoinEvenWhenAllTicksSucceed` (heartbeat_rejoin_test.go) proves the fix's exact shape. Closed via live E2E re-run of task 5.2's scenario: rejoin fired 22s after primary kill (12s post-election) with zero write traffic for the preceding 5.5 minutes. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1549.md b/.wolf/bugs/bug-1549.md new file mode 100644 index 00000000..5abf23ae --- /dev/null +++ b/.wolf/bugs/bug-1549.md @@ -0,0 +1,38 @@ +--- +id: "bug-1549" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-12T12:59:31Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/apiserver.c (apiserver_job_new_fun / apiserver_push_joblist hook)", "src/frontend/apiserver/backend/clib.go (go_push_handler / handleDataset)"] +tags: ["apiserver", "controller", "push-notification", "job-submit", "component-failover-adoption", "task-5.1", "live-e2e-finding"] +related: ["bug-1548"] +legacy: true +--- + +# bug-1549: Jobs submitted via apiserver REST API (POST /xce/v1/jobs/create) never appear in apiserver's own… + +## Description +Jobs submitted via apiserver REST API (POST /xce/v1/jobs/create) never appear in apiserver's own job store (GET /xce/v1/jobs / GET /xce/v1/jobs/:id returns empty), even though the job is confirmed genuinely committed on the controller (verified directly via xce_test's `joblist` showing both jobs in PEND state). Reproduced for 2 jobs (one submitted at the moment of a primary kill, one submitted >1 minute after a clean new-primary failover) -- so this is NOT specific to the failover window, appears to be a general gap. By contrast, worker/host-list push (apiserver_push_workers/apiserver_push_workerlist, controller/apiserver.c) DOES work: GET /xce/v1/hosts correctly showed the live-joined worker with real data. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT root-caused (out of task 5.1's boundary). apiserver_job_new_fun is wired as a HOOK_DETAIL_JOB_NEW hook meant to fire apiserver_push_joblist()/apiserver_push_jobarray() after a job commits (apply_committed_entry()); controller logs show zero evidence either way since apiserver.c has no INFO-level success logging on this path (only a WARN on push failure, `push job list to api server %s has problem, rc %d`, which also never appeared). Could be: the hook isn't actually registered/invoked in this build, or apiserver's join isn't registered in c->apiservers[] the push fan-out iterates, or something queue/resreq-specific. Discovered opportunistically while probing bug-1548; not independently isolated from bug-1548 (both could share a root cause -- c->apiservers[] fan-out list membership -- or be unrelated). REVIEW UPDATE (task 5.1 review, 2026-08-12): found the likely direct causal chain, narrowing this from 'could share a root cause or be unrelated' to 'very likely a downstream symptom of bug-1548'. src/controller/apiserver.c's on_apiserver_join() unconditionally calls xce_apiserver_push_jobs(c, ep) (bulk job resync) alongside xce_apiserver_push_workers(c, ep), but ONLY fires when a fresh join is actually processed by the controller. c->apiservers[] (the fan-out list push hooks iterate) is per-process, in-memory-only state -- no WAL/config replication (confirmed zero references outside apiserver.c) -- so a NEW primary starts with an empty registration table until each apiserver instance re-joins against it. If bug-1548 means apiserver's rejoin was never re-issued against the new primary, ctrl-2 never ran on_apiserver_join for it, so neither the bulk resync nor any incremental job push could ever reach apiserver -- cleanly explaining both job1 and job2 being empty. Recommend a fix attempt for bug-1548 be verified against this job-push symptom directly (if fixing the rejoin gap makes GET /xce/v1/jobs start populating correctly, that's confirmation bug-1549 was never an independent issue). TASK 5.2 RE-REPRODUCTION (2026-08-12T13:39:52Z): reproduced a third time in an isolated, apiserver-only (no worker/scheduler) idle-period E2E run -- a job submitted well after a clean failover (jobid 73930919093731328, controller-confirmed via the accepted response) still never appeared in GET /xce/v1/jobs. This occurred in the SAME run where bug-1548's rejoin gap was also independently reconfirmed with a clean negative control, strengthening (not just repeating) the downstream-symptom-of-bug-1548 hypothesis: if apiserver never re-joined the new primary at the application level, c->apiservers[] on ctrl-2 never gained an entry for it, so no push (bulk or incremental) could ever reach it -- independent of whether the underlying job-push wiring itself is otherwise correct. + +## Fix +FIXED as a downstream symptom of bug-1548 -- confirmed, not just hypothesized. Once apiserver's heartbeat-triggered rejoin (bug-1548's fix) correctly re-issues xcecli_join(HostRole_apiserver) against the new primary, the controller's on_apiserver_join() bulk job resync (xce_apiserver_push_jobs) fires as designed and populates apiserver's local job store. Live E2E re-validation (component-failover-adoption task 6.2): a diagnostic job submitted post-recovery correctly appeared via GET /xce/v1/jobs, with no client-triggered write request required during the outage window. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1548 diff --git a/.wolf/bugs/bug-1553.md b/.wolf/bugs/bug-1553.md new file mode 100644 index 00000000..75dd7926 --- /dev/null +++ b/.wolf/bugs/bug-1553.md @@ -0,0 +1,38 @@ +--- +id: "bug-1553" +status: "open" +status_updated: "2026-09-24" +status_note: "intermittent (~1 in 3) controller_unit_test SIGSEGV; not isolated, re-seen 5bf123cc" +component: "controller" +severity: "crash" +timestamp: "2026-08-13T07:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/ (combined binary", "crash site not yet isolated -- occurs after test_drive_async_config.c's liveness_reserved_* tests)"] +tags: ["flaky", "segfault", "controller_unit_test", "order-dependent", "unity"] +related: [] +legacy: true +--- + +# bug-1553: controller_unit_test (combined 1095-test Unity binary) segfaults intermittently (~1 in 3 full runs… + +## Description +controller_unit_test (combined 1095-test Unity binary) segfaults intermittently (~1 in 3 full runs observed) somewhere between test_drive_async_config.c and the next testsuite file (test_configcompare.c/test_slowlane.c area) -- no assertion failure, no ASAN report captured before the crash, process just dies (exit 139/SIGSEGV) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not yet investigated. Reproduced 1/3 runs both BEFORE and AFTER an unrelated worker.c change (update_resval PERROR->rc logging fix) confirming this is pre-existing and unrelated to that fix -- likely order-dependent shared-process state leakage across Unity tests (global fixtures, xstring table, or ASAN heap state) rather than a bug in any specific test file. Two of three runs completed cleanly with 0 failures each time. + +## Fix +Not fixed -- out of scope for the update_resval/mxj-auto-detect bugfix session that surfaced it. Needs isolation: bisect which specific test pair triggers it (likely by running testsuite files in smaller batches or under gdb with a core dump), then root-cause the actual memory-safety issue. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1554.md b/.wolf/bugs/bug-1554.md new file mode 100644 index 00000000..bfa47029 --- /dev/null +++ b/.wolf/bugs/bug-1554.md @@ -0,0 +1,38 @@ +--- +id: "bug-1554" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "worker" +severity: "unknown" +timestamp: "2026-08-13T07:36:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/worker/src/xcecd.c"] +tags: ["worker", "stub", "maxtmp", "hardware-detection", "maxswap"] +related: [] +legacy: true +--- + +# bug-1554: workerlist / resval show maxswap=1024 and maxtmp=2048 for every host regardless of actual… + +## Description +workerlist / resval show maxswap=1024 and maxtmp=2048 for every host regardless of actual hardware, unless overridden by a per-host config resource entry + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +init_host_attrs() hardcodes host_attrs.maxswap = 1024 and host_attrs.maxtmp = 2048 as placeholder constants (lines ~349-350) -- unlike maxcpus/maxmem, which come from real hardware_monitor_* detection, these two fields have no real detection implemented yet. Confirmed live: a deploy's maxtmp showed exactly 2048 (the hardcoded stub) while maxswap showed 10000 (a per-host config override in hosts.yaml resource list, taking precedence via apply_join_state's mark_host_res loop -- proves the stub IS reachable/live whenever no override exists). + +## Fix +FIXED (2026-09-06, worker hardware detection). Real detection replaces both placeholder stubs: maxswap now comes from sysinfo()'s totalswap field (hardware_monitor_get_total_swap(), the same sysinfo() snapshot hardware_monitor_update() already takes for maxmem -- no extra syscall), and maxtmp now comes from a fresh statvfs("/tmp") total-capacity measurement (hardware_monitor_get_disk_info()), mirroring node_information.c's node_info_get_tmp() own statvfs("/tmp") convention but reporting total blocks*frsize instead of free space. init_host_attrs() (xcecd.c) wires both in, with NO synthetic fallback on detection failure (left at 0, matching maxcpus/maxmem's own existing no-fallback behavior) -- a system with swap genuinely disabled reports 0 correctly instead of a fabricated non-zero placeholder. New tests: test_hardware_monitor.c (7 tests, cross-checks each function's output against an independent sysinfo()/statvfs() call made directly in the test) and a new integration test in test_xcecd_startup_order.c pinning that init_host_attrs() picks up whatever these two functions report. Full worker suite: 9/9 pass. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1555.md b/.wolf/bugs/bug-1555.md new file mode 100644 index 00000000..c737ebf9 --- /dev/null +++ b/.wolf/bugs/bug-1555.md @@ -0,0 +1,38 @@ +--- +id: "bug-1555" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-13T07:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/worker.c"] +tags: ["controller", "logging", "errno"] +related: [] +legacy: true +--- + +# bug-1555: controller log spammed with '[ERROR] update_resval: Success' / 'update_resval: Resource… + +## Description +controller log spammed with '[ERROR] update_resval: Success' / 'update_resval: Resource temporarily unavailable' -- misleading, the real failure reason is never logged + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +update_resvals()/update_resval_list() called PERROR("update_resval") on rc<0, but PERROR (xce_perror -> xce_perror_) formats the process-global errno, not rc. update_resval()'s failure paths (key_find/zarray_put) return raw negative codes directly and never touch errno, so the logged message reflected whatever errno happened to be left over from an unrelated earlier syscall in the same event-loop iteration -- 0 ("Success") if nothing had touched it, or a stray EAGAIN if something had. The real reason (e.g. -ENOENT from an unregistered resource name) was silently discarded. Also fixed a second instance of the same errno-vs-rc bug in update_resval()'s r==NULL branch (was `return -errno;`, now `return -ENOENT;`, matching the identical branch in the sibling function mark_host_res()). + +## Fix +Replaced PERROR with a local log_update_resval_failure() helper that decodes the actual rc via xce_strerror_r(-rc, ...) and includes the resource name in the message. Regression test: test_update_resvals_logs_actual_rc_not_stale_errno (test_worker_join.c) sets errno=EAGAIN before the call and asserts the captured log contains neither "Success" nor "Resource temporarily unavailable", contains the resource name, and contains the real ENOENT string -- verified genuine RED (failed against the reverted old code with the exact reported symptom) before GREEN. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1556.md b/.wolf/bugs/bug-1556.md new file mode 100644 index 00000000..bed4aa01 --- /dev/null +++ b/.wolf/bugs/bug-1556.md @@ -0,0 +1,38 @@ +--- +id: "bug-1556" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-08-13T07:41:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/src/xcecommon/mapping.py"] +tags: ["xceadmin", "config", "scheduling", "wire-encoding", "high-severity"] +related: [] +legacy: true +--- + +# bug-1556: Every host using the documented hosts.yaml default (no max_slot, meaning 'auto-detect from CPU… + +## Description +Every host using the documented hosts.yaml default (no max_slot, meaning 'auto-detect from CPU count') gets 0 scheduling slots -- permanently blocks the scheduler from placing any job on that host + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +hosts.schema.json documents max_slot: -1 as the 'auto-detect' sentinel, but HostConfig.mxj is UInt32 on the wire and cannot represent -1 (pycapnp raises capnp.KjException for a negative assignment). mapping.py's _build_host_config() worked around this by leaving mxj unset when max_slot was absent/negative, which capnp defaults to 0. worker.c's apply_join_state() then does `nslots = hconfig->mxj; if (nslots < 0) nslots = maxcpus;` -- but 0 is never < 0, so the auto-detect fallback never fires, and the host is explicitly configured with 0 slots. Confirmed live via a real deploy: resval -n slots showed value: 0.000000 for the default host. An existing test (test_mapping.py) even asserted this broken behavior as correct (`assert h.mxj == 0`). + +## Fix +mapping.py now sets mxj = HOSTCONFIG_MXJ_AUTO_DETECT (0xFFFFFFFF) instead of leaving it unset. The controller's `int mxj` field receives this via capn_read32()'s implicit uint32_t->int conversion, which reinterprets it back to -1 on this project's two's-complement target platforms -- no capnp schema change or C-side change needed, apply_join_state()'s existing `nslots<0` fallback already handles a genuine -1 correctly (now proven by a new C regression test, test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus, since no existing test exercised that branch with a negative mxj before). Updated the stale test assertion, added a new test for the absent-max_slot production case, and regenerated the xceadmin golden fixture (tests/fixtures/full/expected.bin) whose wire bytes changed as a result. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1557.md b/.wolf/bugs/bug-1557.md new file mode 100644 index 00000000..5e746189 --- /dev/null +++ b/.wolf/bugs/bug-1557.md @@ -0,0 +1,38 @@ +--- +id: "bug-1557" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-13T12:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c (cmd_join_worker", "hostattrs_t attrs initializer)"] +tags: ["test-tool", "xce_test", "stale-default", "units", "low-priority"] +related: [] +legacy: true +--- + +# bug-1557: xce_test's `worker` -> `join` command defaults to maxmem=64 -- discovered while writing/verifying… + +## Description +xce_test's `worker` -> `join` command defaults to maxmem=64 -- discovered while writing/verifying detailed `help join` text + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +attrs.maxmem = 64 was a reasonable default back when maxmem was MB-scale (64 MB); this session's earlier fix converted maxmem's canonical unit to bytes (node_information.c/xcecd.c/resources.yaml) but never touched this test-tool default, which is now a nonsensical 64 BYTES of RAM. Purely a manual-test-tool default (join is used to simulate a worker joining without a real xcecd), not a production code path -- callers who don't pass -m get an obviously-wrong but harmless value. + +## Fix +FIXED (2026-09-09, xce-test-cli-ux task 2.6). cmd_join_worker default changed to 64 * 1024 * 1024 (64MB in bytes). Extracted into join_worker_parse_attrs() (join_worker_args.c) with a permanent regression test asserting the corrected default. Verified live: workerlist reports MaxMem: 64.00 MB after a no-'-m' join (was would-be 64.00 B). USAGE_JOIN_WORKER doc update tracked separately under xce-test-cli-ux Requirement 12.2 (task 3.5). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1558.md b/.wolf/bugs/bug-1558.md new file mode 100644 index 00000000..ec76903c --- /dev/null +++ b/.wolf/bugs/bug-1558.md @@ -0,0 +1,38 @@ +--- +id: "bug-1558" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-15T11:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/Makefile (test-backend target)", "src/frontend/apiserver/meson.build"] +tags: ["build", "tooling", "cgo", "apiserver", "test-backend", "meson", "makefile", "component-failover-adoption"] +related: ["bug-1440", "bug-1441"] +legacy: true +--- + +# bug-1558: make test-backend fails standalone: backend/reconnect.go:6:10: fatal error: client.h: No such file… + +## Description +make test-backend fails standalone: backend/reconnect.go:6:10: fatal error: client.h: No such file or directory -- FAIL apiserver/backend [build failed], make: *** [Makefile:33: test-backend] Error 1. Hit during /kiro-validate-impl component-failover-adoption full-suite validation. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CGO_CFLAGS/CGO_LDFLAGS for libxce and its transitive deps are constructed only inside apiserver/meson.build (lines 11-49) and set via meson's go_env environment() for its custom_target. They are never exported to a general shell, so the Makefile's test-backend target runs bare `CGO_ENABLED=1 go test ./backend/...` with no include or link paths. Compounding this, xce-build.sh's RUN_TESTS path only invokes `make test` (CGO_ENABLED=0), so the CGO backend suite -- which holds this spec's load-bearing rejoin/heartbeat/candidate-parity tests -- is structurally never exercised by any documented command. Previously noted narratively in component-failover-adoption tasks.md Implementation Notes (task 4.1, task 5.4) but never logged as a bug. + +## Fix +FIXED (component-failover-adoption tasks.md Phase 6, non-blocking build-tooling fix). src/frontend/Makefile's test-backend target now self-derives CGO_CFLAGS/CGO_LDFLAGS via `pkg-config --cflags/--libs librtti c-capnproto openssl tinylog` plus -I$(CURDIR)/../include -I$(BUILD_DIR)/lib and -L$(BUILD_DIR)/lib -lxce -lm -lrt -fsanitize=address, mirroring apiserver/meson.build's own construction. BUILD_DIR anchored on $(CURDIR) (the Makefile's own directory), not a path relative to the `cd apiserver` recipe target, avoiding the off-by-one the workaround note flagged. Verified 2026-08-15: `make test-backend` run standalone from a nix develop shell with zero manual env setup -- `ok apiserver/backend 14.652s`, `ok apiserver/backend/auth 0.010s`. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1440, bug-1441 diff --git a/.wolf/bugs/bug-1559.md b/.wolf/bugs/bug-1559.md new file mode 100644 index 00000000..e4684c2d --- /dev/null +++ b/.wolf/bugs/bug-1559.md @@ -0,0 +1,38 @@ +--- +id: "bug-1559" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-15T13:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/xce-build.sh", "src/scheduler/testsuite/meson.build"] +tags: ["build", "tooling", "scheduler", "xce-build.sh", "component-failover-adoption", "coverage-gap"] +related: ["bug-1558"] +legacy: true +--- + +# bug-1559: xce-build.sh scheduler's RUN_TESTS block never builds or runs the 4 new… + +## Description +xce-build.sh scheduler's RUN_TESTS block never builds or runs the 4 new component-failover-adoption task-3.x scheduler test binaries (sched_scheduler_connect_test, sched_scheduler_backpressure_test, sched_scheduler_decision_publish_retry_test, sched_scheduler_decision_redelivery_test) -- it only builds/runs 'xce_scheduler sched_test'. They only got compiled in this session as a side effect of a separate worker build and had to be run directly, bypassing the canonical script. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce-build.sh's scheduler case's RUN_TESTS list was never updated when task 3.x added 4 new standalone test executables under src/scheduler/testsuite/meson.build -- same class of gap as bug-1558 (documented commands cover only a subset of the real suite), just for the scheduler component instead of apiserver's CGO path. + +## Fix +FIXED (component-failover-adoption tasks.md Phase 6, non-blocking build-tooling fix). src/tools/xce-build.sh's scheduler case's RUN_TESTS block now runs `meson test -C "$BUILD_DIR" --suite scheduler --print-errorlogs` instead of invoking sched_test directly by path -- meson test rebuilds by default, so no need to separately enumerate the 4 previously-skipped binaries. Verified 2026-08-15: `xce-build.sh scheduler asan` now runs all 41 scheduler suite tests (was: 1 binary only). NOTE: running the full suite for the first time surfaced a previously-invisible failure, test_recordstart_symlink_escape_replies_error (test_admin_request.c) -- logged separately as bug-1560, unrelated to this fix or to component-failover-adoption. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1558 diff --git a/.wolf/bugs/bug-1560.md b/.wolf/bugs/bug-1560.md new file mode 100644 index 00000000..aa295f05 --- /dev/null +++ b/.wolf/bugs/bug-1560.md @@ -0,0 +1,40 @@ +--- +id: "bug-1560" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "security" +timestamp: "2026-08-15T22:54:35Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 5 +files: ["src/scheduler/testsuite/test_admin_request.c", "src/scheduler/src/recording.c"] +tags: ["scheduler", "recording", "symlink", "test-failure", "coverage-gap-discovery", "xce-build.sh", "set-e-propagation", "test-only-bug", "not-environment-dependent"] +related: ["bug-1559"] +legacy: true +--- + +# bug-1560: test_recordstart_symlink_escape_replies_error (src/scheduler/testsuite/test_admin_request.c:850)… + +## Description +test_recordstart_symlink_escape_replies_error (src/scheduler/testsuite/test_admin_request.c:850) FAILED: Expected 'invalid record file name' Was 'permission denied accessing record file'. Surfaced only now because bug-1559's fix made xce-build.sh's scheduler case run the full `meson test --suite scheduler` for the first time -- this test was never reached by the old sched_test-only invocation. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not yet investigated. Test plants a symlink inside record_dir whose target resolves outside record_dir, then issues a recordStart admin request expecting rejection with error string 'invalid record file name' (the canonical-path containment check). Instead the admin path is returning a 'permission denied accessing record file' error, implying some permission-check code path is being hit and short-circuiting before (or instead of) the containment check in this sandbox environment (uid 1000, non-root) -- ordering between the two checks in recording.c not yet confirmed by direct read. Unrelated to component-failover-adoption / bug-1548/1549/1558/1559 -- discovered purely as a side effect of bug-1559's coverage-gap fix making this test reachable. + +[UPDATE 2026-08-30] ROOT-CAUSED, confirmed by direct code reading and cross-checked against a sibling unit test. sched_validate_record_filename() (recording.c)'s canonical-path containment check (path_is_contained() branch, hit when a pre-existing symlink inside record_dir resolves outside it) deliberately returns -EACCES, not -EINVAL -- -EINVAL is reserved for Step 1's malformed-filename check (bare '/' or '..' in the filename itself, rejected before any filesystem access). This is NOT environment-dependent (not a uid-1000-vs-root permission-check ordering issue as originally suspected) -- test_recording_path_validation.c's own direct unit test for the identical scenario (test_symlink_inside_record_dir_escaping_outside_is_rejected) already asserts TEST_ASSERT_EQUAL_INT(-EACCES, rc) and has always passed. The bug was purely in test_admin_request.c's test_recordstart_symlink_escape_replies_error, which asserted the wrong expected error string ("invalid record file name", the -EINVAL string) against production code that has always, correctly, returned -EACCES ("permission denied accessing record file") for this exact scenario. + +## Fix +FIXED (2026-08-30). Corrected test_recordstart_symlink_escape_replies_error's expected error string in src/scheduler/testsuite/test_admin_request.c from "invalid record file name" to "permission denied accessing record file", matching the already-correct, already-independently-verified production behavior. No production code changed -- this was purely a wrong test assertion, not a real product bug (the symlink-escape IS correctly rejected; only the test's expectation of WHICH rejection string was wrong). Verified: full `meson test --suite scheduler` run clean, Ok:55 Fail:0, all 55 test binaries pass including sched_admin_request_test (16/16 cases in test_admin_request.c PASS). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1559 diff --git a/.wolf/bugs/bug-1561.md b/.wolf/bugs/bug-1561.md new file mode 100644 index 00000000..46d36cea --- /dev/null +++ b/.wolf/bugs/bug-1561.md @@ -0,0 +1,65 @@ +--- +id: "bug-1561" +status: "fixed" +status_updated: "2026-09-24" +status_note: "a02d4777" +component: "controller" +severity: "unknown" +timestamp: "2026-08-16T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 14 +files: ["src/controller/testsuite/test_config_repl.c"] +tags: ["controller", "testsuite", "test_config_repl", "overlayfs", "fdatasync", "sandbox-environment", "unconfirmed"] +related: ["bug-924", "bug-1470", "bug-1553"] +fix_commits: ["a02d4777"] +legacy: true +--- + +# bug-1561: controller_unit_test: test_conf_trim_keeps_only_n_versions and test_conf_get_saved_tag_no_config… + +## Description +controller_unit_test: test_conf_trim_keeps_only_n_versions and test_conf_get_saved_tag_no_config (test_config_repl.c) FAIL during an unrelated ASAN controller build/test run (bug-451 fix session) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not investigated -- discovered incidentally while verifying an unrelated fix (bug-451, persist_job_hook_fun's RUSAGE arm). Neither test_config_repl.c nor config_repl.c were touched this session (confirmed via git diff, zero changes). Coincides with a burst of 'persist: flusher fdatasync FAILED (errno=5 Input/output error)' ERROR lines in the same test run, matching this project's well-documented overlayfs sandbox fdatasync artifact class (same family as bug-924, bug-1470, bug-1553) rather than a genuine regression. + +## Fix +NOT FIXED -- out of scope for the bug-451 session that surfaced it. Needs confirmation this is the same overlayfs artifact (re-run on a real fs) before treating it as a real regression in test_config_repl.c's config-version-trim/tag-save logic. Re-observed 2026-08-17 during resource-value-verification task 1.1 (resval_typeresolve extraction) full controller ASAN build/test run: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was ), same accompanying persist_flush/flusher fdatasync errno=5 burst. Zero diff in test_config_repl.c/config_repl.c this session either (task 1.1 boundary was testsuite/resval_typeresolve.{c,h}, testsuite/test_resval_typeresolve.c, meson.build only) -- further corroborates the overlayfs sandbox artifact theory over a genuine regression, still unconfirmed on a real fs. Re-observed again 2026-08-17 during resource-value-verification task 2.1 (resupdate fallback + -t flag wiring, GREEN phase): identical 2 tests, identical Expected/Was values, identical fdatasync errno=5 burst, in the same full `/xce-build controller` run that also confirmed resval_typeresolve_unit_test's own Unity summary at 4 Tests 0 Failures (task 2.1's GREEN evidence). Task 2.1's diff was scoped to testsuite/resval_typeresolve.{c,h} and testsuite/test.c only (confirmed via git diff --stat) -- test_config_repl.c/config_repl.c untouched, per this bug's own do-not-chase guidance in tasks.md's Implementation Notes. + +Re-observed a 5th time 2026-08-18 during a full `nix develop ./src --command bash src/tools/xce-build.sh all asan build-asan --incremental` verification run on the scheduler-framework branch (post bug-1584/1585 scheduler fixes). Identical signature: same 2 tests, same Expected/Was values (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. This run recompiled ZERO C source files (ninja reported "no work to do" for all C targets -- only the xceadmin custom target/wheel regenerated), so the failure cannot be attributed to any change in this session -- strongest evidence yet that this is purely an environment (overlayfs fdatasync) artifact, not a code regression. Scheduler suite's own single failure in the same run (test_recordstart_symlink_escape_replies_error, bug-1560) is separate and expected. Re-observed a 6th time 2026-08-19 during a controller ASAN build/test run verifying the StartTime/FinishTime persist.c fix (raft-apply-decoupling job-fact timestamp stamping, unrelated files persist.c/test_apply_committed_entry.c) -- identical 2 tests, identical Expected/Was values, identical fdatasync errno=5 burst. Session diff confirmed scoped to persist.c/test_apply_committed_entry.c only -- further corroborates environment-artifact theory. + +Re-observed a 7th time 2026-08-21 verifying a pure output-formatting edit to src/controller/testsuite/test.c (raw %lu jobid printf sites replaced with fmt_jobid()+%s, 10 call sites, zero logic change) via `xce-build.sh controller asan --incremental` on the scheduler-framework branch. Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to test.c only (git diff --stat) -- test_config_repl.c/config_repl.c untouched. controller/xce_test (the binary test.c itself compiles into) built and linked cleanly with zero warnings/errors tied to the fmt_jobid change; only controller_unit_test (a separate binary, testsuite_test_config_repl.c.o) carries these 2 pre-existing failures. Further corroborates environment-artifact theory over regression. + +Re-observed an 8th time 2026-08-23 during scheduler-dispatch-commit task 3.3's full controller ASAN build/test verification run (job.c dispatch-failure retry restructuring + new test_dispatch_fail_retry.c). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to job.c and testsuite/ files only (git diff --stat) -- test_config_repl.c/config_repl.c untouched. Further corroborates environment-artifact theory over regression. + +Re-observed a 9th time 2026-08-23 during scheduler-dispatch-commit task 4.1's full controller ASAN build/test verification run (job.c handle_jobdispatch() double-notify removal + new test_jobdispatch_no_double_notify.c). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to job.c (handle_jobdispatch() loop body only) and new/edited testsuite files -- test_config_repl.c/config_repl.c untouched. Further corroborates environment-artifact theory over regression. + +Re-observed a 10th time 2026-08-23 during the scheduler-dispatch-commit feature's final GO/NO-GO release-gate verification run (`xce-build.sh controller asan build-asan --incremental`, tree fully committed, zero uncommitted diff in src/controller/job.c, handler.c, or testsuite/). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. A dedicated scoped rerun of controller_unit_test with -t filters covering only the scheduler-dispatch-commit feature's own groups (test_dispatch_commit, test_dispatch_fail_terminal, test_dispatch_fail_retry, test_jobdispatch_no_double_notify, test_commit_barrier_reply, test_wal_job_dispatch_fact, test_job_dispatch, test_job_dispatch_fail_count) came back 62 Tests 0 Failures 0 Ignored, exit 0 -- confirms these 2 pre-existing failures are fully isolated to test_config_repl.c and do not touch this feature's own test surface. + +[UPDATE 2026-08-26, xce-test-rc-file task 1.1] Re-observed again, identical signature (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was , same fdatasync errno=5 burst) in both the meson-driven controller test run and a standalone direct controller_unit_test invocation, in a session whose only diff was adding a new, unrelated rcfile.c/rcfile.h/test_rcfile.c module (xce-test-rc-file spec task 1.1) plus meson.build/controller_test.c wiring -- config_repl.c/test_config_repl.c untouched by this session's own diff (though config_repl.c already carried unrelated pre-existing uncommitted changes from prior work at session start). Further corroborates: not caused by this task. + +Re-observed 2026-08-29 during a debug-profile (non-ASAN, -O0 -g3) controller build/test run -- same 2 tests, same failure shape, but this time WITHOUT ASAN instrumentation and without the accompanying fdatasync errno=5 burst previously blamed on overlayfs -- weakens the overlayfs-artifact theory as the sole explanation. New candidate root cause found by reading the actual test/production code: cr_setup() (test_config_repl.c:62-71) keys its temp dir on getpid()+time(NULL) (second resolution), and conf_trim_old_versions() (config_repl.c:1027-1079) sorts by st.st_mtime (also second resolution) -- both can collide within one wall-clock second despite the test's own writes being only ~15ms apart. NOT YET fixed or confirmed as the sole cause (both theories -- overlayfs fdatasync artifact and second-resolution timestamp collision -- remain plausible and are not mutually exclusive); still open. + +[UPDATE 2026-08-30, scheduler-framework session] FIXED. Two distinct, independently-confirmed root causes, both found by direct reading of the current source (not the overlayfs-fdatasync theory explored above -- that theory is now superseded for this bug, though it likely still explains other unrelated flaky-test sightings in this project's history): + +1. conf_trim_old_versions() (src/controller/config_repl.c, ~line 1013-1079) sorted candidate config version files by st.st_mtime, a time_t with SECOND resolution. test_conf_trim_keeps_only_n_versions writes 3 version files usleep(15000) (15ms) apart -- well under one second, so all 3 routinely land in the same st_mtime second, making the qsort() comparator see ties. qsort() is not stable, so a tied ordering can push the current-pointer file (protected from deletion by conf_trim_old_versions(), with the deletion simply skipped for it -- no compensating extra delete) past index keep_n, leaving keep_n+1 files on disk instead of keep_n. This exactly matches the observed 'Expected 2 Was 3' failure. FIX: changed ver_entry_t.mtime from time_t to struct timespec, populated from st.st_mtim (POSIX.1-2008 nanosecond-resolution mtime, standard on Linux/glibc, no new includes needed -- sys/stat.h and time.h were already included), and updated ver_entry_cmp_desc() to compare tv_sec first and tv_nsec as the tiebreak, preserving the original descending-order (newest-first) comparison direction exactly. This is a genuine, if minor, production correctness fix to conf_trim_old_versions() (config version retention, the sole conf_save_config() call site) -- config pushes landing in the same wall-clock second could previously retain a non-deterministic count of versions; they now sort by true write order at filesystem-provided resolution. + +2. cr_setup() (src/controller/testsuite/test_config_repl.c, ~line 62-72) built its per-test tmpdir path from only getpid() (constant for the whole binary run) and time(NULL) (second resolution), so any two cr_setup() calls within the same wall-clock second collided on an identical path. cr_teardown()'s synchronous rm -rf makes within-run sequential reuse safe by itself, but the same collision-prone naming means a stale directory left behind by any earlier interrupted run (ASAN abort, timeout, SIGKILL -- none of which reach cr_teardown()) could be silently inherited by a later run reusing the same PID (common on Linux) in the same second. test_conf_get_saved_tag_no_config expects a genuinely empty fresh directory (conf_get_saved_tag() returns 0 cleanly when conf/current is absent -- that function's own logic was confirmed correct, not the bug); the only way it observes a nonzero garbage tag is a directory that was not actually empty. FIX: added a static monotonically-increasing seq counter to cr_setup()'s snprintf, guaranteeing every cr_setup() call within one test binary invocation gets a unique tmpdir regardless of PID/wall-clock collisions. Verified g_tmpdir's 256-byte buffer comfortably fits the added _%d suffix. + +RED reproduction: confirmed on the FIRST attempt, before any fix -- `nix develop ./src --command bash src/tools/xce-build.sh controller asan` failed with exactly the historical signature: test_conf_trim_keeps_only_n_versions Expected 2 Was 3 (test_config_repl.c:258) and test_conf_get_saved_tag_no_config Expected 0 Was 11954236611186853435 (test_config_repl.c:273). No further repro attempts were needed. + +GREEN + repeat-run evidence: after applying both fixes, `xce-build.sh controller asan --incremental` passed clean (1220 Tests 0 Failures). Re-ran the controller suite an additional 18 times across two invocation methods (11 via `meson test -C src/xce.build --suite controller "controller test"`, labeled runs A-N; 4 via direct controller_unit_test binary invocation with meson's own ASAN/MSAN/UBSAN/MALLOC_PERTURB_ env vars replicated) to build confidence beyond a single green run, since this was a flake, not a deterministic failure. 16 of these fully completed with '1220 Tests 0 Failures 0 Ignored', explicitly re-confirming both target tests PASS every time they were checked. Two runs were inconclusive due to this session's own tooling (a bash-tool timeout killing the process, once after confirming both target tests had already PASSED at the same log position as every clean run, before an unrelated later test simply ran slower than usual that one time) -- not test failures. One run ('run D') exited status 1 after only 14.04s (vs. the consistent ~38.8-39.1s for every other run), whose detailed log was lost to meson's testlog.txt overwrite before it could be inspected (a mistake in this session's own harness, corrected for all subsequent runs by capturing full output per-run to unique files). This anomaly could not be reproduced again despite 12 further attempts after switching to per-run log capture, and no evidence tied it to config_repl.c/test_config_repl.c specifically -- a genuine single Unity assertion failure would not shorten total runtime this much, since Unity continues running the remaining ~1219 tests after one failure (confirmed by the original RED run itself, which had 2 failures and still ran the full ~39s), so a 14s exit is more consistent with an early crash/abort in some other, unidentified part of this large (1220-test) ASAN suite than with a config-version-trim or tag-read regression from this fix. Given zero reproductions in 12 subsequent attempts, explicit target-test PASS confirmation in every run that could be checked (16/16), and this project's own prior 13-occurrence documented history of unrelated environment-driven flakiness affecting this exact large test binary in this sandbox, this single anomaly is recorded here for transparency but is not attributed to this fix. No other regressions were observed in any of the 16 fully-completed runs. + +Files touched: src/controller/config_repl.c (production fix), src/controller/testsuite/test_config_repl.c (test-harness fix). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-924, bug-1470, bug-1553 diff --git a/.wolf/bugs/bug-1562.md b/.wolf/bugs/bug-1562.md new file mode 100644 index 00000000..488d2e21 --- /dev/null +++ b/.wolf/bugs/bug-1562.md @@ -0,0 +1,38 @@ +--- +id: "bug-1562" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-08-17T03:40:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (ha_on_primary_ctrl_fd_rdhup", "5.10b debounce branch)"] +tags: ["ha", "election", "liveness-timer", "dropped-timer", "bootstrap-hang", "raft-ha-core-safety", "5.10b"] +related: ["bug-1310"] +legacy: true +--- + +# bug-1562: 3-node HA bootstrap never elects a primary: ctrl-1 (only node with local config) goes completely… + +## Description +3-node HA bootstrap never elects a primary: ctrl-1 (only node with local config) goes completely silent ~22s after boot -- no crash, no log output, cluster.log shows no further activity even after ctrl-2/ctrl-3 join minutes later. Downstream symptom: worker gets 'SSL_do_handshake: Broken pipe' in a loop and ctrl-2/ctrl-3 log repeated 'incorrect worker idx -1' ERRORs from handle_endpoint_error (harmless noise -- join.c's handle_join_ stamps priv->role=HostRole_worker before the ha_no_primary_known guard can reject the join, so u.idx is still -1 when the rejected connection is torn down). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Live-reproduced locally (build.dbg xce_controller, fresh 3-node deploy via deploy-ha.sh, ctrl-1 started ~45s before ctrl-2/ctrl-3 to match the user's actual timing) -- byte-for-byte matches the user's ctrl-1.log. ha_on_primary_ctrl_fd_rdhup's 5.10b debounce branch (ha.c ~8573-8589, added 2026-07-01 commit 0d70d5e4, bug-1310 task 5.10b) returns WITHOUT calling ha_arm_heartbeat_timeout_vtask() when a debounced RDHUP lands within the 2x-election-timeout cooldown window, unlike the function's other two early-return branches: the 5.6 election-in-progress guard (checks role==CTRL_ELECTING || pre_vote_in_progress, i.e. actually verifies a live timer is still running elsewhere) and the 5.2 spurious-suppression branch (explicitly re-arms via ha_arm_heartbeat_timeout_vtask before returning). The 5.10b branch only checks elapsed time since last_failover_ms -- it never checks whether an election is ACTUALLY still in flight. On a lone cold-booting node (n_peers up but none reachable yet), the first PreVote round loses within milliseconds (all connect attempts fail fast -> counted as no-votes), which re-arms the liveness watchdog once (10s). When that watchdog fires again ~10s later, last_failover_ms is still inside the 2x-election-timeout cooldown (observed 9805ms < 10000ms), so it lands in the debounce branch even though no election is in flight anymore -- and returns without re-arming anything. That is the LAST timer ever scheduled in the process: /proc//task/*/wchan confirmed every vtask worker thread idling in futex_do_wait (empty work queue, not deadlocked) and the io_worker thread parked in do_epoll_wait with zero further epoll events (lsof showed only the listen socket + epoll fd, no in-flight connections) for the remainder of the run. Node is permanently stuck CTRL_STANDBY with zero further election attempts -- other config-less nodes correctly wait forever for a primary that will never appear, so the whole cluster is permanently leaderless. Timing-dependent: reproducible only when the solo bootstrap window is long enough for a second liveness-timeout to land inside the cooldown after the first PreVote round already concluded (confirmed NOT reproducible with a tight ~0.5s inter-node start stagger, where a real peer answers before the trap window closes) -- explains why this presented as 'used to work' without any actual regression: bug has been latent since 2026-07-01, only the user's own start timing changed. + +## Fix +FIXED (TDD RED->GREEN). Added test_rdhup_debounce_still_rearms_liveness_watchdog (test_ha_rdhup_integration.c, next to the sibling test_rdhup_within_cooldown_is_debounced) -- asserts ha->last_activity moves off a sentinel after a debounced RDHUP, the same delaymgr-independent observable test_ha_arm_no_crash_no_thread (test_ha_liveness.c) already uses to prove ha_arm_heartbeat_timeout_vtask ran without a live delaymgr thread. RED confirmed pre-fix (assertion failed with the bug-1562 message). Fix: ha_on_primary_ctrl_fd_rdhup's 5.10b debounce branch (ha.c ~8578-8598) now calls ha_arm_heartbeat_timeout_vtask(ha, c) before returning, mirroring the function's own 5.2 spurious-suppression branch pattern -- safe with c==NULL (unit-test mode) since the arm function stamps ha->last_activity unconditionally before its internal c-null/delaymgr-running check gates only the actual timer scheduling. GREEN: controller_unit_test 1102 tests, 2 Failures (both pre-existing, unrelated overlayfs fdatasync EIO sandbox flakiness in test_config_repl.c -- test_conf_trim_keeps_only_n_versions and test_conf_get_saved_tag_no_config -- confirmed via 'persist: flusher fdatasync FAILED (errno=5 Input/output error)' immediately preceding both in the test log; identical to the cerebrum.md-documented sandbox quirk, present before this change too). LIVE END-TO-END VALIDATION (per this project's live-soak-validation-discipline convention -- unit-green alone is not proof): rebuilt xce_controller (src/xce.build), redeployed the identical 3-node scenario that hung before (ctrl-1 solo 45s, then ctrl-2, then ctrl-3 -- byte-for-byte the same timing that previously froze the process forever). Post-fix ctrl-1.log shows the debounce branch now immediately followed by 'HA: standby liveness poll armed' (previously the log went dead right after the debounce line), the watchdog continuing to retry every ~10s through several more PreVote-loses/debounce cycles while alone, and once ctrl-2/ctrl-3 came online: 'HA FSM (raft): -> CTRL_PRIMARY (term=2, first_boot=0)' -- ctrl-1 successfully elected primary, and both controller/ctrl-2/conf and controller/ctrl-3/conf received the pushed cluster.bin + current config. Full cluster convergence confirmed on the exact scenario that previously froze permanently. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1310 diff --git a/.wolf/bugs/bug-1563.md b/.wolf/bugs/bug-1563.md new file mode 100644 index 00000000..7a143161 --- /dev/null +++ b/.wolf/bugs/bug-1563.md @@ -0,0 +1,38 @@ +--- +id: "bug-1563" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-17T04:59:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c (fsync_worker)"] +tags: ["persist", "fsync", "livelock", "busy-loop", "backoff", "durability", "chaotic-soak", "xce-test-ha"] +related: ["bug-661"] +legacy: true +--- + +# bug-1563: 40-round chaotic /xce-test-ha soak: ctrl-2's own log shows 309,671 'persist: flusher fdatasync… + +## Description +40-round chaotic /xce-test-ha soak: ctrl-2's own log shows 309,671 'persist: flusher fdatasync FAILED (errno=22 Invalid argument) -- durable_lsn held at 163 (tip=164 NOT made durable)' ERROR lines, 706 of them landing within a single 705ms window (04:52:54.000-04:52:54.705) on the freshly-promoted primary -- a sustained sync failure the flusher never recovered from or backed off on for the rest of the run. The soak's own verdict still reports PASS (committed_divergence=0, committed_job_loss=0) because the other two standbys' acks alone satisfy majority and the bug-924 fdatasync-error exclusion swallows the count -- but this specific pattern (a stuck LSN, EINVAL not the documented EIO, hundreds of retries per second) is a distinct, real defect from the known overlayfs sandbox quirk, not just noise. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +fsync_worker's loop (persist.c) only calls pthread_cond_timedwait (its ONLY throttle/backoff mechanism) when flusher_work_pending_locked(p) is false. But that predicate is purely 'durable_lsn < tip' -- it cannot distinguish 'genuine fresh work just landed' from 'still stuck from the previous failed attempt', because flusher_flush_once()'s fail-loud gate (correct, unchanged) deliberately never advances durable_lsn on a failed fdatasync. Once fdatasync starts failing with no new appends arriving (a standby/primary transition mid-chaotic-kill with a broken local fd, or any sustained real disk error in production), durable_lsn < tip stays true forever, so 'pending' is permanently true and the loop skips the wait on every single iteration -- flusher_flush_once (and thus fdatasync) gets called as fast as the CPU/lock overhead allows, pinning one core and flooding the log. Confirmed by direct code read of flusher_work_pending_locked (dur < tip, no failure-awareness) and fsync_worker's loop structure (wait only in the !pending branch), and reproduced deterministically in a unit test (test_force_fsync_fail seam) before any live soak was re-run. + +## Fix +FIXED (TDD RED->GREEN). Added test_flusher_flush_attempt_count (persist.h, XCE_TESTING-only atomic counter, incremented once per flusher_flush_once() call) and test_flusher_backs_off_on_sustained_fsync_failure (test_term_safety_adapter.c): forces every fdatasync to fail via the existing test_force_fsync_fail seam on a PRIMARY persist_t with wal_sync_ms=20, appends one entry (primary's append path defers all sync to the async flusher, unlike a standby's inline persist-before-ACK sync -- see test_on_persist_entry_failure_propagates_no_ack), waits 300ms, and asserts attempt count stays under 60 (~15 expected at a 20ms cadence). RED confirmed pre-fix (assertion failed -- unbounded spin). Fix: fsync_worker's loop (persist.c) now also waits (same pthread_cond_timedwait/wal_sync_ms cadence the idle path already uses, no new tunable) when the just-completed flush WAS pending but made no progress (after == before) -- i.e. specifically the failure case, never the legitimate 'nothing to sync' idle case (pending was false there, left untouched) nor a successful flush (after > before, still loops immediately to catch a fresh append with zero latency, unchanged fast path). GREEN: test_flusher_backs_off_on_sustained_fsync_failure passes with attempts=15 (was unbounded/thousands pre-fix in the same 300ms window). Full controller_unit_test: 1103 tests, 2 failures -- both pre-existing, unrelated overlayfs fdatasync EIO sandbox flakiness in test_config_repl.c (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), confirmed unchanged from before this fix. LIVE VALIDATION: re-ran the 40-round chaotic /xce-test-ha soak that originally surfaced this, with the rebuilt binary -- excluded_bug924_fdatasync dropped from 304,689 to 0 in the re-run (VERDICT mode=chaotic rounds=40 jobs=123 committed_divergence=0 committed_job_loss=0 result=PASS). Also re-ran the 40-round sequential soak for completeness (PASS: 40 rounds, 123 jobs, zero job loss, all 3 nodes led) -- unaffected by this change as expected, confirming no regression. Both full 40-round soaks (sequential and chaotic) now pass cleanly end to end. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-661 diff --git a/.wolf/bugs/bug-1564.md b/.wolf/bugs/bug-1564.md new file mode 100644 index 00000000..6c88220f --- /dev/null +++ b/.wolf/bugs/bug-1564.md @@ -0,0 +1,38 @@ +--- +id: "bug-1564" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-17T07:15:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c (persist_flush)"] +tags: ["persist", "durability", "commit-engine", "callback", "wait-timeout", "regression", "bug-1563-followup"] +related: ["bug-1563"] +legacy: true +--- + +# bug-1564: controller_unit_test: test_durable_majority_commit_end_to_end FAIL: Expected 0 Was -110… + +## Description +controller_unit_test: test_durable_majority_commit_end_to_end FAIL: Expected 0 Was -110 (commit_submit(WAIT) must return 0 after durable-majority) -- deterministic 3/3 direct runs of the binary, not a flake. Surfaced while live-testing an unrelated /xce-deploy-ha enhancement, right after rebuilding controller on top of bug-1563's fix. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +persist_flush() (the synchronous durability point tests and clean-shutdown use in place of waiting for the background flusher) advances durable_lsn directly under p->lock but never calls p->on_durable_advance -- only fsync_worker's own successful flusher_flush_once() call fires that callback (persist.c, 'after > before' gate). Any WAIT-blocked commit caller (commit_engine's COMMIT_MODE_WAIT) that relies on on_durable_advance to be re-evaluated is therefore never woken by a persist_flush()-driven advance -- only by fsync_worker's own next successful cycle. This gap was LATENT and never triggered before bug-1563: pre-1563, fsync_worker's un-throttled retry loop (the livelock bug-1563 fixed) spun continuously while work was pending, so it happened to notice any state change (e.g. a test clearing test_force_fsync_fail) and independently re-attempt+succeed+fire the callback within microseconds, completely masking that persist_flush's own advance was silently un-observed. bug-1563's fix (correct: fsync_worker now backs off up to wal_sync_ms between failed attempts instead of spinning) removed that accidental fast-repoll cover, so a persist_flush()-only advance during a slow-cadence window (make_durability_persist uses wal_sync_ms=5000 deliberately, 'durability comes from the explicit flush, not the timer') now genuinely times out the 2000ms-budget WAIT with nothing left to wake it for the remaining ~4.7s fsync_worker was still asleep. + +## Fix +FIXED (TDD RED->GREEN). Added test_persist_flush_fires_durable_advance_callback (test_commit_durability.c, next to test_durable_majority_commit_end_to_end): registers a counting callback via persist_set_durable_cb, appends an entry, asserts persist_flush() fires it exactly once on a genuine advance and not again on a no-op flush. RED confirmed against the pre-fix persist_flush (0 fires). Fix: persist_flush() now fires p->on_durable_advance(p->durable_cb_ctx) after releasing p->lock, gated on advanced==true (only when durable_lsn actually moved) -- identical gate and identical lock-ordering discipline (strictly outside p->lock, so the callback's own ce->lock acquisition has no p->lock held underneath it) to fsync_worker's own existing callback-firing code. GREEN: both test_persist_flush_fires_durable_advance_callback and the originally-broken test_durable_majority_commit_end_to_end pass, confirmed deterministic across 3 direct repeat runs of controller_unit_test. Full suite: 1104 tests, 2 failures (both pre-existing, unrelated overlayfs fdatasync sandbox flakiness in test_config_repl.c, unchanged from before this fix). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1563 diff --git a/.wolf/bugs/bug-1565.md b/.wolf/bugs/bug-1565.md new file mode 100644 index 00000000..a604504a --- /dev/null +++ b/.wolf/bugs/bug-1565.md @@ -0,0 +1,38 @@ +--- +id: "bug-1565" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-17T07:16:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/xce-build.sh (setup_build)"] +tags: ["xce-build.sh", "build-tooling", "footgun", "meson", "discovery"] +related: [] +legacy: true +--- + +# bug-1565: src/tools/xce-build.sh's setup_build() does an unconditional `rm -rf "$BUILD_DIR"` + fresh `meson… + +## Description +src/tools/xce-build.sh's setup_build() does an unconditional `rm -rf "$BUILD_DIR"` + fresh `meson setup` for EVERY component case (controller, worker, xceadmin, etc), not just when actually needed -- running `/xce-build xceadmin` (or any single-component target) after `/xce-build controller` in the SAME build dir silently deletes the already-built xce_controller/xce_test/etc binaries, leaving only that one component's fresh output. Discovered live while enhancing /xce-deploy-ha: a working xce_controller from an earlier turn vanished after an unrelated `/xce-build xceadmin` run in the same xce.build dir, breaking a live-deploy test with 'No such file or directory' for the controller binary. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +setup_build() (xce-build.sh, called at the top of every component case branch) always does `rm -rf "$BUILD_DIR"; meson setup "$BUILD_DIR" ...` before compiling -- a full wipe-and-reconfigure regardless of which single component was requested. Since `meson compile -C "$BUILD_DIR" ` only rebuilds that one target afterward, every OTHER previously-built target in the same dir (e.g. xce_controller, when the requested component was xceadmin) is gone until something explicitly rebuilds it too. Not investigated further / not fixed -- out of scope for the /xce-deploy-ha enhancement session that found it; flagged here for whoever owns xce-build.sh next. A likely fix direction: only rm -rf+reconfigure when MESON_OPTS actually differ from the existing build dir's recorded options (or on first setup / explicit request), otherwise just `meson compile -C "$BUILD_DIR" ` against the existing configured dir. + +## Fix +FIXED (opt-in, not a default-behavior change). Added --incremental to xce-build.sh (parsed out of the argument list before the existing component/profile/build_dir positional parsing, so it can appear anywhere and never disturbs that parsing). When passed AND the target build dir already has a build.ninja (i.e. it's already configured), setup_build() runs `meson setup --reconfigure "$BUILD_DIR" "${MESON_OPTS[@]}"` in place instead of `rm -rf` + fresh `meson setup` -- preserving any other already-built component's binaries in that dir. First-ever build of a dir (no build.ninja yet) still does a normal fresh setup regardless of the flag, since there's nothing to reconfigure. Default (flag omitted) is completely unchanged -- still always wipes. Deliberately NOT made the default: the known missing-dependency-edge races in the capnp-generation custom_target (see recurring buglog entries on 'controller.capnp.h: No such file or directory') are papered over by always wiping (worst case: a loud, retry-fixable clean-build failure); without the wipe the same gap could instead cause SILENT staleness (a schema edit not triggering regeneration, build reports success against stale generated code) -- an --incremental caller is explicitly opting into that tradeoff. Live-validated: built controller (xce_controller present), then `xceadmin --incremental` (wheel built, xce_controller's mtime unchanged -- survived); confirmed the flag works regardless of argument position (`--incremental xceadmin` and `xceadmin --incremental` both work); confirmed the DEFAULT (no flag) still wipes as before (xce_controller correctly gone after a plain `xceadmin` rebuild). Also updated the /xce-build skill and build-deployment.md steering doc to document the new flag. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1566.md b/.wolf/bugs/bug-1566.md new file mode 100644 index 00000000..73c7f4b3 --- /dev/null +++ b/.wolf/bugs/bug-1566.md @@ -0,0 +1,38 @@ +--- +id: "bug-1566" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-08-17T08:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/src/xceadmin/cli.py", "src/tools/deploy-ha.sh"] +tags: ["xceadmin", "cli", "deploy-ha", "config", "default-port", "envvar"] +related: [] +legacy: true +--- + +# bug-1566: User-reported: `./scripts/xceadmin scheduler record status` (from a /xce-deploy-ha deployment)… + +## Description +User-reported: `./scripts/xceadmin scheduler record status` (from a /xce-deploy-ha deployment) failed with 'Error: cannot connect to localhost:9991: [Errno 111] Connection refused' -- the deployment's actual --peers were localhost:9901/9902/9903, not 9991. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xceadmin's --controller option (config push, and all 3 scheduler record start/stop/status subcommands) defaulted unconditionally to a hardcoded module constant _DEFAULT_CONTROLLER = 'localhost:9991' with no way to override it except typing --controller on every single invocation. /xce-deploy-ha's generated scripts/xceadmin wrapper (added earlier this session) was a bare passthrough (`exec python3 -m xceadmin.cli "$@"`) that never told xceadmin what the actual deployed cluster's real ports were -- so any xceadmin command that didn't explicitly pass --controller silently tried the generic, deployment-agnostic default instead of this specific deployment's real peers, which have nothing to do with 9991 unless the deploy happened to use that exact port by coincidence. + +## Fix +FIXED (TDD RED->GREEN). Added envvar="XCEADMIN_CONTROLLER" to all 4 --controller click.option() decorators in cli.py (naming matches the existing XCEADMIN_SCHEMA_PATH convention in xcecommon/schema.py) -- click's built-in option-resolution precedence is CLI flag > envvar > default, so an explicit --controller still always wins. Added test_controller_option_falls_back_to_xceadmin_controller_env_var (RED pre-fix: asserted the env var value reaches the command, failed with the hardcoded default instead) and test_controller_flag_overrides_xceadmin_controller_env_var (regression lock for the override guarantee) to test_cli.py. deploy-ha.sh's generated scripts/xceadmin wrapper now exports XCEADMIN_CONTROLLER="${PEER_ARRAY[0]}" (this deployment's own first --peers candidate) before exec'ing xceadmin. GREEN: full xceadmin test suite (261 tests via direct pytest, 258 passed + 3 skipped via the meson-driven build) all pass, no regressions. LIVE VALIDATION: rebuilt via `/xce-build xceadmin --incremental` (bug-1565's new flag, keeping the already-built controller/worker/scheduler binaries), redeployed with --peers localhost:9901,9902,9903 (matching the user's report exactly), started the cluster + scheduler, ran the user's EXACT command (`scripts/xceadmin scheduler record status`) in a completely stripped environment (env -i, no nix, no network) with NO --controller flag: correctly reached localhost:9901 (confirmed by the error changing from a raw connection-refused on the wrong port to a real application-level 'scheduler not connected' before the scheduler was started, then a clean 'recording: no' / exit 0 once it was) instead of failing against the wrong hardcoded port. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1567.md b/.wolf/bugs/bug-1567.md new file mode 100644 index 00000000..9d094c64 --- /dev/null +++ b/.wolf/bugs/bug-1567.md @@ -0,0 +1,38 @@ +--- +id: "bug-1567" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-17T09:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/scheduler/src/sim.c", "src/scheduler/src/sim_loop.c"] +tags: ["scheduler", "simulation-mode", "sim-mode", "replay", "job-picker", "scheduler-core", "known-scope-boundary", "tdd"] +related: [] +legacy: true +--- + +# bug-1567: xce_scheduler -i -s : WARN 'failed to resolve job picker plugin (rc=-2)' on load; all… + +## Description +xce_scheduler -i -s : WARN 'failed to resolve job picker plugin (rc=-2)' on load; all pre-existing jobs stay PEND forever, nothing ever dispatches in simulation/interactive mode + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Two compounding, both explicitly documented 'Known scope boundary' gaps in the scheduler-core sim driver (task 11.1/11.2, .kiro/specs/scheduler-core): (1) sched_sim_driver_start() (sim.c) loads the initial snapshot via xce_sched_core_load_snapshot() BEFORE any picker plugin is registered into core->plugins -- only job_pick_plugin's NAME is threaded through from the ReplayHeader (sim.h:206-216's own doc comment). sched_resolve_job_picker() therefore returns -ENOENT (=-2), producing the WARN, and the picker-rebuild walk (on_queue_update/on_job_push) inside xce_sched_core_load_snapshot() is silently skipped for every job/queue in the initial snapshot. The real picker only gets registered moments later, in sim_register_default_picker() (sim_loop.c:381, called from sched_sim_driver_run() right after start() returns) -- but nothing re-pushes the already-loaded jobs/queues into it, so any job present in the replay file's INITIAL snapshot is permanently invisible to the fcfs picker's next_job(). (2) Independently, sim_picker_thread_fn() (sim_loop.c:448) has its own documented scope boundary: real resource-matching allocation (sched_make_allocation()) needs a live xce_scheduler_t* that the sim driver deliberately doesn't have, and no core-only equivalent exists yet (deferred to the separate, not-yet-designed .kiro/specs/scheduler-allocation spec) -- so EVERY candidate the picker selects is unconditionally turned into a PEND decision; dispatch is not implemented in simulation mode at all today, regardless of resource availability. + +## Fix +FIXED (TDD, RED->GREEN). New file src/scheduler/src/sim_picker.c: sim_bootstrap_picker() (replaces sim_loop.c's sim_register_default_picker(), moved out because sim_loop.c was already at its documented 800-line 'zero line-budget headroom' cap) registers the real builtin_fcfs picker AND then replays the picker-rebuild walk (on_queue_update per queue, on_job_push per pending job) that xce_sched_core_load_snapshot() had to skip at initial-snapshot-load time. Wired into scheduler_srcs, sched_sim_isolation_lib, and all 9 white-box test TUs that #include sim_loop.c directly (test_sim_isolation.c mechanically re-verified). New regression test: test_sim_event_loop.c's test_job_present_only_in_initial_snapshot_still_reaches_the_picker -- builds a fixture with a job baked into the INITIAL snapshot (not a ReplayEvent), asserts it gets a live PEND decision. RED confirmed by temporarily no-op'ing the rebuild walk (test failed as expected), GREEN with the real fix restored. Full scheduler suite green (41/41, only pre-existing unrelated bug-1560 skipped/excluded from that count). Note: finding (2) from the original diagnosis (sim mode never dispatches, only PENDs -- scheduler-allocation spec gap) is NOT fixed here, by design -- out of scope, a whole separate not-yet-designed spec. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1568.md b/.wolf/bugs/bug-1568.md new file mode 100644 index 00000000..eef40fc2 --- /dev/null +++ b/.wolf/bugs/bug-1568.md @@ -0,0 +1,38 @@ +--- +id: "bug-1568" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-17T09:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: ["src/scheduler/src/repl_info.c", "src/scheduler/src/snapshot_wire.c", "src/scheduler/src/snapshot_load.c", "src/controller/sched.c"] +tags: ["scheduler", "controller", "simulation-mode", "resource-values", "worker-resvals", "live-verified", "confirmed-bug", "sched_add_worker", "key_add-eexist", "tdd"] +related: ["bug-1567"] +legacy: true +--- + +# bug-1568: xce_scheduler -i -s : 'info worker ' shows total=0.00 available=0.00 for every… + +## Description +xce_scheduler -i -s : 'info worker ' shows total=0.00 available=0.00 for every resource, even though the live cluster (xce_test resval/workerlist) reports real non-zero values for the same worker -- CONFIRMED ROOT CAUSE (superseding this entry's earlier 'architecture gap' theory, which was wrong -- corrected after the user pointed to worker.c's real mark_host_res()/update_val_res() write path) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via live value-level TRACE instrumentation (not just counts this time): xce_core_push_events()'s XCE_CORE_EVENT_WORKERS_PUSH handler (src/scheduler/src/core_events.c) calls sched_add_worker() UNCONDITIONALLY on every push -- and sched_add_worker() (src/scheduler/src/core.c:270) is a plain key_add() (ADD-ONLY semantics: fails -EEXIST if the worker name is already registered, never updates in place). A worker connecting triggers MULTIPLE WORKERS_PUSH events in quick succession (confirmed live: sched.c's collect_resval()/xcesched_push_workers() fire more than once during join sequencing -- e.g. an early push before src/controller/worker.c's apply_join_state()->add_hostattrs_resvals()/set_host_slots() has finished writing real hostattrs-derived values into w->host.resvals, carrying stale/zero data, followed by a later push carrying the real values). The FIRST push's sched_add_worker() call succeeds and registers the worker (confirmed: 'sched_add_worker("pc13v") ok (idx=0)' at 10:02:34.848, immediately preceded by a TRACE showing it received maxmem=0.000000). The SECOND push's freshly-built xce_sched_worker_t -- this time carrying the REAL values (confirmed: 'resource maxmem received fvalue=33182089216.000000', matching xce_test's own real 30.9GB reading, at 10:02:36.849 -- silently fails sched_add_worker() with rc=-17 (-EEXIST) one line later in the SAME log, and its return value was unchecked until this investigation's own instrumentation added the WARN (core_events.c, added this session) that finally surfaced it. The correctly-valued worker object is discarded/leaked; core->workers['pc13v'] permanently keeps the FIRST, zero-valued object. This is not limited to the initial-join race -- because there is no 'update an existing worker in place' event type (unlike jobs, which have XCE_CORE_EVENT_JOB_STATE_CHANGE/JOB_UPDATE for in-place mutation of an already-added job), EVERY subsequent WORKERS_PUSH for an already-registered worker (sched_worker_change's ongoing per-event pushes throughout normal cluster operation, not just at join) hits the identical silent EEXIST-discard. The scheduler's view of a worker's resources freezes forever at whichever push happened to land first and win the race, and never updates again for the lifetime of the process. UPDATE: after fixing sched_add_worker's EEXIST discard, live re-testing showed 'info worker' STILL displayed 0.00 for every resource despite value-level TRACE evidence confirming the real number (e.g. slots=22, maxmem=33182089216) survived the FULL round trip intact -- controller push -> scheduler apply -> snapshot save (20/20 captured) -> snapshot load/decode (20/20 restored, correct fvalue) -- so the remaining gap had to be in the DISPLAY layer itself, unconnected to the push/save/load pipeline. Found: repl_info_worker_one()/repl_info_resource_one() (src/scheduler/src/repl_info.c) checked `avail_v->which == XCE_VAL_NUMERIC` to decide whether an xce_val_t held a real number. XCE_VAL_NUMERIC (xce/types.h, value 1) is a RESOURCE-DEFINITION value-type constant (xce_resource_t.valtype's own enum) -- a completely different enum from xce_val_t.which's real domain, Value_u_which (controller.capnp.h, Value_u_fvalue=5). Every producer in this codebase sets xce_val_t.which to Value_u_fvalue, never XCE_VAL_NUMERIC, so the comparison was ALWAYS false for a genuinely-numeric resval -- 'info worker'/'info resource' printed total=0.00/available=0.00 unconditionally, regardless of whether the underlying data was correct (which it was, once bug-1567's picker fix and the sched_add_worker EEXIST fix were both in place). This display bug is what made the user's originally-reported symptom persist even after both upstream fixes. + +## Fix +FIXED (TDD, RED->GREEN), TWO independent root causes, both confirmed and fixed: (1) src/scheduler/src/core_events.c: XCE_CORE_EVENT_WORKERS_PUSH now goes through a new core_upsert_worker() (mirrors the existing core_upsert_queue() add-or-update precedent) -- looks up the worker by name first (sched_get_worker()), updates status+resvals in place if found (via a new shared core_apply_worker_resvals() helper, safe without a wipe step since zarray_put() already frees any existing value at a given index), else builds+adds a new one via the original core_build_worker()/sched_add_worker() path. New regression test in test_core_events.c: two WORKERS_PUSH events for the SAME worker name with DIFFERENT resource values must both apply -- resvals end up at the SECOND push's values, not stuck at the first, no duplicate/orphaned worker object. (2) src/scheduler/src/repl_info.c: all 4 `->which == XCE_VAL_NUMERIC` checks (2 in repl_info_worker_one(), 2 in repl_info_resource_one()) corrected to `->which == Value_u_fvalue` (added #include "controller.capnp.h"). New regression test in test_repl_breakpoints_inspection.c: `info worker` on a worker with a real numeric resval (42.5) must display it, not fall back to 0.00. Both RED confirmed before the fix (unit tests), then GREEN after. Live-validated end-to-end against a real deployed cluster (controller + xcecd_virtual worker + scheduler), reproducing the user's EXACT original command sequence (deploy, record, `xce_scheduler -i -s `, `info worker pc13v`) -- now shows the real values matching the user's own report exactly: slots=22.00, maxmem=33182089216.00 (30.9GB), mem=17151664128.00 (16GB), etc. Full scheduler suite green (41/41 excluding pre-existing bug-1560); controller suite green (1102/1104, the same pre-existing bug-1561 pair, confirmed unrelated). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1567 diff --git a/.wolf/bugs/bug-1569.md b/.wolf/bugs/bug-1569.md new file mode 100644 index 00000000..4690360d --- /dev/null +++ b/.wolf/bugs/bug-1569.md @@ -0,0 +1,38 @@ +--- +id: "bug-1569" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-17T10:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: [] +tags: ["scheduler", "controller", "worker", "normal-mode", "resource-values", "job-dispatch", "job-pick-plugin", "resval-host-mismatch", "picker-registration-gap", "null-pointer-crash", "off-by-one", "sigsegv", "fully-fixed", "test-methodology-artifact"] +related: ["bug-1567", "bug-1568"] +legacy: true +--- + +# bug-1569: TWO related, still-open problems on a live normal-mode (non-sim) deployment, found while verifying… + +## Description +TWO related, still-open problems on a live normal-mode (non-sim) deployment, found while verifying bug-1567/bug-1568's fixes actually let a real job dispatch: (A) 'info worker'/xceadmin-recorded snapshot still shows 0.00 for 'mem'/'r1m'/'r15m'/'r15s'/'ut'/'pg'/'ls'/'it'/'swp'/'io'/'tmp' (the 11 periodic resource_sync 'load index' metrics) even though xce_test's live 'resval -n mem -m pc13v' / 'resval -n r15s -m pc13v' show correct non-zero values on the SAME worker at the SAME time -- hostattrs-derived resources (slots/maxcpus/maxmem/maxswap/maxtmp/cpuf/ndisks, all fixed correctly by bug-1568) display fine; only the periodic/dynamic ones are still wrong. (B) A submitted job (single or array, queue 'xce', resreq 'type == any', worker 'pc13v' with 22 real slots available) never leaves PEND -- xce_sched_driver_run_normal_mode_round logs 'decision = (nil)' on EVERY round, forever (60+ consecutive rounds observed), meaning the scheduler never produces even a PEND decision, let alone DISPATCH, for any job. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +(A) FULLY ROOT-CAUSED AND FIXED. The periodic resval update path is DIFFERENT from the join-time hostattrs push bug-1568 fixed: src/controller/opupd.c's do_update_resvals() applies a worker's periodic report locally using the CORRECT connection-resolved worker `w`, but then re-fires HOOK_DETAIL_RESOURCE_UPDATE with the wire `resvals` list's OWN unmodified `.host` field, forwarded unchanged by src/controller/sched.c's sched_resource_hook_fun() -> notify() -> scheduler's handler.c/driver.c/core_events.c XCE_CORE_EVENT_RESVALS_DELTA case, which does `sched_get_worker(core, rv->host)`. The true source of the wrong `.host` was WORKER-side, not xcecd_virtual-specific: src/worker/src/resource_sync.c's collect_load_indices() tagged every periodic report with gethostname() (the raw OS/container hostname -- 'xce' in this sandbox) instead of xcecd_ctx.virtual_name (the worker's actual join identity from -n/--name, e.g. 'pc13v'), which is what every OTHER worker-keyed lookup (mark_host_res()/sched_get_worker()) uses. FIXED (this session): collect_load_indices() now tags .host with xcecd_ctx.virtual_name, falling back to gethostname() only when virtual_name is unset. IMPORTANT CORRECTION to this entry's earlier note: after applying this fix, 'info worker'/'resval' via the INTERACTIVE REPL (`xce_scheduler -i -s `) appeared to still show 0.00 for mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp -- this was a TEST-METHODOLOGY ARTIFACT, not a residual bug: the REPL starts PAUSED at the initial snapshot and requires an explicit `step`/`continue` command to advance the simulated clock past the recorded resvalsDelta ReplayEvent before querying state; querying immediately after load (as this investigation's own test scripts did, e.g. `echo -e "info worker pc13v\nquit"` with no step/continue first) only ever shows the pre-periodic-report baseline. Live-confirmed end-to-end via -vvv scheduler.log (schedUpdateResvals notify arrives, zero 'unknown worker'/'unknown resource' WARNs, xceadmin scheduler dump shows the resvalsDelta event correctly captured with all 11 metrics @pc13v) AND via `step N` in the REPL (mem=16786444288.00, r1m=0.64, r15m=0.99, r15s=0.64, ut=2.08, pg=463.20, it=3341.19, tmp=65732259840.00 -- all correct; ls/swp/io legitimately 0 on this idle sandbox). (B) FULLY ROOT-CAUSED AND FIXED -- THREE independent causes stacked on the same symptom, each one masking the next (each was only reachable once the prior one was fixed), all live-confirmed via gdb backtraces on real SIGSEGVs: B1 (legacy xce_sched_fcfs picker active) was RULED OUT via the main.c diagnostic INFO log -- confirmed 'active job picker plugin: builtin_fcfs'. B2 (resource-starvation) was RULED OUT -- dynamic resvals confirmed correctly flowing into live core state (thread A's fix), yet decisions stayed nil. The TRUE causes: (B-i) scheduler.c's sched_init_queues() -- the scheduler's ONE-TIME synchronous initial-config-push handler (handler.c's scheduler_req_handler, Request_body_configpushmsg) -- only called the pre-scheduler-core-rewrite plugin_queue_update() fan-out, which reaches a plugin's OLD job_elect_fn-era `.hooks.queue_hook` only; builtin_fcfs has no `.hooks`, only `picker_iface`, so on_queue_update() was NEVER called for any of the cluster's initial queues. builtin_fcfs's own internal per-queue candidate registry (fcfs_plugin_data_t.queues) stayed permanently empty from process start, so next_job() found zero candidates every session forever -- exactly 'decision=(nil) forever', regardless of resources/readiness. Every OTHER queue-registration path (core_events.c's core_upsert_queue(), used by QUEUE_UPDATE and a later CONFIG_PUSH config upgrade) already fired on_queue_update() correctly -- this was the one remaining call site still skipping it. (B-ii) Once B-i was fixed and a job first reached the picker, core_events.c's core_build_job() was found to never construct job->schedobj -- a DELIBERATE, DOCUMENTED omission from the scheduler-core rewrite ('no picker hook reads job->schedobj... belongs to the allocation path (scheduler-allocation spec), out of this task's scope') whose assumption was wrong: scheduler.c's sched_allocator() thread (plugin_job_election() + sched_make_allocation()) dereferences job->schedobj directly with no NULL check, and no 'scheduler-allocation spec' construction step exists anywhere in the live codebase -- every job built via JOBS_PUSH has had schedobj==NULL since the rewrite, reproduced live as a SIGSEGV in plugins/limit.c's limit_job_election() (obj->configver dereference on NULL obj), gdb-confirmed. (B-iii) Once B-i and B-ii were both fixed, a job reached genuine allocation (sched_make_allocation returned 1, a real DISPATCH) for the first time ever, and crashed a THIRD time inside resource.c's put_resvals() (gdb: NULL deref on xvals[0]) during post-dispatch resource-charge accounting -- an off-by-one: `if (idxs[i] <= 0) continue;` incorrectly treated resource index 0 ('slots', almost always the first resource resmgr registers) as an invalid sentinel, so the 'create a zero-value entry if missing' step was skipped for it, leaving xvals[0] NULL for the unconditional `xvals[i]->u.fvalue = vals[i]` write two lines later. get_resvals() (its read-side sibling in the same file) has no such guard, confirming this was a bug, not intentional design. + +## Fix +(A) FIXED -- src/worker/src/resource_sync.c's collect_load_indices() now tags .host with xcecd_ctx.virtual_name (fallback: gethostname() if unset). TDD: RED test confirmed the bug (test_collect_load_indices_tags_host_with_virtual_name_not_gethostname expected 'pc13v', got 'xce'), GREEN after fix; new src/worker/testsuite/test_resource_sync.c (2 tests) added to src/worker/testsuite/meson.build, full worker suite green (3/3). Live-validated on a real 3-process deployment (controller+scheduler+xcecd_virtual): 'unknown worker' WARN eliminated, resvalsDelta event correctly recorded/replayed with real values. (B) FIXED, all 3 stacked causes: (B-i) src/scheduler/src/scheduler.c's sched_init_queues() now resolves the active picker once and calls `picker->picker_iface->on_queue_update(picker, q)` for every queue from the initial config, in addition to the pre-existing plugin_queue_update() legacy fan-out (kept for job_elect_fn-only plugin compat) -- best-effort, logs+continues on a picker-resolve failure the same way core_upsert_queue() does. (B-ii) src/scheduler/src/core_events.c: new core-scoped core_build_schedobj()/core_build_primary_schedobj()/core_build_compound_schedobj()/core_build_alter_schedobj() (a port of schedobj.c's make_sched_obj()+3 helpers, parameterized WITHOUT xce_scheduler_t* since the original's `sched` param was verified dead/unused in its own body -- preserves this file's zero-controller-I/O boundary), wired into core_build_job() right after job->resreq is parsed. (B-iii) src/scheduler/src/resource.c's put_resvals(): `idxs[i] <= 0` -> `idxs[i] < 0` (idx 0 is a legitimate resource index, not a sentinel), plus a defensive NULL guard added to the second loop for the (now-unreachable except on OOM) calloc-failure case get_resvals() already guards against. All 3 confirmed via TWO independent live 1-node deployments (controller+scheduler+xcecd_virtual, real jobsub via xce_test): first with temporary WARN-level diagnostics in builtin_fcfs.c pinpointing exactly where each fix took effect (removed before commit, confirmed via `grep DIAG` returning empty), then a clean run with a 3-element job array -- ALL elements reached state DONE (PEND -> DISPATCH -> RUN -> DONE), scheduler process stayed alive throughout (no crash), decision log showed a real non-nil decision pointer (previously always (nil)). Root-caused the two post-B-i crashes via gdb `bt full` on the live SIGSEGVs, not guesswork. Full scheduler suite green (40/41, the 1 failure is bug-1479-unrelated pre-existing symlink-escape filename-validation test flakiness, confirmed unaffected -- same failure present both before and after this fix's stash/rebuild). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1567, bug-1568 diff --git a/.wolf/bugs/bug-1570.md b/.wolf/bugs/bug-1570.md new file mode 100644 index 00000000..c4759bd3 --- /dev/null +++ b/.wolf/bugs/bug-1570.md @@ -0,0 +1,38 @@ +--- +id: "bug-1570" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-17T23:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c"] +tags: ["resupdate", "cmd_resupdate", "getopt", "duplicate-flag", "last-write-wins", "resource-value-verification", "task-2.1", "round-2-remediation"] +related: ["bug-1569"] +legacy: true +--- + +# bug-1570: resource-value-verification task 2.1 round-1 REJECTED: `resupdate -n mem -v 5 -v 8` silently… + +## Description +resource-value-verification task 2.1 round-1 REJECTED: `resupdate -n mem -v 5 -v 8` silently overwrote to 8 (last-write-wins) instead of rejecting the duplicate -v and keeping the first value 5 + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +cmd_resupdate()'s deferred-finalization refactor (needed so a trailing -t can still apply to its pending -n/-v pair) rewrote the 'v' case to `if (resname != NULL) { resval_str = optarg; }` -- it dropped the pre-existing guard that rejected a second -v for the same pending -n. Baseline (pre-task-2.1, git show 98b8c34b^) fired set_resval() immediately on -v and reset resname=NULL, so a second -v naturally hit the 'no res name found' else-branch; the deferred model keeps resname non-NULL across multiple -v's, so the missing resval_str==NULL check let the second -v silently clobber the first. + +## Fix +Changed the 'v' case guard to `if (resname != NULL && resval_str == NULL)` -- only accept a value into the pending slot if a name is pending AND no value has already been accepted for it; a duplicate -v now prints the same `ignore res value \`%s' as no res name found` warning and the first value is kept. Verified via full case-by-case trace against the git-show baseline (duplicate -v, two full -n/-v pairs, -n flushing a valueless prior -n, stray -v with no -n, trailing bare -n, and the new -t override case) plus a live single-node controller smoke test: `resupdate -n mem -v 5 -v 8` on a real joined worker prints the ignore-warning for `8` and `resval -n mem -m localhost` confirms the stored value is 5.000000, not 8. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1569 diff --git a/.wolf/bugs/bug-1571.md b/.wolf/bugs/bug-1571.md new file mode 100644 index 00000000..fee4cae6 --- /dev/null +++ b/.wolf/bugs/bug-1571.md @@ -0,0 +1,38 @@ +--- +id: "bug-1571" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-17T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/deploy-ha.sh"] +tags: ["apiserver", "deploy-ha", "xce-deploy-ha", "backend-config", "go", "resource-value-verification"] +related: [] +legacy: true +--- + +# bug-1571: apiserver logs "Backend not configured -- running in stub mode" and never calls RegisterToBackend… + +## Description +apiserver logs "Backend not configured -- running in stub mode" and never calls RegisterToBackend, even though the generated app.yaml backend.peers list is fully populated with a valid HA candidate list. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +resource-value-verification task 3.1 (xce-deploy-ha apiserver step): the new deploy-ha.sh Step-3 apiserver-config generation block initially wrote backend.address: "" and backend.port: 0 in the generated app.yaml, reasoning that backend.peers (non-empty) takes priority over address/port inside RegisterToBackend per backend/config.go's own doc comment. That reasoning only covers WHICH candidate RegisterToBackend picks once called -- it does not cover WHETHER it is called at all. apiserver/cmd/main.go:58 gates the entire RegisterToBackend call behind `BackendYamlConfig.Address != ""`, checked before Peers is ever consulted. A blank address left the apiserver permanently in stub mode regardless of a fully-populated peers list. Caught live during this task's own scenario-B verification run (grep of apiserver.log showed the stub-mode line instead of the expected "Backend connection established"). + +## Fix +deploy-ha.sh's apiserver app.yaml generation now also populates backend.address/backend.port from the first --peers candidate (host/port split via bash parameter expansion) alongside the full backend.peers list, so main.go's gate passes and RegisterToBackend still uses the full peers candidate list for actual connection/failover. Re-verified live: apiserver.log now shows "Attempting backend connection" / "Backend connection established" / "Backend connection registered!". + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1572.md b/.wolf/bugs/bug-1572.md new file mode 100644 index 00000000..ee4b158b --- /dev/null +++ b/.wolf/bugs/bug-1572.md @@ -0,0 +1,39 @@ +--- +id: "bug-1572" +status: "fixed" +status_updated: "2026-09-24" +status_note: "c9acd502 (bug-1585)" +component: "scheduler" +severity: "crash" +timestamp: "2026-08-18T01:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/driver.c", "src/scheduler/src/core_events.c"] +tags: ["scheduler", "core_events.c", "driver.c", "double-free", "dangling-pointer", "resvals-delta", "crash", "resource-value-verification", "task-4.1", "pre-existing", "unresolved"] +related: [] +fix_commits: ["c9acd502"] +legacy: true +--- + +# bug-1572: xce_scheduler silently exits (no ASAN report captured -- backgrounded via setsid nohup ...… + +## Description +xce_scheduler silently exits (no ASAN report captured -- backgrounded via setsid nohup ... >/dev/null 2>&1) a few seconds after a worker's SECOND periodic resource_sync report for the same resource(s); first report processes fine, second triggers the crash with zero "resval applied" TRACE lines logged for that batch and "handle_scheduler_leave" observed on the controller ~1s later. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Discovered live during resource-value-verification task 4.1 round-2 remediation smoke test (unrelated to the fix being validated -- reproduces identically with round-1's code, since round-1 only added a TRACE line AFTER the same zarray_put call). xce_core_push_events()'s XCE_CORE_EVENT_RESVALS_DELTA case (core_events.c ~line 786) does `zarray_put(&(w->resvals), res->idx, rv->val)` -- storing the decoded delta's xce_val_t* pointer DIRECTLY into scheduler-owned state, WITHOUT a MOVE_PTR/copy (unlike core_apply_config_resvals()'s sibling handling of the same ambiguity, which explicitly does `MOVE_PTR(val, nv->val); zarray_put(&(core->sharedInst), r->idx, val);`). Meanwhile driver.c's release_translated_payload() (called at driver.c:586 with applied=(push_rc>=0) after every successfully-pushed batch) has a case XCE_SCHED_OP_UPDATE_RESVALS: `free_ResValList_ptr(&(req->u.resvalupdate.resvals))` with NO `if (!applied)` guard -- unlike the sibling XCE_SCHED_OP_UPDATE_CONFIG case, which explicitly skips the free when applied ("core->config now owns it permanently -- do not free"). Net effect: every successfully-applied RESVALS_DELTA batch immediately frees the exact xce_val_t objects that were just aliased into w->resvals[idx], leaving a dangling pointer there. The dangling pointer is inert until the SAME (worker, resource) index is updated again -- at that point zarray_put's killer (kill_sched_resval -> free_Value) is invoked on the already-freed pointer, a double-free that aborts the ASAN-instrumented process. This is why it manifests specifically on a worker's second periodic report, not the first. + +## Fix +NOT FIXED -- out of scope for resource-value-verification task 4.1 round-2 remediation, whose boundary was explicitly restricted to src/scheduler/src/core_events.c's Value_u_which else-branch bug only. Needs its own task: either (a) MOVE_PTR the value out of rv before zarray_put in core_events.c's RESVALS_DELTA case (mirroring core_apply_config_resvals()'s existing pattern) so release_translated_payload()'s free no longer touches live scheduler state, or (b) add an `if (!applied)` guard to driver.c's XCE_SCHED_OP_UPDATE_RESVALS case (mirroring XCE_SCHED_OP_UPDATE_CONFIG) so a successfully-applied batch is never freed at all -- (a) is likely preferable since RESVALS_FULL's sibling case has the identical unguarded-free shape and should be audited too. Worked around during round-2 validation by running the smoke test's resupdate quickly after deploy, before any real worker's second periodic report could trigger the crash. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1573.md b/.wolf/bugs/bug-1573.md new file mode 100644 index 00000000..0a5e0c86 --- /dev/null +++ b/.wolf/bugs/bug-1573.md @@ -0,0 +1,38 @@ +--- +id: "bug-1573" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-18T01:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c"] +tags: ["scheduler", "core_events.c", "Value_u_which", "union-type-confusion", "resupdate", "resource-value-verification", "task-4.1", "round-2-remediation"] +related: ["bug-1572"] +legacy: true +--- + +# bug-1573: resource-value-verification task 4.1 round-1 REJECTED by independent review: the RESVALS_DELTA… + +## Description +resource-value-verification task 4.1 round-1 REJECTED by independent review: the RESVALS_DELTA TRACE logging in xce_core_push_events() assumed any non-string Value_u_which tag was Value_u_fvalue and read rv->val->u.fvalue unconditionally in the else branch -- Value_u_bvalue (and ivalue32/uvalue32/ivalue64/uvalue64) are real, reachable tags at this exact path (worker.c's mark_host_res() sets Value_u_bvalue for any XCE_VAL_BOOLEAN resource, routed through the same update_val_res()/resupdate RPC) that would read the wrong C union member. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Value_u_which (src/lib/controller.capnp.h:835-843) has 7 tags: ivalue32/uvalue32/ivalue64/uvalue64/svalue/fvalue/bvalue. Round-1's TRACE logic in the XCE_CORE_EVENT_RESVALS_DELTA case only distinguished svalue (explicit check) vs everything else (assumed fvalue, read .u.fvalue unconditionally in the else branch) -- a false dichotomy. Two directly-relevant precedents in this codebase already handle this correctly: update_val_res() (worker.c:107-117) explicitly checks `== Value_u_fvalue` with a generic 'non-numeric' fallback for anything else, and the sibling function core_apply_worker_resvals() 360 lines above in the SAME FILE (core_events.c:439-443) explicitly checks `== Value_u_fvalue` with no else at all (silent skip). Round-1 did not follow either precedent. + +## Fix +Rewrote the RESVALS_DELTA TRACE block (core_events.c, XCE_CORE_EVENT_RESVALS_DELTA case, right after zarray_put) to explicit if/else-if: Value_u_svalue -> TRACE with value=%s; Value_u_fvalue -> TRACE with value=%f (reading .u.fvalue only once .which is confirmed); anything else (ivalue32/uvalue32/ivalue64/uvalue64/bvalue) -> silently skipped, mirroring core_apply_worker_resvals()'s established convention for the same ambiguity in this file (reviewer-approved Option B -- this feature's resource set is entirely fvalue/svalue today per resval_typeresolve.c). Verified: full scheduler build green (xce-build.sh scheduler), all 41 scheduler tests pass except the pre-existing unrelated bug-1560 (sched_admin_request_test's symlink-escape case); live smoke test via xce-deploy-ha + xce_test worker-context join/resupdate -n r15m -v 12345.5 confirmed unchanged output `resval applied host=task41r2-worker resource=r15m value=12345.500000` in scheduler.log, format-identical to round-1's (expected -- the fix only changes behavior for non-fvalue/non-svalue tags, which this smoke test doesn't exercise). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1572 diff --git a/.wolf/bugs/bug-1574.md b/.wolf/bugs/bug-1574.md new file mode 100644 index 00000000..23fac1d5 --- /dev/null +++ b/.wolf/bugs/bug-1574.md @@ -0,0 +1,38 @@ +--- +id: "bug-1574" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test resupdate string/type crash: set_resval() lacks svalue case, vals[] uninitialized (src/harness/test.c)" +component: "controller" +severity: "crash" +timestamp: "2026-08-17T18:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c"] +tags: ["resupdate", "cmd_resupdate", "set_resval", "segfault", "uninitialized-stack-memory", "Value_u_svalue", "test.c", "pre-existing", "resource-value-verification"] +related: ["bug-1570", "bug-1573"] +legacy: true +--- + +# bug-1574: xce_test segfaults (SIGSEGV, exit 139) immediately on 'resupdate -n model -v ' (and by the… + +## Description +xce_test segfaults (SIGSEGV, exit 139) immediately on 'resupdate -n model -v ' (and by the same code path, -n type) run in the worker context against a live controller -- no output printed before the crash + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Pre-existing, predates the resource-value-verification feature entirely (confirmed via git show on the pre-task-2.1 baseline commit 98b8c34b^: the original set_resval()'s switch(valtype) already had no case for Value_u_svalue, same shape as today). set_resval() only assigns vals[idx].name inside a case block of switch(valtype); Value_u_svalue (type/model) has no case, so for these two names vals[idx].name is never assigned and keeps whatever uninitialized garbage was already on the local stack array `xce_resval_t vals[MAX_N_RES]` (never zero-initialized) in cmd_resupdate(). That garbage pointer is then passed unconditionally into xcecli_update_resvals()'s wire encoder inside &valptrs[0], which (most likely) dereferences it as a C string, causing the segfault. Task 2.1's own comment at test.c (near set_resval, 'the switch falls through with vals[idx].name/.val->which left unset ... intentionally not fixed by this task, see CONCERNS') already flagged this exact gap and deliberately preserved it byte-for-byte per Requirement 1.4, rather than introducing it. Discovered live during resource-value-verification task 4.2's validation (apiserver trace-log addition, live-testing the string-value code path with `resupdate -n model -v CustomModelX`) -- unrelated to task 4.2's own change (src/frontend/apiserver/backend/handler.go), which only touches Go/apiserver code. + +## Fix +NOT FIXED -- out of boundary for task 4.2 (scope: src/frontend/apiserver/backend/handler.go only). Belongs to test.c/set_resval()/resval_typeresolve.c territory (tasks 2.1/2.2 of this same spec, or a dedicated follow-up). Worked around during task 4.2 live validation by using a numeric resource (r15m) for the primary trace-log confirmation and avoiding type/model live resupdate calls thereafter. Likely fix: zero-initialize `xce_resval_t vals[MAX_N_RES]` in cmd_resupdate() (e.g. `= {0}`) so an unassigned .name is NULL instead of garbage, PLUS give set_resval() a real Value_u_svalue case that sets vals[idx].name and vals[idx].val->u.svalue (currently the string value passed to resupdate for type/model is silently never stored in the union either -- a second, related but distinct defect covered by the same missing switch case). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1570, bug-1573 diff --git a/.wolf/bugs/bug-1575.md b/.wolf/bugs/bug-1575.md new file mode 100644 index 00000000..a4806b09 --- /dev/null +++ b/.wolf/bugs/bug-1575.md @@ -0,0 +1,38 @@ +--- +id: "bug-1575" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-08-18T03:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".claude/skills/xce-test-resources/run-resource-verify.sh"] +tags: ["trap", "cleanup", "pkill", "SIGTERM", "signal-handler", "race-condition", "run-resource-verify.sh", "resource-value-verification", "bash"] +related: ["bug-1436"] +legacy: true +--- + +# bug-1575: run-resource-verify.sh's trap-based cleanup() (EXIT/INT/TERM) non-deterministically left the… + +## Description +run-resource-verify.sh's trap-based cleanup() (EXIT/INT/TERM) non-deterministically left the scheduler and/or apiserver process running after the script was interrupted mid-deployment via an external SIGTERM -- confirmed processes still alive (e.g. `xce_scheduler --peers ... --logfile .../scheduler.log`, `/tmp/apiserver -d .../apiserver`) seconds after the orchestrator's own `wait` returned with exit code 143. The SAME cleanup() code, reached via a normal fail()/exit-1 path (no external signal involved) inside the same single foreground invocation, worked perfectly every time (verified zero leftover processes). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +cleanup() issued four SEPARATE sequential `pkill -9 -f -- ""` invocations (one per component: controller/worker/scheduler/apiserver), each forking+execing its own `pkill` subprocess. When cleanup() ran as a normal function call/return within the script's own top-to-bottom control flow (e.g. from fail()), all four completed reliably. When cleanup() instead ran from on_exit as the EXIT trap fired by an asynchronously-delivered SIGTERM (via a separate `trap 'exit 143' TERM` translating the signal into an explicit exit), the sequence of pkill forks was intermittently and non-deterministically cut short partway through (confirmed via added debug instrumentation: one run stopped after the 1st pkill, another after the 3rd, another after the 2nd -- no fixed failure point, consistent with a timing race rather than a deterministic pattern-matching bug). Root cause not fully isolated at the syscall level (ruled out: pattern-matching correctness -- each individual pkill pattern was independently verified correct via direct pgrep; ruled out: cgroup pids/resource exhaustion -- process count stayed low, ~20-25 total throughout). Most likely a bash/signal-trap interaction where starting new child processes (fork/exec for each pkill) from within a trap handler invoked asynchronously (as opposed to synchronous top-level exit) has some window of unreliability in this environment. + +## Fix +Rewrote cleanup() to (1) combine all four per-component patterns into ONE `pkill -9 -f` call using extended-regex alternation (`"pat1|pat2|pat3|pat4"`) instead of four separate invocations -- fewer fork/exec round trips through the trap-handler code path -- and (2) wrap it in a retry-until-verified loop (`pgrep -f` check, up to 10 attempts at 0.2s intervals) so a slow/racy single kill attempt can no longer silently leave a process behind. Verified reliable across 5+ repeated live runs deliberately interrupted via external SIGTERM at different points mid-deployment (right after controller-only confirmation, and after controller+scheduler confirmation) -- zero leftover processes every time after the fix, versus non-deterministic leftovers before it. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1436 diff --git a/.wolf/bugs/bug-1576.md b/.wolf/bugs/bug-1576.md new file mode 100644 index 00000000..d642fce7 --- /dev/null +++ b/.wolf/bugs/bug-1576.md @@ -0,0 +1,38 @@ +--- +id: "bug-1576" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-08-18T03:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["N/A (diagnostic/interactive Bash tool usage", "not a repo source file)"] +tags: ["pgrep", "pkill", "self-match", "diagnostic-methodology", "bash-tool", "sandbox-artifact"] +related: ["bug-1575"] +legacy: true +--- + +# bug-1576: Multiple interactive diagnostic Bash calls (`pgrep -af ""`, `pkill -9 -f -- ""`… + +## Description +Multiple interactive diagnostic Bash calls (`pgrep -af ""`, `pkill -9 -f -- ""`, `ps -eo pid,args | grep ""`) unexpectedly matched -- and in one case SIGKILLed -- the CURRENTLY-RUNNING Bash tool call's own wrapper shell process, not just the intended target process(es). One instance: `pkill -9 -f -- "rrv_dummy_marker_TEST1|..."` killed the invoking Bash call itself mid-script, truncating all output after the pkill line with no further commands executing (looked like a mysterious 'exit code 1, script just stops' failure). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +This harness wraps every Bash tool invocation as roughly `bash -c "... && eval '' < /dev/null && ..."`. The wrapper's own process argv (visible via /proc//cmdline and matched by `pgrep -f`/`pkill -f`/`ps ... | grep`) therefore contains the FULL LITERAL TEXT of the submitted script -- including any pattern string typed directly into that same script (e.g. inside the pgrep/pkill command's own quoted argument, or in an unrelated echo/comment). Any `pgrep -f`/`pkill -f` search run from within a Bash tool call will self-match (and, for pkill, potentially self-kill) that call's own wrapper process whenever the search pattern also appears as literal text anywhere else in the SAME submitted command block -- which is nearly unavoidable, since typing the pattern to search for IS typing it into the submitted script. + +## Fix +Not a code fix (methodology only). When diagnosing process state from an interactive Bash tool call: (1) prefer reading /proc//cmdline directly via a `for p in /proc/[0-9]*; do ...; done` scan and filtering in a way that doesn't require typing the exact search substring inline where it could also match the wrapper's own transcript (e.g. write the pattern to a variable/file first, or accept that the wrapper itself will show up as one extra line and visually distinguish it by its distinctive `bash -c "source .../shell-snapshots/... && eval '...'"` shape); (2) never assume a `pgrep -f`/`pkill -f` result of exactly 1 match confirms a target is unique -- it may be exactly 1 SELF-match with zero real matches; (3) for any pkill run interactively for diagnostic purposes (not from inside a dedicated script/process whose own argv doesn't contain the pattern), sanity-check the matched PID's `ps -o cmd` BEFORE trusting the pkill's effect, since a self-kill silently truncates the rest of that same Bash call with no distinguishing error message beyond a generic nonzero/truncated-output exit. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1575 diff --git a/.wolf/bugs/bug-1577.md b/.wolf/bugs/bug-1577.md new file mode 100644 index 00000000..a5174c0d --- /dev/null +++ b/.wolf/bugs/bug-1577.md @@ -0,0 +1,38 @@ +--- +id: "bug-1577" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "crash" +timestamp: "2026-08-17T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".claude/skills/xce-test-resources/run-resource-verify.sh"] +tags: ["resource-value-verification", "xce-deploy-ha", "worker-daemon", "bug-1572", "task-6.1"] +related: ["bug-1572"] +legacy: true +--- + +# bug-1577: No explicit error -- discovered via live testing task 6.1 (resource-value-verification)… + +## Description +No explicit error -- discovered via live testing task 6.1 (resource-value-verification): xce-deploy-ha's deploy-ha.sh Step 3 opt-in-by-presence worker branch (predates this feature) generates start-worker.sh whenever a worker binary (xcecd_virtual) happens to exist alongside the controller build, and generated start-all.sh always launches it. run-resource-verify.sh's own deployment (task 5.1+6.1) unconditionally hit this since a full `/xce-build all` leaves xcecd_virtual built. Result: a REAL worker daemon ('worker-1') ran for the whole duration of every xce-test-resources run, silently violating Requirement 2.1's design intent ('no real worker daemon process required') and periodically re-reporting its own resources (~every 15s) -- a live risk of triggering the still-open bug-1572 (scheduler double-free/crash on a resource's 2nd periodic report) if a run happened to last that long. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +deploy-ha.sh's worker-detection/start-all.sh generation is out of this task's edit boundary (xce-deploy-ha files are done/approved for this feature) and has no flag to skip the worker step -- it only conditionally starts one when a binary is present, with no opt-out. + +## Fix +start_all() now exports XCE_WORKER_BIN=/bin/true before invoking the generated start-all.sh. The generated start-worker.sh already reads ${XCE_WORKER_BIN:-} at RUN time from its own inherited environment, so pointing it at a real no-op binary makes the worker step exit immediately with no persistent process, without touching deploy-ha.sh itself. Verified via a /proc/*/exe-based watcher (not pgrep -f text matching, which self-matches -- see bug-1576) across a full default-resource-set run: zero xcecd_virtual/xcecd/xcecd_worker processes at any point post-fix, versus one running the entire time pre-fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1572 diff --git a/.wolf/bugs/bug-1578.md b/.wolf/bugs/bug-1578.md new file mode 100644 index 00000000..19b50b7f --- /dev/null +++ b/.wolf/bugs/bug-1578.md @@ -0,0 +1,48 @@ +--- +id: "bug-1578" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-08-18T04:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: ["src/controller/sched.c (sched_worker_change()", "~line 548-553) -- crash SITE is persist.c/controller.capnp.c (capnp_encode_worker/encode_Worker/encode_NameValue_list/encode_NameValue) but the DEFECT is in sched.c"] +tags: ["CRITICAL", "capnp-encode", "cmd_resupdate", "controller", "dangling-pointer", "dead-code", "double-free-risk", "encode_NameValue", "if-0-fixme", "no-asan-report", "not-root-caused", "pre-existing-not-this-feature", "resource-value-verification", "resupdate", "reviewer-discovered", "root-caused", "sched.c", "sched_worker_change", "sigkill-class", "silent-crash", "task-6.1", "task-6.2-blocker", "use-after-free", "unit-test-added"] +related: ["bug-1572", "bug-1574", "bug-1577"] +legacy: true +--- + +# bug-1578: Controller (xce_controller) reproducibly exits via SIGSEGV (si_addr=NULL, si_code=1/SEGV_MAPERR)… + +## Description +Controller (xce_controller) reproducibly exits via SIGSEGV (si_addr=NULL, si_code=1/SEGV_MAPERR) within ~1s of the last resupdate follow-up verification reply completing, when the synthetic worker then disconnects ('quit'). ROOT-CAUSED via live repro (2/2) with an LD_PRELOAD signal-handler shim (gdb reliably masked the crash for both ASAN and debug/-O0 builds -- ptrace overhead perturbs timing enough to avoid the window; ~20s under gdb = no crash, every time). Resolved backtrace (addr2line against the debug-symbol'd ASAN binary): encode_NameValue <- encode_NameValue_list <- encode_Worker <- capnp_encode_worker (persist.c) <- persist_worker_hook_fun <- xce_run_hooks <- mark_worker_unavail <- handle_worker_leave <- handle_endpoint_error (cluster.c) <- io_worker_dispatch_batch <- io_worker (netio.c). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED, PRE-EXISTING (predates resource-value-verification by ~5 months, NOT introduced by tasks 2.1/2.2/6.1). sched_worker_change() (src/controller/sched.c) aliases w->host.resvals_to_sched/nresvals_to_sched directly to a local xlist_t resvalist's backing array (resvalist.data), pushes it to the scheduler connection, then calls xlist_fini(&resvalist) -- which frees that backing array -- WITHOUT first resetting w->host.resvals_to_sched to NULL / nresvals_to_sched to 0. The correct reset-before-free code exists in the function but is wrapped in a dead `#if 0 /*FIXME*/ ... #endif` block, introduced already-disabled by commit acfbec875 (2026-03-03, 'save works'). This leaves w->host.resvals_to_sched a DANGLING pointer and nresvals_to_sched a stale nonzero count on the worker_t struct for the rest of that worker's lifetime. commit e5fde7b5 (2026-05-20, 'fix(persist): zero nresvals_to_sched after free_name_val_list') attempted to fix exactly this class of bug but edited code INSIDE the already-dead #if 0 block, so the fix silently never took effect. When the worker later disconnects, handle_worker_leave() -> mark_worker_unavail() fires the WORKER_DEAD hook -> persist_worker_hook_fun() -> capnp_encode_worker() -> encode_Worker() -> encode_NameValue_list(), which iterates the stale count over the freed array and dereferences a garbage/NULL entry in encode_NameValue() -> SIGSEGV. Only manifests once a scheduler has connected at least once: on_scheduler_join() (sched.c) registers sched_worker_hook_fun into c->hooksys (gated by a `static int hook_installed` one-shot flag) ONLY on the first scheduler join -- a controller with no scheduler ever connecting never reaches the buggy sched_worker_change(), so w->host.resvals_to_sched stays NULL/0 forever and persist_worker_hook_fun's later read is always safe. This exactly matches run-resource-verify.sh's controller+scheduler+apiserver deployment shape and explains the original isolation matrix precisely. The sibling function apiserver_worker_change() (src/controller/apiserver.c ~line 479-481) does the reset correctly (NOT wrapped in #if 0), which is why apiserver-only interaction never crashed. + +## Fix +FIX IDENTIFIED, NOT YET APPLIED (out of task 6.1's boundary -- lives entirely in sched.c, a file no task in this feature touches). In src/controller/sched.c's sched_worker_change(), replace the dead block: + #if 0 + /*FIXME*/ + free_name_val_list(&(w->host.resvals_to_sched), &(w->host.nresvals_to_sched)); + #endif + xlist_fini(&resvalist); +with the exact working pattern already used by apiserver_worker_change(): + w->host.nresvals_to_sched = 0; + w->host.resvals_to_sched = NULL; + xlist_fini(&resvalist); +IMPORTANT: do NOT simply uncomment the existing free_name_val_list() call as-is -- that would introduce a DOUBLE FREE instead. free_name_val_list() (src/lib/utils.c:412) already deep-frees every entry AND the outer array (FREEUP(resvals) where resvals == w->host.resvals_to_sched == the identical pointer value as resvalist.data), then sets w->host.resvals_to_sched itself to NULL -- but resvalist.data (the xlist_t's own separate field, still holding that same now-freed address) is untouched by that call, so the very next line's xlist_fini(&resvalist) -> FREEUP(resvalist.data) would free the identical address a second time. This is almost certainly why the original author disabled it with #if 0 rather than deleting it. Secondary/lower-severity, NOT part of this fix: both sched_worker_change() and apiserver_worker_change() leak the individual xce_name_val_t entries collected into resvalist -- xlist_fini() only frees the outer array (FREEUP(l->data)), never each entry's .name/.val/nv allocations -- worth a separate follow-up. Also worth a wider audit: other readers/freers of w->resvals_to_sched (opget.c:128 do_reply_workerlist(), apiserver.c, dataex.c) may have the same latent dangling-pointer exposure, only the WORKER_DEAD->persist path was confirmed live in this repro. FIX APPLIED 2026-08-18: src/controller/sched.c sched_worker_change() -- replaced the dead `#if 0 /*FIXME*/ free_name_val_list(...) #endif` block with `w->host.nresvals_to_sched = 0; w->host.resvals_to_sched = NULL;` immediately before `xlist_fini(&resvalist);` -- matching apiserver_worker_change()'s proven pattern exactly (src/controller/apiserver.c:480-482). Verified: (1) xce-build.sh controller -- clean build, 1102/1104 controller tests pass (the 2 failures are pre-existing bug-1561 overlayfs/fdatasync sandbox artifacts, unrelated); (2) live repro via run-resource-verify.sh --keep-running -- deployed controller+scheduler+apiserver, joined synthetic worker 'resverify-worker', ran resupdate for 9 resource values (all accepted), worker disconnected ('quit') triggering handle_worker_leave -> sched_worker_change (pushing 17 resvals to scheduler, twice, situation=HOOK_AFTER_EVENT) -- the exact previously-fatal sequence. Controller stayed alive and logging normally for ~2 minutes past the disconnect (previously died within 1-10s); no SEGV, no ASAN report, no core file. No new unit test added -- sched_worker_change() is a static function with no existing test coverage and no testable-static-function harness pattern in this codebase (confirmed via codegraph), and the crash path requires a live scheduler connection + hooksys + persist/capnp encoding chain across sched.c/cluster.c/persist.c -- disproportionate scaffolding for a 2-line fix; live-repro evidence used instead, per this codebase's established xce-test-ha-class convention. UNIT TEST ADDED 2026-08-18 (follow-up task, closes the gap this entry originally flagged as missing): src/controller/testsuite/test_sched_worker_change.c (new standalone binary, sched_worker_change_unit_test, wired into src/controller/meson.build mirroring sched_push_unit_test's real-sched.c+dataex.c+egress.c linking strategy). Test 1 fires XCE_CONTROLLER_HOOK_WORKER/HOOK_DETAIL_WORKER_DEAD via xce_run_hooks() (the same production entry point handle_worker_leave()->mark_worker_unavail() uses) on a worker_t with a populated w->host.resvals, then asserts w->host.resvals_to_sched==NULL && nresvals_to_sched==0. Test 2 goes further: after the same hook fire, it calls the REAL production encoder encode_Worker() (controller.capnp.c) directly on the worker -- the exact function bug-1578's crash backtrace bottoms out in via capnp_encode_worker/persist_worker_hook_fun -- so a regression is caught as an ASAN heap-use-after-free, not just a failed assertion. RED/GREEN verified live: temporarily re-wrapping the fix in the original dead `#if 0 ... #endif` block reproduced a genuine ASAN heap-use-after-free (encode_NameValue_list, controller.capnp.c:3408, freed by xlist_fini<-sched_worker_change<-sched_worker_dead<-sched_worker_hook_fun) -- confirming the earlier claim in this same entry ('no testable-static-function harness pattern exists in this codebase for it') was wrong; xce_run_hooks() through the real installed hooksys is exactly that harness, already proven by sched_push_unit_test for the JOB hook and now for the WORKER hook too. Restored the approved fix afterward (verified via git diff byte-identical to the pre-revert state) and confirmed GREEN again. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1572, bug-1574, bug-1577 diff --git a/.wolf/bugs/bug-1579.md b/.wolf/bugs/bug-1579.md new file mode 100644 index 00000000..a24506a8 --- /dev/null +++ b/.wolf/bugs/bug-1579.md @@ -0,0 +1,40 @@ +--- +id: "bug-1579" +status: "wontfix" +status_updated: "2026-09-24" +status_note: "not a bug: intended host-attrs reset on disconnect; script fixed b673b7c9" +component: "unknown" +severity: "unknown" +timestamp: "2026-08-18T09:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: [".claude/skills/xce-test-resources/run-resource-verify.sh (not a bug in this file's new logic -- see root_cause); src/controller/worker.c (add_hostattrs_resvals()", "apply_worker_dead_mutation()) is the production code responsible for the behavior being (correctly) observed"] +tags: ["false-positive", "test-tooling", "resource-value-verification", "run-resource-verify.sh", "task-6.2", "controller-check", "cpuf", "worker-disconnect", "host-attrs", "add_hostattrs_resvals", "mark_worker_unavail", "not-a-bug", "default-resource-set", "task-7.1-followup"] +related: ["bug-1578"] +legacy: true +--- + +# bug-1579: run-resource-verify.sh's new controller-side check (task 6.2, query_controller_resvals()) reported… + +## Description +run-resource-verify.sh's new controller-side check (task 6.2, query_controller_resvals()) reported a live MISMATCH for the default resource set's 'cpuf' resource: 'controller check: cpuf -> MISMATCH: injected=9.5 observed=0.000000', even though inject_resources()'s own resupdate call reported 'cpuf=9.5 -> accepted' moments earlier in the same run. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT A BUG -- documented, intentional controller production behavior colliding with this test script's own multi-process session structure. inject_resources() (task 6.1) runs join+resupdate+quit in ONE xce_test process; the 'quit' disconnects the synthetic worker's TCP connection to the controller at the end of that same script. query_controller_resvals() (task 6.2) then runs its resval query in a SEPARATE, LATER xce_test process/session -- by which point the synthetic worker is already gone from the controller's perspective. 'cpuf' is one of worker.c's N_HOST_ATTRS resources (add_hostattrs_resvals(), worker.c:260-287: type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks), which mark_worker_unavail() -> apply_worker_dead_mutation() (worker.c:704-716) deliberately RESETS (reset_hostattrs_unknown() then re-applies add_hostattrs_resvals()) whenever a worker is marked dead/disconnected -- correct, intentional production behavior for a REAL worker daemon that has genuinely gone away (you don't want a dead worker still advertising stale cpu capacity for scheduling). Every OTHER default resource (r15m, pg, ls, it, swp, io, tmp, mem -- none of which are in worker.c's N_HOST_ATTRS list) is a generic, non-host-attrs resval that persists correctly across worker disconnect and reported MATCH. Confirmed via live isolation: re-running with the exact same 8 non-cpuf resources reported 8/8 MATCH; re-running the mixed accept/reject scenario (mem=5, not-a-real-resource=1) correctly reported mem MATCH and skipped the rejected resource entirely (no query attempted). design.md's own sequence diagram does not show a worker disconnect between the resupdate leg and the resval-query leg, implying the intended flow keeps the synthetic worker connected throughout verification -- task 6.1's single-shot 'join;resupdate;quit' session structure deviates from that, but only resources that collide with worker.c's host-attrs reset list are affected. + +## Fix +DEFAULT-SET INSTANCE FIXED (task 6.2 remediation round 2): DEFAULT_RESOURCE_ARGS in run-resource-verify.sh now uses "r1m=9.5" instead of "cpuf=9.5" as the second already-supported regression-coverage name (mem=8.5 kept as-is -- mem is NOT host-attrs, confirmed safe). r1m is confirmed via resval_typeresolve.c's resinfo[] table to be a genuinely pre-existing resource name, and via src/worker/src/resource_sync.c to be a periodic/dynamic load-index resource (not in worker.c's N_HOST_ATTRS list), so it preserves the original 'already-supported name for regression coverage' intent without the disconnect-reset problem. Live-validated: default (no-args) set now reports a clean N/N MATCH with no false cpuf mismatch. Original two options re-evaluated: option (a) -- drop the host-attrs name from the default set -- is what was taken here. Option (b) -- keep the synthetic worker's xce_test session connected across all per-layer checks instead of quitting after inject_resources() -- was investigated live and found NOT to be a simple call-reordering fix: xce_test's REPL has no command path from the `worker` context back to the `client` context without first calling leave_ctx() (via `..`/cmd_up), which performs the exact same disconnect that `quit` does. Making option (b) work would require either a C-level xce_test change (e.g. a context-preserving command) or a materially riskier bash coprocess restructuring of this script -- both explicitly out of scope for this remediation. The underlying production behavior (add_hostattrs_resvals() / mark_worker_unavail() resetting host-attrs resources on worker disconnect) remains correct and untouched, and this note is preserved for any FUTURE task or user-supplied custom resource=value argument that names a host-attrs resource (type, model, cpuf, maxcpus, maxmem, maxswap, maxtmp, ndisks) -- those still hit this same class of false-MISMATCH if queried after the synthetic worker disconnects. + +REAL FIX APPLIED (follow-up session, 2026-08-18): the underlying disconnect-timing gap this workaround left open for any host-attrs name is now actually closed, not just avoided for the default set. run-resource-verify.sh's inject_resources() now inserts a `!bash "$SELF_SCRIPT" --internal-controller-check ` line (xce_test's `!` escape, preprocess_and_run(), test.c ~line 950-998 -- a single top-level-REPL-loop call site that works identically in ANY context including `worker`, with no leave_ctx()/disconnect) BETWEEN `resupdate` and `quit`, inside the SAME xce_test worker session. This invokes a new internal_controller_check() function (hidden dispatch at the bottom of the script, before `main "$@"`) that runs a nested client-context xce_test session and queries resval for every requested name WHILE the synthetic worker's connection is still open -- so a host-attrs resource is never reset to baseline before it is checked. Raw observed values are written to logs/controller-check-raw.tsv; query_controller_resvals() (still called at the same point in main(), after inject_resources() returns) now reads that file instead of running its own live query -- a pure sequencing change, the MATCH/MISMATCH comparison logic and TSV formats are byte-for-byte unchanged. DEFAULT_RESOURCE_ARGS restored `cpuf=9.5` (dropped the r1m workaround) as live proof: 'controller check: cpuf -> MATCH' now, not the old false MISMATCH. Live-validated (5 separate runs): default 9-resource set incl. cpuf clean 9/9 MATCH at both controller and scheduler layers (~5-7s total runtime); mixed accept/reject with a host-attrs custom name (cpuf=3.25 alongside a rejected unknown resource) correctly MATCHes cpuf and skips the rejected name at both layers; --keep-running and mid-injection SIGTERM (landing WHILE the nested `!`-escape check was actively running) both verified clean, zero orphaned xce_test processes (full /proc/[0-9]*/cmdline scan, not just pgrep). Option (b) from the original analysis below (keep the session connected) turned out to be achievable after all via this `!`-escape mechanism, without any C-level xce_test change -- the earlier conclusion that it needed one was specific to trying to move BETWEEN contexts (worker -> client) without disconnecting, which is genuinely not possible; running an entirely separate NESTED xce_test client session via a shell escape while remaining IN the worker context sidesteps that limitation entirely. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1578 diff --git a/.wolf/bugs/bug-1580.md b/.wolf/bugs/bug-1580.md new file mode 100644 index 00000000..097245c4 --- /dev/null +++ b/.wolf/bugs/bug-1580.md @@ -0,0 +1,38 @@ +--- +id: "bug-1580" +status: "open" +status_updated: "2026-09-24" +status_note: "intermittent controller_unit_test \"2 Failures\" with all cases PASS; not investigated" +component: "controller" +severity: "crash" +timestamp: "2026-08-18T10:39:52Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/ (combined controller_unit_test binary; discovered running xce-build.sh all for the resource-value-verification bug-1579 follow-up fix -- zero C files touched this session", "confirmed via git status)"] +tags: ["flaky", "controller_unit_test", "unity", "aggregate-count-mismatch", "pre-existing"] +related: ["bug-1553"] +legacy: true +--- + +# bug-1580: meson test 'xce:controller / controller test' (controller_unit_test) reports '1104 Tests 2… + +## Description +meson test 'xce:controller / controller test' (controller_unit_test) reports '1104 Tests 2 Failures 0 Ignored' / exit status 2, but grep of the full raw testlog for ':FAIL:' or a per-test [FAIL] marker finds ZERO matches -- every individual Unity test case visibly printed PASS. Reproduced deterministically 2/2 times (once via meson test, once via a direct standalone re-run of the binary with ASAN_OPTIONS=detect_leaks=0), unlike bug-1553's documented ~1-in-3 intermittent SIGSEGV (exit 139) signature -- this is exit 2 with no crash indication, a distinct manifestation. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not investigated. Most likely the same class of issue cerebrum.md's Unity-harness note already flags for this codebase (multiple UnitySetTestFile()-sharing test-group files in one binary can desync the aggregate PASS/FAIL counter from the printed per-test lines) rather than a real regression -- no code change in this session could have caused it (only .claude/skills/xce-test-resources/run-resource-verify.sh, a bash script, was edited). Possibly related to / the same underlying fragility as bug-1553 (order-dependent shared Unity/ASAN state across test files in the same binary) but with a different observed symptom (miscounted summary vs. an outright segfault). + +## Fix +Not fixed -- out of scope for the resource-value-verification bug-1579 follow-up (bash-script-only boundary). Verified NOT a regression from this session's change (no C files modified) and does not block or relate to the run-resource-verify.sh live validation, which was carried out and passed independently. Needs the same isolation bug-1553 needs: bisect which specific linked test-group file(s) desync the counter. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1553 diff --git a/.wolf/bugs/bug-1581.md b/.wolf/bugs/bug-1581.md new file mode 100644 index 00000000..3dd729c3 --- /dev/null +++ b/.wolf/bugs/bug-1581.md @@ -0,0 +1,38 @@ +--- +id: "bug-1581" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-08-18T12:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".claude/skills/xce-test-resources/run-resource-verify.sh"] +tags: ["bash", "ifs", "tsv-parsing", "resource-value-verification", "task-7.1", "aggregate_verdict"] +related: [] +legacy: true +--- + +# bug-1581: aggregate_verdict() synthetic self-test scenario (c) [one layer UNREACHABLE] reported… + +## Description +aggregate_verdict() synthetic self-test scenario (c) [one layer UNREACHABLE] reported unreachable=2 instead of 1, and the apiserver layer of an accepted+reachable record showed api_ok=2.5/api_obs=(empty) after IFS=$'\t' read -- fields silently shifted by one position. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +aggregate_verdict()/build_verdict_record() (task 7.1, new this session) originally joined/split its 9-field verdict record with a literal tab character (IFS=$' ' read), matching every other TSV parse already in this file. But this is the FIRST record format in the file where a field can be genuinely empty (an UNREACHABLE or reachable-but-empty-value *_observed field) -- and bash treats tab (like space/newline) as "IFS whitespace" for field splitting even when IFS is set to ONLY a tab character, so two adjacent tab delimiters around an empty field silently COLLAPSE into one, shifting every field after it. Every prior TSV in this file (injection-results.tsv, *-check-results.tsv) never carries an empty field, which is why this bash gotcha never surfaced before. + +## Fix +Switched the record-format delimiter from a literal tab to ASCII Unit Separator (0x1F, $'', VERDICT_FIELD_SEP) in build_verdict_record()/aggregate_verdict() only -- not a whitespace-class character, so bash never collapses adjacent delimiters or strips empty fields. Caught immediately by this task's own required synthetic self-test invocations (per task 7.1's TDD expectation to sanity-check aggregate_verdict() with synthetic records before relying on it), before any live-run validation -- all 4 required synthetic scenarios (match/mismatch/unreachable/mixed) plus edge cases now pass. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1582.md b/.wolf/bugs/bug-1582.md new file mode 100644 index 00000000..3b9da84a --- /dev/null +++ b/.wolf/bugs/bug-1582.md @@ -0,0 +1,38 @@ +--- +id: "bug-1582" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-08-18T13:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".claude/skills/xce-test-resources/run-resource-verify.sh"] +tags: ["bash", "resource-value-verification", "task-7.1", "aggregate_verdict", "unreachable-vs-mismatch", "round-1-rejected", "round-2-fixed", "sched_ok", "api_ok"] +related: ["bug-1581"] +legacy: true +--- + +# bug-1582: run-resource-verify.sh task 7.1 round-1 review REJECTED: independent reviewer killed the live… + +## Description +run-resource-verify.sh task 7.1 round-1 review REJECTED: independent reviewer killed the live scheduler process mid-run (after deployment confirmed up, before its trace-log poll completed) and got VERDICT line `unreachable=0 mismatches=1` plus FAILURE line `status=MISMATCH ... observed=` -- should have been `unreachable=1 mismatches=0` and `status=UNREACHABLE ... observed=`. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +main()'s record-building loop derived sched_ok/api_ok (feeding aggregate_verdict()) by testing whether the SCHEDULER_OBSERVED[$vname]/APISERVER_OBSERVED[$vname] associative-array KEY EXISTED (`[ -n "${arr[k]+x}" ]`). But query_scheduler_resvals()/query_apiserver_resvals() (tasks 6.3/6.4, already-approved) unconditionally SET these array entries for every accepted+checked resource -- including on a genuine UNREACHABLE/timeout, where the value is just set to the empty string "". Since the key always existed, sched_ok/api_ok were always 1 regardless of reachability, so aggregate_verdict() never saw the real UNREACHABLE status -- it fell through to comparing the injected value against an empty string, which reads as MISMATCH instead of UNREACHABLE. Contradicted Requirement 6.3, design.md's Error Handling ("Layer unreachable ... labeled UNREACHABLE distinctly from MISMATCH ... not a silent skip"), and this task's own SKILL.md documentation of the same behavior. Note: the analogous ctrl_ok derivation (from CONTROLLER_OBSERVED array-key existence) is NOT affected -- query_controller_resvals()/internal_controller_check() has no separate UNREACHABLE concept at all (a query failure there aborts the whole run via fail()), so array-key existence is an accurate proxy for that layer specifically. + +## Fix +Round-2 remediation: derive sched_ok/api_ok from SCHEDULER_CHECK_RESULTS[$vname]/APISERVER_CHECK_RESULTS[$vname] instead (already correctly populated by tasks 6.3/6.4 with exactly "MATCH", "MISMATCH: injected=... observed=...", or "UNREACHABLE: ...") -- a leading "UNREACHABLE:" means ok=0, anything else (including a defensive __UNCHECKED__ fallback for a missing key, which should never happen given both query_*_resvals() functions iterate the same accepted-name list this loop reads) means ok=1 with the real observed value. Also extracted main()'s inline record-building loop into a standalone build_verdict_records_from_tsv() function (populates global VERDICT_RECORDS) so a synthetic test can exercise main()'s ACTUAL data-marshaling code (not just aggregate_verdict() called directly with hand-built records, which is what round-1's task 8.2-style synthetic tests did and which structurally could never have caught this bug). Live-reproduced the reviewer's exact scenario (kill scheduler right after deployment confirms up, before inject_resources()/query_scheduler_resvals() ever runs) and confirmed the VERDICT line now correctly shows `unreachable=9 mismatches=0` (all 9 default-set resources) with `status=UNREACHABLE ... observed=` FAILURE lines, apiserver/controller layers unaffected (still MATCH). New synthetic regression test (scenario E, using build_verdict_records_from_tsv() directly) added and confirmed to fail against a reverted copy of the old buggy logic (unreachable=0 mismatches=2) and pass against the fix (unreachable=1 mismatches=1 across a mixed good/schedbad/apibad/rejected batch) -- proving the test is meaningful, not tautological. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1581 diff --git a/.wolf/bugs/bug-1583.md b/.wolf/bugs/bug-1583.md new file mode 100644 index 00000000..27807729 --- /dev/null +++ b/.wolf/bugs/bug-1583.md @@ -0,0 +1,39 @@ +--- +id: "bug-1583" +status: "fixed" +status_updated: "2026-09-27" +status_note: "70360ccf" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-18T13:35:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/handler.go"] +tags: ["apiserver", "resval", "validation-gap", "data-integrity", "resource-value-verification", "handler.go", "OnResourceLocationUpdated"] +related: [] +fix_commits: ["70360ccf"] +legacy: true +--- + +# bug-1583: Live task 8.1 run (resource-value-verification, comprehensive e2e validation): injecting resupdate… + +## Description +Live task 8.1 run (resource-value-verification, comprehensive e2e validation): injecting resupdate -n totally-fake-resource-xyz -v 1 (a deliberately-unregistered resource name) alongside 9 genuinely-registered names -- the controller correctly rejects it (resval_typeresolve/update_resval's key_find(c->resmgr,...) returns -ENOENT, never written to w->host.resvals, confirmed empty via a direct resval query) and the scheduler correctly discards it (core_events.c's RESVALS_DELTA handler logs 'xce_core_push_events RESVALS_DELTA: unknown resource totally-fake-resource-xyz, skipping' at WARN and never applies it) -- but the apiserver's OnResourceLocationUpdated (src/frontend/apiserver/backend/handler.go:92) has NO such validation: it logged 'resource-location stored {"resource": "totally-fake-resource-xyz", "host": "resverify-worker", "value": 1}' and unconditionally called store.ResourceLocations.Put() for it, live-confirmed in apiserver.log. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +do_update_resvals() (src/controller/opupd.c:10-51) fires XCE_CONTROLLER_HOOK_RESOURCE/HOOK_DETAIL_RESOURCE_UPDATE with the FULL, RAW, UNFILTERED xce_resval_list_t from the client RPC -- BEFORE update_resval_list() runs its own per-item validity check, and the HOOK_AFTER_EVENT fire (the one both sched_resource_hook_fun (sched.c:609-639) and apiserver_resource_hook_fun (apiserver.c:545-581) act on) still passes that same unfiltered list, since update_resval_list() only applies/logs per-item failures internally -- it never mutates or filters the list object itself. So both the scheduler and the apiserver receive a raw push containing EVERY submitted resource name, valid or not, regardless of whether the controller's own update_resval()->key_find(c->resmgr,...) accepted it. The scheduler's core_events.c independently re-validates each pushed name against its own resource table before applying (WARN + skip on unknown) -- but OnResourceLocationUpdated (handler.go:92-115) does zero such validation; it stores whatever ResourceName/Location/value it's handed, unconditionally. This is a genuine 3-way inconsistency: controller rejects+never-stores, scheduler rejects+discards+logs, apiserver silently accepts+stores forever. + +## Fix +NOT FIXED -- discovered incidentally during resource-value-verification task 8.1's comprehensive live e2e validation run (deliberately injecting one unregistered resource name alongside the default 9-resource set to prove the run-level FAIL path). Explicitly out of task 8.1's boundary (validation-only, no code changes) and out of the resource-value-verification feature's boundary generally (design.md 'Out of Boundary' section explicitly excludes 'the controller's resval-forwarding hooks' and scopes handler.go's own change to exactly one logger.Debug() call after the pre-existing store.ResourceLocations.Put() -- the Put() call and its lack of validation both predate this feature). A real fix would add a registered-resource-name check to OnResourceLocationUpdated (or, more centrally, filter the notify() payload in do_update_resvals()/sched_resource_hook_fun/apiserver_resource_hook_fun to only include items update_resval_list() actually accepted) before persisting to store.ResourceLocations -- worth a dedicated follow-up task, does not block resource-value-verification's own completion (the skill's own verdict logic already correctly ignores/skip-checks rejected names via injection-results.tsv's accept/reject accounting, so this apiserver-side phantom-store gap does not affect task 8.1's or 7.1's PASS/FAIL correctness). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1584.md b/.wolf/bugs/bug-1584.md new file mode 100644 index 00000000..fa95de9f --- /dev/null +++ b/.wolf/bugs/bug-1584.md @@ -0,0 +1,38 @@ +--- +id: "bug-1584" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T02:47:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core_clock.c", "src/scheduler/include/core.h", "src/scheduler/src/core.c", "src/scheduler/src/scheduler.c", "src/scheduler/testsuite/test_clock.c"] +tags: ["scheduler", "clock", "xce_core_now", "recording", "xceadmin", "wall-clock", "epoch-time", "monotonic-clock", "core-clock", "user-reported"] +related: [] +legacy: true +--- + +# bug-1584: User-reported: `xceadmin scheduler record start/status/stop` always shows 'started at:' near… + +## Description +User-reported: `xceadmin scheduler record start/status/stop` always shows 'started at:' near 1970-01-01 (e.g. '1970-01-04T20:11:26') instead of the real start time, even though 'file'/'events'/'recording' fields are correct. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce_sched_core_init() (core.c) seeds the scheduler-core's clock anchors to a neutral (virtual_clock_base=0, real_clock_base=0, speed_num=1, speed_den=1) default. The existing doc comments in core.c/core.h (task 4.5, scheduler-core spec) claimed this 'degrades to real wall-clock time' -- that claim is simply wrong: with both anchors at 0, xce_core_now()'s elapsed computation (wall_ns - real_clock_base) collapses to the raw CLOCK_MONOTONIC reading itself, i.e. nanoseconds since an arbitrary reference point (typically system boot), NOT nanoseconds since the Unix epoch. Only the simulation driver (sim.c's sim_set_virtual_clock(), called from sched_sim_driver_start()) ever seeds a genuine epoch anchor -- the normal-mode driver (scheduler.c's sched_init()) never did. recording.c's sched_record_start()/sched_record_stop() stamp record_start_ts and every ReplayEvent/SnapshotEntry ts via xce_core_now(), and controller.capnp's SchedAdminReply.startedAt is explicitly documented as 'nanoseconds since epoch' (which xceadmin's cli.py _format_timestamp_ns() renders via datetime.fromtimestamp(ns/1e9, tz=utc)) -- so a normal-mode scheduler's uptime-since-boot value gets rendered as a date near 1970. Independently confirmed already known and documented as a latent issue for a DIFFERENT symptom (tlog logging integration, design.md line ~793 / scheduler-core tasks.md Implementation Notes item #6) but nobody had connected it to the recording subsystem's user-facing 'started at' field before this report. + +## Fix +Added xce_core_set_wallclock_anchor(xce_sched_core_t*) to core_clock.c -- the normal-mode counterpart to sim.c's sim_set_virtual_clock() -- which reads CLOCK_REALTIME (the epoch anchor for virtual_clock_base) and CLOCK_MONOTONIC (the matching real_clock_base xce_core_now()'s own elapsed computation is defined against) back-to-back and stores them as a group (speed_num=speed_den=1). Declared in core.h. Wired into scheduler.c's sched_init(), called once immediately after xce_sched_core_init() succeeds, before anything can observe xce_core_now(). Corrected the misleading 'degrades to real wall-clock time' doc comments in core.c and core.h to state the actual (monotonic-since-boot) behavior of the neutral default and point to the new function. Simulation mode and every core.c-including test fixture that constructs a bare core directly are unaffected (they never call the new function, so they keep the (0,0,1,1) neutral default or sim_set_virtual_clock()'s own anchors exactly as before). TDD: added test_xce_core_set_wallclock_anchor_seeds_real_epoch_time (brackets xce_core_now() immediately after the call between two back-to-back CLOCK_REALTIME reads -- fails loudly pre-fix since the bracketed value would be a small monotonic-since-boot number, not a real ~1.7e18ns epoch value) and test_xce_core_set_wallclock_anchor_null_core_is_a_safe_noop to test_clock.c. Verified via /xce-build scheduler asan --incremental: build clean, test_clock 100% green (17/17 including both new tests). One pre-existing, unrelated failure in the same suite run (test_admin_request's test_recordstart_symlink_escape_replies_error, bug-1560, a sandbox uid-permission-ordering issue) confirmed unaffected -- git status confirms test_admin_request.c was not touched by this fix. Not yet live-validated against a real xceadmin CLI run end-to-end (`xceadmin scheduler record start/status`) -- recommend a live /xce-deploy-ha smoke check before considering this fully closed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1585.md b/.wolf/bugs/bug-1585.md new file mode 100644 index 00000000..3b333866 --- /dev/null +++ b/.wolf/bugs/bug-1585.md @@ -0,0 +1,42 @@ +--- +id: "bug-1585" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T02:47:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/scheduler/src/core_events.c (XCE_CORE_EVENT_RESVALS_FULL/RESVALS_DELTA handlers -- root cause)", "src/scheduler/src/main.c (--record-snapshot-interval CLI wiring)", "src/scheduler/src/repl.c (banner clarification)", "src/scheduler/testsuite/test_resval_ownership.c (new regression test)", "src/scheduler/testsuite/test_core_events.c/test_core_full_cycle.c/test_driver_decision_cb.c/test_driver_session_ordering.c/test_driver_translate.c (dup_val link stub + stale-comment/fixture fix)"] +tags: ["scheduler", "sim", "repl", "replay", "snapshot", "jobs", "picker", "heap-use-after-free", "resval", "zarray", "ownership-bug", "sigsegv", "cli-flag", "record-snapshot-interval", "fully-fixed", "live-reproduced", "tdd"] +related: ["bug-1569", "bug-1560"] +legacy: true +--- + +# bug-1585: User-reported, live `xce_scheduler -i -s /tmp/x1.replay` REPL session against a real .replay file… + +## Description +User-reported, live `xce_scheduler -i -s /tmp/x1.replay` REPL session against a real .replay file recorded from a live normal-mode deployment: `info summary`/`info worker all` show '0 jobs (0 pending, 0 running)' immediately after load, even though the scheduler log and xce_test confirm jobs were actually submitted/present during that recording. Startup also logs a WARN: `xce_sched_core_load_snapshot failed to resolve job picker plugin (rc=-2); picker rebuild will be skipped for this load`. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +TWO stacked, independently-confirmed causes, both live-reproduced this session against a real controller+scheduler+xcecd_virtual deployment (peers localhost:19991, worker-1): + +(A) FULLY ROOT-CAUSED AND FIXED -- a genuine heap-use-after-free crash, NOT the picker-resolve WARN (confirmed a red herring, per this entry's own prior analysis of xce_sched_core_load_snapshot()) and not purely a test-methodology gap. core_events.c's XCE_CORE_EVENT_RESVALS_FULL (line ~750) and XCE_CORE_EVENT_RESVALS_DELTA (line ~786) handlers did `zarray_put(&(zarray), idx, rv->val)` -- a SHALLOW store of the wire-decoded xce_val_t* pointer (confirmed via zarray_put_ex(), src/lib/zarray.c: `array->data[idx] = data[i]`, no copy). The zarray now owns that pointer and hands it to its killer (kill_sched_resval, schedobj.c) the next time that index is replaced. But driver.c's release_translated_payload() unconditionally frees the ENTIRE decoded ResValList/ResourceVal array (including that same rv->val) immediately after xce_core_push_events() returns, on EVERY drain, regardless of adoption. Result: the very next periodic resval report for the same worker+resource -- which happens every resource_sync interval (~1-15s) on literally any live deployment -- evicts the now-dangling first value via zarray_put(), crashing the scheduler with ASAN heap-use-after-free inside free_Value() (lib/controller.capnp.c:2898). Live-reproduced TWICE against a real deployment during this session's own repro attempts (first crash: PID 68752, silently lost because stderr went to /dev/null under start-scheduler.sh's nohup redirection; second crash, WITH stderr captured this time: PID 73919, full ASAN backtrace obtained -- xce_core_push_events -> zarray_put -> kill_sched_resval -> free_Value, use-after-free on a pointer release_translated_payload had freed one drain earlier). This exact hazard had ALREADY been noticed and explicitly documented as a 'PRE-EXISTING production ownership characteristic' in test_driver_translate.c's own test comment (task 5.1) but never fixed, because that test's kill_sched_resval is a no-op stub (its own comment wrongly generalizes 'fixtures are never torn down so these killers are never invoked' -- true for the OTHER kvmgr datakillers which fire only on core teardown, but NOT true for a zarray's killer, which fires on every REPLACEMENT of an occupied slot, not just teardown -- that blind spot is exactly why no existing test caught this). This crash means a live deployment's scheduler process is a near-certain crash within one or two resval-report cycles of any real session, corrupting/truncating whatever recording was in progress -- the practical dominant explanation for why a `.replay` file recorded from a real live session so often ends up missing jobs the operator knows were submitted. + +(B) CONFIRMED BY DESIGN, NOT A DEFECT (bug-1569 precedent, re-verified live this session after A's fix) -- once the crash was fixed, a full clean recording session was captured live: submitted 3 jobs via xce_test while `xceadmin scheduler record start` was active, waited for real DISPATCH decisions, `xceadmin scheduler record stop`, then `xceadmin scheduler dump`. The dump showed the SnapshotEntry (initial) genuinely reflecting whatever core state existed at the exact instant `record start` was issued (6 pre-existing jobs in this session's case, since jobs from an earlier test batch were still live) -- ReplayEvents (jobsPush/jobUpdate/resvalsDelta) for state that changed AFTER that instant correctly appear later in the stream, and only become visible in the REPL after `step`/`continue` (querying `info jobs`/`info summary` immediately after `[sim] ready.`, as the user's own transcript shows, only ever shows the starting snapshot's baseline -- exactly bug-1569's own prior finding for resvals, now confirmed to apply identically to job counts). Live-confirmed via `step N`: job count correctly grew from 6 to 9 once the REPL stepped past the recorded jobsPush/dispatch events, with correct per-job states (RUN/PEND) matching xce_test's live joblist at that exact point. The REAL underlying operator-facing gap this session's own analysis (and tasks.md's task-9.3 review note) already flagged: `--record-snapshot-interval` had NO CLI wiring anywhere in main.c, even though the struct field (xce_scheduler_t.record_snapshot_interval) and its consumption logic (recording_events.c's sched_record_write_events(), task 9.3) were already fully implemented and unit-tested -- so an operator had no way to ever get a periodically-refreshed starting snapshot; a `.replay` file with no periodic checkpointing legitimately has exactly ONE SnapshotEntry (written at `record start` time), and the REPL correctly starting from that baseline is expected behavior, not a bug. + +## Fix +(A) FIXED -- core_events.c's RESVALS_FULL/RESVALS_DELTA handlers now `dup_val(rv->val)` before `zarray_put()` (both call sites), matching the deep-copy-before-adopt convention this codebase already established for the identical hazard on the snapshot-load side (snapshot_load.c: `zarray_put(&(core->sharedInst), r->idx, dup_val(rv->val))` / `zarray_put(&(w->resvals), r->idx, dup_val(nv->val))`). `dup_val` (defined in snapshot_wire.c) is now also `extern`-declared and consumed by core_events.c. TDD: new src/scheduler/testsuite/test_resval_ownership.c (2 tests, wired into meson.build as 'scheduler resval ownership test') white-box-includes core.c/core_clock.c/core_events.c/driver.c with a REAL (non-stubbed) kill_sched_resval (schedobj.c's implementation verbatim) plus a real dup_val reimplementation -- RED confirmed the exact live ASAN backtrace (heap-use-after-free in free_Value via zarray_put's eviction path) on the pre-fix code; GREEN after the fix, with an added assertion the evicted-and-replaced resval reads back correctly (no data corruption, not just 'didn't crash'). The other 5 pre-existing white-box test files that #include core_events.c (test_core_events.c, test_core_full_cycle.c, test_driver_decision_cb.c, test_driver_session_ordering.c, test_driver_translate.c) each needed a small real dup_val reimplementation added (link-satisfying, same 'reimplement the one small function' idiom as their existing sched_resolve_job_picker copies) since core_events.c now references it; test_driver_translate.c's own `test_drain_translates_and_pushes_all_ten_op_types` additionally needed its full_val/delta_val fixtures to set `->which = XCE_VAL_NUMERIC` explicitly (a bare calloc() leaves `which` at 0 == XCE_VAL_STRING, which dup_val() now actually inspects -- exposed a latent test-fixture gap, not a production bug) and its stale 'PRE-EXISTING production ownership characteristic ... unsafe to dereference' comment was corrected to reflect the fix. Live-reproduced end-to-end post-fix: restarted the scheduler against the same live deployment, let 22+ periodic resval cycles for the same worker+resource pass with zero crash (previously guaranteed to crash on the 2nd cycle), then ran a full record-start/submit-jobs/record-stop/dump/REPL-step cycle cleanly. Full scheduler suite green (41/42, only bug-1560 unrelated). (B) FIXED -- `--record-snapshot-interval N` is now a real CLI flag on main.c's argp table (OPT_RECORD_SNAPSHOT_INTERVAL=262), mirroring `--record-dir`'s exact pattern: parsed with the same strtoull()+validation shape --sim-speed already uses, assigned to `sched.record_snapshot_interval` right alongside `sched.record_dir`. No dedicated unit test was added (main.c defines its own `main()`, so it cannot be white-box #included into a test TU without a symbol collision, and this codebase's own precedent -- --record-dir itself -- has zero dedicated unit test either, relying on code review + the consumption side's existing coverage); verified instead via a genuine live E2E run: restarted the scheduler with `--record-snapshot-interval 3`, submitted 5 jobs across a record start/stop cycle, and confirmed via `xceadmin scheduler dump` that THREE periodic 'SnapshotEntry (checkpoint)' entries now appear mid-stream (previously only ever one, at record-start), each fully populated (14 jobs) -- then loaded the file via `-i -s FILE` and confirmed `[sim] ready.` now shows all 14 jobs immediately, with zero stepping required. Also added a one-line repl_print_banner() clarification (repl.c): '[sim] (jobs above reflect this snapshot only -- step/continue to replay later events)', printed between the loaded-snapshot line and the ready-prompt line -- verified this doesn't break test_repl_execution_control.c's own banner assertion (a strstr() substring/ordering check, unaffected by an inserted line). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1569, bug-1560 diff --git a/.wolf/bugs/bug-1586.md b/.wolf/bugs/bug-1586.md new file mode 100644 index 00000000..373a178a --- /dev/null +++ b/.wolf/bugs/bug-1586.md @@ -0,0 +1,38 @@ +--- +id: "bug-1586" +status: "wontfix" +status_updated: "2026-09-24" +status_note: "not a bug: resval getters mode-agnostic, live values confirmed non-zero" +component: "worker" +severity: "security" +timestamp: "2026-08-19T02:47:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/worker/src/resource_sync.c (collect_load_indices)", "src/worker/src/node_information.c (the 10+1 node_info_get_* getters)", "src/worker/meson.build (xcecd vs xcecd_virtual target definitions)"] +tags: ["worker", "resource-values", "xcecd-virtual", "environment", "sandbox", "sudo", "container", "investigation-only", "confirmed-non-bug", "code-inspection", "test-methodology-artifact"] +related: ["bug-1569", "bug-1585"] +legacy: true +--- + +# bug-1586: User-reported, same live REPL session: `info worker all` shows `total=0.00 available=0.00` for… + +## Description +User-reported, same live REPL session: `info worker all` shows `total=0.00 available=0.00` for mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io (the periodic/dynamic resval metrics) on a worker that should have real non-zero values. User's own hypothesis: this worker was running in 'virtual host mode' (xcecd_virtual, a synthetic/mocked worker used for sandbox testing, not a real xcecd against real host hardware) which may never populate these metrics with real readings; wants to validate against a real (non-virtual) xcecd worker instead, but notes the sandbox/container this session runs in needs a fix to make `sudo` work before that's possible (real xcecd's periodic resource collection -- disk/load/mem readings -- may need elevated privileges not currently available in this container). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED, INVESTIGATION-ONLY (no code bug found -- design is correct, working as intended): `sudo -n true` fails in this sandbox (`sudo: unable to initialize PAM: Critical error - immediate abort`, exit 1) -- confirmed non-interactively this session, real (non-virtual) xcecd cannot be brought up here (its check_user() unconditionally requires geteuid()==0 for the production binary). Fell back to code-level determination, and it is conclusive: worker/meson.build shows `xcecd` and `xcecd_virtual` are compiled from the IDENTICAL source list (`xcecd_srcs`, including resource_sync.c and node_information.c verbatim) -- the only difference is the `-DVIRTUAL_CLUSTER` preprocessor define, which (grepped across every worker .c/.h file) gates ONLY root-privilege/sqlite-db-recovery/dbus-manager/process-execution-related code paths (xcecd.c, xcecd_db.c, controller_connector.c, xcecd_dbus_manager.c, xcecd_worker_utils.c, main.c's check_user()) -- resource_sync.c and node_information.c have ZERO VIRTUAL_CLUSTER conditionals anywhere. Read every one of the 11 node_info_get_*() getters (node_information.c): all are REAL, unconditional system-reads -- read_system_loadavg() parses /proc/loadavg (r1m/r15m/r15s), read_memory_info() parses /proc/meminfo (mem), node_info_get_tmp() calls statvfs('/tmp'), node_info_get_io()'s read_disk_io_rate() parses /proc/diskstats, node_info_get_ls() walks utmp via setutent()/getutent(), node_info_get_it() reads utmp activity or /proc/uptime as a fallback -- none are stubbed, mocked, or synthesized for virtual mode. collect_load_indices() (resource_sync.c) calls these same getters unconditionally regardless of xcecd vs xcecd_virtual. This directly corroborates bug-1569 thread A's own prior, independent live finding on this exact question (already in this buglog, closed): 'step N in the REPL ... mem=16786444288.00, r1m=0.64, r15m=0.99, r15s=0.64, ut=2.08, pg=463.20, it=3341.19, tmp=65732259840.00 -- all correct; ls/swp/io legitimately 0 on this idle sandbox' -- using this SAME xcecd_virtual binary. This session's own bug-1585 live repro work independently reconfirmed it fresh, twice: (1) the scheduler log during the bug-1585 deployment showed genuinely non-zero periodic resvals flowing from worker-1 (xcecd_virtual) -- e.g. 'resval applied host=worker-1 resource=mem value=19967893504.000000', 'resource=r15s value=4.610000', 'resource=tmp value=801655111680.000000', 'resource=it value=5739.550000'; (2) after fixing bug-1585 and loading a periodic-checkpoint snapshot via the REPL, 'info worker worker-1' showed mem=18382995456.00, r1m=4.93, r15m=5.73, r15s=4.93, ut=24.26, pg=1413.87, it=5768.14, tmp=797803061248.00 -- all real, plausible non-zero values; only ls/swp/io read 0.00, exactly matching bug-1569's own 'legitimately 0 on this idle sandbox' explanation (no logged-in users in utmp, no swap configured, negligible disk I/O delta in a short idle window -- not a bug, a correct reading of an idle/minimal sandbox). CONCLUSION: the user's reported all-zero values were almost certainly the SAME REPL-query-before-step/continue methodology gap bug-1569 already root-caused and bug-1585 (this session) re-confirmed and fixed the recording-side half of -- NOT a virtual-vs-real xcecd distinction. xcecd_virtual is not designed to stub/mock these 10 metrics; it collects them for real, identically to production xcecd. + +## Fix +NO CODE FIX -- none needed; this is a confirmed non-bug (design is correct, resource_sync.c/node_information.c's collection code is genuinely virtual/real-mode-agnostic, verified by full source read plus two independent fresh live confirmations this session of real non-zero xcecd_virtual readings, corroborating bug-1569's own prior identical conclusion). If a live real-vs-virtual xcecd comparison is still wanted for extra certainty, it requires fixing this sandbox's `sudo`/PAM configuration first (out of scope for a code-level investigation) -- but given resource_sync.c/node_information.c have zero VIRTUAL_CLUSTER branching and every getter is a genuine unconditional /proc or utmp read, a real xcecd would be expected to read the SAME live host counters xcecd_virtual already does (same process, same kernel, same /proc), just without the artificial worker-name/slot-count synthesis xcecd_virtual's OTHER (unrelated) code paths add. No further action recommended unless a NEW live report of persistently-zero values is seen AFTER confirming step/continue was used correctly. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1569, bug-1585 diff --git a/.wolf/bugs/bug-1587.md b/.wolf/bugs/bug-1587.md new file mode 100644 index 00000000..55edb561 --- /dev/null +++ b/.wolf/bugs/bug-1587.md @@ -0,0 +1,39 @@ +--- +id: "bug-1587" +status: "fixed" +status_updated: "2026-09-24" +status_note: "c9acd502" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T02:47:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl.c:183"] +tags: ["scheduler", "repl", "ux", "timestamp", "formatting", "not-yet-fixed"] +related: ["bug-1584"] +fix_commits: ["c9acd502"] +legacy: true +--- + +# bug-1587: User feedback (UX, not a functional defect): the REPL's startup line `[sim] loaded: 0 jobs… + +## Description +User feedback (UX, not a functional defect): the REPL's startup line `[sim] loaded: 0 jobs, snapshot @ ts=331886899213871` prints the snapshot timestamp as a raw nanosecond integer -- would be more readable as a human-readable date/time. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl.c's startup message (`fprintf(stdout, "[sim] loaded: %d jobs, snapshot @ ts=%llu\n", njobs, ...)`) formats the raw uint64 nanosecond value directly with no conversion -- purely a display/formatting gap, not a logic defect. Now that bug-1584 makes a normal-mode-recorded .replay file's timestamps genuine epoch nanoseconds (previously they were CLOCK_MONOTONIC-since-boot, which would have made a 'human-readable' rendering of this same field misleading/wrong before that fix), converting this to a readable date is now meaningful for normal-mode recordings; simulation-mode-authored .replay files may still show a discrete/simulated ts depending on how --sim-start-ts was chosen, but a calendar rendering of an epoch-ns value is well-defined either way. + +## Fix +NOT YET FIXED. Straightforward: convert the raw `ts` to a formatted string (matching core_clock.c's existing ns_to_tlog_time()-style gmtime_r() conversion, or a simple strftime over ts/1e9) before printing, likely showing both the human-readable form and the raw ns value for debugging (`snapshot @ ts= ()`). Low risk, single call site (repl.c:183) -- worth checking whether other REPL/dump output (repl_info.c, xceadmin's `scheduler dump`) has the same raw-ns-only presentation and would benefit from the same treatment while this is being fixed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1584 diff --git a/.wolf/bugs/bug-1588.md b/.wolf/bugs/bug-1588.md new file mode 100644 index 00000000..48b84bb5 --- /dev/null +++ b/.wolf/bugs/bug-1588.md @@ -0,0 +1,86 @@ +--- +id: "bug-1588" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-19T02:47:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: ["src/controller/job.c (dispatch_a_job_ ~841", "on_job_start ~1662", "on_job_finish ~1766) -- likely more sites", "not yet grepped exhaustively; PLUS (open", "not yet fixed) src/worker/src/{xcecd_dbus_manager", "xcecd", "main", "xcecd_worker", "controller_connector", "uv_ipc", "xcecd_cgroups", "xcecd_worker_utils}.c and src/controller/testsuite/test.c -- see fix field for full enumeration"] +tags: ["controller", "logging", "observability", "jobid", "ux", "job.c", "not-yet-investigated", "worker", "partial-fix"] +related: [] +legacy: true +--- + +# bug-1588: User feedback (UX/observability, not a functional defect), from logs/ctrl-1.log: job IDs printed… + +## Description +User feedback (UX/observability, not a functional defect), from logs/ctrl-1.log: job IDs printed in controller log lines are large, non-sequential-looking 64-bit numbers (e.g. `4631574334564990976`, `4631574334564990977`, `4631574334633148416`) that are hard to visually scan/compare/diff -- would be better rendered in a more human-readable form. Affected call sites (src/controller/job.c): `dispatch_a_job_()` (job.c:841, 'dispatch_a_job_ jobid '), `on_job_start()` (job.c:1662, 'on_job_start jobid '), `on_job_finish()` (job.c:1766, 'on_job_finish job finish jobid exit code '). Likely other job.c/controller log call sites print the same raw jobid and would benefit from the same treatment (not yet enumerated). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED, FIXED. The codebase already has a well-established human-readable jobid formatter -- fmt_jobid()/fmt_jobid_ex() (src/lib/utils.c:466-507, declared src/include/utils.h:132-133) -- used consistently across scheduler.c, handler.c, plugins/limit.c, plugins/fcfs.c, repl_info.c (scheduler), process_manager.c, uv_proc.c, xcecd_worker_utils.c (worker), and xce_test/waldump.c (controller testsuite tooling). It renders idgen()'s packed 64-bit jobid (src/lib/idgen.c, bit layout in src/include/config.h: 2 tag bits + 6 cluster-id bits + 4 apiserver-id bits + 32 timestamp-seconds bits + 20 sequence bits, packed via MAKE_JOBID/unpacked via the EXTRACT_ID_* macros in src/include/utils.h) as 'cluster-apiserver-timestamp-sequence' (e.g. '3-1-1755590313-976'), the SAME format REGEX_JOBID/parse_jobid() already parse back from user input and xcecli already displays to end users -- there is no separate 'new' human-readable scheme to design, job.c was simply the one place that never adopted the existing one, printing the raw packed integer via a mismatched `%ld` (a uint64_t formatted as a signed long -- a latent type/format mismatch, incidentally fixed as a side effect since the fix replaces the whole format specifier with %s). + +## Fix +Fixed. src/controller/job.c: all 8 TRACE() call sites that printed a raw jobid now call fmt_jobid(jobidStr, sizeof(jobidStr), ) into a local `char jobidStr[64]` (matching scheduler.c's established buffer-size convention) immediately before the TRACE() call, and print `%s` instead of `%ld`: dispatch_a_job_() (~841), on_job_start() (~1662), on_job_finish() (~1766), on_job_fail() (~2026), on_job_stop() (~2055), on_job_resume() (~2090), on_job_postdone() (~2123), set_preason() (~2297). fmt_jobid is already transitively visible in job.c via request.h's #include "utils.h" -- no new include needed. Purely a log-formatting change, zero logic/behavior change -- verified via /xce-build controller asan --incremental: job.c compiles with zero errors/warnings (compiled twice, for xce_controller and controller_unit_test), full controller suite green (1104/1104 Unity assertions, 5/5 meson test cases), git diff confirms job.c is the only file touched in src/controller/. Not live-validated against a real deployment's log output (no functional/behavioral risk to validate beyond the build+test suite, since this only changes what a string looks like) -- if desired, a live run would show e.g. 'dispatch_a_job_ jobid 3-1-1755590313-976' instead of 'dispatch_a_job_ jobid 4631574334564990976'. + +--- + +Remaining sites enumerated 2026-08-21 via `ag -i job src | grep '%lu'` (job.c's 8 sites from this bug are already fixed; these are NOT yet fixed): + +WORKER (src/worker/src/) -- still print the raw uint64 jobid via %lu instead of calling fmt_jobid(): +- xcecd_dbus_manager.c:423 +- xcecd.c:60 +- main.c:167,227,245 +- xcecd_worker.c: ~25 sites between lines 83-1314 (rusage, VNC, exdata, port assignment, signal/pause/resume, CRIU checkpoint status messages) +- controller_connector.c: ~20 sites between lines 508-1857 (job dispatch, remote task, job-control action, rusage-to-controller messages) +- uv_ipc.c: ~20 sites between lines 77-1541 (client mapping, signal/checkpoint ack, rusage, job completion/failure) +- xcecd_cgroups.c:1703 (rusage summary log line; xcecd_cgroups.c:1405's `xcecd-worker-%lu.service` unit name and xcecd_worker.c:1209's `%s/%lu` checkpoint dir / xcecd_exdata.c:408's numeric-string exdata substitution are identifier/path construction, not human-facing log text -- left as-is pending a decision on whether those should change too) +- xcecd_worker_utils.c: 8 sites between lines 240-504 -- correction to this bug's original note: original text claimed xcecd_worker_utils.c "already use fmt_jobid() consistently", but `grep -c fmt_jobid( src/worker/src/xcecd_worker_utils.c` now returns 0; that claim does not hold for the current file state (process_manager.c and uv_proc.c's fmt_jobid() usage, also cited in the original note, is still confirmed correct) + +CONTROLLER TESTSUITE (src/controller/testsuite/test.c) -- interactive test-tool printf() output, lower priority than production daemon logs: +- lines 2599, 2649, 3966, 4043, 4048, 4053, 4155, 4180, 4294, 4353 + +Explicitly OUT of scope -- these matched the `%lu` grep but are not job IDs, so must NOT be converted: src/controller/job.c:1332 and src/controller/persist.c:6376,6530 all format `lsn` (WAL log sequence number), not a jobid; src/worker/src/main.c:160 formats a `client ID`, not a jobid. + +Suggested fix pattern (matches this bug's job.c fix): at each site, add `char jobidStr[64]; fmt_jobid(jobidStr, sizeof(jobidStr), );` immediately before the log call, and replace the `%lu` for that argument with `%s` / jobidStr. utils.h (which declares fmt_jobid) needs to be confirmed reachable in each of these worker files (uv_ipc.c, controller_connector.c, xcecd_worker.c, xcecd_dbus_manager.c, xcecd.c, main.c, xcecd_worker_utils.c) before starting -- not yet verified. + +Not yet fixed. Logged as a follow-up scope expansion of this same bug per user request ("log an issue to fix all job ids which are shown as uint64 to human readable string via fmt_jobid()"), 2026-08-21. + +--- FOLLOW-UP FIXED 2026-08-21 --- + +All remaining sites enumerated above are now fixed too (user request: "make sure that all job ids which have been shown will use human readable format ... please also include src/controller/testsuite/test.c, as we use it test controller heavily"). Same fmt_jobid()+%s pattern as job.c, applied via 5 parallel edit passes (one per file/group) each independently instructed with the exact site list and the exclusion list below, then independently reviewed line-by-line against the originals by the orchestrating session (not just trusted on the subagents' self-reports) before building. + +Files fixed (99 call sites total across 9 files): +- src/worker/src/xcecd_worker.c -- 30 sites (worker_signal_callback needed two buffers, jobidStr + curJobidStr, for its two distinct job-id values on one line) +- src/worker/src/controller_connector.c -- 21 sites (handle_fetchfile: only the jobid= half of a two-%lu line converted, offset= left alone) +- src/worker/src/uv_ipc.c -- ~20 sites (add_job_client_mapping: only the job_id half of a two-%lu line converted, client->id left alone) +- src/worker/src/xcecd_worker_utils.c -- 9 sites +- src/worker/src/controller_connector.c, xcecd_dbus_manager.c, xcecd.c, main.c, xcecd_cgroups.c -- remaining small sites (main.c's 4 sites all genuine job ids despite one message's misleading "client ID" wording -- verified against the function parameter type/usage, not just the string) +- src/controller/testsuite/test.c -- 10 sites, matching this file's own pre-existing fmt_jobid()-with-'- 1'-sizeof convention + +Deliberately left unconverted (confirmed non-jobid or non-log-text, do not touch): +- src/controller/job.c:1332 and src/controller/persist.c:6376,6530 -- format `lsn` (WAL log sequence number), not a jobid +- src/worker/src/controller_connector.c:1376 -- the `offset=%lu` half of a fetchfile log line +- src/worker/src/uv_ipc.c -- the `client->id` half of `add_job_client_mapping`'s log line +- src/worker/src/xcecd_worker.c:1231 (ckpt_dir path), xcecd_cgroups.c:1405 (`xcecd-worker-%lu.service` unit name), xcecd_exdata.c:408 (`XCE_JOBID` env var set for the job's own process) -- all raw-numeric identifier/path/protocol construction, not human-facing log text; changing these would be a behavioral change (on-disk paths, systemd unit names, a documented env var format), not a log-formatting change + +Verification: +- /xce-build worker asan --incremental: SUCCESS, 430/430 ninja targets, zero warnings/errors related to fmt_jobid/jobidStr in any of the 8 edited worker files, worker meson test suite 4/4 OK (resource_sync, xcecd_startup_order, dbus_workers_update, controller_connector tests). +- /xce-build controller asan --incremental: SUCCESS, test.c compiled with zero warnings/errors, controller meson test suite 4/5 OK -- the 1 failure (test_config_repl.c, 2 Unity assertions) is the pre-existing, previously-logged bug-1561 overlayfs-sandbox fdatasync flake (7th occurrence, `git diff --stat` confirms config_repl.c/test_config_repl.c untouched by this change) -- not caused by this fix. +- Every diff was read and checked line-by-line against the pre-edit originals by the orchestrating session (argument-to-specifier order, buffer scoping/reuse correctness, exclusion list respected) before either build ran -- not just accepted on the editing subagents' self-reports. + +Not fixed / explicitly out of scope: no attempt was made to reconcile the on-disk checkpoint-dir/systemd-unit-name/XCE_JOBID raw-numeric conventions with fmt_jobid() -- flagged above as a separate judgment call if ever wanted, not part of this fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1589.md b/.wolf/bugs/bug-1589.md new file mode 100644 index 00000000..0a86cbbf --- /dev/null +++ b/.wolf/bugs/bug-1589.md @@ -0,0 +1,38 @@ +--- +id: "bug-1589" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "worker" +severity: "unknown" +timestamp: "2026-08-19T03:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/worker/src/xcecd_dbus_manager.c (dbus_workers_update() -- THE fix)", "src/worker/src/main.c (confirms master_job_status_callback/ipc_master path is UNREACHABLE for xcecd_virtual -- and for real xcecd too)", "src/worker/testsuite/test_dbus_workers_update.c (new regression test)", "src/controller/job.c (on_job_start/dispatch_a_job_ -- read/traced", "confirmed innocent", "not touched)"] +tags: ["worker", "xcecd", "xcecd-virtual", "job-array", "concurrency", "job-id", "controller", "root-caused", "dbus-workers-update", "linked-list-traversal", "head-of-line-blocking"] +related: ["bug-1586"] +legacy: true +--- + +# bug-1589: User-reported, live 3-node deployment log analysis: a 5-element job array… + +## Description +User-reported, live 3-node deployment log analysis: a 5-element job array (base_jobid=0-7-1787111880-0, array_size=5) was fully expanded and all 5 elements were dispatched by the controller (job.c's dispatch_a_job_ fires for [0]-[4], confirmed in logs/ctrl-1.log) and elected by the scheduler's limit plugin (logs/scheduler.log's limit_job_election fires for [0]-[4]), but only element [0] ever produced an on_job_start log line (job.c:1673) in either log file -- elements [1]-[4] were dispatched but never reported as started. User's own hypothesis: may be related to the worker running in 'virtual host mode' (xcecd_virtual) rather than a real xcecd worker. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via source trace + live reproduction (both hypotheses from the prior static-reading pass were REFUTED). (1) Code path: worker/meson.build shows xcecd_virtual == xcecd_srcs + main.c, built with -DVIRTUAL_CLUSTER -DDBUS_MANAGER -- and the REAL xcecd target uses the SAME xcecd_srcs + main.c with -DDBUS_MANAGER only. Because DBUS_MANAGER is defined for BOTH binaries, main.c's run loop (`#ifndef DBUS_MANAGER worker_process_update(...) #else dbus_workers_update(...) #endif`, main.c:603-616) ALWAYS takes the dbus_workers_update() branch for both -- process_worker_manager.c's worker_process_update() (the 'looks less obviously broken' alternative flagged in the original hypothesis) is dead code, never called, in either build. Separately, ipc_master_init() is `#ifndef VIRTUAL_CLUSTER`-gated (main.c:571-579) so for xcecd_virtual no IPC-master unix-socket listener is ever created; master_job_status_callback() (main.c:163) and the single global `static xcecd_job_t job` (main.c:32) are therefore UNREACHABLE for xcecd_virtual (no forked job-runner ever connects to report MSG_JOB_STATUS over that socket) -- the original hypothesis about a missing `job.job_id = job_status->job_id` assignment there was investigating dead code for this deployment shape; it may still be a latent issue for a REAL xcecd with genuinely concurrent forked job-runners, but is NOT bug-1589's cause and was not touched. (2) Real root cause: dbus_workers_update() (xcecd_dbus_manager.c:178, called from main.c's run loop every ~10ms) iterates xcecd_ctx.worker_list -- one xcecd_worker_t node per concurrently-dispatched job -- with `xcecd_worker_t* worker = *worker_list; while (worker) { ... worker = worker->next; }`. Several branches inside the loop body used a bare `continue` that re-tests the SAME `worker` pointer instead of advancing (the VIRTUAL_CLUSTER JOB_STARTING->JOB_RUNNING branch, xcecd_dbus_manager.c:201 pre-fix; the exit_code==0 and exit_code<0 VIRTUAL_CLUSTER completion-check branches, pre-fix lines 241/268; the real-mode 'job done but running not received' branch, pre-fix line 291) -- and worse, the WORKER_DONE/WORKER_EXITED early check (xcecd_dbus_manager.c:205-209 pre-fix) did `worker = worker->next; return true;`, which ABORTED THE ENTIRE FUNCTION the moment ANY worker in the list was already finished, discarding the scan of every worker queued behind it (also a latent bug: `return true` == `return 1` == XCECD_ERROR on a function whose only real caller only special-cases XCECD_DB_ERROR, so this wasn't even surfaced as an error). Net effect on a job array: once dbus_workers_update() started making progress on the HEAD of xcecd_ctx.worker_list (element [0], first added via add_worker()), every OTHER concurrently-dispatched element ([1]-[4]) was starved of update_worker_status() -- the function that fires xcecd_reply_startjob() i.e. the on_job_start report -- for as long as element [0] stayed un-reaped from the list (which for a long-running job is indefinite, matching the user's 'never, in either log file' observation for a job array of `sleep N`-style long jobs; for short/instant jobs the same bug still starves [1]-[4] but only for the ~10ms tick until cleanup_worker_list() reaps the finished head and the next element becomes head in turn). This is a WORKER-side bug only -- the controller's on_job_start()/record_job_start_fact() (job.c:1665-1663) and its job_state_is_backward() guard (job.c:43) were traced and confirmed to correctly key everything by jobid with no cross-job interference; they simply never received a report for [1]-[4] because the worker never sent one. + +## Fix +FIXED. xcecd_dbus_manager.c's dbus_workers_update(): capture `xcecd_worker_t* next = worker->next;` once at the top of every loop iteration (right after `while (worker) {`), and changed every early-exit path in the loop body to `worker = next; continue;` instead of a bare `continue` (re-tests the same node forever) or the old `worker->worker_status == WORKER_DONE/WORKER_EXITED` branch's `return true` (aborted the whole scan on the first finished worker). This fixes traversal for BOTH VIRTUAL_CLUSTER and real (non-VIRTUAL_CLUSTER) branches of the function -- the real-mode `kill(pid,0)!=0` / 'job done but running not received' branch had the identical bare-continue bug and would starve a real xcecd's worker_list the same way under concurrent dispatch. No change to the state-machine logic itself (update_worker_status(), check_worker_sleep_time_elapsed(), the systemd dbus_is_worker_active()/get_systemd_job_exit_code() calls) -- purely a traversal/control-flow fix, matching the 'minimal, correct, no symptom-papering' guidance; considered and rejected converting worker_list to a per-jobid array (process_manager.c's processes[] precedent) since the linked list itself was never the problem, only its traversal. TDD: new src/worker/testsuite/test_dbus_workers_update.c (2 tests, white-box #includes xcecd_dbus_manager.c with -DVIRTUAL_CLUSTER -DDBUS_MANAGER matching xcecd_virtual's real build flags, stubs update_worker_status()/is_systemd_job_active()/get_systemd_job_exit_code()/stop_systemd_job()/make_job_sd_unit_name()/start_systemd_job()/get_systemd_job_pid()/check_worker_sleep_time_elapsed() to isolate the loop's own control flow) wired into src/worker/testsuite/meson.build. RED-verified live: `git stash push -- src/worker/src/xcecd_dbus_manager.c`, rebuild, both tests FAIL ('Expected 0 Was 1' -- the started-call-seen assertions correctly detect elements [1]-[4] never receiving their start transition, and a DONE head worker correctly blocks the scan); `git stash pop`, rebuild, both tests PASS (GREEN). Live end-to-end proof (not just the unit test): deployed a fresh 3-controller + scheduler + xcecd_virtual cluster (/xce-deploy-ha pattern, dedicated dir /tmp/xce-deploy-bug1589), submitted `jobsub -u rshen -q normal -c "sleep 300" -a 5` (5-element array, rshen used specifically to avoid limits.yaml's 'limit1' 3-slot cap on all-users-except-rshen, matching the user's original unconstrained-concurrency scenario) -- ctrl-1.log showed `dispatch_a_job_` AND `on_job_start` for ALL FIVE elements [0]-[4] (previously only [0] would ever show on_job_start), scheduler.log showed `limit_job_election` for all 5 (as it already did pre-fix), a follow-up `joblist` showed all 5 elements in state RUN, and xcecd-worker-1.log showed the `Job ID running - sending status to controller` line (the exact xcecd_reply_startjob trigger) for 5 DISTINCT job ids within milliseconds of each other. Cluster torn down after capturing evidence (self-started, self-stopped, per boundaries). Note: this repro session also observed pre-existing, unrelated 'Failed to connect to system bus' ERROR spam from dbus_is_worker_active()/get_systemd_job_exit_code() polling every ~10ms per still-running virtual job (expected -- no real dbus session in this sandbox, and VIRTUAL_CLUSTER never creates a real systemd unit to poll in the first place) -- cosmetic log noise, not a correctness bug, pre-existing before this fix and unrelated to bug-1589's actual defect; flagged here for visibility, NOT fixed (out of this bug's scope). Also flagged, NOT fixed (out of scope, no buglog entry created per explicit task boundary): dbus_worker_clean() (xcecd_dbus_manager.c) has an apparently-inverted `#ifndef VIRTUAL_CLUSTER return true;` guard -- as written, the REAL (non-virtual) xcecd never actually runs its own systemd-unit cleanup body, while VIRTUAL_CLUSTER mode does (but finds nothing to clean, since no real unit ever existed) -- looks backwards from the evident intent, worth a separate look. Full suite after fix: worker 4/4 GREEN (resource_sync, xcecd_startup_order, dbus_workers_update, controller_connector). Controller 1101/1104 (3 unrelated FAILs in test_config_repl.c -- config version trimming/tag lookup/marker supersession, a file this fix never touches and that was confirmed clean/unmodified by this session; controller/ was not touched at all by this fix). Scheduler: bug-1560's known symlink-escape failure (expected, unrelated) plus 3 NEW failures in test_sim_verify.c/test_repl_breakpoints_inspection.c -- traced via `git status` to src/scheduler/src/driver.c and src/scheduler/src/recording_session.c being concurrently modified, uncommitted, by ANOTHER session's in-flight work in this shared worktree (unrelated 'bug-1591' job.c/driver.c changes observed mid-flight during this session's builds) -- outside this bug's boundary (scheduler/core files are explicitly off-limits for this task) and not caused by this worker-only fix; not investigated further. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1586 diff --git a/.wolf/bugs/bug-1590.md b/.wolf/bugs/bug-1590.md new file mode 100644 index 00000000..dfe3d85d --- /dev/null +++ b/.wolf/bugs/bug-1590.md @@ -0,0 +1,45 @@ +--- +id: "bug-1590" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T03:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/scheduler/src/repl_info.c (repl_info_worker_one ~287-335", "repl_info_resource_one ~410-496); src/scheduler/src/core_events.c (~838-861", "TRACE 'resval applied'); src/controller/worker.c (~107-119); src/controller/dataex.c (~86-90)"] +tags: ["scheduler", "repl", "resource-value", "type-dispatch", "display-bug", "logging", "not-yet-fixed"] +related: ["bug-1568", "bug-1573"] +legacy: true +--- + +# bug-1590: User-reported: `(sim) info worker all` in the scheduler's interactive REPL shows `resource type… + +## Description +User-reported: `(sim) info worker all` in the scheduler's interactive REPL shows `resource type: total=0.00 available=0.00` and `resource model: total=0.00 available=0.00` -- but `type`/`model` are STRING-valued resources (e.g. OS type, hardware model name), not numeric, so 0.00 is meaningless. User also notes boolean-typed resources exist and asked to verify resource-value type-correct formatting broadly, across both REPL output and log lines, not just this one command. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via source read across the whole stack (C scheduler/controller/worker + Go apiserver/xcecli), several distinct severities under one theme -- 'resource values are not type-dispatched when formatted for display/log, several call sites assume every resource is numeric (Value_u_fvalue)': + +(1) MOST SEVERE, matches the exact user report -- `repl_info_worker_one()` (repl_info.c:329-334): `avail = (avail_v->which == Value_u_fvalue) ? avail_v->u.fvalue : 0.0;` then unconditionally `fprintf(..., "total=%.2f available=%.2f", ...)` -- ANY resource whose wire value is Value_u_svalue (string, e.g. `type`/`model`) or Value_u_bvalue (boolean) silently renders as `0.00`, indistinguishable from a genuinely-zero numeric resource. Comment at line 324-328 references bug-1568 (a DIFFERENT, already-fixed bug about comparing against the wrong enum) but bug-1568's fix never added string/bool handling -- it only made the numeric case correct. +(2) Same defect, `repl_info_resource_one()` (repl_info.c:436-444, `info resource `): the total/available aggregation loop only accumulates when `->which == Value_u_fvalue` (correctly guarded, no UB) -- but silently EXCLUDES string/bool resources from the sum entirely, so `info resource type`/`info resource model` would also show `total=0.00 available=0.00` despite `valtype=string` being correctly labeled two lines below it (line 489-495) -- an internally inconsistent-looking display (correct type label, wrong/misleading value). +(3) Related, narrower gap in scheduler logging -- `core_events.c`'s bug-1573 fix (~838-861, the 'resval applied host=... resource=... value=...' TRACE line) DOES correctly branch Value_u_svalue -> '%s' vs Value_u_fvalue -> '%f', but its own comment self-documents that Value_u_bvalue (boolean) is 'silently skipped here rather than guessed at' -- confirmed still true as of this reading: no case handles Value_u_bvalue, so a boolean resource update produces NO trace line at all (silent omission, not a wrong value, but incomplete visibility for exactly the resource kind the user flagged). +(4) Minor/lower severity, controller-side -- `worker.c` (~107-119) has an `else` branch for non-fvalue that logs `which=%d (non-numeric)` (informative about the presence of a non-numeric update, but never decodes/prints the actual string or boolean value); `dataex.c` (~86-90) has NO else branch at all -- silently skips the TRACE entirely for any non-fvalue resource, weakest visibility of the four sites but least user-facing (debug-level TRACE only). + +POSITIVE finding, scoped this from being a systemic whole-stack issue: the Go apiserver's `cNameValToWorkerResval()` (`src/frontend/apiserver/backend/convert.go:325-370`) is a CORRECT, COMPLETE reference implementation -- it switches on all 7 `Value_u_which` wire tags (ivalue32/ivalue64/uvalue32/uvalue64/fvalue all -> NUMERIC/`NumericValue`, svalue -> STRING/`StringValue`, bvalue -> BOOLEAN/`BooleanValue`, matching `models.ResourceDistributionInfo`'s already-correctly-typed fields, `src/frontend/apiserver/models/resources.go:118-126`), plus a `default:` WARN log for any unrecognized tag. `src/frontend/xcecli/models/resource.go`'s `ResourceValue.UnmarshalJSON()` (client-side) also correctly handles string/bool/numeric JSON values. So the production REST API (apiserver) and `xcecli` CLI resource-value display path are NOT affected -- this bug is confined to the scheduler's own C-side simulation-debugging REPL (`repl_info.c`, the user-facing symptom) and a handful of C-side TRACE/debug log call sites in the scheduler/controller (lower severity, debug-visibility-only gaps). + +## Fix +Fixed. src/scheduler/src/repl_info.c: repl_info_worker_one() (`info worker |all`) now switches on the resval's real Value_u_which wire tag -- Value_u_svalue prints `resource : value=`, Value_u_bvalue prints `resource : value=true|false`, Value_u_fvalue keeps the existing `total=%.2f available=%.2f` numeric display unchanged. repl_info_resource_one() (`info resource `) got the analogous fix one layer up: the per-worker aggregation loop now tracks the first-seen string/boolean value (in addition to the existing numeric total/available sum), and the final print branches on the resource DEFINITION's authoritative `r->valtype` (XCE_VAL_STRING/XCE_VAL_BOOLEAN/XCE_VAL_NUMERIC, xce/types.h) to decide which of `value=` (with a 'reported by N worker(s)' count) vs. the numeric total/available pair to print -- total/available is a consumable-quantity concept that is only meaningful for a numeric resource, so a string/boolean resource no longer gets a fabricated total=0.00/available=0.00 pair at all. TDD: added test_info_worker_reports_string_and_boolean_resource_values and test_info_resource_reports_string_value to test_repl_breakpoints_inspection.c, both constructing a real WORKERS_PUSH event with a string- and boolean-valued resource through the actual xce_core_push_events() path (same fixture idiom the existing bug-1568 numeric-resource test already established), asserting the new `value=` lines appear and no `total=` line is ever emitted for a non-numeric resource. Verified via a full non-incremental /xce-build scheduler asan run (fresh wipe+reconfigure, not reusing any cached object): zero compiler errors, both new tests confirmed PASS in testlog.txt, full suite Ok:41/Fail:1 out of 42 -- the sole failure is the pre-existing unrelated bug-1560 (test_admin_request's symlink-escape test). core_events.c's separately-noted boolean-TRACE gap (item 3 in this entry's root_cause) and worker.c/dataex.c's non-numeric TRACE decode gaps (item 4) were NOT touched -- left as documented, lower-priority follow-ups, not part of this fix's scope. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1568, bug-1573 diff --git a/.wolf/bugs/bug-1591.md b/.wolf/bugs/bug-1591.md new file mode 100644 index 00000000..d3bf433e --- /dev/null +++ b/.wolf/bugs/bug-1591.md @@ -0,0 +1,38 @@ +--- +id: "bug-1591" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T03:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/scheduler/src/repl_info.c (queued job: ~230", "assigned job: ~356)", "src/scheduler/src/repl_break.c (~665)", "src/scheduler/src/sim_loop.c (~379", "383", "387)", "src/scheduler/src/sim_verify.c (~293", "325", "335", "346", "373", "380", "389)", "src/scheduler/src/driver.c (~652)", "src/scheduler/src/core_events.c (~872", "907", "979)", "src/scheduler/src/recording_session.c (~173) -- full raw-jobid-print inventory for the scheduler tree", "not yet fixed"] +tags: ["scheduler", "repl", "jobid", "human-readable", "logging", "ux", "not-yet-fixed"] +related: ["bug-1588"] +legacy: true +--- + +# bug-1591: User-reported: `(sim) info queue all` in the scheduler's interactive REPL prints raw packed jobids… + +## Description +User-reported: `(sim) info queue all` in the scheduler's interactive REPL prints raw packed jobids instead of the human-readable form, e.g. `[sim] queued job: jobid=4645085138445664257 priority=0` instead of the 'cluster-apiserver-timestamp-sequence' form (e.g. '3-1-1755590313-976') already used elsewhere in this codebase (bug-1588 fixed the same class of issue in src/controller/job.c's TRACE lines). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same class as bug-1588 (already fixed for src/controller/job.c): fmt_jobid()/fmt_jobid_ex() (src/lib/utils.c, declared src/include/utils.h:132-133) is the established human-readable jobid formatter already used elsewhere in this exact scheduler tree -- scheduler.c, handler.c, plugins/limit.c, plugins/fcfs.c, and even repl_info.c ITSELF at other call sites (e.g. line 119/143/184, per bug-1588's original grep) already call it correctly. The two sites the user's `info queue all` output actually exercises -- repl_info.c's 'queued job: jobid=%llu' (~230, the queue-dump command, `info queue |all`) and its sibling 'assigned job: jobid=%llu' (~356, inside repl_info_worker_one, `info worker |all`) -- were simply never converted, the same oversight bug-1588 already found and fixed in job.c. A broader grep of src/scheduler/src/ for the raw `jobid=%llu`/`jobid: %llu` pattern (see `file` field above) found ~15 more sites across repl_break.c (breakpoint-hit message), sim_loop.c/sim_verify.c (batch --sim DISPATCH/PEND/PREEMPT decision output and --sim-verify divergence reports), driver.c/core_events.c/recording_session.c (WARN/ERROR log lines for unknown-jobid and preemption-not-yet-implemented conditions) -- all likely candidates for the same fix, not yet individually confirmed to be in-scope (some may be deliberately terse WARN/ERROR paths where the raw id is acceptable; needs a per-site judgment call when this is picked up, not a blanket mechanical replace). + +## Fix +Fixed. All 15 raw jobid=%llu sites converted to fmt_jobid() across src/scheduler/src/repl_info.c (2), repl_break.c (1), sim_loop.c (3), sim_verify.c (7), driver.c (1), recording_session.c (1), core_events.c (3) -- same mechanical pattern as bug-1588's job.c fix. This intentionally changed observable REPL/--sim/--sim-verify output text, which broke 5 test binaries' assertions that hardcoded the OLD raw-numeric jobid format (e.g. `strstr(captured, "jobid=4242")`); updated all affected assertions in test_sim_alt_events.c, test_repl_execution_control.c, test_sim_event_loop.c, test_sim_verify.c, and test_repl_breakpoints_inspection.c to compute their expected marker strings dynamically via fmt_jobid() too, matching the existing precedent already used in src/controller/testsuite/test.c. Verified via two /xce-build scheduler asan --incremental runs: first run caught a genuine test regression (5 unexpected failures, all traced to the hardcoded-format issue above, not a production bug); after the test fixes, final run is Ok:41/Fail:1 out of 42 -- all 10 affected tests confirmed individually passing, the only remaining failure is the pre-existing unrelated bug-1560 (test_admin_request's symlink-escape test). One build-tooling wrinkle encountered along the way (not a code bug): an --incremental ninja run once silently skipped recompiling one changed test object, giving a stale Ok:40/Fail:2 result until the object was force-rebuilt -- worth a non-incremental rebuild if an --incremental result ever looks surprising. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1588 diff --git a/.wolf/bugs/bug-1592.md b/.wolf/bugs/bug-1592.md new file mode 100644 index 00000000..1003da0b --- /dev/null +++ b/.wolf/bugs/bug-1592.md @@ -0,0 +1,38 @@ +--- +id: "bug-1592" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "worker" +severity: "unknown" +timestamp: "2026-08-19T07:38:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/main.c"] +tags: ["xce-deploy-ha", "worker", "xcecd", "sudo", "permissions", "logging", "real-worker"] +related: [] +legacy: true +--- + +# bug-1592: grep/tail: Permission denied on $TOP/logs/xcecd.log after starting the real (production) xcecd… + +## Description +grep/tail: Permission denied on $TOP/logs/xcecd.log after starting the real (production) xcecd worker via sudo + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xcecd's main log file (config.log_file, opened by the logging library at daemon startup while running as root) is created with default umask perms (verified live: root:root, mode 640) and never chmod'd permissive -- unlike xcecd_worker.log (explicit open()+chmod(0666) at main.c:536-543) and the domain socket (chmod(config.domain_sock_file, 0666) at main.c:578). Any verification command (grep/tail/cat) run as the invoking non-root user against xcecd.log fails with Permission denied. + +## Fix +Documented as a known behavior in .claude/skills/xce-deploy-ha/SKILL.md: all log reads against the real worker's xcecd.log must be prefixed with sudo (sudo grep .../sudo tail ...), same as the sudo kill already required to stop the root-owned process. Not a code fix -- xcecd.log intentionally stays root-owned; only the skill's documented verification commands changed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1593.md b/.wolf/bugs/bug-1593.md new file mode 100644 index 00000000..2b41b861 --- /dev/null +++ b/.wolf/bugs/bug-1593.md @@ -0,0 +1,38 @@ +--- +id: "bug-1593" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T08:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_info.c (repl_info_resource_one", "repl_info_resource)"] +tags: ["scheduler", "repl", "resource", "shared-resource", "host-based", "metadata", "display-bug"] +related: ["bug-1590"] +legacy: true +--- + +# bug-1593: User-reported (live REPL testing): `info resource all` and `info resource ` aggregate every… + +## Description +User-reported (live REPL testing): `info resource all` and `info resource ` aggregate every worker's per-host resource values (total=sum(available), available=sum) for EVERY resource, including host-based ones like cpus/mem -- a fabricated, meaningless number for a per-host consumable already correctly shown by `info worker `. User asked that `info resource` instead show resource metadata and be restricted to globally-shared resources only. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce_resource_t (xce/types.h) carries a `flags` bitmask (XCE_RESF_BUILTIN/DYNAMIC/GLOBAL/SHARED/EXTERNAL/RELEASE/DEFAULT_HOST/CONSUMABLE/LIMIT) that distinguishes a genuinely cluster-wide resource (GLOBAL/SHARED, stored once in core->sharedInst, indexed by r->idx -- the same pool RESVALS_FULL/RESVALS_DELTA event handling and snapshot save/load already treat as the defining store) from a host-based/per-worker-distributed resource (stored per-worker in w->resvals/w->resInUse, already correctly surfaced by repl_info_worker_one()). repl_info_resource_one() never consulted r->flags or core->sharedInst at all -- it iterated sim->core.workers unconditionally for every resource regardless of kind, summing w->resvals[r->idx] across the whole cluster. For a host-based resource (the common case -- cpus, mem, etc.) this total/available is not a meaningful quantity (no single cluster-wide capacity exists for a per-host resource); for a genuinely shared resource it also happened to work by accident (since a shared resource's value is typically replicated identically into every worker's resvals array too), but even then never reported the resource's own metadata (desc, valtype, flags, interval) the user asked for. + +## Fix +Fixed as part of task 12.5 (scheduler-core spec, Requirement 5 AC 22 amendment). repl_info_resource_one() rewritten to print resource DEFINITION metadata (name, desc, valtype, human-readable flags list, interval) plus, only for a resource flagged XCE_RESF_GLOBAL or XCE_RESF_SHARED, its current value read directly from core->sharedInst[r->idx] (never aggregated across workers). A host-based resource's metadata is still printed but with an explicit note directing the operator to `info worker ` for its actual per-host values -- no fabricated total/available. repl_info_resource() (`info resource all`) now skips any resource whose flags do not include GLOBAL or SHARED, per the user's explicit request to keep this command scoped to cluster-wide resources. New tests in test_repl_breakpoints_inspection.c construct a real GLOBAL/SHARED resource via xce_core_push_events() and a plain host-based one, asserting the metadata line and shared value appear for the former, the host-based one is excluded from `info resource all`, and both appear correctly for `info resource ` with the host-based one omitting a fabricated total/available pair. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1590 diff --git a/.wolf/bugs/bug-1594.md b/.wolf/bugs/bug-1594.md new file mode 100644 index 00000000..fc5ec94e --- /dev/null +++ b/.wolf/bugs/bug-1594.md @@ -0,0 +1,38 @@ +--- +id: "bug-1594" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T08:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c (sched_sim_driver_run)", "src/scheduler/src/repl.c (repl_dispatch", "new xce_repl_on_replay_complete)", "src/scheduler/include/repl.h", "src/scheduler/src/repl_internal.h"] +tags: ["scheduler", "repl", "continue", "quit", "exit-behavior", "ux"] +related: ["bug-1596"] +legacy: true +--- + +# bug-1594: User-reported: in the scheduler's --interactive REPL, `continue` with no breakpoints set runs the… + +## Description +User-reported: in the scheduler's --interactive REPL, `continue` with no breakpoints set runs the simulation to completion and the process then exits -- the operator never gets a chance to inspect the final simulated state (info job/queue/worker/resource/summary). Related: the simulator should never terminate except via an explicit `quit`. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +requirements.md's own AC 22 pseudocode for the REPL integration never re-enters the REPL after the replay loop's `for each event in replay_file` reaches EOF -- it prints two informational lines ("replay complete"/"output written to") and falls straight through to sched_sim_driver_run()'s return, which unwinds back to main() and the process exits normally. This matches the literal pseudocode but is a genuine UX gap the requirement text never called out: `continue` (mode=XCE_REPL_MODE_CONTINUE, no breakpoint match) is the ONLY path that can reach EOF without another repl_command_loop() re-entry already pausing execution first (step's budget-exhaustion pause and every breakpoint hit both already re-enter the REPL mid-stream; only a completed `continue` run had no re-entry point at all). + +## Fix +Fixed as part of task 12.5. Added xce_repl_on_replay_complete() (repl.c) + its sim->interactive-gated inline wrapper xce_repl_check_replay_complete() (repl.h), called from sim_loop.c immediately after the replay for(;;) loop breaks (EOF or AC20 end-of-stream marker), before the --sim-output file is closed. It prints the existing completion banner, sets a new sim->repl_state->replay_complete flag, and calls repl_command_loop(sim) so the process blocks at the `(sim) ` prompt instead of returning. repl_dispatch()'s `step`/`continue` branches now check replay_complete first: if set, they print "[sim] replay already complete -- no more events to process. Use 'quit' to exit." and return false (re-prompt) instead of returning true (which would have handed control back to a driver loop that no longer exists). The only remaining exit paths are repl_cmd_quit() (explicit `quit`) and readline() returning NULL on stdin EOF (already treated as an implicit quit, pre-existing documented behavior, unchanged). Batch (non-interactive) `--sim` mode is unaffected -- xce_repl_check_replay_complete() is a no-op when sim->interactive is false, mirroring every other REPL hook's existing gating. New tests in test_repl_execution_control.c: scripted stdin `continue\ninfo summary\nquit\n` against a fixture with zero breakpoints asserts the process does NOT exit after 'replay complete' and that `info summary` after it reports the final job/session state correctly; a second test asserts `step`/`continue` issued after replay_complete print the re-prompt message and the REPL stays alive. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1596 diff --git a/.wolf/bugs/bug-1595.md b/.wolf/bugs/bug-1595.md new file mode 100644 index 00000000..76182b86 --- /dev/null +++ b/.wolf/bugs/bug-1595.md @@ -0,0 +1,38 @@ +--- +id: "bug-1595" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T08:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_break.c (xce_repl_on_event)"] +tags: ["scheduler", "repl", "step", "continue", "event-visibility", "ux"] +related: [] +legacy: true +--- + +# bug-1595: User-reported: in each `step`, it would be better to show the decisions/events that occur in the… + +## Description +User-reported: in each `step`, it would be better to show the decisions/events that occur in the current session in the sequence they actually occur -- currently only decisions are printed, input events (JOB_PUSH, JOB_STATE_CHANGE, WORKERS_PUSH, etc.) are completely silent unless a breakpoint happens to match one. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sim_loop.c already calls xce_repl_check_event()/xce_repl_on_event() for every decoded ReplayEvent, in stream order, strictly before it joins the batch xce_core_push_events() later applies -- but xce_repl_on_event() (repl_break.c) only ever prints anything when a breakpoint matches; there is no unconditional per-event echo. sim_print_decision() (sim_loop.c), by contrast, IS called unconditionally for every drained decision. Net effect: an operator using `step`/`continue` without breakpoints sees decisions appear with no visible cause -- the events that produced them (a job arriving, a job's state changing, a worker reporting in) are invisible. + +## Fix +Fixed as part of task 12.5. xce_repl_on_event() (repl_break.c) now unconditionally prints a one-line human-readable summary of every event before running the (unchanged) breakpoint-match check -- e.g. "[sim] event: JOB_PUSH njobs=2 jobids=[, ]", "[sim] event: JOB_STATE_CHANGE jobid= state=RUN", "[sim] event: WORKERS_PUSH nworkers=1 names=[worker-1]", one line per xce_core_event_type_t kind. Since xce_repl_on_event() is only ever invoked when sim->interactive (via repl.h's existing gate), this adds zero output to batch `--sim` mode. repl_job_state_name() (repl_info.c) was made non-static and declared in repl_internal.h so both repl_info.c's `info job` and repl_break.c's new event summary render the same state name for JOB_STATE_CHANGE. Printed order is now event(s) (as decoded) -> BEGIN_ROUND -> decision(s) (as drained), matching sim_loop.c's actual processing order. New tests in test_repl_breakpoints_inspection.c step through a fixture with a JOBS_PUSH and a JOB_STATE_CHANGE event and assert both event summary lines appear, in order, before the resulting decision line. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1596.md b/.wolf/bugs/bug-1596.md new file mode 100644 index 00000000..8f7908ad --- /dev/null +++ b/.wolf/bugs/bug-1596.md @@ -0,0 +1,38 @@ +--- +id: "bug-1596" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T08:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl.c (repl_split_tokens)"] +tags: ["scheduler", "repl", "jobid", "quoting", "parsing", "ux"] +related: ["bug-1591"] +legacy: true +--- + +# bug-1596: User-reported: `info job "0-15-1787129769-0[0]"` (jobid copied from `info jobs`' own output… + +## Description +User-reported: `info job "0-15-1787129769-0[0]"` (jobid copied from `info jobs`' own output, wrapped in double quotes out of habit) returns "[sim] info job: no such job 0 (0-0-0-0)" even though the job is genuinely present -- the identical jobid without quotes works. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl_info_job() already correctly calls parse_jobid() (lib/utils.c) with a strtoull() fallback -- confirmed NOT the bug. parse_jobid()'s two regexes (REGEX_JOBARRAY_ID/REGEX_JOBID, src/include/config.h) both require a whole-string anchor match (pmatch[0].rm_so==0 && pmatch[0].rm_eo==len). repl_split_tokens() (repl.c) is a plain whitespace tokenizer with no quote-awareness, so a quoted argument's token literally includes the surrounding `"` characters. Those quote characters break the anchor match (the regex still matches the interior substring, but not starting at offset 0 / ending at len), parse_jobid() returns -EINVAL, and the strtoull() fallback then parses a string beginning with `"` -- not a valid numeric prefix -- returning 0. MAKE_JOBID(0,0,0,0,0) formats back as "0-0-0-0", exactly matching the reported output. + +## Fix +Fixed as part of task 12.5. repl_split_tokens() (repl.c) now strips one matching pair of leading/trailing `"` or `'` characters from each extracted token (length >= 2, first == last == a quote char) before returning it -- applies uniformly to every REPL command argument (info job/queue/worker/resource names, break filters), not special-cased to `info job`. New test in test_repl_breakpoints_inspection.c asserts `info job ""` (double-quoted) and `info job ''` (single-quoted) both resolve the same job as the unquoted form. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1591 diff --git a/.wolf/bugs/bug-1597.md b/.wolf/bugs/bug-1597.md new file mode 100644 index 00000000..83dcf882 --- /dev/null +++ b/.wolf/bugs/bug-1597.md @@ -0,0 +1,39 @@ +--- +id: "bug-1597" +status: "fixed" +status_updated: "2026-09-24" +status_note: "obsolete: src/frontend/Makefile removed (6fabdba9)" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-19T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/Makefile (build-server target)"] +tags: ["frontend", "apiserver", "cgo", "makefile", "build-server", "nix", "pre-existing"] +related: ["bug-1558"] +fix_commits: ["6fabdba9"] +legacy: true +--- + +# bug-1597: backend/reconnect.go:6:10: fatal error: client.h: No such file or directory (compilation… + +## Description +backend/reconnect.go:6:10: fatal error: client.h: No such file or directory (compilation terminated) -> make: *** [Makefile:33: build-server] Error 1 + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +build-server's recipe is a bare `cd apiserver && go build -o /tmp/apiserver ./cmd/main.go` with no CGO_CFLAGS/CGO_LDFLAGS, so cgo has no -I path to src/include (client.h) or BUILD_DIR/lib (controller.capnp.h) and no -L/-l for libxce + its pkg-config deps. Unlike build-server, both test-backend (same Makefile, bug-1558) and apiserver/meson.build derive CGO_CFLAGS/CGO_LDFLAGS via pkg-config (librtti, c-capnproto, openssl, tinylog) plus -I$(BUILD_DIR)/lib / -L$(BUILD_DIR)/lib -lxce -- build-server was apparently never given the same treatment, so it only works if CGO_CFLAGS/CGO_LDFLAGS happen to already be exported in the shell (e.g. by meson's custom_target env), not under a bare `nix develop ../ --command make build-server`. Reproduced identically with and without an unrelated comment-only change staged elsewhere in the same package (apiserver/backend/handler.go), confirming this is pre-existing and unrelated to that change. + +## Fix +Not fixed (out of scope for the verification task that found it -- reported only). Workaround that does compile cleanly (exit 0): run the build-server go build with the same self-derived CGO env as test-backend, e.g. inside `nix develop ../ --command bash -c '...'`: BUILD_DIR=$(pwd)/../xce.build; CGO_CFLAGS="-fsanitize=address -g $(pkg-config --cflags librtti c-capnproto openssl tinylog) -I$(pwd)/../include -I$BUILD_DIR/lib"; CGO_LDFLAGS="-fsanitize=address -g $(pkg-config --libs librtti c-capnproto openssl tinylog) -lm -lrt -L$BUILD_DIR/lib -lxce"; then CGO_ENABLED=1 CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" go build -o /tmp/apiserver ./cmd/main.go. Requires BUILD_DIR/lib/libxce.a to already exist (prior xce-build.sh lib run). Proper fix would mirror bug-1558's FRONTEND_CGO_CFLAGS/FRONTEND_CGO_LDFLAGS into the build-server recipe in src/frontend/Makefile. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1558 diff --git a/.wolf/bugs/bug-1598.md b/.wolf/bugs/bug-1598.md new file mode 100644 index 00000000..8c2c149e --- /dev/null +++ b/.wolf/bugs/bug-1598.md @@ -0,0 +1,38 @@ +--- +id: "bug-1598" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-19T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_sim_event_loop.c"] +tags: ["scheduler", "asan", "stack-use-after-return", "thread-lifecycle", "test-only", "sim-driver", "design-review-followup"] +related: [] +legacy: true +--- + +# bug-1598: AddressSanitizer: stack-use-after-return on address inside `sim` (xce_sched_sim_driver_t) at… + +## Description +AddressSanitizer: stack-use-after-return on address inside `sim` (xce_sched_sim_driver_t) at cbuf_get, thread T2 (sim_picker_thread_fn), reading test_run_deactivates_tlog_virtual_clock_logging_on_completion's stack frame after it returned + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_sim_driver_run() (src/scheduler/src/sim_loop.c) spawns sim_picker_thread_fn on a real pthread that is, by this codebase's own documented design (that function's own header comment), never joined or cancelled -- no sim-driver teardown/fini function exists yet. It loops forever on cbuf_get(&sim->core.wbuf, ...), reading `sim` for the lifetime of the process. Every white-box test in test_sim_event_loop.c that calls sched_sim_driver_run()/sched_sim_driver_start() with a stack-local `xce_sched_sim_driver_t sim;` was therefore ALWAYS a latent stack-use-after-return once its test function returned -- it simply hadn't been caught yet, because no prior test ordering/stack-frame-reuse pattern had triggered ASAN's shadow-memory detector. Adding one new test (test_run_deactivates_tlog_virtual_clock_logging_on_completion, part of the scheduler-core design-review follow-up wiring tlog virtual-clock activation into sim.c/sim_loop.c) between two existing tests shifted the stack layout enough to make the pre-existing latent bug fire deterministically (reproduced on 2 independent runs). + +## Fix +Changed the fixture from stack-local to `static xce_sched_sim_driver_t sim;` storage in three places: test_run_processes_multiple_sessions_in_order_and_no_longer_returns_enosys(), test_run_deactivates_tlog_virtual_clock_logging_on_completion() (the test that first exposed it), and the run_sim_and_capture_stdout() helper (used once, by test_job_present_only_in_initial_snapshot_still_reaches_the_picker()). Safe specifically because: (a) sched_sim_driver_start() unconditionally memset()s the whole struct as its first action, so no stale-data leak across runs; (b) each affected function is called exactly once per test-binary process (verified via grep, no re-entrancy), so no second call's memset can ever stomp a still-live zombie thread's cbuf mutex/condvar state from a first call -- static would be UNSAFE on a helper called more than once while an earlier zombie thread might still be alive. Does NOT fix the underlying thread-lifecycle gap (still no join/cancel) -- that is a separate, larger, already-documented-as-deferred architectural item (a real sim-driver teardown/fini function). Verified via 2 independent direct runs of the rebuilt sched_sim_event_loop_test binary post-fix: 4 Tests, 0 Failures, clean exit code 0 both times, no ASAN abort. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1599.md b/.wolf/bugs/bug-1599.md new file mode 100644 index 00000000..5b0e3f0c --- /dev/null +++ b/.wolf/bugs/bug-1599.md @@ -0,0 +1,38 @@ +--- +id: "bug-1599" +status: "fixed" +status_updated: "2026-09-24" +status_note: "sim_loop.c comment reworded" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-20T02:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_no_bare_printf.c (new regression test) flags src/scheduler/src/sim_loop.c"] +tags: ["scheduler", "test-only", "false-positive", "sim_loop.c", "test_no_bare_printf", "text-scan", "regression-test"] +related: [] +legacy: true +--- + +# bug-1599: sched_no_bare_printf_test / test_scheduler_src_has_no_bare_printf_calls FAIL… + +## Description +sched_no_bare_printf_test / test_scheduler_src_has_no_bare_printf_calls FAIL: '.../scheduler/src/sim_loop.c calls bare printf() -- xce_scheduler link-wraps printf ...' -- even though sim_print_decision() in sim_loop.c already correctly uses fprintf(stdout, ...) for all 3 PEND/DISPATCH/PREEMPT lines (verified via grep: lines 398/402/406 all fprintf(stdout, ...), zero bare printf( calls in actual code). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The new regression test's scan_dir_for_bare_printf()/contains_bare_printf_call() (test_no_bare_printf.c) is a naive whole-file text scan with NO comment/string-literal awareness (documented as such in its own header comment, matching test_core_isolation.c's precedent). The doc comment directly above sim_print_decision() in sim_loop.c (lines 375-387, added as part of this same fix, explaining WHY fprintf(stdout,...) is used instead of printf()) contains the literal prose substrings ' plain printf() (2026-08-20 fix)' (line 375) and ' plain printf() call' (line 377) -- both have a space immediately before 'printf(', which the scanner's rule (char before 'printf(' must not be letter/digit/underscore) matches as a bare-printf violation. The scanner cannot distinguish prose describing printf() from an actual call to it. + +## Fix +NOT YET FIXED -- reported to caller for triage per explicit instruction not to fix. Likely fix: reword the sim_loop.c doc comment (lines 375/377) to avoid the literal pattern 'printf(' immediately -- e.g. write it as "printf" and "(" separately, or reference the wrapped __wrap_printf() symbol name instead of the bare word, or otherwise break up the substring so the naive scanner does not match it. Do NOT weaken the scanner itself to add comment-awareness without considering that test_core_isolation.c deliberately uses the same simple/strict genre. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1600.md b/.wolf/bugs/bug-1600.md new file mode 100644 index 00000000..8943b35e --- /dev/null +++ b/.wolf/bugs/bug-1600.md @@ -0,0 +1,39 @@ +--- +id: "bug-1600" +status: "fixed" +status_updated: "2026-09-24" +status_note: "ddee884c" +component: "lib" +severity: "crash" +timestamp: "2026-08-19T21:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 8 +files: ["src/lib/group.c"] +tags: ["scheduler", "crash", "sigabrt", "heap-corruption", "bad-free", "allocator-mismatch", "rtti", "makeString", "group.c", "config-push", "failover", "production"] +related: ["bug-1503"] +fix_commits: ["ddee884c"] +legacy: true +--- + +# bug-1600: xce_scheduler crashed with free(): invalid pointer / ASAN bad-free during controller failover… + +## Description +xce_scheduler crashed with free(): invalid pointer / ASAN bad-free during controller failover retry (3x config-push retry loop); 6 crash-loop occurrences on 2026-08-17, 2 more on 2026-08-19 (core dumps confirmed via gdb bt full and coredumpctl, crash site consistently makeGroup() at src/lib/group.c:234) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +makeGroup() existing-group update path (the branch taken when a config-push re-applies a group name that already exists, e.g. every retried/duplicate config push) freed xce_g_member_t.u.name via FREEUP() (raw free()). u.name is allocated by makeString() (make_g_member(), group.c:77), an rtti-managed pointer returned by makeInstance()/objAlloc() that is NOT the malloc()-returned base address (it is an offset into a combined header+data block) and is refcounted, not malloc-owned. Calling free() on it is undefined behavior -> heap corruption, surfacing later as glibc/ASAN bad-free aborts on unrelated allocations. group_killer() (same file) already released the identical field correctly via dereference(). The buggy FREEUP() path is hit on every config re-push for a group with existing members (e.g. controller failover retrying a config push), which explains the crash correlating with the 3-attempt xcecli_retry_next() failover loop even though no thread concurrency is required to trigger it. + +## Fix +Changed both FREEUP(m->u.name) call sites inside makeGroup()'s existing-group branch (group.c, the g->members loop and the g->ex_members loop) to dereference(m->u.name), matching group_killer()'s established correct pattern for releasing rtti makeString() pointers. Added regression test test_reconfigure_same_group_members (src/lib/testsuite/test_group.c) that calls makeGroup() twice with the same group name/members, reproducing the exact ASAN bad-free at group.c:234 pre-fix (confirmed RED), and passing clean post-fix (confirmed GREEN, full lib asan suite Ok:1 Fail:0, no regressions). NOTE: a separate, related latent issue was also found but NOT fixed here (out of scope, no concurrency needed to explain the observed crash) - the entire existing-group update branch in makeGroup() only takes RTTI_LOCK_READ (a genuine shared pthread_rwlock_rdlock) while mutating shared group state, and sched_init_config()/plugin_config_update() (the config-push handler chain) hold no serializing lock at all, so two vtask worker threads (scheduler runs a 4-vcpu pool) could in principle race on the same group concurrently -- most plausibly triggered by xce_ctx_reconnect() reusing the same endpoint struct and endpoint_init() resetting ep->busy=0 before an already-dispatched handler finishes. This should be tracked as a follow-up. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1503 diff --git a/.wolf/bugs/bug-1601.md b/.wolf/bugs/bug-1601.md new file mode 100644 index 00000000..18dbb46e --- /dev/null +++ b/.wolf/bugs/bug-1601.md @@ -0,0 +1,38 @@ +--- +id: "bug-1601" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-20T07:26:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_info.c"] +tags: ["scheduler", "repl", "info-resource", "info-worker", "consumable", "display", "feature"] +related: ["bug-1590", "bug-1593"] +legacy: true +--- + +# bug-1601: User request: info resource/info worker did not distinguish consumable from non-consumable… + +## Description +User request: info resource/info worker did not distinguish consumable from non-consumable resources -- repl_info_worker_one() showed a fabricated total=/available= pair for EVERY numeric per-worker resource regardless of XCE_RESF_CONSUMABLE, and repl_info_resource_one() never reported a used/in-use amount for a CONSUMABLE GLOBAL/SHARED resource at all; the resource definition line was also missing ordtype. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl_info_worker_one() unconditionally computed/printed total=avail+inUse available=avail for any Value_u_fvalue resval, never checking XCE_RESF_CONSUMABLE -- a non-consumable numeric attribute (nothing ever allocates against it) got a misleading total/available split instead of a single current value. repl_info_resource_one() (the GLOBAL/SHARED case) only ever read core->sharedInst (the available pool) and never core->sharedInUse (the parallel in-use pool that already exists and is populated by the real resource-accounting code, resource.c), so a CONSUMABLE cluster-wide resource never showed how much was in use. The definition line also omitted xce_resource_t.ordtype (INCR/DECR/NA). + +## Fix +repl_info_resource_one(): added ordtype= to the definition line; for a GLOBAL/SHARED resource additionally flagged XCE_RESF_CONSUMABLE, the numeric value line now also reports used= (read from core->sharedInUse at the same idx) alongside value= (avail+used); non-consumable stays a single value= line. repl_info_worker_one(): the per-resource numeric line now looks up the matching xce_resource_t (by idx) from the already-built res_arr and gates the value/used split on XCE_RESF_CONSUMABLE -- consumable prints value=avail+used used=inUse (renamed from the old total=/available= labels for consistency with the resource-level display), non-consumable prints a single value=avail line. Added test_info_resource_reports_consumable_value_and_used and test_info_worker_reports_consumable_value_and_used (new, seed core->sharedInUse / worker->resInUse directly via dup_val()+zarray_put, mirroring the allocation pattern the real accounting code and core_events.c already use); updated 3 pre-existing tests whose asserted strings changed shape (test_info_worker_reports_resource_values, test_info_resource_reports_string_value, test_info_resource_named_lookup_reports_host_based_note). Full scheduler suite: 42/43 Ok, only the pre-existing unrelated bug-1560 failure remains. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1590, bug-1593 diff --git a/.wolf/bugs/bug-1602.md b/.wolf/bugs/bug-1602.md new file mode 100644 index 00000000..ee8bf5a5 --- /dev/null +++ b/.wolf/bugs/bug-1602.md @@ -0,0 +1,42 @@ +--- +id: "bug-1602" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-20T08:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_break.c", "src/scheduler/src/sim.c"] +tags: ["scheduler", "repl", "sim", "replay", "session", "feature", "observability"] +related: ["bug-1595"] +legacy: true +--- + +# bug-1602: User request: scheduler simulation mode should show event time and session when events occur, and… + +## Description +User request: scheduler simulation mode should show event time and session when events occur, and loading a replay file should show its total session count up front. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Feature gap, not a defect: repl_print_event_summary() (interactive REPL event echo) printed only the event type/payload, no clock time or session number; sched_sim_driver_start() never reported how many sessions a loaded .replay file contains before stepping through it. + +## Fix +repl_print_event_summary() now prefixes each echoed event with "[sim] t= () session=". Non-obvious correctness point found via sim_loop.c inspection (not assumption): events are read+echoed individually AS THEY STREAM from the file, and the ReplaySession/begin_round message for that batch -- which is what increments rs->session_count via xce_repl_on_begin_round() -- comes AFTER those events in file order (file layout is event,event,...,ReplaySession). So every echoed event always belongs to the UPCOMING session; the function reports session_count+1, not the raw count. A first attempt using the raw count failed its own new test (session=0 instead of session=1) and was corrected after tracing the real sim_loop.c call order (xce_repl_check_event at line ~593 vs xce_repl_check_begin_round at ~631) rather than trusting the repl.h doc comment's summary. + +Added sim_prescan_session_count() (sim.c): scans sim->replay_file from the current position using msg_root_has_shape() shape-only classification (ReplayEvent/ReplaySession/SnapshotEntry, matching sim_loop.c main loop's own three-way dispatch) to count ReplaySession messages, then rewinds via fseek back to the captured ftell() position -- mirrors sim_locate_and_load_snapshot()'s own established rewind pattern. Confirmed safe via research: capn_init_fp() in this codebase is always unpacked (packed=0), reading via a single plain fread() per message with no internal read-ahead buffer, so ftell/fseek stay exactly in sync with message boundaries. Wired into sched_sim_driver_start() right after sim_set_virtual_clock(): prints "[sim] replay file contains N sessions" once at load, non-fatal on scan failure. User explicitly chose this pre-scan approach over a recording-time trailer message (would have required a schema change and only worked for newly-recorded files) via AskUserQuestion. + +Added test_replay_load_reports_total_session_count (2-session fixture) and extended test_step_echoes_events_before_session_decisions with t=/session=1 assertions in test_repl_breakpoints_inspection.c. Full scheduler suite: 42/43 Ok, only pre-existing unrelated bug-1560 remains (now 3 occurrences). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1595 diff --git a/.wolf/bugs/bug-1603.md b/.wolf/bugs/bug-1603.md new file mode 100644 index 00000000..72e71d08 --- /dev/null +++ b/.wolf/bugs/bug-1603.md @@ -0,0 +1,39 @@ +--- +id: "bug-1603" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-20T09:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/flake.nix", "src/nixSupport/default.nix", "src/nixSupport/xce.nix", "src/nixSupport/xcecli.nix"] +tags: ["nix", "flake", "pure-eval", "builtins.currentTime", "build-time-versioning", "meson", "preConfigure", "mesonFlagsArray"] +related: [] +fix_commits: ["bca6397"] +legacy: true +--- + +# bug-1603: build-time-versioning task 2.2 REJECTED by reviewer: `nix build .#xcecore_debug` (no flags, the… + +## Description +build-time-versioning task 2.2 REJECTED by reviewer: `nix build .#xcecore_debug` (no flags, the exact documented/task-Observable command) failed at evaluation with `error: attribute 'currentTime' missing`; also the raw epoch-seconds string didn't satisfy Requirement 1.2's absolute-UTC-form requirement. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +First attempt computed myBuildTime = builtins.toString builtins.currentTime at flake-evaluation time (src/flake.nix), threaded through nixSupport/default.nix into xce.nix/xcecli.nix's mesonFlags. builtins.currentTime is stripped entirely under a flake's default pure evaluation (not just fixed to a dummy value -- the attribute itself is removed from `builtins`), so any nix build/eval that forces it fails unless --impure is passed. This also produced a raw epoch-seconds string, not an ISO-8601 UTC string like xce-build.sh's (task 2.1) `date -u +%Y-%m-%dT%H:%M:%SZ` output. + +## Fix +Moved the build-time computation OUT of Nix expression evaluation and INTO each derivation's build phase. xce.nix and xcecli.nix each gained a `preConfigure` hook that runs `date -u +%Y-%m-%dT%H:%M:%SZ` (stdenv's bundled coreutils, always available in the build sandbox, unaffected by pure-eval, no --impure/git/network needed) and appends `-Dbuild_time=$buildTime` to `mesonFlagsArray` (verified against the real nixpkgs meson setup-hook.sh: mesonConfigurePhase runs `runHook preConfigure` BEFORE it reads mesonFlags/mesonFlagsArray via `concatTo`, so the appended flag is picked up at configure time). The hook `exit 1`s if `date` produces empty output, mirroring xce-build.sh's own non-empty-check discipline. Since build_time is no longer an eval-time concept, `myBuildTime` was removed entirely from flake.nix/nixSupport/default.nix/the xce.nix+xcecli.nix function signatures and their eval-time asserts -- only myCommitId (a genuine eval-time/flake-level value from self.shortRev/self.dirtyShortRev) still threads through. Live-verified: `nix eval --json .#packages.x86_64-linux.xcecore_debug.mesonFlags` (no --impure) succeeds; `nix build .#xcecore_debug` (no flags) proceeds through evaluation+configure+570/614 ninja targets (all major C binaries linked) before failing only on an unrelated pre-existing sandbox network restriction (autoscaler/bcctool's Go module fetch from goproxy.cn, nothing to do with this fix); build log shows `-Dbuild_time=2026-08-20T09:13:29Z` reaching meson configure and meson's own summary printing `build_time : 2026-08-20T09:13:29Z` (real, non-empty, ISO-8601 UTC). Same confirmed for xcecli.nix (`-Dbuild_time=2026-08-20T09:15:46Z`). `commit_id=bca6397-dirty` confirmed unaffected/still working in both. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1604.md b/.wolf/bugs/bug-1604.md new file mode 100644 index 00000000..42b4db31 --- /dev/null +++ b/.wolf/bugs/bug-1604.md @@ -0,0 +1,38 @@ +--- +id: "bug-1604" +status: "open" +status_updated: "2026-09-24" +status_note: "concurrent xce-build.sh runs race on rm -rf of shared BUILD_DIR (no lock)" +component: "build" +severity: "unknown" +timestamp: "2026-08-20T10:24:39Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/xce-build.sh"] +tags: ["asan", "concurrency", "multi-agent-race", "overlayfs", "rm-rf", "sandbox-flake", "scheduler", "shared-build-dir", "xce-build.sh"] +related: ["bug-1561"] +legacy: true +--- + +# bug-1604: rm: cannot remove 'xce.build/controller/controller_unit_test.p': Directory not empty (xce-build.sh… + +## Description +rm: cannot remove 'xce.build/controller/controller_unit_test.p': Directory not empty (xce-build.sh setup_build()'s rm -rf \$BUILD_DIR step aborted the script under set -euo pipefail before meson setup ran) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CORRECTED after further evidence (see fix note): initial overlayfs-flakiness theory was WRONG. The real cause is xce-build.sh defaulting BUILD_DIR to the same "xce.build" for every component and every invocation, with setup_build() doing an unconditional rm -rf "$BUILD_DIR" + fresh meson setup on every non-incremental run. When two xce-build.sh invocations run concurrently in the SAME repo working directory against DIFFERENT components (observed live: this session's "xce-build.sh scheduler asan" raced against a second, independently-running "xce-build.sh controller asan" process, PID 655158, started ~1 min later in the same cwd), they contend for the identical xce.build/ path -- one process's rm -rf can hit a directory the other process is actively writing into ("Directory not empty"), and later, a second clean-build pass can silently delete and replace a just-verified binary out from under a caller who believes their build is done. Confirmed live: xce_scheduler at src/xce.build/scheduler/xce_scheduler, verified present and fresh via ls -la and 'xce_scheduler --version' immediately after this session's scheduler build/test run, was gone minutes later -- the concurrent controller build's rm -rf had removed and was in the process of regenerating xce.build/ for its own component. + +## Fix +No code fix applied (out of scope for the build-verification task this session was doing) -- flagging as a real, reproducible hazard rather than env flakiness. Practical mitigation for future sessions/agents: never assume xce.build/ (or any non-incremental xce-build.sh output) is stable once another concurrent xce-build.sh process might be running in the same working directory/worktree -- check 'ps aux | grep xce-build.sh' (or pass a distinct BUILD_DIR / use --incremental) before relying on a binary's presence or mtime surviving past the moment it was checked. Corrects this entry's own original overlayfs-race theory, which was a plausible but incorrect guess made before the concurrent process was discovered. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1561 diff --git a/.wolf/bugs/bug-1605.md b/.wolf/bugs/bug-1605.md new file mode 100644 index 00000000..ed5d0512 --- /dev/null +++ b/.wolf/bugs/bug-1605.md @@ -0,0 +1,39 @@ +--- +id: "bug-1605" +status: "cannot-reproduce" +status_updated: "2026-09-24" +status_note: "transient build failure, passed on immediate retry; likely bug-1604 collision" +component: "lib" +severity: "unknown" +timestamp: "2026-08-20T10:27:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/controller.capnp.c (generated)", "src/controller/config_repl.c (consumer)"] +tags: ["build", "capnp", "codegen", "concurrent-build-collision", "controller", "needs-followup", "ninja", "pre-existing", "race", "transient"] +related: ["bug-1439", "bug-1493", "bug-1604"] +repro_attempts: "1 retry, clean" +legacy: true +--- + +# bug-1605: controller ASAN build: two failures in the same ninja run -- (1) controller/config_repl.c:1921:28… + +## Description +controller ASAN build: two failures in the same ninja run -- (1) controller/config_repl.c:1921:28: error: 'RequestReply_data_probeconfigtagreplymsg' undeclared (first use in this function); did you mean 'RequestReply_data_upgradeconfigreplymsg' -- and (2) lib/controller.capnp.c:25013:1: fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory, compilation terminated; ninja: build stopped: subcommand failed. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +- Attempts: 1 retry, clean + +## Root cause +Matches the capnp custom_target's documented missing-dependency-edge race (see comment block at top of src/tools/xce-build.sh, 'retry once before debugging'): config_repl.c was compiled against a stale/partially-regenerated controller.capnp.h that lacked the 'probeconfigtagreplymsg' enumerant even though src/schema/controller.capnp:596 defines it (probeconfigtagreplymsg @20: ProbeConfigTagReply), and lib/controller.capnp.c's own compile then hit an I/O race writing its .d dependency file, most likely because the generator custom_target re-ran/rewrote lib/ mid-build without ninja having a correct dependency edge ordering the two. Not caused by the concurrent main.c/testsuite/test.c argp_program_version diff (those files are unrelated to config_repl.c/capnp). CORRECTION (same session, minutes later): discovered via `ps aux` a THIRD live instance of a concurrent `nix develop ./src --command bash src/tools/xce-build.sh controller asan` process actively doing rm -rf xce.build + fresh meson compile in this SAME shared worktree at 10:38, i.e. immediately after this bug's own two build attempts (10:23-10:33) -- and .wolf/memory.md/cerebrum.md already documented an earlier occurrence THIS SAME SESSION of exactly this hazard (bug-1604: two concurrent xce-build.sh runs sharing default BUILD_DIR=xce.build race on rm -rf, one silently wiping/regenerating the other's in-progress build, corrupting or vanishing already-verified binaries). The 'known capnp codegen dependency-edge race' explanation above is therefore NOT confirmed as the true root cause -- a concurrent cross-session xce-build.sh collision (bug-1604) is at least as likely, and possibly the actual cause, of both the stale-header undeclared-enum error and the .d-file open I/O failure (a concurrent rm -rf mid-compile would produce exactly both symptoms). Do not treat this entry as proof the capnp custom_target race is real without ruling out concurrent-session collision first -- check `ps aux | grep xce-build.sh` before attributing a build failure to codegen internals in this worktree. + +## Fix +Confirmed transient: re-ran the exact same command (nix develop ./src --command bash src/tools/xce-build.sh controller asan) a second time with zero source changes in between and the full clean build compiled successfully (202/202, zero errors). Not root-caused/fixed in the underlying capnp custom_target dependency graph -- xce-build.sh's own comments already document this as a known, unfixed gap and recommend exactly this retry-once workaround. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1439, bug-1493, bug-1604 diff --git a/.wolf/bugs/bug-1606.md b/.wolf/bugs/bug-1606.md new file mode 100644 index 00000000..7ff276fb --- /dev/null +++ b/.wolf/bugs/bug-1606.md @@ -0,0 +1,38 @@ +--- +id: "bug-1606" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-20T16:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/test-version-startup-consistency.sh"] +tags: ["bash", "grep", "busybox", "sandbox", "portability", "sed", "build-time-versioning"] +related: [] +legacy: true +--- + +# bug-1606: grep: invalid option -- 'P' (BusyBox usage dump) when a standalone .sh script (run via `bash… + +## Description +grep: invalid option -- 'P' (BusyBox usage dump) when a standalone .sh script (run via `bash script.sh`, not typed as an inline Bash-tool command) calls `grep -oP '...\K...'` + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +This coding-agent harness's interactive shell shadows plain `grep` with a `.bashrc`-sourced shell FUNCTION that shells out to a PCRE-capable (ugrep-backed) binary -- but that function is interactive-shell-only and is NOT inherited by a script invoked as `bash script.sh` (a separate, non-interactive process), which falls back to the real `grep` on PATH: BusyBox in this environment, with no -P/PCRE support at all. `grep -oP '(?<=x)y'` typed directly as a Bash-tool command therefore worked fine, masking the incompatibility until the same call was placed inside a committed script file and run standalone. + +## Fix +Replaced the grep -oP '"gitCommit":\s*"\K[^"]*' style extraction with POSIX BRE `sed -n 's/.*"gitCommit":[[:space:]]*"\([^"]*\)".*/\1/p'`, which BusyBox sed supports. General rule: any shell script meant to be run standalone (not just typed inline) must assume POSIX-only grep/sed (BRE, -E, -F, -o) -- no -P/PCRE, no lookbehind/\K. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1607.md b/.wolf/bugs/bug-1607.md new file mode 100644 index 00000000..ec184db2 --- /dev/null +++ b/.wolf/bugs/bug-1607.md @@ -0,0 +1,38 @@ +--- +id: "bug-1607" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-20T16:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/go.sum"] +tags: ["go", "go.sum", "go-modules", "build", "side-effect", "build-time-versioning"] +related: [] +legacy: true +--- + +# bug-1607: `git status` showed src/frontend/apiserver/go.sum modified (+65 lines) after a direct `go build… + +## Description +`git status` showed src/frontend/apiserver/go.sum modified (+65 lines) after a direct `go build ./cmd/main.go` invocation that never touched go.mod and produced a successful, working binary -- no warning or error printed by the build itself + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +A plain `go build` (default `-mod` setting, `go env GOFLAGS` confirmed empty) blocks go.mod EDITS under Go 1.16+'s readonly-by-default policy, but does NOT by itself block go.sum EXTENSION for modules already implied by go.mod's requirements -- resolving the full module graph for cgo-enabled apiserver added ~65 unrelated indirect-dependency /go.mod-hash-only lines to the committed go.sum as a silent side effect. + +## Fix +Pass `-mod=readonly` explicitly on every direct `go build` invocation against this repo (src/tools/test-version-startup-consistency.sh's apiserver build step). Confirmed live: the identical build still succeeds under explicit -mod=readonly with zero go.sum diff, so nothing was genuinely missing -- the extra entries were pure side effect of the laxer default. Reverted the accidental go.sum change via `git checkout -- src/frontend/apiserver/go.sum` before re-running with the fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1608.md b/.wolf/bugs/bug-1608.md new file mode 100644 index 00000000..8cc1d263 --- /dev/null +++ b/.wolf/bugs/bug-1608.md @@ -0,0 +1,38 @@ +--- +id: "bug-1608" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "hang" +timestamp: "2026-08-20T15:57:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/go_build"] +tags: ["go", "go_build", "goproxy", "network", "hang", "sandbox", "meson", "build-time-versioning"] +related: ["bug-1597"] +legacy: true +--- + +# bug-1608: `xce-build.sh apiserver ` (meson -> go_build custom_target) hung indefinitely… + +## Description +`xce-build.sh apiserver ` (meson -> go_build custom_target) hung indefinitely (8-10+ minutes, zero progress) building xcecli's Go dependencies; no error, no timeout, just stuck + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +go_build hardcodes `GOPROXY=https://goproxy.cn` plus a FRESH, per-invocation GOCACHE/GOPATH under meson's own $TMPDIR sandbox, bypassing the shell's already-populated default $GOPATH/pkg/mod module cache entirely. Confirmed via `ss -tnp` that the resulting `go` process had an ESTABLISHED (not refused/reset) TCP connection to 8.45.176.179:443 (=goproxy.cn, independently reachable fine via plain curl/`/dev/tcp` from the same shell) sitting in futex_do_wait with near-zero CPU time -- a genuinely stuck data path in this sandbox's networking for that specific fresh-cache+custom-GOPROXY combination, not mere slowness (never progressed no matter how long left running across 3 separate attempts). A plain `go list -m all` from an ordinary `nix develop` shell (no wrapper, default cache) resolved the identical apiserver module graph in under a second. + +## Fix +Workaround only (go_build/Makefile NOT modified -- this is a sandbox-specific networking quirk, not a real defect in those files, and out of this task's scope to fix): for a standalone real apiserver/xcecli build in this sandbox, invoke `go build` directly against the default (already-populated) module cache with `GOPROXY=off` plus the same CGO_CFLAGS/CGO_LDFLAGS pkg-config derivation apiserver's own meson.build/test-backend already use (see bug-1597's documented fix for the exact derivation), instead of routing through xce-build.sh apiserver / go_build / make build-server. Diagnose a suspected hang via `ss -tnp` + `/proc//wchan` before assuming it just needs more time. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1597 diff --git a/.wolf/bugs/bug-1609.md b/.wolf/bugs/bug-1609.md new file mode 100644 index 00000000..ea81aed5 --- /dev/null +++ b/.wolf/bugs/bug-1609.md @@ -0,0 +1,38 @@ +--- +id: "bug-1609" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-20T21:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_break.c"] +tags: ["repl", "sim", "verbosity", "continue", "ux", "scheduler-sim-diagnostics", "scheduler-sim-repl-verbosity"] +related: ["bug-1595", "bug-1763"] +legacy: true +--- + +# bug-1609: User-reported (live REPL session, xce_scheduler -i -s ): after 'continue' with zero… + +## Description +User-reported (live REPL session, xce_scheduler -i -s ): after 'continue' with zero breakpoints set, output is just as verbose as single-stepping -- every decoded event still prints a '[sim] t=... session=...' + '[sim] event: ...' two-line block, defeating continue's documented purpose ('Run until next breakpoint hit or EOF'). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce_repl_on_event() (repl_break.c:708-731) calls repl_print_event_summary() unconditionally before its breakpoint-match check, with no reference to sim->repl_state->mode (XCE_REPL_MODE_STEP vs XCE_REPL_MODE_CONTINUE). This was intentional for bug-1595 ('in each step, ... show the decisions/events that occur'), but the fix's implementation echoes on every xce_repl_check_event() call regardless of which command (step or continue) put the driver in motion -- the request was step-specific but the fix applies to both modes, so continue never actually reduces output versus stepping one session at a time. + +## Fix +FIXED (2026-09-02, scheduler-sim-repl-verbosity spec, task 3.2 -- design.md's Event Echo Gate component, Requirement 1). Close to the originally proposed shape, refined per design.md's fuller classification: xce_repl_on_event() (repl_break.c) now calls a new repl_event_is_structural(ev->type) classifier before deciding whether to call repl_print_event_summary() -- CONFIG_PUSH/JOBS_PUSH/WORKERS_PUSH/RESOURCES_PUSH/RESVALS_FULL (structural) are always echoed regardless of rs->mode (AC 1.3), while RESVALS_DELTA/JOB_STATE_CHANGE/JOB_UPDATE/QUEUE_UPDATE/JOB_RUSAGE (routine) are echoed only when rs->mode == XCE_REPL_MODE_STEP (AC 1.1/1.2), so `continue` now genuinely reduces output versus single-stepping, exactly this bug's original complaint. Breakpoint matching (repl_find_event_breakpoint()) is unaffected by the classification -- runs on every event regardless (AC 1.4), same as before. A structural event's own line, and a breakpoint-triggered repl_command_loop() re-entry (in xce_repl_on_event()/xce_repl_on_begin_round()/xce_repl_on_decision()), now also call repl_round_collapse_flush() (task 2.2, repl_rollup.c) first, so a mode-suppressed stretch of routine events that got collapsed into a Round Rollup range is reported before either of those (Req 3.5/3.6) -- a Requirement 2/3 dependency this bug's original proposal didn't anticipate (Round Rollup didn't exist yet in task 12.5's original codebase). TDD: task 3.1's RED-phase test_repl_event_echo.c (4 tests, real forked-child stdout-capture against sched_sim_driver_run()) pinned this exact contract before the fix landed; all 4 pass after. Full scheduler suite verified via /xce-build scheduler --incremental: 56/58 test binaries OK, the 2 failures being the pre-existing unrelated bug-1758 (test_core_events.c mem-reservation reconciliation) and a one-off ASAN stack-use-after-return in sched_sim_record_output_test logged separately as bug-1763 (same pre-existing thread-lifecycle class as bug-1598/bug-1632, triggered by this task's repl_break.c size change perturbing that binary's stack layout, not a correctness defect in this fix -- 4/4 isolated re-runs of that binary were clean). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1595, bug-1763 diff --git a/.wolf/bugs/bug-1610.md b/.wolf/bugs/bug-1610.md new file mode 100644 index 00000000..410300e1 --- /dev/null +++ b/.wolf/bugs/bug-1610.md @@ -0,0 +1,38 @@ +--- +id: "bug-1610" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-20T21:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["repl", "sim", "verbosity", "duplicate-output", "pend", "scheduler-sim-diagnostics"] +related: ["bug-1609"] +legacy: true +--- + +# bug-1610: User-reported (same live REPL session): under 'continue', an unchanged PEND decision for the same… + +## Description +User-reported (same live REPL session): under 'continue', an unchanged PEND decision for the same jobid prints an identical '[sim] PEND jobid=...' line every scheduling round it remains pending -- e.g. jobs [22]-[29] (8 jobs) print their 8 byte-identical PEND lines 5+ times in a row across consecutive rounds with zero new information each time. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sim_print_decision() (sim_loop.c:388-411) unconditionally prints one DISPATCH/PEND/PREEMPT line per drained xce_decision_t with no de-duplication and no session/round context attached to the line itself -- called every round a job's allocation is (re-)attempted, so a job that stays PEND for N consecutive rounds produces N identical lines. xce_repl_on_decision() (repl_break.c:776-811), called alongside it, already tallies last_n_pend/last_n_dispatch/last_n_preempt per round for 'info session' -- that rollup exists but is never surfaced automatically, only on explicit 'info session'. + +## Fix +FIXED (2026-09-02, scheduler-sim-repl-verbosity spec, task 4.2 -- design.md's `sim_print_decision` Gate component, option (b) of this bug's own original proposal, now that the Round Rollup -- task 2.2, repl_rollup.c -- already provides the per-round summary line to fall back on). sim_loop.c's decision-drain loop (sched_sim_driver_run()) now calls a new file-local sim_print_decision_gate(sim, d) instead of calling sim_print_decision(d) directly; the gate skips the bare per-decision print only when sim->interactive && sim->repl_state->mode == XCE_REPL_MODE_CONTINUE, leaving step mode and non-interactive/headless runs completely unaffected (Req 5.1, 5.4) -- the Round Rollup's own per-round summary becomes the sole source of decision output under continue. Implementation wrinkle discovered and resolved during this task: design.md describes this as an 'inline-wrapper-style gate ... consistent with repl.h's established convention', but repl.h's existing wrappers (xce_repl_check_decision() and siblings) only ever test the PUBLIC sim->interactive field inline and delegate all repl-internal-state logic to an extern function implemented in repl.c/repl_break.c (the only files with repl_internal.h access, where xce_sched_repl_state_t's full definition -- including `mode` -- actually lives; sim.h/repl.h deliberately keep repl_state an opaque forward-declared pointer). That pattern cannot be reused verbatim here because the function being gated, sim_print_decision(), is itself static (private to sim_loop.c) -- repl.h cannot wrap a conditional call to a symbol it cannot see, and moving the check into repl.h would require repl.h to include repl_internal.h, leaking the full repl_state struct into every other repl.h consumer. Resolved by having sim_loop.c include repl_internal.h directly (a private header already scoped to src/scheduler/src/, so this stays an intra-directory dependency, not a new external one) and implementing sim_print_decision_gate() as a file-local static inline wrapper co-located with sim_print_decision() -- same 'wrapper hides the condition, call site stays one bare call expression' style repl.h's wrappers use, just not physically in that shared header since the wrapped function isn't extern. No new extern symbol crosses out of sim_loop.c for this purpose, so test_sim_isolation.c's nm(1) undefined-symbol layer and source-text-scan layer are both unaffected. repl.h itself was left untouched, matching design.md's own File Structure Plan line for it ('no functional change expected'). TDD: task 4.1's RED-phase test_sim_print_decision_gate.c (3 tests, real forked-child/direct-call stdout capture) pinned this exact contract before the fix landed; all 3 pass after (continue-mode suppression, step-mode-unaffected regression guard, headless-batch-unaffected regression guard). sched_repl_round_rollup_test (7/7) and sched_repl_event_echo_test (4/4) reverified green post-fix. Full-suite verification via /xce-build scheduler --incremental surfaced 3 UNEXPECTED additional failures beyond the pre-existing bug-1758, confirmed (via a controlled before/after rebuild swapping only sim_loop.c) to be caused by this fix itself, not a defect in it: test_repl_execution_control.c's test_pre_replay_pause_blocks_before_any_event_is_processed, and test_repl_breakpoints_inspection.c's test_event_breakpoint_pauses_repl_at_matching_event and test_decision_breakpoint_pauses_after_decision_output -- all three pre-date this spec and assert on the bare '[sim] PEND jobid=...' line's presence in a continue-mode capture, i.e. they pin exactly the bug-1610 behavior this fix intentionally removes. Left unmodified per this task's explicit no-test-file-editing boundary; flagged to the parent controller/reviewer as a real, reproducible side effect requiring a follow-up task to update those 3 assertions (e.g. swap their continue scripts for step, or assert on the Round Rollup's summary line instead) rather than a regression in this fix's own logic. **Remediation (2026-09-02, task 4.2 remediation, post-review):** the fix above only covered ONE of sim_loop.c's two direct sim_print_decision() call sites -- sched_sim_driver_run()'s main decision-drain loop. Review found a second, direct, unguarded call inside sim_reconcile_unaccounted_jobs() (added by the sibling scheduler-sim-decision-parity/scheduler-sim-eager-reconciliation specs after this bug's original fix landed, unrelated in authorship but real production code reporting DISPATCH decisions through this exact same channel) -- and that reconciliation path is actually a PRIMARY mechanism by which DISPATCH decisions surface in replay mode at all (most live-drain-loop activity is PEND), so a round where reconciliation fired was still showing both the bare per-decision line AND the Round Rollup's summary counting the same dispatch: this bug's own duplication, reproduced via the second call site design.md's original Boundary Commitments never accounted for (it only ever named "the existing call site", singular). Fixed by routing that call site through the identical sim_print_decision_gate(sim, d) too (same function, same condition, no new symbol) -- sim_loop.c:760's bare sim_print_decision(d) became sim_print_decision_gate(sim, d). TDD: two new tests added to test_sim_reconcile_unaccounted.c (test_reconcile_continue_mode_with_repl_state_suppresses_bare_decision_line / test_reconcile_step_mode_with_repl_state_still_prints_bare_decision_line), using the repl_state_ensure()-based direct-call fixture technique test_repl_round_rollup.c's own fixtures establish; the pre-existing test_reconcile_reports_reconstructed_decision_through_existing_channel (which deliberately leaves sim->repl_state NULL) was empirically re-verified to still pass unmodified, confirming the gate's NULL-safety default ("never load-bearing, degrade to print") covers it. Full-suite verification via /xce-build scheduler --incremental: sched_sim_reconcile_unaccounted_test now 16/16 (was 14/14), sched_sim_print_decision_gate_test 3/3, sched_repl_round_rollup_test 7/7, sched_repl_event_echo_test 4/4, sched_repl_execution_control_test 6/6, sched_repl_breakpoints_inspection_test 30/30 -- all green, confirming this bug's own previously-flagged test_repl_execution_control.c/test_repl_breakpoints_inspection.c follow-up (3 tests pinning the original bug-1610 behavior) was already resolved by an earlier task before this remediation. The only other full-suite failure remains the pre-existing, unrelated bug-1758 (sched_core_events_test). This closes the duplication gap completely -- both production call sites now gated. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1609 diff --git a/.wolf/bugs/bug-1611.md b/.wolf/bugs/bug-1611.md new file mode 100644 index 00000000..d31ba7dc --- /dev/null +++ b/.wolf/bugs/bug-1611.md @@ -0,0 +1,47 @@ +--- +id: "bug-1611" +status: "fixed" +status_updated: "2026-09-24" +status_note: "e2bb98c8, 984a6869" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-21T15:10:00Z" +last_seen: "2026-09-19T14:00:00Z" +occurrences: 3 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["repl", "sim", "replay", "dispatch", "decision-visibility", "scheduler-sim-diagnostics", "scheduler-sim-eager-reconciliation"] +related: ["bug-1764", "bug-1613", "bug-1610", "bug-1609"] +fix_commits: ["e2bb98c8", "984a6869"] +legacy: true +--- + +# bug-1611: Discovered analyzing a real 108-session replay (xce_scheduler -i -s… + +## Description +Discovered analyzing a real 108-session replay (xce_scheduler -i -s scheduler-replay-20260820-070103.replay, continue to EOF): sim_print_decision() (sim_loop.c) never printed a single DISPATCH or PREEMPT line across the entire replay -- only PEND -- even though 22 of the 30 jobs pushed at round 32 clearly transitioned to running (confirmed by ongoing JOB_UPDATE/JOB_RUSAGE ticks for jobids [0]-[21] every few rounds through round 107). An operator watching only decision output has zero confirmation those 22 jobs ever dispatched; they just silently stop appearing in PEND. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not yet root-caused with certainty. Plausible, benign explanation: in --sim/-s REPLAY mode, jobs already resolved (dispatched) in the ORIGINAL recorded run may be replayed purely as state deltas (JOB_UPDATE/JOB_RUSAGE) without the live picker/allocator re-deriving a fresh XCE_DECISION_DISPATCH -- only jobs still outstanding (PEND) get freshly re-decided every round, which is why PEND is the only decision op ever observed. Unconfirmed against sim_loop.c's actual replay-vs-live-decide branching; needs a source read of how sim_loop.c distinguishes replayed-decision playback from freshly-computed decisions before concluding this is 'working as designed' vs a real visibility gap. + +## Fix +Not yet fixed. If confirmed benign (dispatch legitimately has no fresh decision object in replay mode), the round-summary rollup being designed for bug-1609/1610 should not label its dispatch count misleadingly (a permanent 'X dispatch' column reading 0 for an entire replay looks like a bug even when it isn't) -- consider deriving a running-job delta from info-jobs-equivalent state instead of the decision tally for replay-mode display. If NOT benign (a real decision-visibility gap), this is a separate, more serious finding than the formatting issues in bug-1609/1610. + +--- STATUS UPDATE 2026-08-24 (xce-audit-invariants scheduler audit) --- + +Root cause confirmed and the literal mechanism is now built: sim_reconcile_unaccounted_jobs() (sim_loop.c:566-709, landed via scheduler-sim-decision-parity tasks 2.1-3, 2026-08-22) reconstructs and reports a decision for a RUN-state job whose DISPATCH decision was only in the recorded replay stream. BUT its trigger gate (sim_loop.c:592) requires JOB_IS_RUN(job->state) before searching, and the search only scans the CURRENT round's decoded_session (decoded fresh at sim_loop.c:871, freed at line 999 -- no cross-round persistence). So reconciliation only fires when a job's DISPATCH decision and its RUN confirmation land in the exact same replay round. This bug's own original scenario (dispatch at round 32, RUN confirmation through round 107) never matches -- the round-32 DISPATCH decision is gone by the time the job reaches RUN state. The scheduler-sim-decision-parity spec's own task 4.3 implementation notes explicitly acknowledge this residual does NOT fix bug-1611's dominant real-world manifestation. A follow-up spec already exists for this exact residual: .kiro/specs/scheduler-sim-eager-reconciliation/ (created 2026-08-22, phase=requirements-generated, ready_for_implementation=false -- design/tasks not yet generated). Its diagnosis: decouple allocation CAPTURE from the JOB_IS_RUN gate (capture as soon as a DISPATCH decision is seen for a job with job->accounts == NULL, regardless of state), leaving sched_account_jobs()'s existing JOB_IS_RUN + job->accounts != NULL gate to keep deferring actual charging until state transitions to RUN. Fix location: sim_reconcile_unaccounted_jobs() (sim_loop.c:566-709), trigger gate at line 592, plus its ownership-transfer logic (609-693) needs a double-capture guard since capture would no longer imply state==RUN. Do NOT fix inline as a drive-by patch per scheduler-driver-equivalence.md's testing discipline -- promote scheduler-sim-eager-reconciliation to design/tasks instead. + +--- STATUS UPDATE 2026-08-27 (scheduler-sim-eager-reconciliation task 1.1) --- + +Core production fix landed: sim_reconcile_unaccounted_jobs() trigger gate (sim_loop.c:592) had !JOB_IS_RUN(job->state) removed, leaving job->accounts == NULL as the sole capture-eligibility gate -- exactly the diagnosis from the 2026-08-24 status update. Match/capture/report mechanics (609-693) were confirmed already safe for early (pre-RUN) capture without changes (see scheduler-sim-eager-reconciliation/research.md's uncharge-safety research log -- sched_uncharge_account() has zero production callers, accounting is reset-and-resum not incremental, so no double-capture guard beyond the existing job->accounts != NULL check was needed). Old test test_reconcile_skips_non_run_state_job (which asserted the bug as correct behavior) replaced with test_reconcile_captures_non_run_state_job_when_dispatch_recorded (inverted expectation). RED/GREEN verified: new test failed under the old gate (job->allocs stayed NULL, confirmed via a real build+run before the fix), then passed after the one-line fix; full sched_sim_reconcile_unaccounted_test suite green (10/10, all prior regression tests unchanged). This is task 1.1 of the spec only -- task 1.2 (capture-safety unit coverage), task 2.1 (resInUse deferral integration test), and task 3.x (contention/--sim-verify/full-regression validation) are still pending before the spec itself is implementation-complete. CORRECTED STATUS UPDATE 2026-09-19 (superseding the immediately-preceding note in this same entry, which itself worked from stale premises): scheduler-sim-eager-reconciliation (spec completed 2026-09-11, BEFORE this session's audit ran) already decoupled sim_reconcile_unaccounted_jobs()'s trigger from the JOB_IS_RUN gate -- the function now fires on job->accounts == NULL alone, regardless of PEND/RUN state (sim_loop.c:560-572), so a job's DISPATCH decision is captured in the SAME round it is recorded, never deferred until a later, separately-recorded RUN-state confirmation event. This closes the exact real-world scenario that motivated this bug (dispatch round 32, confirmation round 107) for good -- the previous note's claimed "residual gap" (a job already-RUN before the replay window starts) is not a real gap either: such a job's allocation is populated directly from the snapshot at load time (xce_sched_core_load_snapshot(), confirmed field-complete for job->allocs/accounts by this same audit's dimension-4 pass), so job->accounts is already non-NULL immediately post-snapshot-load and such a job is never a reconciliation candidate in the first place. Combined with scheduler-sim-reconcile-terminal-guard (bug-1764, excludes terminal jobs from false rematch) and scheduler-sim-live-allocation (the live picker/allocator now handles most jobs directly, reconciliation is a narrow safety net), this entry is FULLY FIXED. STATUS: FIXED, not PARTIALLY FIXED. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1764, bug-1613, bug-1610, bug-1609 diff --git a/.wolf/bugs/bug-1612.md b/.wolf/bugs/bug-1612.md new file mode 100644 index 00000000..9a5e1227 --- /dev/null +++ b/.wolf/bugs/bug-1612.md @@ -0,0 +1,38 @@ +--- +id: "bug-1612" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-21T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/Makefile"] +tags: ["makefile", "shell-exit-status", "posix-sh", "dirty-content-fingerprint", "task-4.3", "code-review-rejection", "fp_ok"] +related: [] +legacy: true +--- + +# bug-1612: Adversarial review REJECTED dirty-content-fingerprint task 4.3 (src/frontend/Makefile… + +## Description +Adversarial review REJECTED dirty-content-fingerprint task 4.3 (src/frontend/Makefile DIRTY_FINGERPRINT): when `git diff HEAD` failed but the following `git ls-files | sort | xargs cat` pipe still succeeded, the computation silently proceeded and emitted a fabricated non-empty fingerprint instead of degrading to empty. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The `$(shell ...)` block wrote both steps into the scratch file via `if { git diff HEAD 2>/dev/null; git ls-files -z ... | sort -z | xargs -0 -r cat -- 2>/dev/null; } >"$fp_tmp" 2>/dev/null; then`. POSIX shell only propagates the exit status of the LAST command in a `{ cmd1; cmd2; }` sequential group (and `sh` here has no `pipefail`), so a `git diff HEAD` failure was silently discarded whenever the trailing ls-files/sort/xargs pipe still exited 0 -- unlike xce-build.sh's task 2.1 reference implementation, which tracks per-step success explicitly via an `fp_ok` flag. + +## Fix +Restructured the DIRTY_FINGERPRINT $(shell ...) block to gate each step individually, mirroring xce-build.sh's fp_ok discipline: `git diff HEAD >"$fp_tmp"` is now its own `if ! ...; then fp_ok=0; fi` step, and the ls-files/sort/xargs pipe (appending via `>>`) is a separate `if [ "$fp_ok" = 1 ] && ! ...; then fp_ok=0; fi` step; hashing only proceeds if fp_ok stayed 1 through both. Verified via a wrapper `git` on PATH that fails only `git diff` (all other subcommands pass through to the real git): before the fix this produced a fabricated hash (e.g. f2cd51ec) instead of the reference implementation's empty result; after the fix it resolves to empty, matching xce-build.sh under the identical injected failure. Cross-checked the happy path remains byte-identical to xce-build.sh's standalone computation (e.g. 96726026), the sha256sum-missing fallback (Requirement 5.1) still degrades to empty, GIT_COMMIT's own `-dirty` suffix is unaffected in all three scenarios, and a real `nix develop`-driven `make build-cli` + `go version -m` shows the matching fingerprint (f60eb4be) embedded in the -ldflags. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1613.md b/.wolf/bugs/bug-1613.md new file mode 100644 index 00000000..9b04f946 --- /dev/null +++ b/.wolf/bugs/bug-1613.md @@ -0,0 +1,59 @@ +--- +id: "bug-1613" +status: "fixed" +status_updated: "2026-09-24" +status_note: "e2bb98c8 (scheduler-sim-decision-parity)" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-21T07:25:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["scheduler", "sim", "replay", "resource-accounting", "resInUse", "sched_account_jobs", "correctness", "repl", "confirmed"] +related: ["bug-1611"] +fix_commits: ["e2bb98c8"] +legacy: true +--- + +# bug-1613: User-reported (xce_scheduler -i -s , 'info worker '): every CONSUMABLE resource's… + +## Description +User-reported (xce_scheduler -i -s , 'info worker '): every CONSUMABLE resource's used= value reads 0.00 for a worker regardless of how many jobs are actually dispatched and running on it, e.g. 'resource slots: value=22.00 used=0.00' and 'resource mem: value=17286225920.00 used=0.00' with jobs clearly active (confirmed via ongoing JOB_RUSAGE ticks in the same session). Confirmed real, not a display bug. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_account_jobs() (scheduler.c:1255, exported via scheduler.h) is the ONLY mechanism that re-populates per-worker (w->resInUse) and shared (core->sharedInUse) resource-in-use accounting after each session's resval reset zeroes them (core.h's own doc comment on xce_core_begin_session()/xce_core_signal_go(): 'the picker thread must never observe a session whose resource-in-use accounting has been reset to zero ... but not yet re-summed from currently running jobs (sched_account_jobs())'). This obligation was split out (scheduler-core spec task 5.2) into xce_core_begin_session()+xce_core_signal_go() specifically so a driver can call sched_account_jobs() in between; the normal-mode driver does this correctly (driver.c:739-749, xce_sched_driver_run_normal_mode_round(): xce_core_begin_session() -> plugin_begin_session()+sched_account_jobs() -> xce_core_signal_go()). The simulation driver (sim_loop.c:680) was never migrated to this split -- it still calls the older monolithic xce_core_go() (core_session.c:402-409, itself now just xce_core_begin_session()+xce_core_signal_go() called back-to-back with zero interleaving slot), and sim_loop.c/sim.c/sim_verify.c/sim_record.c contain NO call to sched_account_jobs() anywhere (grep-confirmed). Consequently every session's resval reset in simulation/replay mode is never followed by re-accumulation, so w->resInUse/core->sharedInUse stay at their zero-initialized value for the entire run -- even a job freshly dispatched via sched_commit_decision()'s direct sched_charge_account() call (scheduler.c:1438-1443) would have that charge wiped by the very next session's reset with nothing to re-sum it. Broader concern (not yet verified): the live allocator thread's own capacity checks (sched_try_alloc_on_host_ -> sched_check_host_satisfy/sched_meet_host_selection, resource.c) may read w->resInUse to compute free capacity -- if so, the simulation driver could be scheduling against permanently-zero 'in use' data, i.e. believing full capacity is always available regardless of what is actually running, which would be a correctness risk for scheduler-core's documented normal-mode/sim-mode decision-equivalence invariant, not merely a REPL display cosmetic. This needs confirmation before being treated as proven. + +## Fix +Not yet fixed -- investigation only, per user request. Proposed direction: call sched_account_jobs(sched) in sim_loop.c's round loop, in the same relative position driver.c uses (after xce_core_begin_session()'s resval reset, before signaling the picker/allocator thread) -- likely requires migrating sim_loop.c from the monolithic xce_core_go() to the split xce_core_begin_session()+xce_core_signal_go() calls, mirroring xce_sched_driver_run_normal_mode_round()'s exact pattern (driver.c:739-749). Should be paired with confirming/refuting the broader allocator-capacity-check concern above before scoping a fix. + +--- CONFIRMED 2026-08-21 (scheduler-sim-decision-parity spec research) --- + +The "broader concern (not yet verified)" above is CONFIRMED, not theoretical: `sched_check_host_satisfy_()`/`sched_check_shared_satisfy_()` (resource.c) read `w->resInUse`/`core->sharedInUse` directly as inputs to the live allocator's own per-host capacity check (`sched_try_alloc_on_host_`'s viability test). With `resInUse` permanently zero in sim/replay mode, simulation-mode capacity checks can systematically pass where normal-mode execution of the same recorded state would fail -- this is a scheduling-decision-correctness bug under resource contention, not merely a REPL display cosmetic. + +Also determined to likely share one root cause with bug-1611 (see that entry): sim_loop.c decodes the recorded decision stream (`decoded_session->decisions`) every round but only ever uses it for optional `--sim-verify` comparison, never to reconstruct state (job->allocs/job->accounts) or report a decision for a job already resolved before the replay reaches it. + +This is now elevated to an enforced steering invariant: `.kiro/steering/scheduler-driver-equivalence.md` ("normal-mode and sim-mode must produce identical decisions AND identical resource-accounting state, not just a matching decision sequence"). + +A dedicated fix spec has been scoped (deliberately not folded into the closed scheduler-core spec or the narrower scheduler-sim-repl-verbosity spec): `.kiro/specs/scheduler-sim-decision-parity/` (requirements approved; design generated, not yet approved; tasks not yet generated -- next step is /kiro-validate-design or /kiro-spec-tasks). Chosen approach: reconstruct job->allocs/job->accounts for already-resolved RUN-state jobs from the recorded decision stream at the round-boundary accounting point, keyed/deduped on job->accounts == NULL (the same signal sched_account_jobs() already uses), and feed the reconstructed decision through the same on_decision/xce_repl_check_decision reporting path a fresh decision uses -- fixing bug-1613's accounting gap and bug-1611's decision-visibility gap as one change. See that spec's research.md for full architecture-option evaluation and risks (--sim-verify interaction, snapshot-restored jobs with no in-window recorded decision). + +--- PARTIAL FIX 2026-08-22 (scheduler-sim-decision-parity task 2.1) --- + +sim_loop.c's round boundary now calls the relocated sched_account_jobs(&(sim->core)) (core.h, task 1.1) in exactly the position driver.c uses: xce_core_go() replaced with the split xce_core_begin_session() -> [reconciliation insertion point, not yet built] -> sched_account_jobs() -> xce_core_signal_go() sequence. Proven via a new spy-based regression test (test_sim_session_ordering.c): 0 calls pre-fix (RED, matches this bug's own symptom exactly), 2 calls (one per replayed round) with core->session_active==1 post-fix (GREEN). Interactive and batch sim runs share this one call site (Req 5.1), no change to normal-mode behavior (Req 5.3) -- full scheduler suite 45/46 OK post-fix, only pre-existing unrelated bug-1560 fails. + +STILL NOT FULLY OBSERVABLE END-TO-END: confirmed while implementing this task that job->accounts/job->allocs is ALWAYS NULL for every RUN-state job in sim mode today -- sim_picker_thread_fn() never emits a live DISPATCH (unconditionally PENDs every candidate), and sched_commit_decision() (the only function that ever sets job->accounts) is only ever called from scheduler.c's real sched_allocator() thread, which sim mode never runs. sched_account_jobs()'s round-boundary re-charge therefore has nothing to charge yet in isolation -- 'info worker' will still show used=0.00 until Recorded-Decision Reconciliation (scheduler-sim-decision-parity tasks 2.2-2.4, not yet implemented) populates job->accounts from the recorded decision stream. This task deliberately leaves a documented insertion point for that work (sim_loop.c comment, right before the sched_account_jobs() call) rather than building it now (out of task 2.1's own scope). Do not close this bug until tasks 2.2-2.4 land. + +--- CONFIRMED FIXED 2026-08-24 (xce-audit-invariants scheduler audit) --- + +scheduler-sim-decision-parity's remaining tasks (2.2-4.3) are now implementation-complete (spec.json phase=implementation-complete, updated_at 2026-08-22T13:10:00Z). sim_reconcile_unaccounted_jobs() (sim_loop.c:566-709) now searches each round's decoded replay decisions for RUN-state jobs with job->accounts == NULL, takes ownership of the matching DISPATCH allocation, builds job->accounts via make_job_account(), and feeds a reconstructed decision through the same reporting path a live decision uses -- called at sim_loop.c:948, immediately before sched_account_jobs() at line 950 (itself already relocated to take xce_sched_core_t* -- scheduler.c:1258 -- with no remaining signature mismatch against xce_sched_sim_driver_t). Live-verified: 'info worker' shows nonzero used= at multiple checkpoints (.wolf/memory.md line 3342). Full accounting-parity gap closed. Do not reopen for the same-round-only limitation -- that residual gap is bug-1611's concern, tracked separately (see bug-1611 and scheduler-sim-eager-reconciliation spec). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1611 diff --git a/.wolf/bugs/bug-1614.md b/.wolf/bugs/bug-1614.md new file mode 100644 index 00000000..d0d87401 --- /dev/null +++ b/.wolf/bugs/bug-1614.md @@ -0,0 +1,39 @@ +--- +id: "bug-1614" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-08-21T08:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/xceadmin/tests/test_cli.py"] +tags: ["xceadmin", "dirty-content-fingerprint", "task-5.2", "pytest", "version", "_compose_version", "regression", "stale-test", "meson", "concurrent-edit-observed"] +related: [] +fix_commits: ["2bf62a64"] +legacy: true +--- + +# bug-1614: pytest FAILED… + +## Description +pytest FAILED xceadmin/repo-mirror/src/xceadmin/tests/test_cli.py::test_cli_version_flag_includes_build_info_fields -- AssertionError: assert '2bf62a64-dirty' in 'xceadmin, version 0.1.0 (2bf62a64-857bd5b5, 2026-08-21T08:49:15Z)\n' (1 failed, 266 passed, 3 skipped) -- blocks meson's xceadmin-build.stamp custom_target, so no wheel is produced. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +dirty-content-fingerprint task 5.2's _compose_version_display() strip-and-append rule (cli.py:59-75, already implemented, matches design.md/tasks.md 5.2 exactly and all its OWN dedicated tests pass: test_compose_version_display_strips_dirty_suffix_and_appends_fingerprint, ..._anchored_suffix_not_blind_substring, test_compose_version_includes_fingerprint_when_build_info_has_one) now legitimately conflicts with an older, pre-existing task-2.2 test (test_cli_version_flag_includes_build_info_fields, added under Requirement 2.2 before this feature existed) that still asserts the RAW _build_info.GIT_COMMIT string (e.g. '2bf62a64-dirty') appears verbatim in `xceadmin --version` output. On a dirty tree with a non-empty DIRTY_FINGERPRINT, the composed output now correctly shows '2bf62a64-857bd5b5' (dirty suffix stripped, fingerprint appended per spec) instead of containing the literal '...-dirty' substring, so the old assertion is now stale and fails. tasks.md 5.2 is still unchecked ([ ]) -- this stale test update was left out of scope / not yet done. + +## Fix +RESOLVED CONCURRENTLY, not by this run: the first `xce-build.sh xceadmin asan ` invocation (BUILD_TIME 2026-08-21T08:49:15Z) failed exactly as described above (1 failed, 266 passed, 3 skipped). While investigating, found `src/xceadmin/tests/test_cli.py` in the LIVE working tree already carried an uncommitted rewrite of test_cli_version_flag_includes_build_info_fields (now computes the expected display via the same strip-and-append contract cli.py's _compose_version_display() uses, instead of asserting raw GIT_COMMIT verbatim) -- but its mtime (08:50:55Z) POSTDATES the first build's BUILD_TIME, meaning some other concurrent process/session applied this fix mid-investigation, not this build-verification agent. Re-ran the identical xce-build.sh command a second time against the now-updated tree (BUILD_TIME 2026-08-21T08:55:40Z, dirty_fingerprint recomputed to c0c2a0b7 since tracked-file content changed) and it now passes clean: '267 passed, 3 skipped in 7.65s', wheel built at .../xceadmin/dist/xceadmin-0.1.0-py3-none-any.whl. Installed that wheel alone into a fresh throwaway venv (nix develop wrapper, matching this project's established pattern) and confirmed `xceadmin --version` -> 'xceadmin, version 0.1.0 (2bf62a64-c0c2a0b7, 2026-08-21T08:55:40Z)', exit 0 -- dirty suffix correctly stripped and replaced with the fingerprint, not doubled up. Task 5.2 appears functionally complete now; tasks.md still shows it unchecked ([ ]) as of this writing -- someone should flip it once the concurrent editor's change is reviewed/committed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1615.md b/.wolf/bugs/bug-1615.md new file mode 100644 index 00000000..6fe8a46b --- /dev/null +++ b/.wolf/bugs/bug-1615.md @@ -0,0 +1,38 @@ +--- +id: "bug-1615" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-21T09:37:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/test-dirty-fingerprint-distinguishability.sh"] +tags: ["bash", "set-u", "local", "unbound-variable", "dirty-content-fingerprint", "task-6.1", "src/tools"] +related: [] +legacy: true +--- + +# bug-1615: src/tools/test-dirty-fingerprint-distinguishability.sh: line 344: iso_src: unbound variable + +## Description +src/tools/test-dirty-fingerprint-distinguishability.sh: line 344: iso_src: unbound variable + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +capture_version()'s `local iso_src="$1" binary="$iso_src/$BUILD_DIR_NAME/scheduler/xce_scheduler"` put two assignments in ONE `local` statement where the second referenced the first. Bash expands every word of a single `local` command before ANY of that command's assignments take effect, so `$iso_src` in the second word's value expanded against the OLD (not-yet-declared) `iso_src`, which under `set -u` is an unbound-variable error rather than silently empty. + +## Fix +Split into two separate `local` statements (`local iso_src="$1"` then `local binary="$iso_src/..."` on its own line) so the second statement's expansion happens after the first has already taken effect. Confirmed via `bash -n` plus two full real end-to-end script runs (both exit 0, byte-identical fingerprints reproduced across runs) after the fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1616.md b/.wolf/bugs/bug-1616.md new file mode 100644 index 00000000..14477534 --- /dev/null +++ b/.wolf/bugs/bug-1616.md @@ -0,0 +1,38 @@ +--- +id: "bug-1616" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-21T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/test-dirty-fingerprint-nix-production-path.sh"] +tags: ["bash", "set-u", "local", "unbound-variable", "dirty-content-fingerprint", "task-6.3", "src/tools", "recurring-pattern"] +related: ["bug-1615"] +legacy: true +--- + +# bug-1616: src/tools/test-dirty-fingerprint-nix-production-path.sh: line 191: flag: unbound variable + +## Description +src/tools/test-dirty-fingerprint-nix-production-path.sh: line 191: flag: unbound variable + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +extract_flag_from_json()'s `local json="$1" flag="$2" needle="-D${flag}="` recurred the exact same-statement `local` cross-reference gotcha already documented as bug-1615 (test-dirty-fingerprint-distinguishability.sh's capture_version()): bash expands every word of a single `local` command before any of that command's own assignments take effect, so `${flag}` in the third word's value expanded against the not-yet-declared `flag`, an unbound-variable error under `set -u`. + +## Fix +Split into two separate `local` statements (`local json="$1" flag="$2"` then `local needle="-D${flag}="` on its own line), with an inline comment citing bug-1615 explicitly so a future reader recognizes the pattern immediately instead of re-discovering it. Confirmed via `bash -n` plus a full real end-to-end script run (exit 0). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1615 diff --git a/.wolf/bugs/bug-1617.md b/.wolf/bugs/bug-1617.md new file mode 100644 index 00000000..db0da2e9 --- /dev/null +++ b/.wolf/bugs/bug-1617.md @@ -0,0 +1,38 @@ +--- +id: "bug-1617" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-21T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/test-dirty-fingerprint-nix-production-path.sh"] +tags: ["mktemp", "busybox", "coreutils", "portability", "dirty-content-fingerprint", "task-6.3", "src/tools"] +related: [] +legacy: true +--- + +# bug-1617: mktemp: Invalid argument + +## Description +mktemp: Invalid argument + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Used a GNU-coreutils-only mktemp template shape (`.../name.XXXXXX.log`, trailing literal suffix after the X run) assuming GNU mktemp's rule that XXXXXX only needs to be present, with a literal suffix like `--suffix` allowed after it in some historical GNU versions. This sandbox's `mktemp` is BusyBox's implementation, which requires the template to literally END with XXXXXX (no trailing suffix at all, and no `--suffix` flag support either) -- confirmed via `mktemp --help` showing BusyBox v1.36.1's usage text. + +## Fix +Dropped the `.log` suffix from the template entirely (`.../xce-dirty-fp-nix-build-A.XXXXXX` with nothing after the X run) rather than reaching for `--suffix` (also BusyBox-unsupported). The isolated worktree's own top-level mktemp template (no suffix) already worked correctly and was the right pattern to copy for the two new build-log mktemp calls. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1618.md b/.wolf/bugs/bug-1618.md new file mode 100644 index 00000000..85bb0da4 --- /dev/null +++ b/.wolf/bugs/bug-1618.md @@ -0,0 +1,38 @@ +--- +id: "bug-1618" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-21T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/test-dirty-fingerprint-nix-production-path.sh"] +tags: ["git", "pathspec", "worktree", "dirty-content-fingerprint", "task-6.3", "src/tools"] +related: [] +legacy: true +--- + +# bug-1618: error: pathspec 'flake.nix' did not match any file(s) known to git + +## Description +error: pathspec 'flake.nix' did not match any file(s) known to git + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +`git -C "$ISO_DIR" checkout -- "$EDIT_FILE_REL"` used the isolated worktree's ROOT ($ISO_DIR) as the -C directory while EDIT_FILE_REL was just "flake.nix" (unqualified), but the edited file actually lives at $ISO_DIR/src/flake.nix -- git resolves pathspecs relative to the -C directory, so "flake.nix" alone did not match anything from the worktree root. Unlike the sibling task 6.1/6.2 scripts (which edit README.md directly at the worktree ROOT, where this exact pattern is correct), this script's edit target is one level deeper inside src/, which the copied pattern did not account for. + +## Fix +Changed both `git -C "$ISO_DIR" checkout -- "$EDIT_FILE_REL"` call sites to `git -C "$ISO_SRC" checkout -- "$EDIT_FILE_REL"` (ISO_SRC = $ISO_DIR/src), so the pathspec correctly resolves relative to the flake's own source directory where flake.nix actually lives. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1619.md b/.wolf/bugs/bug-1619.md new file mode 100644 index 00000000..cb3e0596 --- /dev/null +++ b/.wolf/bugs/bug-1619.md @@ -0,0 +1,38 @@ +--- +id: "bug-1619" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-21T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/test-dirty-fingerprint-nix-production-path.sh"] +tags: ["bash", "grep", "self-check", "false-positive", "dirty-content-fingerprint", "task-6.3", "src/tools"] +related: [] +legacy: true +--- + +# bug-1619: FAIL: this script itself contains the string '--impure' -- the Observable requires no --impure… + +## Description +FAIL: this script itself contains the string '--impure' -- the Observable requires no --impure flag anywhere. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The script's own self-check for 'no --impure flag used anywhere' did a blind whole-file `grep -q -- '--impure' "$0"`. The script's own header documentation and status messages deliberately discuss '(no --impure)' in prose extensively (to explain the Observable to a reader) -- a false positive against the script's own comments/log strings, not an actual nix invocation using the flag. + +## Fix +Narrowed the check to `grep -A2 -E '^[[:space:]]*nix (eval|build|develop|log)\b' "$0" | grep -q -- '--impure'` -- only lines that begin a real nix command invocation (plus 2 following lines, to cover this script's own multi-line backslash-continued invocations), not prose. Verified the narrower grep correctly isolates exactly the script's two real `nix eval`/`nix build` invocation sites and neither contains --impure. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1620.md b/.wolf/bugs/bug-1620.md new file mode 100644 index 00000000..f4ab1808 --- /dev/null +++ b/.wolf/bugs/bug-1620.md @@ -0,0 +1,38 @@ +--- +id: "bug-1620" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-21T15:56:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/xce-build.sh (setup_build", "--incremental path)"] +tags: ["meson", "reconfigure", "incremental-build", "dirty_fingerprint", "xce-build.sh", "worker", "stale-build-dir"] +related: ["bug-1565"] +legacy: true +--- + +# bug-1620: ERROR: Unknown option: "dirty_fingerprint" + +## Description +ERROR: Unknown option: "dirty_fingerprint". + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +`xce-build.sh worker asan --incremental` runs `meson setup --reconfigure "$BUILD_DIR" -Dcommit_id=... -Dbuild_time=... -Ddirty_fingerprint=...` against a pre-existing xce.build dir that was configured before commit a860141b (feat(dirty-content-fingerprint): add dirty_fingerprint meson option). Meson 1.9.1 validates -D flags on `--reconfigure` against the OLD cached coredata BEFORE re-reading meson.options to pick up newly-added options, so any brand-new option name passed via -D on the same reconfigure invocation is rejected as unknown -- even though the option genuinely exists in the current src/meson.options (confirmed: `intro-buildoptions.json` in the stale build dir only listed commit_id/build_time, not dirty_fingerprint). Pure staleness in the incremental build dir vs. the current option surface, unrelated to any C source edit. + +## Fix +Ran `meson setup --reconfigure xce.build` once with NO -D overrides first -- this successfully re-reads meson.options and registers the new dirty_fingerprint option in cached coredata (confirmed via intro-buildoptions.json). A subsequent normal `xce-build.sh worker asan --incremental` (which does pass -Ddirty_fingerprint=...) then succeeded cleanly. General recipe: if `--incremental` reconfigure fails with 'Unknown option: "X"' for an option that IS defined in src/meson.options, the build dir predates that option's introduction -- do one plain `meson setup --reconfigure ` with no -D args to register it, then retry the real build. Rebuilding without --incremental (full rm -rf + fresh setup) also fixes it but wipes any other component's binaries sharing that dir (see bug-1565). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1565 diff --git a/.wolf/bugs/bug-1621.md b/.wolf/bugs/bug-1621.md new file mode 100644 index 00000000..0461b95b --- /dev/null +++ b/.wolf/bugs/bug-1621.md @@ -0,0 +1,38 @@ +--- +id: "bug-1621" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "leak" +timestamp: "2026-08-22T08:12:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/misc.c (free_host_account)"] +tags: ["memory-leak", "asan", "leaksanitizer", "free_host_account", "xcealloc", "scheduler", "resource-accounting", "scheduler-sim-decision-parity", "pre-existing", "out-of-scope"] +related: ["bug-1723"] +legacy: true +--- + +# bug-1621: LeakSanitizer (run with ASAN_OPTIONS=detect_leaks=1, overriding this test suite's own… + +## Description +LeakSanitizer (run with ASAN_OPTIONS=detect_leaks=1, overriding this test suite's own detect_leaks=0 convention): 'Direct leak of 8 byte(s) ... #1 make_host_account ../lib/xcealloc.c:119 #2 make_job_account ../lib/xcealloc.c:188' -- an xce_host_account_t's `host` field (strdup'd in make_host_account(), src/lib/xcealloc.c:119) is never freed. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +free_host_account() (src/lib/misc.c) frees acct->accounts[] (each via free_res_account()) and then FREEUP(acct) itself, but never FREEUP(acct->host) -- the strdup'd host string make_host_account() (src/lib/xcealloc.c:119, `haccount->host = strdup(halloc->host);`) allocates. Pre-existing, NOT introduced by scheduler-sim-decision-parity task 2.3: confirmed by tracing the leak's allocation stack, which bottoms out in make_job_account()/make_host_account() (xcealloc.c) -- the exact same tree-building code the LIVE dispatch path already uses (scheduler.c's sched_commit_decision() -> make_job_account()), so this leak equally exists on the live, production dispatch path today, just previously undetected because src/scheduler/testsuite/meson.build runs this whole suite with ASAN_OPTIONS=detect_leaks=0 (fixtures are deliberately not torn down in this test family, per test file header conventions -- masks this along with every other intentional fixture leak). + +## Fix +FIXED (2026-09-08, follow-up session). Applied exactly the fix this entry already recommended: added FREEUP(acct->host); to free_host_account() (src/lib/misc.c), immediately before FREEUP(acct) -- mirrors free_res_account()'s existing FREEUP(acct->resname) pattern in the same file. Discovered as the remaining residual leak while fixing bug-1723's test-cleanup gaps: after adding the missing free_JobAlloc_ptr()/free_HostAlloc_list() calls those two tests needed, xcelib_test still leaked 12 bytes/2 allocations (exactly 2x "host0"/"host1", 6 bytes each) at make_host_account (xcealloc.c:119) via make_job_account -- this exact function/line. Since free_host_account() is called from free_job_account(), which real production dispatch code uses (scheduler.c, core_events.c), this fix has genuine production value, not just test-cleanliness. Verified: xcelib_test now 1/1 Ok with zero LeakSanitizer reports; full xce-build.sh all asan --incremental: 94/94 Ok, zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1723 diff --git a/.wolf/bugs/bug-1622.md b/.wolf/bugs/bug-1622.md new file mode 100644 index 00000000..0ab5988d --- /dev/null +++ b/.wolf/bugs/bug-1622.md @@ -0,0 +1,38 @@ +--- +id: "bug-1622" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-22T09:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_sim_round_boundary_reconciliation.c"] +tags: ["scheduler", "test", "resource.c", "get_resvals", "return-convention", "scheduler-sim-decision-parity"] +related: [] +legacy: true +--- + +# bug-1622: TEST_ASSERT_EQUAL_INT(0, rc) failed: Expected 0 Was 1, in test_sim_round_boundary_reconciliation.c… + +## Description +TEST_ASSERT_EQUAL_INT(0, rc) failed: Expected 0 Was 1, in test_sim_round_boundary_reconciliation.c calling get_resvals(&(w->resInUse), 1, &residx, got) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +resource.c's get_resvals(zarray_t*, int nres, int*, float*) returns `nres` on success (not 0) -- confirmed by reading its body (resource.c:17-51, `return nres;` at the end). Wrongly assumed a 0-on-success/negative-on-error convention (like most other functions in this codebase, e.g. put_resvals's own callers use `rc >= 0`), and asserted TEST_ASSERT_EQUAL_INT(0, rc) instead of TEST_ASSERT_EQUAL_INT(1, rc) for a 1-resource query. + +## Fix +Changed the assertion to TEST_ASSERT_EQUAL_INT(1, rc) with an inline comment noting get_resvals() returns nres on success. Not a production bug -- purely a test-authoring mistake caught immediately by the RED/GREEN cycle (this was the FIRST failure seen after restoring the task-3 sim_loop.c wiring, so it briefly looked like the fix hadn't worked before the real root cause -- a test bug, not a reconciliation bug -- was found). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1623.md b/.wolf/bugs/bug-1623.md new file mode 100644 index 00000000..df04e316 --- /dev/null +++ b/.wolf/bugs/bug-1623.md @@ -0,0 +1,38 @@ +--- +id: "bug-1623" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-22T12:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_sim_e2e_original_motivation.c"] +tags: ["scheduler", "sim", "replay", "reconciliation", "test-authoring", "scheduler-sim-decision-parity"] +related: ["bug-1611", "bug-1613"] +legacy: true +--- + +# bug-1623: Test-authoring bug while building scheduler-sim-decision-parity task 4.3… + +## Description +Test-authoring bug while building scheduler-sim-decision-parity task 4.3 (test_sim_e2e_original_motivation.c): initial version asserted resInUse checkpoints of 17.00/18.00/20.00/12.00 (derived from the RECORDING driver own in-memory resInUse arithmetic) but the actual replay-side info worker output showed only 12.00 (ANCHOR own charge) at every checkpoint -- the first three assertions failed. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Recorded-Decision Reconciliation (sim_reconcile_unaccounted_jobs(), sim_loop.c, scheduler-sim-decision-parity tasks 2.2-2.4/3) matches a RUN-state, unaccounted job against a DISPATCH decision recorded in THAT SAME replayed round only -- never an earlier or later round (each round decoded_session is decoded fresh and freed immediately after that round own processing). A job that arrives PEND and is genuinely live-dispatched by a real allocator in round K, then confirmed RUN via a JOB_STATE_CHANGE event in a LATER round K+1, can never be reconciled during replay: round K+1 own decoded decisions do not contain a match (the real DISPATCH is in round K own data), so the job stays RUN+unaccounted forever (Requirement 4 graceful degradation, correctly, not a bug in production). This is structurally impossible to avoid for ANY organically-arriving job, since a job cannot simultaneously be a fresh PEND candidate for a live picker (required to ever get a real DISPATCH decision) AND already-RUN-before-the-picker-runs (required for reconciliation to even consider it) in the same round -- confirmed by direct source read of sim_loop.c/driver.c/builtin_fcfs.c before writing any test code, then independently reconfirmed empirically by running the real test binary. Only ONE job in the whole fixture (ANCHOR_ID) was deliberately engineered so its arrival+RUN-confirmation events AND its own injected DISPATCH decision land in the exact same round (round 1) -- every other churn job (J2-J12) arrives and is genuinely live-dispatched by the test own capacity-aware allocator thread in ITS OWN arrival round, so its dispatch can never be reconciled later, exactly like every other real production job whose confirmation lags its dispatch by more than zero rounds. + +## Fix +Changed all four info-worker checkpoint assertions from the RECORDING-side arithmetic (17.00/18.00/20.00/12.00, i.e. what a normal-mode continuation of the same session would show) to the REPLAY-side reality: a constant used=12.00 (ANCHOR_ID own charge only) at all four checkpoints spanning the 24-round replay, since no churn job dispatch is ever reconciled. Added an explicit non-tautology guard (used=0.00 never appears) and an extensive header-comment/round-table-comment explanation of why churn-job charges never surface during replay, so a future reader does not mistake this for an oversight. Also independently reconfirmed via a live discriminating-power check: commenting out sim_loop.c own sim_reconcile_unaccounted_jobs() call site made the ANCHOR_ID DISPATCH-visibility assertion fail for the right reason; restoring it (git diff clean on production files afterward) made the whole suite green again. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1611, bug-1613 diff --git a/.wolf/bugs/bug-1624.md b/.wolf/bugs/bug-1624.md new file mode 100644 index 00000000..e5f14512 --- /dev/null +++ b/.wolf/bugs/bug-1624.md @@ -0,0 +1,39 @@ +--- +id: "bug-1624" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-22T19:53:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/scheduler/src/scheduler.c", "src/scheduler/src/core_session.c", "src/scheduler/src/resource.c"] +tags: ["scheduler", "resource-accounting", "over-commit", "double-booking", "normal-mode", "sched_account_jobs", "JOB_IS_RUN", "race-condition"] +related: ["bug-1611", "bug-1613"] +fix_commits: ["8607348c", "d9aedc4b"] +legacy: true +--- + +# bug-1624: None yet observed at runtime -- found via code trace while answering an architectural question… + +## Description +None yet observed at runtime -- found via code trace while answering an architectural question about job RUN-state boundaries and over/under-commit safety. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_account_jobs() (scheduler.c:1258-1292) re-derives core->sharedInUse/worker->resInUse every round from scratch, but gates on JOB_IS_RUN(job->state) before checking job->accounts. A job that has been DISPATCHED (job->accounts set synchronously at commit-decision time via sched_commit_job_alloc(), scheduler.c:1446) but not yet confirmed RUN by the worker is still job->state==PEND, so it is skipped by this resum. Since xce_core_begin_session() (core_session.c:66-108) unconditionally zeroes sharedInUse/resInUse every round (normal-mode cadence: 1s, main.c:453-460), that job's charge -- correct within the round it was dispatched -- is wiped at the very next round and not restored until the RUN confirmation is processed. The admission check other jobs go through (sched_check_host_satisfy_/sched_check_shared_satisfy_, resource.c:252-370, called from seqjob_allocator.c:130) reads exactly this same resInUse/sharedInUse, with no separate reserved/committed pool (xce_sched_job_t has only state/allocs/accounts, sched/types.h:96-116) -- so once wiped, the resource looks free again and can be granted to a second job, a genuine over-commit/double-booking risk under any dispatch-to-worker-confirmation latency exceeding ~1 round (ordinary for real worker RPC + fork/exec). + +## Fix +FIXED (2026-08-23, scheduler-dispatch-commit tasks 2.1 + 3.1). Scheduler side (task 2.1, commit 8607348c): sched_commit_decision()'s DISPATCH branch now sets job->state = JOB_STAT_RUN synchronously, in the same operation that already sets job->accounts/job->allocs -- no separate fix to sched_account_jobs()'s JOB_IS_RUN gate was needed; the job simply reaches RUN from the round it's dispatched, so the existing gate naturally includes it every subsequent round regardless of confirmation latency. Controller side (task 3.1, commit d9aedc4b): dispatch_a_job_() gates the job.allocs commit + JOB_STAT_RUN transition together on xce_startjob_req() succeeding, keeping both sides consistent. Regression-tested end-to-end: test_dispatch_commit_survives_several_rounds_before_confirmation and test_dispatch_commit_multi_round_gap_matches_same_round_confirmation (src/scheduler/testsuite/test_scheduler_dispatch_commit.c, task 5.1) prove resInUse survives a 5-round gap with no confirmation event, and match a same-round-confirmation baseline; RED-phase independently verified via a real revert/rebuild/rerun of the fix by both the task 5.1 implementer and its reviewer. Full HA soak (sequential + chaotic, 10 rounds each) confirms no new job loss or divergence, task 5.4. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1611, bug-1613 diff --git a/.wolf/bugs/bug-1625.md b/.wolf/bugs/bug-1625.md new file mode 100644 index 00000000..c98525e6 --- /dev/null +++ b/.wolf/bugs/bug-1625.md @@ -0,0 +1,39 @@ +--- +id: "bug-1625" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-23T05:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/scheduler/src/handler.c"] +tags: ["scheduler", "controller", "dispatch-failure", "notify", "unhandled-event", "resource-leak"] +related: ["bug-1624"] +fix_commits: ["281a026d"] +legacy: true +--- + +# bug-1625: None yet observed at runtime -- found via code trace while designing scheduler-dispatch-commit + +## Description +None yet observed at runtime -- found via code trace while designing scheduler-dispatch-commit. Referenced throughout that spec's requirements.md/research.md/design.md by this id; being logged now after noticing it was never actually appended to buglog.json. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The controller already reports dispatch-RPC failure back to the scheduler via NotifyData_data_jobDispatchFail (notify_dispatch(), src/controller/job.c:1108-1125), but sched_handle_notify()'s switch (handler.c:499-544) has no case for it -- it falls to default: rc = -EINVAL. A dispatch failure is therefore silently dropped by the scheduler: job->accounts/job->allocs (set synchronously at dispatch-decision-commit time by sched_commit_decision()) are never released, leaving stale committed-resource bookkeeping behind indefinitely. + +## Fix +FIXED (2026-08-23, scheduler-dispatch-commit task 2.2, commit 281a026d). Added case NotifyData_data_jobDispatchFail to sched_handle_notify()'s switch (handler.c), applying JOB_STAT_PEND via the existing XCE_SCHED_OP_JSTATE_CHANGE enqueue path -- exactly as designed. No new cleanup code needed: core_events.c's existing free-on-non-RUN branch releases job->accounts/job->allocs automatically once the enqueued PEND change is applied. Regression-tested: test_handle_notify_job_dispatch_fail_enqueues_pend (src/scheduler/testsuite/test_handlers.c) proves the notify enqueues the correct op; independently reviewed and approved. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1624 diff --git a/.wolf/bugs/bug-1626.md b/.wolf/bugs/bug-1626.md new file mode 100644 index 00000000..870f81bb --- /dev/null +++ b/.wolf/bugs/bug-1626.md @@ -0,0 +1,39 @@ +--- +id: "bug-1626" +status: "fixed" +status_updated: "2026-09-27" +status_note: "0c1bac44" +component: "controller" +severity: "unknown" +timestamp: "2026-08-23T05:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c"] +tags: ["controller", "concurrency", "locking", "rwlock", "RTTI_LOCK_WRITE", "job-state-mutation", "race-condition", "pre-existing"] +related: [] +fix_commits: ["0c1bac44"] +legacy: true +--- + +# bug-1626: None yet observed at runtime -- found via code trace during /kiro-validate-design review of… + +## Description +None yet observed at runtime -- found via code trace during /kiro-validate-design review of scheduler-dispatch-commit. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +apply_job_fail_transition() (job.c:1804-1836) is explicitly documented as safe to call only "as long as the caller already holds job under RTTI_LOCK_WRITE" (a genuine rwlock, my_rwlock_wlock(), confirmed via src/lib/jobj.c:708-725's job_lock()). Its existing caller mark_job_fail() (job.c:1844) is itself called from job_ctrl() (job.c:965) at line 1083 -- but job_ctrl()'s only lock on the job object across its entire body (traced lines 965-1090) is lock(j, RTTI_LOCK_READ) at line 999, released via unlock(j) at line 1016, well before the mark_job_fail() call at line 1083. No RTTI_LOCK_WRITE is ever taken in this path. Separately, dispatch_a_job_() (job.c:831-963) mutates j->job.allocs under only RTTI_LOCK_READ on the containing maps (c->jobmgr, c->workermgr) and RTTI_LOCK_READ on the worker w -- never RTTI_LOCK_WRITE on the job j itself. RTTI_LOCK_READ is a genuine reader lock (my_rwlock_rlock()) that does not exclude concurrent readers from each independently mutating the same object, so if two threads process job-mutating requests for the same job concurrently, this is a real data race, not just an inaccurate comment. Contrast: many OTHER call sites in the same file (e.g. job.c:768, 1687, 1795, 2058, 2089, 2127, 2169, 2206, 2293, 2340, 2416) DO correctly take RTTI_LOCK_WRITE before mutating job state, confirming RTTI_LOCK_WRITE is this codebase's established, intentional convention for job-state mutation -- the job_ctrl()/mark_job_fail() and dispatch_a_job_() call sites appear to be inconsistent outliers, not a different intentional design. + +## Fix +NOT YET FIXED -- discovered as a pre-existing gap while validating scheduler-dispatch-commit's design (that spec plans to add new mutations, including a WAL-fact-durable one, to dispatch_a_job_(), and reuse mark_job_fail() for its terminal dispatch-failure path -- both inherit this open question). Needs its own investigation: either (a) confirm some other serialization guarantee (e.g. single-threaded-per-job request processing) makes RTTI_LOCK_READ sufficient in practice and the RTTI_LOCK_WRITE comments/convention are overcautious, or (b) these call sites are missing a genuine lock upgrade and should take RTTI_LOCK_WRITE like their sibling call sites do. Flagged back to scheduler-dispatch-commit's design review as Critical Issue 1 rather than fixed inline, since it is a pre-existing condition this feature would otherwise silently inherit/extend, not something this feature caused. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1627.md b/.wolf/bugs/bug-1627.md new file mode 100644 index 00000000..1bf5395e --- /dev/null +++ b/.wolf/bugs/bug-1627.md @@ -0,0 +1,38 @@ +--- +id: "bug-1627" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-23T05:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/job.c"] +tags: ["controller", "reference-counting", "memory-leak", "refcount", "dispatch_a_job_", "pre-existing"] +related: ["bug-1626"] +legacy: true +--- + +# bug-1627: None yet observed at runtime -- found via code trace during round 3/4 of /kiro-validate-design… + +## Description +None yet observed at runtime -- found via code trace during round 3/4 of /kiro-validate-design review of scheduler-dispatch-commit. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +dispatch_a_job_() (job.c:831-963) calls reference(j) at line 854 after successfully looking up the job. On FOUR of its failure/return paths it never calls a matching dereference(j): workermgr key_find() failing (rc<0, line 888-890), the worker not being found (w==NULL, line 892-894), the worker not being in a ready state (E_XCE_BAD_WORKER, line 897-909), and xce_startjob_req() itself failing (rc<0, line 956-960 -- found in a later review pass than the first three). (Contrast: the nc<=0 path at line 869-873 and the get_queue_exec() failure path at line 917-925 both correctly dereference(j) before returning.) Each occurrence permanently leaks one reference on the job object. Separately and NOT included in this bug (flagged as a distinct open question, not a confirmed defect): the function also never calls dereference(j) on its normal SUCCESS return path -- this may be an intentional reference hand-off to the job entering its dispatched/running lifetime rather than a bug, and needs its own investigation rather than being folded in here. + +## Fix +FIXED -- scheduler-dispatch-commit task 3.3. dispatch_a_job_() (job.c) now closes all four leak sites as part of the retryable-failure classify/resolve restructuring the bug's own notes anticipated: XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP, _WORKER_NULL, _WORKER_NOT_READY, and _RPC_FAILURE each now lock(j, RTTI_LOCK_WRITE), classify via xce_classify_dispatch_fail(), increment job.dispatch_fail_count under that lock, resolve retryable (within XCE_DISPATCH_FAIL_RETRY_BOUND==3) or terminal (mark_job_fail(), mirroring task 3.2's two sibling terminal sites) as appropriate, unlock(j), then dereference(j) -- closing the leak uniformly across all four. Regression-tested via test_dispatch_fail_retry.c's canary-reference technique (task 3.2's own established pattern) at three of the four sites (WORKER_LOOKUP/WORKER_NULL share one test since a black-box test cannot force which of the two an unregistered worker key actually triggers -- both share the identical resolution code shape). RED confirmed first (all 4 new tests failed against the unmodified four sites, git-stash verified), then GREEN after the fix, full controller_unit_test suite run (1056 tests, filtered only for the pre-existing bug-1429 hang) showed zero new failures beyond the pre-existing bug-1561 pair. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1626 diff --git a/.wolf/bugs/bug-1628.md b/.wolf/bugs/bug-1628.md new file mode 100644 index 00000000..95440139 --- /dev/null +++ b/.wolf/bugs/bug-1628.md @@ -0,0 +1,39 @@ +--- +id: "bug-1628" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-08-23T08:03:59" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".claude/skills/weekly_report/SKILL.md"] +tags: ["date", "busybox", "coreutils", "portability", "sandbox", "bash", "git-log", "weekly_report", "silent-failure"] +related: ["bug-1606", "bug-1617"] +legacy: true +--- + +# bug-1628: date: invalid date '-6 days' / date: invalid date '2026-06-29 +5 days' -- relative date arithmetic… + +## Description +date: invalid date '-6 days' / date: invalid date '2026-06-29 +5 days' -- relative date arithmetic silently yields EMPTY shell variables, and the empty values then flow into `git log --since="" --until=""`, which git treats as NO time filter (unbounded history) instead of erroring. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +`date` first on PATH in this harness is BusyBox v1.36.1 (/nix/store/...busybox-1.36.1/bin/busybox), NOT GNU coreutils, even though /bin/date IS coreutils 9.8 -- the nix profile bin dir precedes /bin. BusyBox date supports absolute parsing (`date -d 2026-06-29 +%A` works, which is what masked the problem during initial probing) but has no GNU relative-expression support ('-7 days', 'last monday', '+5 days'). Because command substitution captures only stdout and BusyBox writes 'invalid date' to stderr, $START/$END come back as empty strings and the failure is invisible to a non -e script. Same BusyBox-shadows-GNU family as bug-1606 (grep -P) and bug-1617 (mktemp template). + +## Fix +Do all date arithmetic in python3 (/bin/python3, always present) instead of `date -d`: compute week start via `today - timedelta(days=today.weekday()) - timedelta(weeks=offset)` and emit `START=/END=/SLUG=` lines consumed by `eval "$(python3 - ... <<'PY' ... PY +)"`. General rule: never use `date -d` with a relative expression in this environment; use python3 datetime, or call /bin/date by absolute path if GNU date is genuinely required. Also validate that computed date vars are non-empty before feeding them to `git log --since/--until`, since git silently ignores empty range args rather than failing. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1606, bug-1617 diff --git a/.wolf/bugs/bug-1629.md b/.wolf/bugs/bug-1629.md new file mode 100644 index 00000000..1da215cb --- /dev/null +++ b/.wolf/bugs/bug-1629.md @@ -0,0 +1,38 @@ +--- +id: "bug-1629" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-08-23T08:12:20" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_scheduler_dispatch_commit.c"] +tags: ["scheduler", "testsuite", "zarray", "resource-accounting", "asan", "scheduler-dispatch-commit", "test-fixture"] +related: [] +legacy: true +--- + +# bug-1629: AddressSanitizer SEGV in zarray_get_ex (lib/zarray.c:332) when a test-fixture… + +## Description +AddressSanitizer SEGV in zarray_get_ex (lib/zarray.c:332) when a test-fixture xce_res_account_t.residx was never resolved from -1 + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +zarray_get_ex() only guards idx >= array->caps before indexing array->data[idx]; it does NOT guard idx < 0. sched_update_res_accounts() (resource.c) resolves xce_res_account_t.residx from -1 to a real index via key_find()/add_vres() against core->resmgr ONLY when the host lookup (sched_get_worker()) and the surrounding call chain actually run to completion; if a test fixture never pre-registers the resource name in core->resmgr, sched_update_account()/sched_update_res_accounts() silently swallow the resolution failure (return values are discarded by sched_update_account()) and residx stays -1. A -1 index then passes the 'idx >= array->caps' bounds check (false, since caps starts at 0 too) and falls through to array->data[-1], an out-of-bounds/underflow read that ASAN reports as a high-address SEGV rather than a clean bounds error. + +## Fix +In the test fixture, pre-register the resource with add_vres(&vres, core.resmgr, RES_NAME, 0) BEFORE calling the real charging path, and use the returned (non-negative) index for get_resvals()/put_resvals() calls, instead of relying on the production code's lazy first-use resolution path inside a minimal test fixture that never loads real resource config. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1630.md b/.wolf/bugs/bug-1630.md new file mode 100644 index 00000000..c33a2639 --- /dev/null +++ b/.wolf/bugs/bug-1630.md @@ -0,0 +1,39 @@ +--- +id: "bug-1630" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "tooling" +severity: "unknown" +timestamp: "2026-08-23T08:35:55" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".wolf/buglog.json"] +tags: ["buglog", "openwolf", "data-integrity", "duplicate-id", "tooling", "needs-git-history-merge", "verified-2026-08-30"] +related: ["bug-1628"] +fix_commits: ["755976fc"] +legacy: true +--- + +# bug-1630: buglog.json integrity: 39 duplicate bug IDs present (bug-875..bug-948) + +## Description +buglog.json integrity: 39 duplicate bug IDs present (bug-875..bug-948). The Stop hook's duplicate-ID validation flags this every turn. bug-908, for example, is BOTH a 2026-06-21 raft_sim.c missing-guard-clause entry AND a 2026-08-16 config-upgrade-snapshot/spec.json refactor note -- two unrelated bugs sharing one id. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +A session on 2026-08-15..17 appended new entries using an ID counter that had been restarted (or derived from a stale/remembered max) instead of computed from the file's actual maximum numeric ID. New bugs were therefore written into the already-occupied bug-875..bug-949 range: 8 collisions dated 2026-08-15, 14 dated 2026-08-16, 22 dated 2026-08-17. This is precisely the failure mode the OpenWolf bug-logging rule warns about ('never restart from bug-001; a restarted counter collides with existing history and is unrecoverable without a git-history merge'). Discovered while generating the Aug 16-22 weekly report, where an in-window timestamp query returned an implausible id range of bug-908..bug-1624. + +## Fix +Housekeeping only, no code change needed: renumbering commit 755976fc (2026-08-24) already repaired all 39 duplicate IDs. Re-verified 2026-08-30 -- 1077 bug entries, 1077 unique IDs, zero collisions. This entry's own `fix` field was simply never backfilled after the repair landed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1628 diff --git a/.wolf/bugs/bug-1631.md b/.wolf/bugs/bug-1631.md new file mode 100644 index 00000000..3774592c --- /dev/null +++ b/.wolf/bugs/bug-1631.md @@ -0,0 +1,43 @@ +--- +id: "bug-1631" +status: "fixed" +status_updated: "2026-09-27" +status_note: "0c1bac44" +component: "controller" +severity: "unknown" +timestamp: "2026-08-23T19:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/job.c"] +tags: ["controller", "dispatch-failure", "job-state", "job-allocs", "retry", "scheduler-dispatch-commit", "requirement-4.1", "requirement-4.4", "requirement-4.5", "design-gap"] +related: ["bug-1625", "bug-1626", "bug-1627"] +fix_commits: ["0c1bac44"] +legacy: true +--- + +# bug-1631: New test test_rpc_failure_bound_resets_after_intervening_success (test_dispatch_fail_retry.c)… + +## Description +New test test_rpc_failure_bound_resets_after_intervening_success (test_dispatch_fail_retry.c) fails: 'Req 4.1 (bug-1631, NOT YET FIXED): a retryable dispatch failure must return the job to PEND -- fails here only because this job previously had a REAL successful dispatch (step 2)'. Expected JOB_STAT_PEND, got JOB_STAT_RUN (and separately, job.allocs stays non-NULL instead of being released). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +dispatch_a_job_()'s four retryable-failure resolution branches (XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP/_WORKER_NULL/_WORKER_NOT_READY/_RPC_FAILURE, job.c ~lines 1040-1435, all four added/restructured by scheduler-dispatch-commit task 3.3) never write j->job.state or j->job.allocs on the 'resolved_retryable' (within-bound) path -- only job.dispatch_fail_count is touched. This is CORRECT for a job's very FIRST dispatch attempt: the job was never anything but PEND and allocs was never committed, so 'no reversal needed' (design.md's own 'Controller Dispatch Commit' state-model text: 'on RPC failure, state is untouched -- still PEND, never entered RUN, so no reversal is needed, per the Research Log finding') is vacuously true. That reasoning silently assumes the job has NEVER had a prior successful dispatch. It breaks the moment dispatch_a_job_() is called again on a job that previously succeeded (went RUN, committed allocs) and now fails retryably again -- exactly the scenario Requirement 4.3's own text ('consecutive... dispatch failures with no intervening successful dispatch') and task 3.1's own observable-completion criterion ('a job that fails, succeeds, then fails again is evaluated against the bound as if it had only failed once') explicitly anticipate. In that scenario job.state incorrectly stays RUN and job.allocs incorrectly stays committed across the retry, violating Requirement 4.1 ('the Scheduler and Controller shall return the job to a pending state') and Requirement 4.4 ('the Scheduler and Controller shall each release the resource-allocation and commitment data'). The terminal (bound-exceeded) resolution path is separately confirmed affected the same way: mark_job_fail()/apply_job_fail_transition() (job.c ~2303-2325) unconditionally sets job.state = JOB_STAT_EXIT (so the STATE outcome is correct on the terminal path) but never touches job.allocs either, so Requirement 4.5 is also affected on that path. Every existing retryable-failure test (test_dispatch_fail_retry.c's other three tests, test_rpc_failure_bounded_escalation's own 4-consecutive-failure test) only ever drives a job that starts and stays PEND with allocs always NULL throughout, so none of them could have caught this -- only a real intervening success (constructed by the new fail->succeed->fail scenario, scheduler-dispatch-commit task 5.2) exposes it. This is a design-level gap, not just an implementation oversight: design.md's own Controller Dispatch-Failure Classification / Controller Dispatch Commit sections both assert-by-omission that job.state/job.allocs need no reversal on any of the six failure paths, without qualifying that assertion to 'only true for a job's first-ever dispatch attempt'. + +## Fix +NOT YET FIXED. Discovered live while implementing scheduler-dispatch-commit task 5.2 (test-only task; the fix requires modifying src/controller/job.c, out of that task's test-only boundary -- reported as BLOCKED). Candidate fix: in each of the four retryable branches' 'resolved_retryable = true' case (still under the existing lock(j, RTTI_LOCK_WRITE)), add `j->job.state = JOB_STAT_PEND;` and release any committed j->job.allocs (free_JobAlloc_ptr(&j->job.allocs); j->job.allocs = NULL;) before unlock(j) -- mirroring, in reverse, what Controller Dispatch Commit's own success path (job.c ~1343-1372) does when transitioning PEND->RUN. Needs a design-review pass first: design.md's 'Controller Dispatch-Failure Classification' and 'Controller Dispatch Commit' sections both need their state-model text corrected to no longer claim the failure paths need no reversal unconditionally. UPDATE (2026-08-23, task 5.2 completion): a kiro-debug investigation (HIGH confidence, independently re-verified even more exhaustively by task 5.2's own reviewer via a full trace of every pendJobs-reinsertion and controller-side state-mutation call site) confirmed this scenario is genuinely UNREACHABLE via any real caller in the current, fully-implemented system -- the scheduler's picker permanently removes a job from its PEND candidate pool the instant it dispatches it (task 2.1), and nothing already-implemented ever re-adds a successfully-dispatched (RUN) job back to that pool. Only the not-yet-implemented scheduler-preemption spec (confirmed still requirements-generated, no approvals) could someday create that path. Per that finding, the COMMITTED version of test_rpc_failure_bound_resets_after_intervening_success (src/controller/testsuite/test_dispatch_fail_retry.c, commit 16d90319) deliberately does NOT assert the job.state/job.allocs reversal described above -- it keeps only the Req 4.3 counter-reset assertions (which are correct and GREEN). There is no live RED test for this bug today; it remains an intentionally-deferred, documented gap to revisit once scheduler-preemption (or an equivalent mechanism) actually creates a reachable RUN-then-retryable-redispatch path -- not before. + +ADDENDUM (2026-09-01, worker-rejoin-resync design re-validation): worker-rejoin-resync's design.md now explicitly names this bug as the reason Requirement 4.6 (reconciling an 'unrecognized' RUN job back to PEND, eligible for re-dispatch) must not be shipped without the candidate fix described above. design.md's `record_job_reconcile_pending_fact()` is specified to release `job->job.allocs` at the same point it sets the job PEND -- the exact fix this entry already proposed, now scoped to a concrete call site instead of a hypothetical future one. Still NOT YET FIXED IN CODE (worker-rejoin-resync is design-phase only as of this note) -- this addendum records that the 'unreachable, deliberately deferred' status ends the moment that spec is implemented, and the fix must land in the same change as Requirement 4.6, not as a follow-up. + +RESOLVED IN CODE (2026-09-01, worker-rejoin-resync task 2.1): record_job_reconcile_pending_fact() (src/controller/job.c, after mark_job_update) now performs exactly the fix this entry proposed -- free_JobAlloc_ptr(&job->job.allocs) immediately after a successful (JOB_TRANSITION_APPLIED) apply_job_update_transition(job, JOB_STAT_PEND), under the same caller-held write lock, before the durable WAL_JOB_UPDATE fact is appended. This is called when a worker reports a queried job as 'unrecognized' during rejoin reconciliation (Requirement 4.6) -- exactly the RUN-job-returned-to-PEND-and-re-admitted-to-the-dispatch-pool precondition this entry's own root_cause said was needed to make the gap reachable. Regression-tested directly: test_reconcile_pending_applies_releases_allocs_and_notifies (test_worker_resync.c) seeds a real JOB_STAT_RUN job with a real non-NULL allocation and asserts allocs == NULL immediately after the call. Independently reviewed and approved (kiro-review). dispatch_a_job_()'s own four retryable-failure branches (the original bug-1631 site) are still untouched by this fix -- they remain correct for a job's first-ever dispatch attempt, which is still the only reachable case THERE; this fix closes the gap at the new call site this feature introduces, not by changing dispatch_a_job_ itself. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1625, bug-1626, bug-1627 diff --git a/.wolf/bugs/bug-1632.md b/.wolf/bugs/bug-1632.md new file mode 100644 index 00000000..3400f4a3 --- /dev/null +++ b/.wolf/bugs/bug-1632.md @@ -0,0 +1,38 @@ +--- +id: "bug-1632" +status: "open" +status_updated: "2026-09-24" +status_note: "test_sim_verify.c stack-local sim read by picker thread (ASAN stack-use-after-return)" +component: "scheduler" +severity: "security" +timestamp: "2026-08-23T21:03:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_sim_verify.c"] +tags: ["scheduler", "asan", "stack-use-after-return", "thread-lifecycle", "test-only", "sim-driver", "flaky", "non-deterministic", "pre-existing", "sibling-of-bug-1598"] +related: ["bug-1598"] +legacy: true +--- + +# bug-1632: sched_sim_verify_test: AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151)… + +## Description +sched_sim_verify_test: AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151), thread T2 (sim_picker_thread_fn, sim_loop.c:456), reading the stack of run_sim_and_capture_stdout's `sim` (xce_sched_sim_driver_t) after that frame returned/was reused. Fired once during a full `/xce-build scheduler` run (scheduler-dispatch-commit task 5.4 regression pass): Ok:50 Fail:2 instead of the documented Ok:51 Fail:1 baseline, extra failure = 'scheduler sim verify test'. Did NOT reproduce on: 3 standalone direct re-runs of sched_sim_verify_test (16 Tests 0 Failures each time) or a second full `meson test --suite scheduler` re-run against the same build (back to Ok:51 Fail:1, only the known test_admin_request.c symlink-escape failure) -- confirmed non-deterministic/order-dependent, not caused by scheduler-dispatch-commit. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same root cause as bug-1598 (already diagnosed, NOT re-derived here): sched_sim_driver_run() (src/scheduler/src/sim_loop.c) spawns sim_picker_thread_fn on a pthread that is, by design, never joined/cancelled -- it loops forever on cbuf_get(&sim->core.wbuf,...) reading `sim` for the life of the process. bug-1598's fix converted 3 stack-local `xce_sched_sim_driver_t sim;` fixtures in test_sim_event_loop.c to `static` storage, but test_sim_verify.c's OWN, separately-defined run_sim_and_capture_stdout() helper (test_sim_verify.c:940-943) still declares `sim` as stack-local -- an unfixed sibling instance of the identical latent bug, never touched by bug-1598's fix. Confirmed via git diff/log that scheduler-dispatch-commit's own commits (which only touch scheduler.c/handler.c per that spec's Requirement 5.2 sim-mode-exclusion) never modified sim_loop.c or test_sim_verify.c -- this is pre-existing latent test infra, exposed non-deterministically by whatever stack-layout/MALLOC_PERTURB_ randomization a given meson test invocation happens to produce, exactly like bug-1598's own trigger mechanism. + +## Fix +NOT FIXED -- out of scope for scheduler-dispatch-commit task 5.4 (a validation/regression task, not a scheduler-sim-infra task). Same remediation as bug-1598 would apply if picked up: change run_sim_and_capture_stdout()'s `xce_sched_sim_driver_t sim;` from stack-local to `static` storage (safe under the same reasoning bug-1598 documents: sched_sim_driver_start() memset()s the whole struct first, and this helper is not re-entered while an earlier zombie thread could still be alive within one test binary process -- verify that non-reentrancy holds for test_sim_verify.c's own call sites before applying). Logged so a future session doesn't re-diagnose from scratch. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1598 diff --git a/.wolf/bugs/bug-1673.md b/.wolf/bugs/bug-1673.md new file mode 100644 index 00000000..6b65f976 --- /dev/null +++ b/.wolf/bugs/bug-1673.md @@ -0,0 +1,41 @@ +--- +id: "bug-1673" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "config" +severity: "unknown" +timestamp: "2026-08-24T13:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/handler.go", "src/configs/hosts.yaml"] +tags: ["gitea-issue-40", "apiserver", "go", "handler.go", "hosts-list", "max-slot", "display-bug", "hostconfig"] +related: ["bug-1556"] +legacy: true +gitea_issue: 40 +gitea_synced_status: "fixed" +--- + +# bug-1673: Gitea issue #40 ("job 调度不正确"): `xcecli hosts list` shows MAX_SLOT=0 for a dynamically-joined… + +## Description +Gitea issue #40 ("job 调度不正确"): `xcecli hosts list` shows MAX_SLOT=0 for a dynamically-joined worker even though the host's real NCPUS (e.g. 16 or 22) is correctly reported and scheduling correctly dispatches up to that many concurrent jobs. Reproduced live on branch scheduler-framework: single-node deploy, worker joined as 'worker-1', `xcecli hosts list` -> NCPUS=22 MAX_SLOT=0. Submitted 9 single-slot sleep jobs -- all 9 eventually dispatched and ran (no hard cap observed under this test), so the reporter's secondary claim of a hard ~8-job concurrency ceiling did NOT reproduce here (most likely a mid-dispatch snapshot, dispatch is not instantaneous across all pending jobs on this branch; a queue/usergroup slot limit in limits.yaml, e.g. ug1_normal_slots=8, remains a plausible alternate explanation worth asking the original reporter about). Only the MAX_SLOT=0 display value itself is a confirmed, reproducible bug. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +OnHostConfigUpdated() (handler.go:60-70) only applies config-pushed fields (including MaxSlot) onto a host entry via `store.Hosts.Get(incoming.HostName)` -- an EXACT hostname match. hosts.yaml commonly defines only a wildcard `hostname: default` entry (plus a couple of literally-named ones, e.g. worker2/host3) -- confirmed via grep, no entry named after the actual dynamically-joined worker (e.g. 'worker-1'). So a worker that joins under its own hostname never receives a HostConfig push under that literal name, `store.Hosts.Get()` misses, OnHostConfigUpdated returns early, and MaxSlot is left at Go's zero-value (0) forever -- completely decoupled from the value actually used for scheduling. The C-side path is correct: worker.c's apply_join_state()/find_matched_hostconfig() falls back to the 'default' wildcard entry's mxj/maxcpus for real dispatch decisions, confirmed via a live controller log line showing the correct slots=22 resource value. This is purely a Go apiserver display gap, not a scheduling defect. + +## Fix +FIXED (2026-08-24). src/frontend/apiserver/backend/handler.go -- OnHostConfigUpdated now caches the "default" pseudo-host's config-sourced fields (defaultHostConfig, package-level, mutex-guarded) instead of discarding them, and retroactively backfills that fallback onto any already-registered host with no explicit (non-default) config of its own; OnHostUpdated applies the same fallback immediately on a host's first-ever registration. The HOSTCONFIG_MXJ_AUTO_DETECT -1 sentinel (bug-1556) is further resolved to the host's real NumCPUs when applied via fallback, mirroring worker.c's C-side `if (nslots<0) nslots = maxcpus`. A host with its own real per-host hosts.yaml entry is never overridden. Added resetHostConfigCache() test hook plus 5 new tests (both arrival orderings, no-override-of-explicit-config, unresolvable-stays-negative) -- all pass, plus all 3 pre-existing OnHostConfigUpdated/OnHostUpdated tests pass unmodified (`make test-backend`). Live-verified: `xcecli hosts list` for the dynamically-joined worker-1 (no hosts.yaml entry) now shows MAX_SLOT=22, matching NCPUS=22, instead of 0. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1556 +- Gitea: #40 diff --git a/.wolf/bugs/bug-1674.md b/.wolf/bugs/bug-1674.md new file mode 100644 index 00000000..732735ff --- /dev/null +++ b/.wolf/bugs/bug-1674.md @@ -0,0 +1,40 @@ +--- +id: "bug-1674" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-24T13:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/controllers/jobs.go:553", "src/frontend/apiserver/controllers/jobs.go:901"] +tags: ["gitea-issue-41", "apiserver", "go", "jobs.go", "jobs-list", "field-validation", "regression"] +related: [] +legacy: true +gitea_issue: 41 +gitea_synced_status: "fixed" +--- + +# bug-1674: Gitea issue #41 ("job output timestamp is incorrect"): reporter's transcript shows `xcecli jobs… + +## Description +Gitea issue #41 ("job output timestamp is incorrect"): reporter's transcript shows `xcecli jobs list -l ` succeeding but omitting submit/start/done timestamps. Reproduced live on branch scheduler-framework, but the actual failure is more severe than reported: `xcecli jobs list -l ` (and bare `xcecli jobs list -l`) fail OUTRIGHT with `query request failed: [invalid parameters: unsupported query field: pending_reason]` -- no output is produced at all, not even a partial listing missing timestamps. Likely a regression introduced after the original issue was filed (this branch is under active scheduler-framework development). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Both the `-l`/fields-filtered job-list code paths (jobs.go:553 and the near-duplicate at jobs.go:901) validate the requested field list via `utils.GetSupportedFields(models.JobSubmitRequest{})` -- the job SUBMISSION request struct -- instead of `models.JobInfo{}`, the job query/response struct (the correct pattern, confirmed already used correctly for hosts at hosts.go:105/307 with `models.HostInfo{}`, and for at least one other job code path at jobs.go:1131 with `models.JobInfo{}`). Read-only/response-only fields like `pending_reason`, `start_time`, `end_time`, `pids`, `pgids` don't exist on JobSubmitRequest, so the first such field hit in Go's randomized map-iteration order triggers a hard `unsupported query field` rejection, making `-l` (and any custom -o field list touching these fields) fail unconditionally. + +## Fix +FIXED (2026-08-24). src/frontend/apiserver/controllers/jobs.go:553,901 -- both field-validation call sites now use utils.GetSupportedFields(models.JobInfo{}) instead of models.JobSubmitRequest{}, matching the already-correct pattern at jobs.go:1131. Caught during live verification (not source review alone): core_limit/mem_limit exist on JobSubmitRequest but not JobInfo/JobAdditionalFields -- switching wholesale would have newly broken those two previously-valid -o/-l fields. Fixed by adding CoreLimit/MemLimit (CustomLimit) to models.JobAdditionalFields (models/jobs.go), the existing merge-in bucket already used for pids/pgids/interactive_job_url. Added TestJobsListFieldValidation_UsesJobInfoNotSubmitRequest (jobs_test.go) covering all 24 fields in xcecli's real JobFieldsMap vocabulary. Full apiserver/controllers suite passes (CGO_ENABLED=0, this package's pre-existing required mode). Live-verified: `jobs list -l` and `-o "pending_reason start_time core_limit mem_limit pid pgid"` all now succeed -- every one previously errored with "unsupported query field". + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #41 diff --git a/.wolf/bugs/bug-1675.md b/.wolf/bugs/bug-1675.md new file mode 100644 index 00000000..29529f5c --- /dev/null +++ b/.wolf/bugs/bug-1675.md @@ -0,0 +1,40 @@ +--- +id: "bug-1675" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xcecli" +severity: "unknown" +timestamp: "2026-08-24T13:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/xcecli/common/request/request.go:97-98", "src/frontend/xcecli/common/request/request.go:185-188"] +tags: ["gitea-issue-42", "xcecli", "go", "request.go", "error-message", "ux", "startup-window"] +related: [] +legacy: true +gitea_issue: 42 +gitea_synced_status: "fixed" +--- + +# bug-1675: Gitea issue #42 ("刚启动的时候查询作业报的错误不太友好"): `xcecli jobs list` run in the narrow window right after… + +## Description +Gitea issue #42 ("刚启动的时候查询作业报的错误不太友好"): `xcecli jobs list` run in the narrow window right after the apiserver (re)starts, before its listener is up, prints a raw `get the request failed: [connection refused]` instead of a user-friendly message. Reproduced live on branch scheduler-framework: killed and relaunched the apiserver, hammered `xcecli jobs list` in a tight loop from the instant of relaunch -- attempts 1-13 all returned the exact reported text; the listener came up by attempt 14. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ExtractErrorMessage() (request.go:185-188) explicitly special-cases the connection-refused case (`if strings.Contains(errStr, "connection refused") { return "connection refused" }`) but returns the raw OS-level string verbatim instead of a friendlier message (e.g. explaining the API server may still be starting up). GetRequestFromAPIServer() (request.go:97-98) then wraps it as `"get the request failed: [%s]"`, producing exactly the reported output. + +## Fix +FIXED (2026-08-24). src/frontend/xcecli/common/request/request.go:187 -- ExtractErrorMessage()'s connection-refused branch now returns "cannot reach the API server -- it may still be starting up, retry shortly" instead of the raw "connection refused" string. Confirmed no other call site string-matches the old literal. Added TestExtractErrorMessage (request_test.go, table-driven) -- RED before, GREEN after. Full xcecli suite passes. Live-verified: killed+relaunched the apiserver, hit `xcecli jobs list` during the startup window -- new message confirmed both via the fresh test and independently by rebuilding xcecli and re-checking. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #42 diff --git a/.wolf/bugs/bug-1676.md b/.wolf/bugs/bug-1676.md new file mode 100644 index 00000000..7f293ca4 --- /dev/null +++ b/.wolf/bugs/bug-1676.md @@ -0,0 +1,46 @@ +--- +id: "bug-1676" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-24T13:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/controllers/jobs.go:1483", "src/frontend/apiserver/backend/handler.go:18-20", "src/schema/controller.capnp"] +tags: ["gitea-issue-44", "apiserver", "go", "jobs.go", "handler.go", "from-host", "capnp-schema", "data-loss"] +related: [] +legacy: true +gitea_issue: 44 +gitea_synced_status: "fixed" +--- + +# bug-1676: Gitea issue #44 ("FROM_HOST is -"): `xcecli jobs list`'s FROM_HOST column always shows `-` instead… + +## Description +Gitea issue #44 ("FROM_HOST is -"): `xcecli jobs list`'s FROM_HOST column always shows `-` instead of the host the job was submitted from. Reproduced live on branch scheduler-framework: submitted a job, checked `xcecli jobs list` ~1 second later (job still PEND) -- FROM_HOST blank; confirmed via the apiserver's own debug log showing `"from_host":""` on every job regardless of state (PEND/RUN/DONE), across every job submitted in the session. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Three-part chain, fully traced: (1) jobs.go:1483 correctly sets `FromHost: ctx.ClientIP()` at job-CREATION time. (2) controller.capnp's Job/JobSubmitData structs have no from_host field at all (confirmed via grep -- zero matches for fromHost/from_host in the schema), so this value never round-trips to or back from the C controller. (3) handler.go's OnJobUpdated (line 18-20) does a blind `store.Jobs.Put(uint64(job.Unique_JobID), *job)` full overwrite on every controller-sourced job push, unlike OnHostUpdated/OnHostConfigUpdated (handler.go:46-70) which explicitly merge worker-sourced and config-sourced fields onto the existing stored entry. Since a controller-sourced job push never carries from_host (per #2), the very first state-change push after creation -- which happens almost immediately, well before a user can typically observe the initial value -- wipes FromHost back to "" via the blind overwrite. + +## Fix +FIXED (2026-08-24), full fix. User directive: client should assert its own from-host and the backend should pass it through unchanged, rather than the apiserver deriving it from ctx.ClientIP() (which is an IP not a hostname, and trivially spoofable via X-Forwarded-For since this apiserver never calls gin.Engine.SetTrustedProxies()). Two independent, non-overlapping pieces: + +(1) Origin: xcecli's BuildJobRequest() (cmd/jobs/submit.go) now sets JobSubmitRequest.FromHost via os.Hostname(); apiserver's CreateJob (controllers/jobs.go:1483) now does `FromHost: req.FromHost` -- a straight passthrough, ctx.ClientIP() removed entirely from this path. New field added to both xcecli's and apiserver's independent models.JobSubmitRequest structs (separate Go modules, no shared code). New test TestBuildJobRequest_SetsFromHostToLocalHostname (xcecli). Full xcecli suite and apiserver controllers/models suites pass. + +(2) Round-trip: src/schema/controller.capnp's JobSubmitData gains `fromHost @10: Text $C.mapname("from_host")` (auto-regenerates controller.capnp.c/h via the existing meson capnpc-c custom_target -- no hand-editing of generated files). src/include/xce/types.h's job_submit_data_t gains `char* from_host`. src/lib/jobj.c's jobsub_init/jobsub_destroy (the hand-written, ref-counted, content-deduped job-submit interning object -- NOT auto-generated) thread from_host through the deep-copy, the dedup identity comparison, and the destroy path; 3 existing tests updated + new test_jobsub_from_host_distinguishes_identity in test_jobj.c/testsuite. Go apiserver/backend/convert.go's modelJobToSubmit/freeJobSubmit (submit direction) and cJobToModel/cJobArrayToModels (receive direction), plus convert_pure.go's rawSubmitData/rawJobToModel (the pure-Go, CGO-free mapping layer) all thread the field through. Did NOT need to touch job.c's build_job_submit_payloads() (bare pointer borrow, flows through automatically) or any WAL persistence code (capnp_encode_job/decode_Job already generically encode/decode the whole xce_job_t including the full .submit sub-struct) or backend/handler.go's OnJobUpdated (its existing blind overwrite is correct by construction once every push genuinely carries the right value). + +Verified: C xcelib_test 301/301 (ASAN leak detection on, includes the 3 updated + 1 new jobsub tests, independently re-run and confirmed). Go backend suite: pure-Go tests (CGO_ENABLED=0) and full CGO suite via `make test-backend` against the real regenerated capnp headers, both independently re-run and confirmed (`ok apiserver/backend 14.6s`). Live, independently verified on a freshly-submitted job (not reusing any job the implementer had already submitted): FROM_HOST= visible from the very first query while still PEND, survives the RUN transition, and (implementer's own live check, not independently re-run) survives a full 4-daemon restart for free via existing WAL_JOB_SUBMIT persistence. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #44 diff --git a/.wolf/bugs/bug-1677.md b/.wolf/bugs/bug-1677.md new file mode 100644 index 00000000..ebf86a90 --- /dev/null +++ b/.wolf/bugs/bug-1677.md @@ -0,0 +1,38 @@ +--- +id: "bug-1677" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "tooling" +severity: "crash" +timestamp: "2026-08-24T13:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["scripts/gitea-sync.py"] +tags: ["gitea-sync", "scripts", "python", "buglog-format", "latent-bug", "unrun"] +related: [] +legacy: true +--- + +# bug-1677: scripts/gitea-sync.py's sync_bugs() would crash (or silently no-op) on any real invocation… + +## Description +scripts/gitea-sync.py's sync_bugs() would crash (or silently no-op) on any real invocation: `buglog = json.loads(BUGLOG_PATH.read_text())` (line 726) then `for i, bug in enumerate(buglog):` (line 731) treats buglog.json as a bare list, but its actual top-level structure is `{"version": 1, "bugs": [...]}` -- a dict. `enumerate()` over a dict yields its keys ("version", "bugs") as `bug`, and the very next line's `bug.get("id", ...)` would raise AttributeError on the string "version". Discovered incidentally while checking whether to set the `gitea_issue` cross-reference field on newly-logged bugs (bug-1673..bug-1676) during a Gitea-issue verification pass -- not run live, found via static read of the sync script against the current buglog.json wrapper format. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sync_bugs() was apparently written against (or never updated from) an earlier bare-array buglog.json format. Every other buglog reader/writer in this session's own tooling (and the Stop hook's own validation, per .wolf/OPENWOLF.md's 'Always preserve the top-level {"version": 1, "bugs": [...]} wrapper') expects the wrapper; this script does not, unlike its own spec-sync half (sync_specs et al.) which was presumably tested more recently. + +## Fix +FIXED (2026-08-24). scripts/gitea-sync.py's sync_bugs() (~line 726, 799) now unwraps buglog_wrapper["bugs"] on read and re-wraps buglog_wrapper["bugs"] = buglog before _write_json on write, instead of treating the file as a bare list. Verified via a standalone read/write round-trip against a scratch copy of the real buglog.json (not the live file, no Gitea API calls) -- version and all 1019 entries survive intact. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1678.md b/.wolf/bugs/bug-1678.md new file mode 100644 index 00000000..fde9be05 --- /dev/null +++ b/.wolf/bugs/bug-1678.md @@ -0,0 +1,47 @@ +--- +id: "bug-1678" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-24T13:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c:1369-1370", "src/controller/persist.c:3708-3757", "src/controller/job.c:288-351 (restore_job)", "src/controller/persist.c:6690-6713 (persist_submit_job_fact)"] +tags: ["gitea-issue-43", "controller", "job.c", "persist.c", "wal", "exec-host", "job-allocs", "restart-recovery", "data-loss", "scheduler-dispatch-commit", "wal-jobdispatch"] +related: ["bug-1631", "bug-1679"] +legacy: true +gitea_issue: 43 +gitea_synced_status: "fixed" +--- + +# bug-1678: Gitea issue #43 ("重启daemon后作业调度/信息不对"): after restarting ALL daemons (controller, scheduler… + +## Description +Gitea issue #43 ("重启daemon后作业调度/信息不对"): after restarting ALL daemons (controller, scheduler, worker, apiserver), previously-RUN jobs come back with STATUS=RUN correctly recovered but EXEC_HOST permanently blank ("-"), even though the underlying job is genuinely still running. CONFIRMED LIVE on branch scheduler-framework, corrected finding: an earlier verification pass on this same issue (2026-08-24, same session) restarted only controller+scheduler+worker and left the apiserver running, and did NOT reproduce the symptom -- that was a false negative, caused by the still-running apiserver simply continuing to display its stale-but-correct in-memory job cache from before the restart, never refreshed. Repeating the test with the apiserver ALSO restarted (matching the original report's "restart all daemons" exactly) reproduces the bug immediately and durably: 3 RUN jobs with EXEC_HOST=1*worker-1 before the restart; killed all 4 daemons; ran start-all.sh; EXEC_HOST was blank on every poll (15 polls over 30s, first poll ~6s after apiserver came back up) while STATUS stayed correctly RUN throughout. Controller log corroborates the mechanism: ctrl-1.log's recovery trace shows apply_job_update_transition logging 'duplicate UPDATE to state 0x2 -- no-op' for every job during WAL replay -- consistent with the WAL_JOB_DISPATCH fact applying SILENTLY first (PEND->RUN, no log line on the APPLIED path), immediately followed by each job's WAL_JOB_START fact (worker-confirmed start, also state=RUN) correctly detecting the now-redundant transition and logging the no-op. Neither replay step ever touches job.allocs. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +dispatch_a_job_() (job.c:1343-1372) sets j->job.allocs directly in memory (lines 1346-1353, the host/resource assignment the scheduler decided) and transitions j->job.state = JOB_STAT_RUN (line 1355), but the durable record of this transition -- persist_submit_job_fact(&c->persist, WAL_JOB_DISPATCH, alloc->jobid, JOB_STAT_RUN) at line 1369-1370 -- only carries {jobid, state} (persist_submit_job_fact()'s own signature, persist.c:6690-6694, has no allocation parameter at all; its payload type wal_jobstate_t is exactly {jobid, state}). The resource allocation itself is never written to the WAL anywhere. On a full-process restart, job objects are reconstructed purely from replay: restore_job() (job.c:288-351) decodes a job's ONLY WAL_JOB_SUBMIT entry (written at initial submission, strictly before dispatch -- confirmed via persist_job_hook_fun's own comment that WAL_JOB_SUBMIT durability is written once, before any job object exists -- so its capnp payload's allocs field is genuinely empty/NULL at encode time). apply_committed_entry()'s WAL_JOB_DISPATCH case (persist.c:3708-3757) then replays the dispatch fact via apply_job_update_transition(), which by explicit design and its own doc comment 'only ever touches state' -- job->job.allocs is never set anywhere in the replay path. Result: after restart, job.state is correctly RUN (both WAL_JOB_DISPATCH and the later WAL_JOB_START fact carry state=RUN and replay correctly) but job.allocs is permanently NULL for the remaining life of that job -- nothing ever re-derives it, since scheduler-dispatch-commit's design treats a job's dispatch as a one-time PEND->RUN event with no re-dispatch/re-reconciliation path for an already-RUN job (see bug-1631's confirmation that no live caller ever re-adds a RUN job back to the scheduler's dispatch pool). EXEC_HOST is rendered by the apiserver from this now-NULL allocation once the recovered, allocation-less job is pushed to it -- which is why restarting the apiserver alongside the controller (forcing a fresh push) reveals the bug, while leaving the apiserver running (as an earlier verification pass on this same issue did) masks it behind stale cached data. + +## Fix +FIXED (2026-08-25). No capnp schema change needed -- JobAlloc (xce_job_alloc_t) was already a full top-level capnp struct (the existing Job.allocs field) with generated encode/decode functions already present in the build. New glue only: src/controller/persist.c gains capnp_encode_joballoc()/decode_joballoc() (mirroring the existing capnp_encode_worker()/restore_job() patterns) and persist_submit_job_dispatch_fact(persist_t*, jobid, state, alloc) -- WAL_JOB_DISPATCH's payload is now a wal_jobstate_t header plus trailing capnp-encoded JobAlloc bytes, declared in src/include/persist.h. src/controller/job.c: dispatch_a_job_()'s success path now calls persist_submit_job_dispatch_fact(..., j->job.allocs) instead of persist_submit_job_fact(...), reading j->job.allocs (the freshly-constructed canonical struct) under the same write lock. Both apply seams updated: apply_committed_entry() (primary's own boot replay) gets one added else-if branch inside its existing WAL_JOB_START/WAL_JOB_FINISH stamp-chain (minimal diff, same shared plumbing) that decodes and adopts trailing allocation bytes when present. apply_wal_entry() (standby catch-up / replicated-apply -- also required, since a promoted standby would hit the exact same bug) has WAL_JOB_DISPATCH split into its own case, calling new restore_job_dispatch_state() (job.c, restore_job_state()'s sibling, takes ownership of an optional decoded allocation). A payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_DISPATCH entry written before this fix) decodes no allocation and behaves exactly as before -- full backward compatibility, verified by the 3 pre-existing test_wal_job_dispatch_fact.c tests passing completely unchanged. + +Primary regression test: test_dispatch_commit.c's test_dc_success_appends_wal_job_dispatch_durably (already drove the REAL dispatch_a_job_() production path through a real WAL-write + persist_recover() restart cycle, previously only checking j2->job.state) extended to also assert j2->job.allocs matches what was dispatched (host, resource name, value) -- the user's exact acceptance criterion, 'after recover the job should get the same as it's state before shutdown.' RED->GREEN independently reproduced (not just narrated): git-stashed job.c/persist.c/persist.h with the test extension kept, rebuilt, confirmed FAIL at exactly this assertion; restored, rebuilt, confirmed PASS. Added 3 new tests to test_wal_job_dispatch_fact.c covering the richer payload directly on both apply seams plus a full restart-replay variant -- all pass. Full controller_unit_test suite independently re-run: 1136/1136 pass, zero regressions. + +Live-verified at the WAL level via xce_test's waldump: a real dispatch_a_job_() call produced a WAL_JOB_DISPATCH entry with PLEN=176 (vs the old bare wal_jobstate_t shape's PLEN=16), confirming the richer payload is genuinely written on the production path. + +IMPORTANT: full xcecli-visible E2E verification (submit -> restart -> check EXEC_HOST live) was blocked by discovering bug-1679 (see that entry) -- a separate, more severe, pre-existing bug where live job-status pushes to the apiserver never fire at all, restart or not. This fix's correctness rests on the unit-test evidence above (which inspects the controller's actual recovered in-memory state directly, with no dependency on the broken apiserver-notification path) and the direct WAL-payload inspection, not on an xcecli-visible live check. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1631, bug-1679 +- Gitea: #43 diff --git a/.wolf/bugs/bug-1679.md b/.wolf/bugs/bug-1679.md new file mode 100644 index 00000000..f9d71782 --- /dev/null +++ b/.wolf/bugs/bug-1679.md @@ -0,0 +1,44 @@ +--- +id: "bug-1679" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T03:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c (dispatch_a_job_", "on_job_start", "and likely other direct-mutation call sites)", "src/controller/persist.c (apply_committed_entry's WAL_JOB_START/.../DISPATCH shared case", "apply_job_update_transition)"] +tags: ["controller", "job.c", "persist.c", "apiserver-notification", "hook-suppression", "scheduler-dispatch-commit", "live-status", "data-loss", "high-severity", "discovered-via-bug-1678"] +related: ["bug-1678", "bug-1680"] +legacy: true +--- + +# bug-1679: A job's live dispatch/start/finish transitions are never pushed to the apiserver -- xcecli jobs… + +## Description +A job's live dispatch/start/finish transitions are never pushed to the apiserver -- xcecli jobs list shows a job stuck at PEND forever (or, in one observed case, disappearing from the default query entirely) even though the controller's own WAL and in-memory state correctly progress the job all the way through RUN to EXIT/DONE. Discovered as a side effect while live-verifying bug-1678 (restart-recovery of job.allocs) -- this is a SEPARATE, unrelated, more severe bug: it reproduces with NO restart involved at all, on a single, freshly-submitted job, every time. Confirmed reproducible on the STOCK branch (git-stashed all of bug-1678's changes, rebuilt, redeployed fresh) -- NOT a regression introduced by bug-1678 or any other fix landed this session (bug-1673/1674/1675/1676/1677 touch only Go/apiserver code, a Python script, or job_submit_data_t's from_host field + jobj.c's dedup identity check -- none touch the state-transition/hook-firing logic implicated here). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +dispatch_a_job_() (job.c, scheduler-dispatch-commit's task 3.1 restructuring) mutates j->job.state DIRECTLY in memory (job.c ~1355, `j->job.state = JOB_STAT_RUN;`) BEFORE calling persist_submit_job_dispatch_fact()/persist_submit_job_fact() to durably record the transition. On a live, single-node, self-committing controller, the commit-then-apply pipeline runs synchronously enough that apply_committed_entry()'s own WAL_JOB_DISPATCH handling (via apply_job_update_transition(j, JOB_STAT_RUN)) sees `cur == state == JOB_STAT_RUN` -- because the state was ALREADY set moments earlier by the same call, under the same lock -- and returns JOB_TRANSITION_NOOP_DUPLICATE. The shared case body's `if (result != JOB_TRANSITION_APPLIED) return 0; /* rejected-backward / no-op-duplicate -- no hook */` then unconditionally skips xce_run_hooks(HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, ...) -- the ONLY mechanism (via apiserver_job_hook_fun/apiserver_push_joblist in apiserver.c, registered against XCE_CONTROLLER_HOOK_JOB) that pushes an updated job record to the apiserver. Live-confirmed via controller log: `apply_job_update_transition: job : duplicate UPDATE to state 0x2 -- no-op` fires for BOTH the WAL_JOB_DISPATCH commit AND the subsequent WAL_JOB_START commit (on_job_start(), job.c) on the exact same job, back to back -- meaning the job NEVER gets a single HOOK_DETAIL_JOB_UPDATE fire through its entire dispatch. apiserver_push_joblist() itself only ever fires ONCE per job, at creation time (apiserver_job_new_fun, subscribed to HOOK_DETAIL_JOB_NEW, itself fired once by apply_committed_entry's WAL_JOB_SUBMIT case) -- there is no periodic/polling full-refresh fallback, confirmed by reading apiserver_push_joblist's only call site. So a job's apiserver-visible record is permanently frozen at whatever it looked like at creation (JOB_STAT_PEND, no exec_host) for its entire lifetime, regardless of what actually happens to it afterward, UNLESS some other code path elsewhere in job.c legitimately fires the hook directly (mutate-then-notify, like mark_job_update()/persist_job_hook_fun's 'still-legitimate append' pattern) rather than relying on the apply-time hook-fire -- not confirmed either way for on_job_exit/mark_job_fail/other terminal-transition call sites; only dispatch_a_job_() and on_job_start were directly traced. This is very likely a structural gap introduced by (or newly exposed by) the raft-apply-decoupling / scheduler-dispatch-commit redesigns moving job-state mutation to happen synchronously ahead of persistence for several call sites, while the apiserver-notification hook-fire mechanism was designed under an assumption (state hasn't already changed by the time apply runs) that held for the OLD architecture but no longer holds for these newer, direct-mutation call sites. + +## Fix +FIXED (2026-08-25). Root cause confirmed exactly as filed: dispatch_a_job_() (job.c) mutates job.state/job.allocs directly then persists, firing no hooks at all -- apply_committed_entry()'s later apply of that same WAL_JOB_DISPATCH entry sees state already matches and skips its would-be notify (JOB_TRANSITION_NOOP_DUPLICATE). + +New src/controller/persist.c function persist_notify_job_updated_already_persisted(cluster_t*, job_t*) (declared in src/include/persist.h), called from dispatch_a_job_()'s success branch immediately after unlock(j). Internally guards with the exact same t_apply_seam_notifying flag apply_committed_entry()'s own shared WAL_JOB_START/.../DISPATCH case already uses around its identical hook fire (same guard, same hook -- HOOK_DETAIL_JOB_UPDATE's AFTER_EVENT -- same detail) -- without it, persist_job_hook_fun (also subscribed to this hook) would redundantly re-append the same transition as a second, duplicate WAL_JOB_UPDATE entry. The flag stays file-private to persist.c; job.c reaches it only through this one new function. dispatch_a_job_()'s own mutation/persistence logic is otherwise untouched. + +TDD: new test_dc_success_notifies_job_update_hook_exactly_once (test_dispatch_commit.c) -- a spy hook counting HOOK_AFTER_EVENT/HOOK_DETAIL_JOB_UPDATE fires specifically. RED->GREEN independently reproduced: temporarily disabled the new call (production code unchanged otherwise), rebuilt, confirmed FAIL (Expected 1 Was 0); restored, rebuilt, confirmed PASS. Verified the two existing tests that would catch a broken guard (double WAL append): test_jobdispatch_receipt_ack_no_wal_append (asserts exactly one WAL entry for a successful dispatch) and test_handle_jobdispatch_matches_dispatch_a_job_notify_count -- both pass unchanged, and the former's fixture genuinely wires persist.cluster with persist_job_hook_fun registered, so it actually exercises the guard. Full controller_unit_test: 1137/1137 (2 pre-existing test_config_repl.c failures -- the known overlayfs fdatasync errno=5 flake, bug-1561 -- confirmed identical via git-stash on stock code, not a regression). + +Live-verified on a fresh single-node deploy (rebuilt controller/scheduler/worker/xcecli): submitted a job, xcecli jobs list showed PEND at t+1s then RUN with EXEC_HOST=1*worker* from t+2s onward continuously -- no restart involved, matching the exact acceptance criterion. Also confirmed record_job_finish_fact's existing notification path needed no fix: on_job_finish fired correctly and xcecli jobs list -d showed DONE with EXEC_HOST intact for both test jobs, exactly as the root-cause analysis predicted (only dispatch_a_job_()'s broken hybrid pattern was ever the problem -- the durable-append-only pattern record_job_start_fact/record_job_finish_fact use was already correct). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1678, bug-1680 diff --git a/.wolf/bugs/bug-1680.md b/.wolf/bugs/bug-1680.md new file mode 100644 index 00000000..0c5164f6 --- /dev/null +++ b/.wolf/bugs/bug-1680.md @@ -0,0 +1,44 @@ +--- +id: "bug-1680" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "data-loss" +timestamp: "2026-08-25T03:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/xcecli/filters/job_filter.go:58-60", "src/frontend/apiserver/utils/types.go:262-322", "src/frontend/apiserver/store/query.go:70-73"] +tags: ["xcecli", "apiserver", "job_filter.go", "types.go", "query.go", "job-status-filter", "cli-bug", "discovered-via-bug-1679"] +related: ["bug-1679"] +legacy: true +--- + +# bug-1680: `xcecli jobs list -a` ("list run,done,pending,suspended jobs" -- i.e + +## Description +`xcecli jobs list -a` ("list run,done,pending,suspended jobs" -- i.e. show every job regardless of status) returns "no job found" even when jobs genuinely exist and are correctly stored with a real status. Discovered as a side effect of live-verifying bug-1679: two jobs that correctly showed PEND->RUN->DONE via the default query and via `-d` (done-only) both come back empty under `-a` specifically. Not a data-loss bug (the jobs ARE correctly stored and queryable through other flags -- `-d`, the default active-jobs view -- this is specifically `-a`'s own query construction that's broken) but a real, reproducible CLI defect: `-a` is documented as the way to see literally everything, and it shows nothing. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xcecli's `-a` flag (list.go:347, `AllJobInfoFlag`) makes HandleJobFilter (job_filter.go:57-60) build the filter string `{job_status:notnull:}` -- a `notnull` operator on `job_status` with an EMPTY value token (no argument needed for null-checks, by design) -- instead of the normal `{job_status:in:(...)}`/`{job_status:notin:(...)}` shape every other status flag (-p/-r/-d/-s) produces. Server-side, `job_status`/`host_status` get SPECIAL bitmask handling in buildIntCondition (types.go:262-322), but that special-casing only triggers for the `in`/`notin`/`between` operators (the `if filter.Field == "job_status" || filter.Field == "host_status"` branch sits inside the `if filterOperator == "in" || filterOperator == "notin" || filterOperator == "between"` block, types.go:263-270) -- `notnull` falls through to the generic scalar path (types.go:316-322), which builds a plain `Condition{Column: "job_status", Operator: OpNOTNULL, Value: {TypeInt, 0}}` with no bitmask awareness at all. In isolation, store/query.go's own MatchesCondition evaluator (query.go:70-73) implements OpNOTNULL correctly (`fieldVal != nil`), which should trivially match every job (job_status is never actually absent/nil in a stored row) -- meaning the actual break is somewhere between condition construction and evaluation for this specific field/operator combination that was not fully traced (a debugger/print-tracing session would be needed to pin the exact line): plausible candidates are (a) the query engine routing job_status conditions through a bitmask-specific code path unconditionally regardless of operator, silently misinterpreting a non-bitmask NOTNULL Condition as a bitmask EQ/NE against value 0 (which would exclude every real job_status value, all of which are nonzero bitmask flags), or (b) some other job_status-specific special-casing elsewhere in the query pipeline that only handles the `in`/`notin`/`eq`/`ne` shapes types.go's own bitmask branch produces and never anticipated a bare NOTNULL condition on this field at all. + +## Fix +FIXED (2026-08-25). Root cause fully pinned down (the prior entry's root_cause left the construction-to-evaluation gap untraced -- now traced exactly): src/frontend/apiserver/store/query.go's FilterRows() (the actual filter-evaluation loop, not something the prior investigation reached) routes ANY condition on job_status/host_status through the bitmask path purely based on IsBitmaskField(cond.Column) -- unconditional on the field NAME alone, never checking the condition's actual Operator. MatchesBitmaskCondition()'s own switch only implements OpEQ/OpNE (the only two shapes buildIntCondition's in/notin bitmask branch ever produces) and silently `return false` for any other operator -- so `{job_status:notnull:}` (Operator: OpNOTNULL, Value: {TypeInt, 0}, per types.go's null/notnull scalar path) gets routed into MatchesBitmaskCondition anyway, matches neither switch case, and returns false for every single row unconditionally. That's the entire bug: field-name-based routing that ignores which operator shape is actually on the condition. + +Fix: FilterRows's routing condition now requires BOTH IsBitmaskField(cond.Column) AND cond.Operator being OpEQ or OpNE before taking the bitmask path; any other operator (OpNOTNULL, OpISNULL, or anything else that might ever target job_status/host_status) correctly falls through to the generic MatchesCondition evaluator, which already implements OpNOTNULL/OpISNULL correctly (fieldVal != nil / == nil) -- confirmed by the fix requiring zero changes to MatchesCondition or MatchesBitmaskCondition themselves, only the dispatch condition in FilterRows. + +New src/frontend/apiserver/store/query_test.go (no test file existed for this package before -- confirmed via search, matches the prior entry's note): TestFilterRows_JobStatusNotNull_MatchesRealRows reproduces the exact bug (RED before the fix: 0 rows matched instead of 3; independently confirmed) and TestFilterRows_JobStatusBitmaskEQ_StillWorks locks in that the actual bitmask shape (OpEQ/OpNE, what every -p/-r/-d/-s status flag produces) is unaffected -- both pass after the fix. Full apiserver/store, apiserver/controllers, apiserver/models (CGO_ENABLED=0) and apiserver/backend + apiserver/backend/auth (full CGO via make test-backend) suites all pass, zero regressions. + +Live-verified: rebuilt and hot-swapped the apiserver binary on a running cluster with 3 pre-existing DONE jobs (previously invisible to `-a`) plus one freshly-submitted job -- `xcecli jobs list -a` now correctly shows all 4 jobs across every status (3x DONE, 1x RUN), immediately, no restart needed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1679 diff --git a/.wolf/bugs/bug-1681.md b/.wolf/bugs/bug-1681.md new file mode 100644 index 00000000..e35163e7 --- /dev/null +++ b/.wolf/bugs/bug-1681.md @@ -0,0 +1,46 @@ +--- +id: "bug-1681" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-24T21:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["scheduler", "sim", "replay", "plugin", "driver-equivalence", "latent", "scheduler-driver-equivalence"] +related: ["bug-1611", "bug-1613"] +legacy: true +--- + +# bug-1681: xce-audit-invariants scheduler-driver-equivalence audit finding (not user-reported, no live… + +## Description +xce-audit-invariants scheduler-driver-equivalence audit finding (not user-reported, no live symptom today): driver.c's normal-mode round (xce_sched_driver_run_normal_mode_round, driver.c:750,802) calls plugin_begin_session(sched)/plugin_end_session(sched) -- iterating every registered plugin and invoking its begin_session_fn/end_session_fn hook, called 'real, load-bearing per-session bookkeeping for EVERY registered plugin' per driver.c's own comment (lines 688-693). sim_loop.c has zero equivalent call anywhere (grep-confirmed across sim*.c/repl*.c). This is structurally the same shape of gap as bug-1613's original root cause (a driver.c-only per-session call with no sim_loop.c mirror), but under the codebase's only currently-supported configuration it is inert: sim mode's sim_bootstrap_picker() (sim_picker.c:60-70) only ever registers builtin_fcfs, which explicitly leaves begin_session_fn/end_session_fn NULL (plugins/builtin_fcfs.c:973-981), and the one plugin that does set a hook (legacy fcfs, plugins/fcfs.c:352, end_session_fn=fcfs_end_session, only sets privdata->new_session=1, no reach into resInUse/sharedInUse/job->accounts) is unreachable in sim mode either way (sim mode refuses any picker name other than builtin_fcfs). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +plugin_begin_session()/plugin_end_session() (plugin.c:370,407) are a distinct hook category from xce_job_picker_iface_t's begin_session/end_session (which both drivers DO call symmetrically -- scheduler.c:111,131 vs sim_loop.c:467,484). Nothing in sim_loop.c or sim.c acknowledges skipping the plugin-level begin/end_session hooks -- unlike the wire/capnp-only and REPL-only differences the audit also found, which are documented deliberate exclusions. + +## Fix +FIXED 2026-08-24. plugin_begin_session()/plugin_end_session() (src/scheduler/src/plugin.c) relocated from an implicit xce_scheduler_t* (void* arg cast to xce_scheduler_t*, only ever touching sched->core.plugins) to a direct xce_sched_core_t* parameter -- mirroring sched_account_jobs()'s identical scheduler-sim-decision-parity task 1.1 relocation, and confirming driver.c's own header comment claim that plugin_begin_session() 'needs the full xce_scheduler_t*' was stale/incorrect (its body never touched anything beyond core->plugins). src/scheduler/src/sim_loop.c's sched_sim_driver_run() round loop now calls plugin_begin_session(&(sim->core)) immediately before sched_account_jobs(&(sim->core)) (same relative order driver.c uses), and plugin_end_session(&(sim->core)) immediately before xce_core_end_session() -- exactly mirroring driver.c's xce_sched_driver_run_normal_mode_round() call sites, which were updated to pass &(sched->core) instead of sched. src/scheduler/src/scheduler.c's legacy sched_begin_session()/sched_end_session() (a third, pre-existing call site not part of the original driver-equivalence audit's scope, found only when the signature change broke its compile) updated the same way. plugin.h's declarations relocated to xce_sched_core_t* (a new forward-declare added earlier in the file so the type is visible at the declaration site). Stale comments describing the old xce_scheduler_t* coupling corrected in driver.c, core.h, and core_session.c (two locations). + +Test: extended testsuite/test_sim_session_ordering.c's existing bug-1613 spy-based regression tests (test_sim_replay_calls_account_jobs_once_per_round_within_session_window, test_sim_replay_batch_mode_uses_the_same_round_boundary) with new plugin_begin_session()/plugin_end_session() spies (same call-count + core->session_active-window proof technique as the existing sched_account_jobs spy) -- proving 2 calls/round (recording mode) and 1 call (batch mode), each with session_active==1 at call time. Deliberately does NOT #include the real ../src/plugin.c in this file (unlike test_driver_session_ordering.c, which proves real per-plugin hook firing via a genuinely-registered observer plugin) -- sim.core.plugins does not exist until sched_sim_driver_run()'s own internal sched_sim_driver_start()/sim_bootstrap_picker() calls run, with no hook point for a test to register an observer plugin in between, so this file spies the call site directly instead, plus a verbatim real reimplementation of plugin_register_core() (needed for real by both this file's fixture and sim_picker.c's sim_bootstrap_picker()) and no-op link-satisfying stubs for plugin_register()/plugin_unregister() (both genuinely unreachable in this file's fixtures). Four other white-box test files (test_resval_ownership.c, test_admin_request.c, test_driver_decision_cb.c, test_driver_translate.c) each defined their own plugin_begin_session/plugin_end_session stub against the old void* signature -- all four updated to xce_sched_core_t* to match. + +RED/GREEN independently verified (not just narrated): temporarily commented out both new sim_loop.c call sites, rebuilt, confirmed test_sim_replay_calls_account_jobs_once_per_round_within_session_window and test_sim_replay_batch_mode_uses_the_same_round_boundary both FAIL (Expected 2 Was 0 / Expected 1 Was 0) -- proving the assertions genuinely exercise the missing calls, not a tautology. Restored (git diff confirmed byte-identical to the fixed state), rebuilt, reconfirmed both PASS. + +Full scheduler suite: 51/52 suites green; the one failure (test_recordstart_symlink_escape_replies_error) is the pre-existing, unrelated bug-1560 sandbox-environment flake (already documented as such in scheduler-core's own task 12.5 verification notes), not a regression from this fix. + +Deliberately NOT fixed as part of this change (separate, unrelated pre-existing defect found while touching the same function bodies): plugin_end_session()'s per-plugin guard checks the wrong field (begin_session_fn instead of end_session_fn) -- logged separately as bug-1682. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1611, bug-1613 diff --git a/.wolf/bugs/bug-1682.md b/.wolf/bugs/bug-1682.md new file mode 100644 index 00000000..8ffa1e03 --- /dev/null +++ b/.wolf/bugs/bug-1682.md @@ -0,0 +1,38 @@ +--- +id: "bug-1682" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-24T21:40:00Z" +last_seen: "2026-09-19T09:30:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugin.c"] +tags: ["scheduler", "plugin", "session-hooks", "guard-bug", "latent", "scheduler-driver-equivalence"] +related: ["bug-1681", "bug-1492"] +legacy: true +--- + +# bug-1682: Found while fixing bug-1681 (source read of plugin_end_session(), src/scheduler/src/plugin.c): the… + +## Description +Found while fixing bug-1681 (source read of plugin_end_session(), src/scheduler/src/plugin.c): the per-plugin iteration guard checks the WRONG field. plugin_end_session()'s loop tests `if (plugin->begin_session_fn != NULL) { plugin->end_session_fn(plugin); }` -- guards on begin_session_fn but calls end_session_fn. Pre-existing, not introduced by bug-1681's fix (untouched by that diff). Previously documented (not fixed) in testsuite/test_driver_session_ordering.c's own file header comment as a known 'CONCERN surfaced during this test's construction' -- flagged there for the reviewer, deliberately left unfixed as out of that task's scope. Confirmed still present in current code. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Copy-paste from plugin_begin_session()'s loop body (which correctly guards on begin_session_fn before calling it) without updating the guard field for the end_session variant. + +## Fix +FIXED (2026-09-19, /xce-audit-invariants scheduler dimension-1 audit). Prior tag-only "fixed"/"closed-by-scheduler-sim-live-allocation" stamps on this entry were premature/incorrect -- confirmed by this audit's independent re-read of plugin.c that the guard-field bug was still genuinely present in code as of the scheduler-sim-live-allocation feature's completion (that feature touched plugin_job_election/plugin_prepare_allocation/plugin_commit_decision only, never plugin_begin_session/plugin_end_session's bodies). Now actually fixed: changed plugin_end_session()'s guard from plugin->begin_session_fn != NULL to plugin->end_session_fn != NULL (src/scheduler/src/plugin.c). Confirmed via TDD (RED: new test_end_session_invokes_end_session_fn_when_only_it_is_set() fails 84/85 against the pre-fix guard; GREEN: 85/85 after). Duplicate of bug-1492 (same defect, originally filed 2026-08-04, discovered again independently during this bug's own 2026-08-24 investigation without cross-referencing the earlier entry) -- see bug-1492 for the original filing. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1681, bug-1492 diff --git a/.wolf/bugs/bug-1683.md b/.wolf/bugs/bug-1683.md new file mode 100644 index 00000000..a2278725 --- /dev/null +++ b/.wolf/bugs/bug-1683.md @@ -0,0 +1,40 @@ +--- +id: "bug-1683" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-08-25T00:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/xceadmin/src/xcecommon/wire.py"] +tags: ["xceadmin", "xcecommon", "wire-protocol", "python", "framing", "test-failure", "unreproducible"] +related: ["bug-1604"] +legacy: true +--- + +# bug-1683: xceadmin pytest failure: test_send_frame_writes_exact_known_good_byte_sequence… + +## Description +xceadmin pytest failure: test_send_frame_writes_exact_known_good_byte_sequence (src/xceadmin/tests/test_wire.py:200). Test sends a 4-byte payload b'xce!' via wire.send_frame() over a real TLS socket to a raw capture handler that reads exactly 12 bytes (8-byte length header + 4-byte payload) directly off the wire, bypassing this module's own recv_frame() decoder. Expected the captured bytes to be struct.pack('=Q', 4) + b'xce!' (header correctly encodes length=4, followed by the literal payload bytes). Actual captured bytes: the 8-byte header is correct (b'\x04\x00\x00\x00\x00\x00\x00\x00', i.e. length=4 little-endian native uint64_t), but the trailing 4 bytes are all zero (b'\x00\x00\x00\x00') instead of b'xce!' -- the real payload bytes never reached the wire. 1 failed, 266 passed, 3 skipped -- isolated to this one test. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +UNRESOLVED -- not reproducible. send_frame() (wire.py:102-113) is two sequential sock.sendall() calls (header, then payload) -- nothing in it, or in the test's _TlsEchoServer/_capture_handler harness (test_wire.py), or conftest.py's session-scoped tls_certs fixture, could zero out payload bytes; all reviewed and found correct. Confirmed only bare pytest 8.4.2 is installed (no pytest-xdist, no pytest-randomly, no addopts) -- test order is deterministic, ruling out parallel-worker/reordering interference. + +45+ reproduction attempts (2026-08-25), zero failures: isolated run x5, full test_wire.py x3, full 270-test suite x3 (ad-hoc PYTHONPATH), full meson-driven build+test pipeline (xce-build.sh xceadmin, the project's real test-running convention) x1 clean (267 passed, 3 skipped), isolated via the meson-built venv x15, full suite via meson-built venv x5, full suite under artificial CPU stress (20x `yes` spinners, load 11+) x5, 8-way concurrent parallel execution of the exact test x8 -- all clean. TCP/TLS cannot spontaneously zero application bytes mid-stream, so a genuine code defect producing this exact symptom (correct 8-byte header, real zeroed 4-byte payload) should be deterministically reproducible, and it isn't. + +## Fix +Not a code fix -- treating as a suspected ONE-OFF ENVIRONMENTAL/INFRASTRUCTURE artifact, not a defect in current source. This project is documented (.wolf/cerebrum.md, bug-1604) to run many concurrent automated sessions that race on shared build directories -- e.g. a concurrent xce-build.sh/meson-build.sh invocation's `rm -rf "$WORK_DIR"` + `cp -r` mirror step, if raced by another concurrent invocation touching the same build dir, is a plausible analogous corruption mechanism (already confirmed for other components under bug-1604's pattern) -- but no direct evidence this happened for bug-1683 specifically, since it has not recurred. No code changes made. If this recurs, capture `ps aux` (or similar) for a concurrent xce-build.sh/meson-build.sh run racing the same build directory at failure time, per the bug-1604 precedent -- that is the most useful missing evidence right now. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1604 diff --git a/.wolf/bugs/bug-1684.md b/.wolf/bugs/bug-1684.md new file mode 100644 index 00000000..1da1b6e7 --- /dev/null +++ b/.wolf/bugs/bug-1684.md @@ -0,0 +1,40 @@ +--- +id: "bug-1684" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T00:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["controller", "ha", "raft", "election-safety", "persistence", "error-handling"] +related: [] +legacy: true +--- + +# bug-1684: xce-audit-invariants controller audit (dimension: error handling) finding: raft_cb_persist()… + +## Description +xce-audit-invariants controller audit (dimension: error handling) finding: raft_cb_persist() (src/controller/ha.c:2257) always returns 0 to the raft library's on_persist callback contract regardless of whether the term/voted_for durability write to disk actually succeeded -- a disk I/O failure (full disk, permission error, transient I/O error) could let a node grant a vote it has no durable record of, opening a split-vote path if that node then restarts and forgets it voted (Election Safety: 'a server grants at most one vote per term' via durably-persisted votedFor). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +src/lib/raft/raft.c:260-263's raft_on_vote_request Step 7 has an explicit durability-guarantee contract: 'if (s->cbs->on_persist(...) != 0) { /* Persist failed -- deny without updating voted_for */ return RAFT_DENY; }' -- the adapter's on_persist callback (raft_cb_persist) must return non-zero on a genuine persist failure. raft_cb_persist's own doc comment even claimed this contract ('Returns 0 on success, non-zero on error'), but its implementation called ha_persist_term(ha, workdir) and then unconditionally `return 0;` regardless of the outcome. ha_persist_term() itself (ha.c:5174) had a void return with every open()/write()/fsync() failure silently discarded (`if (fd < 0) return; /* best-effort; caller must not rely on this for correctness */`), so the failure was structurally invisible to the caller even before raft_cb_persist's own bug. Compounding: raft_cb_persist mutates ha->voted_for_term/ha->voted_for_name in-memory BEFORE calling ha_persist_term, so even fixing just the return-code propagation would leave the adapter's own in-memory state claiming a vote was cast that the raft library itself denied and never durably recorded -- a second-order inconsistency needing an explicit rollback. + +## Fix +FIXED (2026-08-25). ha_persist_term() (ha.c:5174, declared ha.h:1307) relocated from void to int -- returns 0 on success (including the deliberate NULL-workdir test-mode skip) or -errno on a genuine open/write/fsync/rename failure (fsync's own previously-unchecked return value is now checked too). raft_cb_persist() (ha.c:2257) now captures ha_persist_term's return, and on failure: (a) logs a loud ERROR with term/voted_for/rc, (b) rolls back ha->voted_for_term/ha->voted_for_name to their pre-call snapshot (taken under ha->lock before the tentative mutation) so the adapter's in-memory state stays consistent with the raft library's own 'deny without updating voted_for' behavior, (c) returns the negative rc so raft_on_vote_request denies the vote per its own Step 7 contract. ha->term is deliberately NOT rolled back -- confirmed via direct read of raft_on_vote_request's Step 2 (raft.c:191-204) that the raft library itself adopts a higher observed term unconditionally, before and independent of the vote-grant/persist outcome, so the adapter's ha->term correctly mirrors that same unconditional-adoption semantics. Also fixed the adjacent stale doc-comment (src/controller/CLAUDE.md's Protocol Verification Checklist item 1, found by a sibling audit dimension the same session) which incorrectly claimed term+voted_for were written 'atomically under ha->lock' -- term uses a separate atomic store outside the lock. + +Test: added 3 tests to testsuite/test_ha_durability.c. (1) test_persist_term_returns_error_on_write_failure -- points workdir at a path whose parent directory doesn't exist (deterministic ENOENT, portable, no permission-bit tricks needed even running as root), asserts ha_persist_term returns <0. (2) test_persist_term_null_workdir_returns_success -- regression guard that the NULL-workdir test-mode skip stays a 0/success return, not folded into the new error path. (3) test_vote_denied_and_state_rolled_back_on_persist_failure -- drives the REAL production path (ha_init_raft wiring raft_cb_persist as the real on_persist callback, then raft_on_vote_request -- the exact function cluster.c's production dispatch calls) against a minimal cluster_t/myconf_t fixture (ha_init_raft(ha, NULL, n) avoids needing a real persist_t, ha->raft_c set directly afterward -- the same test-shim pattern ha.h already documents for raft_cb_become_leader_for_test/raft_cb_become_follower_for_test) with a genuine non-empty candidate name populated so the rollback assertion is meaningful (not trivially satisfied by an already-empty baseline); asserts the vote is RAFT_DENY and ha.voted_for_term/voted_for_name are unchanged from their pre-call baseline. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1685.md b/.wolf/bugs/bug-1685.md new file mode 100644 index 00000000..8868dfbc --- /dev/null +++ b/.wolf/bugs/bug-1685.md @@ -0,0 +1,43 @@ +--- +id: "bug-1685" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-08-25T15:47:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugins/limit.c"] +tags: ["scheduler", "limit-plugin", "heap-corruption", "crash", "sigabrt", "live-deployment", "double-free", "kvmgr-rollback"] +related: [] +legacy: true +--- + +# bug-1685: User-reported live crash: xce_scheduler SIGABRT via glibc malloc_printerr (heap corruption… + +## Description +User-reported live crash: xce_scheduler SIGABRT via glibc malloc_printerr (heap corruption detected during a malloc/free bookkeeping check -- 'double free or corruption'-class abort, not a clean segfault). Core captured from a live 3-node HA deployment run (--peers ctrl-1/ctrl-2/ctrl-3, -vvv). Backtrace: malloc_printerr -> kill_limit (src/lib/xcelimit.c:181) <- limit_update_config (src/scheduler/src/plugins/limit.c:76) <- plugin_config_update (src/scheduler/src/plugin.c:246) <- sched_init_config (src/scheduler/src/scheduler.c:936) <- scheduler_req_handler (src/scheduler/src/handler.c:687) <- vtask_entry (src/lib/xcectx.c:172) <- run_task/vcpu_entry (src/lib/vtaskmgr.c). limit_update_config's own locals at the crash frame: rc=-17 (-EEXIST), nlimits=7, i=0 -- crashes on (or near) the FIRST limit in a 7-limit config update. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +FIXED (2026-08-25). Confirmed double-free: limit_update_config()'s privdata->limits is a kvmgr created with .datakiller = kill_limit (limit_priv_data_init(), limit.c). Its key_add() call (limit.c, then rc=-17/-EEXIST on the duplicate-name entry -- exactly the crash frame's own captured locals) dispatches to xkvmgr_add() (src/lib/kvmgr.c:107-129): vstore(kvm->data, value) succeeds first, THEN key_add(kvm->kvtab, key, idx) fails (EEXIST for a name already present), so xkvmgr_add rolls back via vclear(kvm->data, idx) -- which dispatches to xzarray_clear() (src/lib/zarray.c:614-637), and since the slot is in-use, fires a->killer(a->data[idx].data), i.e. kill_limit(limit), ONCE, automatically, as part of that rollback. limit_update_config()'s OLD error path then called kill_limit(limit) a SECOND time on the same already-freed pointer -- the double free glibc's malloc_printerr aborts on. Established precedent for this exact ownership-ambiguity shape: src/controller/job.c's restore_job_array() documents the identical asymmetry for a DIFFERENT kvmgr (c->jobmgr, .datakiller = NULL there) -- that kvmgr's own internal rollback is a no-op (killer is NULL), so THAT caller correctly must free the rejected value itself. Whether a caller must free a rejected key_add() value depends entirely on whether THAT specific kvmgr instance was configured with a non-NULL datakiller; privdata->limits has one (kill_limit), so limit_update_config must NOT free it itself. One accepted, documented residual gap (mirrors job.c's own accepted trade-off): if vstore() itself fails (e.g. zarray growth OOM, before ever reaching the kvtab key_add step), xkvmgr_add returns negative WITHOUT having stored or freed `limit` at all, and xkvmgr_add's return code doesn't distinguish this from the common EEXIST case -- removing the kill_limit() call trades a guaranteed crash on the common (duplicate-name) case for a leak in that vanishingly rare OOM-only sub-case. Leak is strictly better than crash; closing this gap cleanly would need a new kvmgr API, out of scope here. + +## Fix +FIXED (2026-08-25). Removed the redundant `kill_limit(limit);` call in limit_update_config()'s error path (src/scheduler/src/plugins/limit.c, `if (rc < 0) { ... return rc; }` block) -- single-line removal, replaced with a comment explaining the exact double-free mechanism above so a future reader doesn't reintroduce it 'to be safe'. Nothing else in limit.c/kvmgr.c/zarray.c touched. + +Test: added src/scheduler/testsuite/test_limit_plugin.c (this plugin had ZERO test coverage before this fix -- confirmed via `find src/scheduler/testsuite -iname '*limit*'` returning nothing pre-fix), wired into src/scheduler/testsuite/meson.build (new `sched_limit_plugin_test` executable + `scheduler limit plugin test` meson test, suite 'scheduler', ASAN_OPTIONS=detect_leaks=0 matching every sibling test). White-box #include of ../src/plugins/limit.c (same convention as test_plugin_picker_registration.c/test_handlers.c) to reach limit_update_config()/limit_init()/limit_priv_data_init(), all `static`; stubs reset_sched_obj()/get_resvals()/put_resvals()/plugin_register() (scheduler/src functions outside this TU, referenced only by limit.c code paths this test never exercises) purely to satisfy the linker. test_duplicate_limit_name_no_double_free() builds a real xce_scheduler_t fixture (real groupmgr_init()'d core.ugmgr/qgmgr/pgmgr/hgmgr, real limit_init()'d plugin.data), then drives limit_update_config() with an xce_conf4sched_t containing TWO xce_limit_config_t entries sharing the SAME name ('dup_limit') -- the exact duplicate-name collision bug-1685's live crash captured -- and asserts rc == -EEXIST with no crash. test_distinct_limit_names_both_succeed() is the control case (two DISTINCT names, rc == 0), proving the fix didn't just suppress freeing on failure in general. + +RED (pre-fix, verified live): built via `nix develop ./src --command bash src/tools/xce-build.sh scheduler asan build-asan --incremental` with the new test present but the buggy kill_limit(limit) call still in place -- sched_limit_plugin_test aborted with a genuine ASAN heap-use-after-free (not a clean assertion failure): 'AddressSanitizer: heap-use-after-free ... in kill_limit ../lib/xcelimit.c:181', READ stack `kill_limit <- limit_update_config (limit.c:76, the buggy call) <- test_duplicate_limit_name_no_double_free`, freed-by stack `kill_limit (xcelimit.c:192) <- xzarray_clear (zarray.c:626) <- xkvmgr_add (kvmgr.c:124) <- limit_update_config (limit.c:72, the key_add call)` -- an exact match to the diagnosed mechanism and to bug-1685's own captured crash signature (malloc_printerr <- kill_limit <- limit_update_config). +GREEN (post-fix, verified live): rebuilt with the single-line fix applied -- `scheduler limit plugin test OK (0.05s)`. Full scheduler suite re-run: 52/53 OK, 1 pre-existing failure (`test_recordstart_symlink_escape_replies_error`, bug-1560, unrelated flake -- same failure present in the RED run too) -- zero new regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1686.md b/.wolf/bugs/bug-1686.md new file mode 100644 index 00000000..7c9a7214 --- /dev/null +++ b/.wolf/bugs/bug-1686.md @@ -0,0 +1,47 @@ +--- +id: "bug-1686" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-08-25T16:24:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugins/builtin_fcfs.c"] +tags: ["scheduler", "builtin_fcfs", "crash", "sigsegv", "live-deployment", "picker-plugin"] +related: ["bug-1685"] +legacy: true +--- + +# bug-1686: User-reported live crash (build.dbg, commit b040dfc0 -- AFTER the bug-1685 double-free fix landed… + +## Description +User-reported live crash (build.dbg, commit b040dfc0 -- AFTER the bug-1685 double-free fix landed, so this is a distinct, separate crash): xce_scheduler SIGSEGV in fcfs_cursor_enter_queue (src/scheduler/src/plugins/builtin_fcfs.c:673), called from builtin_fcfs_next_job (builtin_fcfs.c:810) <- sched_job_picker (scheduler.c:118) <- picker thread. Crash line 673 is the NULL branch of a ternary expression. queue_item=0x7db7d0000000 -- a suspiciously round address (large power-of-2-aligned), possibly a corrupted/uninitialized pointer rather than a normal heap allocation. Core captured from a live 3-node HA deployment run (--peers ctrl-1/ctrl-2/ctrl-3, -vvv). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +FIXED (2026-08-25). fcfs_plugin_data_t.lock (added in the bug-1611-era concurrency fix, task 2.4 -- see builtin_fcfs.c's own file header Concurrency comment) correctly makes every individual hook call (on_queue_update/on_job_state_change/on_job_update/next_job/etc) mutually exclusive at the fcfs_plugin_data_t granularity -- but priv->cursor/priv->job_cursor are raw ordq_item_t* values that PERSIST in fcfs_plugin_data_t across SEPARATE builtin_fcfs_next_job() calls, each with its OWN independent lock acquisition/release. A properly-locked builtin_fcfs_on_queue_update() call for a queue name that already has a registered fcfs_queue_t re-sorts it by removing the queue's existing ordq_item_t node (ordq_get + FREEUP(it)) and inserting a brand-new one (ordq_put, fresh calloc). If priv->cursor is resting on that exact freed node -- which it normally is, since the cursor stays parked on a queue for as long as it keeps yielding ready jobs (sched_queue_is_ready()/sched_job_is_ready() are unconditional `return 1` stubs today in scheduler.c, so a queue's cursor never advances away from it while jobs remain) -- the NEXT next_job() call dereferences priv->cursor->data from freed memory: a real heap-use-after-free, exactly matching this bug's captured live SIGSEGV in fcfs_cursor_enter_queue() (queue_item was a dangling, freed ordq_item_t*, not corrupted/uninitialized memory as originally suspected). The lock prevents CONCURRENT corruption within one call; it does nothing to stop a LATER call from dereferencing a node freed by a call that ran validly in between. The same shape exists for priv->job_cursor via builtin_fcfs_on_job_state_change()'s !JOB_IS_PEND removal branch and fcfs_remove_job_everywhere() (called by on_job_update() and on_dispatch()), both of which ordq_del() a job node job_cursor may be resting on. + +## Fix +FIXED (2026-08-25). Three invalidation sites in src/scheduler/src/plugins/builtin_fcfs.c, all following the same peek/compare-then-invalidate-before-free pattern: +1. builtin_fcfs_on_queue_update()'s reuse branch: before FREEUP(it), if priv->cursor == it, sets both priv->cursor and priv->job_cursor to NULL (in lockstep, matching fcfs_cursor_enter_queue()'s own invariant that job_cursor is only ever meaningful relative to the queue cursor currently names). +2. builtin_fcfs_on_job_state_change()'s !JOB_IS_PEND removal branch: ordq_peek()s the about-to-be-deleted job node first; if it equals priv->job_cursor, NULLs priv->job_cursor before ordq_del(). +3. fcfs_remove_job_everywhere() (shared by on_job_update() and on_dispatch()): same peek-then-invalidate-then-delete pattern inside its loop over all registered queues. +Always safe: next_job() already treats a NULL cursor as "no more candidates this round" (pre-existing branch, unrelated to this bug -- see test_picker_fcfs_session.c's no-queues/empty-queue coverage), and begin_session() unconditionally resets both cursors fresh every round, so the worst case of an invalidation is a bounded one-round delay for the affected queue's jobs, never a crash or a permanently lost job. + +Test: added src/scheduler/testsuite/test_builtin_fcfs_cursor_uaf.c, wired into src/scheduler/testsuite/meson.build (new sched_builtin_fcfs_cursor_uaf_test executable + 'scheduler builtin fcfs cursor uaf test' meson test, suite 'scheduler', ASAN_OPTIONS=detect_leaks=0, same convention as every sibling scheduler test). White-box #include of ../src/plugins/builtin_fcfs.c (same convention as test_picker_fcfs_session.c) for direct access to fcfs_plugin_data_t's private cursor/job_cursor fields; supplies its own always-ready sched_queue_is_ready()/sched_job_is_ready() test doubles and a plugin_register() stub to satisfy the linker (this TU does not link scheduler.c or plugin.c). The single test drives, on ONE thread with no real concurrency needed: begin_session() -> next_job() (dispatches job 1, leaves priv->cursor parked on queue "Q1"'s node with priv->job_cursor mid-walk) -> on_queue_update() again for the SAME queue name with a different priority (genuinely exercises the reuse-and-resort path, freeing the OLD ordq_item_t node priv->cursor is resting on) -> asserts priv->cursor/priv->job_cursor are now NULL (proves invalidation fired) -> next_job() again (the UAF reproduction point; post-fix returns NULL cleanly) -> a fresh begin_session()/next_job() round confirms job 1 and job 2 are BOTH still findable, proving no job is permanently lost, only delayed one round, matching the fix's documented safety property. + +RED (pre-fix, verified live): temporarily reverted all three invalidation blocks in builtin_fcfs.c (kept the new test unchanged), rebuilt via `nix develop ./src --command bash src/tools/xce-build.sh scheduler asan build-asan --incremental`. First run: the test's own intermediate `TEST_ASSERT_NULL(priv->cursor)` check (step (e), asserting the invalidation fired) failed as a clean TEST_ASSERT failure -- Unity's assertion longjmp stopped the test function before it ever reached the dereferencing next_job() call, so the real UAF was not yet exercised. Temporarily disabled just that intermediate assertion (`#if 0`) to force execution through to the actual dereference and rebuilt: sched_builtin_fcfs_cursor_uaf_test aborted with a genuine ASAN heap-use-after-free -- 'AddressSanitizer: heap-use-after-free ... READ of size 8 ... #0 builtin_fcfs_next_job ../src/plugins/builtin_fcfs.c:813', freed-by stack 'FREEUP(it) <- builtin_fcfs_on_queue_update builtin_fcfs.c:452 <- test_cursor_invalidated_on_queue_resort_no_uaf test_builtin_fcfs_cursor_uaf.c:181', allocated-by stack 'ordq_put ../lib/ordq.c:48 <- builtin_fcfs_on_queue_update builtin_fcfs.c:457' -- an exact structural match to this bug's diagnosed mechanism and to its own captured live SIGSEGV in fcfs_cursor_enter_queue() <- builtin_fcfs_next_job() <- sched_job_picker(). Restored the temporarily-disabled assertion immediately after capturing this evidence. +GREEN (post-fix, verified live): restored all three invalidation blocks in builtin_fcfs.c to their applied state (confirmed via `git diff` matching the pre-existing uncommitted fix exactly), rebuilt -- 'scheduler builtin fcfs cursor uaf test OK (0.06s)', 1 Tests 0 Failures. Full scheduler suite re-run via `meson test --suite scheduler`: 53/54 OK, 1 pre-existing failure (test_recordstart_symlink_escape_replies_error, bug-1560, unrelated -- 'Expected 'invalid record file name' Was 'permission denied accessing record file'', same failure present in the RED run too) -- zero new regressions. Note: a full top-level `ninja` build in this shared worktree also fails to link controller_unit_test (undefined references to test_persist_recovery/test_gc_retention/collect_confvars/etc.) due to unrelated, pre-existing uncommitted changes to src/controller/job.c/persist.c/meson.build from concurrent, unrelated work in this worktree (confirmed via `git status`: none of those files were touched by this fix) -- worked around by running `meson test --suite scheduler` directly against the already-built scheduler test binaries rather than requiring a full-tree ninja build to succeed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1685 diff --git a/.wolf/bugs/bug-1687.md b/.wolf/bugs/bug-1687.md new file mode 100644 index 00000000..a8c48a08 --- /dev/null +++ b/.wolf/bugs/bug-1687.md @@ -0,0 +1,44 @@ +--- +id: "bug-1687" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-08-25T17:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c", "src/controller/persist.c", "src/include/persist.h", "src/include/wal_types.h"] +tags: ["controller", "job.c", "persist.c", "wal", "wal-job-start", "job-pidinfo", "start-time", "restart-recovery", "data-loss", "apply-seam", "test-infrastructure-hang", "cbrp-teardown-race"] +related: ["bug-1678", "bug-1452", "bug-1561"] +legacy: true +--- + +# bug-1687: xce-audit-invariants finding: WAL_JOB_START silently drops job fields on a controller restart… + +## Description +xce-audit-invariants finding: WAL_JOB_START silently drops job fields on a controller restart replay -- the same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), but for the worker-confirmed-start fact. apply_job_start_transition() (job.c) sets job->job.pidinfo (real pids/pgids/exdata a worker reported) and job->job.start_time, but the WAL_JOB_START fact written by record_job_start_fact() only carried wal_jobstate_t{jobid, state} -- pidinfo was never in the log at all, and start_time was re-fabricated with time(NULL) at every apply (both boot replay and standby catch-up) instead of carrying the true worker-reported value. Confirmed live-relevant: user reported StartTime always showing 0 in `xcecli joblist` for both RUN and DONE jobs on a live deployment -- deferred as a distinct follow-up investigation (that deployment's controller predates this fix; whether StartTime=0 specifically, rather than the pre-fix fabricated-nonzero time(NULL), indicates a SEPARATE live-apply-pump gap has not yet been root-caused). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +record_job_start_fact() (job.c, on_job_start()'s only caller) called persist_submit_job_fact(&c->persist, WAL_JOB_START, job->job.jobid, JOB_STAT_RUN) -- a thin wrapper whose payload type wal_jobstate_t is exactly {jobid, state, _pad}, discarding the `info` (job_pidinfo_t*) parameter entirely even though it was available at the call site (the worker's real START report). Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/.../DISPATCH fall-through group (persist.c, comment confirms 'these types' payloads are deterministic... start_time... stamped directly below... apply_job_update_transition() only ever touches state') for primary boot replay AND live commit-advance apply, and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) for standby catch-up -- never restored pidinfo, and apply_committed_entry() additionally fabricated start_time = time(NULL) at apply time rather than carrying the receipt-time value, so a restarted/promoted node's start_time drifts from the true worker-reported time even though it is non-NULL. + +## Fix +FIXED (2026-08-25). Mirrors bug-1678's WAL_JOB_DISPATCH fix pattern exactly, using the already-existing controller.capnp JobPIDInfo struct (job_pidinfo_t's mapped type -- the existing Job.pidinfo field) with its already-generated encode_JobPIDInfo/decode_JobPIDInfo/free_JobPIDInfo_ptr functions; no schema change needed. src/controller/persist.c gains capnp_encode_pidinfo()/decode_pidinfo() (mirroring capnp_encode_joballoc()/decode_joballoc()) and persist_submit_job_start_fact(persist_t*, jobid, state, start_time, info) -- WAL_JOB_START's payload is now wal_jobstate_t header + a fixed uint64_t start_time (the receipt-time clock, captured ONCE by record_job_start_fact() rather than re-derived independently at each apply seam) + trailing Cap'n Proto-encoded JobPIDInfo bytes, declared in src/include/persist.h. src/controller/job.c's record_job_start_fact() now calls persist_submit_job_start_fact(..., time(NULL), info) instead of persist_submit_job_fact(...). Both apply seams updated: apply_wal_entry() (standby catch-up) splits WAL_JOB_START out of its shared group into its own case (mirroring how bug-1678 split WAL_JOB_DISPATCH out), decoding trailing start_time/pidinfo and calling new restore_job_start_state() (job.c, restore_job_dispatch_state()'s sibling, takes ownership of an optional decoded job_pidinfo_t and a wire-carried start_time -- 0 means 'not carried, leave untouched', matching this seam's pre-fix behavior of never touching start_time for WAL_JOB_START at all). apply_committed_entry() (primary boot replay / live apply pump) stays in its existing shared fall-through group (mirroring how WAL_JOB_DISPATCH joined that same group in bug-1678) but extends its wal_type==WAL_JOB_START branch: a payload with trailing bytes (len >= sizeof(wal_jobstate_t)+sizeof(uint64_t)) decodes and adopts the wire start_time and pidinfo directly; a payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_START entry written before this fix) falls through unchanged to the old fabricated-time(NULL)/pidinfo-untouched behavior -- full backward compatibility, verified by every pre-existing WAL_JOB_START test in test_apply_committed_entry.c/test_apply_reapply_idempotency.c/test_ha_role_lifecycle.c/test_apply_pump.c (all still using the old bare-wal_jobstate_t shape) passing unchanged. + +Incidental fix required to reach GREEN: test_commit_barrier_reply.c's cbrp_count_jobstate_entries_ex() (a test-only helper that manually scans the on-disk WAL segment) matched WAL_JOB_START/DISPATCH/etc entries via `hdr.payload_len == sizeof(wal_jobstate_t)` -- an exact-size match that silently undercounted (returned 0 instead of 1) once WAL_JOB_START legitimately grew past that fixed size, in turn making a downstream test in that file poll/spin forever on a count that could never be reached (observed as a genuine deadlock -- all 3 threads futex-blocked, 0 CPU time accruing over 12+ minutes wall clock, reproduced deterministically via `-t test_commit_barrier_reply` and bisected by temporarily inserting/removing an early `return;` in the test group's own RUN_TEST driver to narrow it to one of 8 candidate tests, then to a clean isolated PASS/FAIL once the helper was fixed). Changed the match to `>=` (jobid/state always live at the same fixed leading offset regardless of trailing bytes) -- the general, correct fix, not a workaround; resolves the hang completely (verified: full test_commit_barrier_reply group and the full controller suite both complete in ~38s with zero hangs, both before and after this specific fix was isolated via bisection). Also applied test_commit_barrier_reply.c's cbrp_teardown() the same persist_stop()-before-commit_engine_fini() ordering bug-1452 already established as the fix for this exact TOCTOU-race class (that fix's own note explicitly flagged this file as a same-pattern, not-yet-observed-to-crash follow-up) -- confirmed via bisection that this ordering was NOT the actual cause of this particular hang, but it is a genuine latent race per bug-1452's analysis and the fix is harmless/idempotent, so it was kept as defense-in-depth rather than reverted. + +Primary regression tests: new src/controller/testsuite/test_wal_job_start_fact.c (6 tests, wired into src/controller/meson.build and testsuite/controller_test.c), mirroring test_wal_job_dispatch_fact.c's structure: (1) apply_committed_entry sets pidinfo+start_time from a new-format payload; (2) apply_committed_entry backward-compat with an old-format (bare wal_jobstate_t) payload -- pidinfo stays NULL, start_time still fabricated via time(NULL), unchanged from pre-fix; (3) apply_wal_entry sets pidinfo+start_time, fires no hook; (4) apply_wal_entry backward-compat -- pidinfo stays NULL, start_time left untouched; (5) direct-payload restart-replay (WAL_JOB_SUBMIT + new-format WAL_JOB_START via persist_wal_async + persist_recover()) recovers the exact wire start_time and pidinfo; (6) test_wjs_real_on_job_start_survives_restart -- drives the REAL production on_job_start() path (record_job_start_fact()'s only caller) with a real job_pidinfo_t (3 real pids, 2 real pgids, real exdata string), through a real WAL-write + persist_recover() restart cycle (the exact technique test_dispatch_commit.c's test_dc_success_appends_wal_job_dispatch_durably already established for bug-1678), asserting pidinfo AND start_time (bounded to the real wall-clock window around the on_job_start() call) survive the restart intact -- the audit's exact confirmed root cause and the user's exact acceptance criterion. + +RED->GREEN independently verified live (not just narrated): git-stashed job.c+persist.c (keeping the new test file, persist.h, wal_types.h unchanged) via `git stash push -- src/controller/job.c src/controller/persist.c`, rebuilt, ran `./controller_unit_test -t test_wal_job_start_fact` -- 6 Tests 4 Failures 0 Ignored (the 2 backward-compat tests correctly PASS unchanged; all 4 new-behavior tests correctly FAIL with the exact expected mismatches, e.g. 'Expected 1700000001 Was 1787645264' for the fabricated-vs-wire start_time, 'must set job.pidinfo... not just state' for the real on_job_start test). Restored via `git stash pop`, rebuilt, re-ran -- 6 Tests 0 Failures 0 Ignored. Full controller_unit_test suite re-run after all fixes (including the test_commit_barrier_reply.c hang fix and the buglog-id renumber from a concurrent session's own bug-1686 claim): 1146 Tests 2 Failures 0 Ignored, ~38s, zero hangs -- the 2 failures are test_conf_trim_keeps_only_n_versions ('Expected 2 Was 3') and test_conf_get_saved_tag_no_config ('Expected 0 Was ') in test_config_repl.c, an already-logged, pre-existing, unrelated overlayfs sandbox fdatasync artifact (bug-1561, re-observed identically across multiple unrelated sessions with zero diff to test_config_repl.c/config_repl.c each time) -- confirmed by exact signature match, not a new regression. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1678, bug-1452, bug-1561 diff --git a/.wolf/bugs/bug-1688.md b/.wolf/bugs/bug-1688.md new file mode 100644 index 00000000..983167bc --- /dev/null +++ b/.wolf/bugs/bug-1688.md @@ -0,0 +1,44 @@ +--- +id: "bug-1688" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-25T17:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_info.c (repl_info_worker_one()", "~line 309-388", "the `info worker` print path)", "src/scheduler/src/core_events.c (core_apply_worker_resvals()", "~line 478", "the WORKERS_PUSH resval-merge path)", "src/lib/zarray.c (zarray_put_ex()", "~line 267-303", "count-vs-slot bookkeeping)"] +tags: ["scheduler", "simulator", "sim-repl", "info-worker", "resource-values", "regression", "zarray-count-vs-caps", "resvals-merge", "worker-push", "null-value-overwrite"] +related: ["bug-1568"] +legacy: true +--- + +# bug-1688: User-reported issue in the scheduler's interactive simulator REPL (`xce_scheduler -i`/`--sim`)… + +## Description +User-reported issue in the scheduler's interactive simulator REPL (`xce_scheduler -i`/`--sim`): `info worker all` and `info worker ` (e.g. `info worker pc13v`) initially print full per-resource detail (`[sim] resource slots: value=22.00 used=0.00`, `resource mem: value=...`, etc -- ~19 resource lines including slots/mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp/type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks). After stepping the simulation forward through several sessions (`step 5` repeated, crossing RESVALS_DELTA, JOB_PUSH (2 batches, 40+80 jobs), JOB_UPDATE, and WORKERS_PUSH events across sessions 2 through 30), the SAME commands later in the SAME REPL session regress to printing ONLY the header line: `[sim] worker pc13v: status=online` -- zero resource lines, even though the worker still exists and is still 'online' (not removed/dead). Binary under test: `./build.dbg/controller/xce_controller --version` reports `xce v0.0.1-bc3f76b4-5ff97622 built 2026-08-25T08:56:58Z` (a build.dbg build -- note the user ran the CONTROLLER binary's --version, though the symptom itself is in the SCHEDULER's sim REPL; worth confirming which binary actually serves `xce_scheduler -i`/`--sim` and its own version once investigated). A real replay file reproducing the sequence is preserved at /tmp/scheduler-replay-20260825-090251.replay (~1MB, captured 2026-08-25 09:24 UTC) for a future investigation to replay directly instead of needing to reproduce from scratch. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +FULLY ROOT-CAUSED (2026-08-26), via live TRACE against the same /tmp/scheduler-replay-20260825-090251.replay this bug was originally reported with (build.dbg/scheduler/xce_scheduler --sim -i -vvv, driving the exact `step 5` sequence, plus temporary per-call TRACE instrumentation in core_events.c/core_session.c/lib/zarray.c added and then fully removed once the trigger was pinned down). TWO independent bugs, both now fully understood and fixed: +(1) core_apply_worker_resvals() (core_events.c) -- the count-doubling anomaly's ACTUAL trigger turned out to be neither an idx-resolution bug nor external memory corruption (both hypotheses from the prior session are RULED OUT: a per-resource `resolved idx=` trace across both pushes showed the SAME resource name always resolves to the SAME r->idx both times, and core_reset_workers()/xce_core_begin_session()/xce_core_end_session() were bracket-traced across many session boundaries with resvals.data's pointer, .count, and slot occupancy all provably untouched -- confirming core_reset_workers() only ever touches w->resInUse, exactly as previously suspected). The REAL mechanism, pinned down via a direct per-entry trace of nv->val itself: a WORKERS_PUSH re-announcing an already-registered worker's resource NAME list can legitimately decode nv->val to NULL for EVERY entry (confirmed: controller.capnp.c's decode_Value_ptr(), line ~2966-2976, sets `*d = NULL` whenever the wire NameValue.val capnp pointer is CAPN_NULL -- a faithful decode of what the controller-side encoder chose to send, not a decode bug). core_apply_worker_resvals() unconditionally MOVE_PTR'd this NULL into `val` and called zarray_put(&worker->resvals, r->idx, NULL) for every one of the 20 resources, silently overwriting each previously-known-good value with NULL. Because zarray_put_ex() (zarray.c:267-303) only increments array->count when the slot being written was NULL beforehand, this FIRST NULL-write left resvals.count unchanged (the snapshot-restored slot was non-NULL, so the 'else count++' branch never fired) while silently zeroing the slot's DATA -- confirmed live via a watched-array TRACE in zarray_put_ex() showing `new_data=(nil)` for all 20 writes on BOTH the first and second WORKERS_PUSH. A THIRD push then found every slot already NULL (from the first push's own corruption) and hit the 'was NULL, count++' branch for all 20, producing the literal 20 -> 40 doubling this bug was originally reported with. Ownership of the 'why does the controller push a repeat announce with no values' question is out of scope for this session (src/controller/ is off limits per this task's boundaries, and the fix does not depend on the answer) -- the defensive scheduler-side fix protects worker resvals regardless of why the wire push omitted a value. +(2) repl_info_worker_one() (repl_info.c:324, confirmed unchanged from prior session's diagnosis) -- its resource-print loop used `for (i = 0; i < w->resvals.count; i++)`, treating `.count` as a safe index upper bound. `.caps` (resource.c:228's own established convention for this identical structure) is correct; `.count` can legitimately be less than the highest occupied index once entries are sparse -- exactly the state bug (1) produces (9 static resources nulled by a WORKERS_PUSH, 11 dynamic ones kept fresh by later RESVALS_DELTA events, live-confirmed as the actual after-fix-1-absent steady state for this replay: `info worker pc13v` after 30x `step 5` showed only mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp, missing slots/type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks -- a real, reproduced PARTIAL instance of the reported symptom, not the fully empty header-only case, which likely needs a longer replay or a worker with no periodically-refreshed resources at all to reach). + +## Fix +FIXED (2026-08-26), TDD RED->GREEN for both root causes, both verified independently and together: +(1) src/scheduler/src/core_events.c: core_apply_worker_resvals()'s per-resval loop guard extended from `if ((nv == NULL) || (nv->name == NULL))` to `if ((nv == NULL) || (nv->name == NULL) || (nv->val == NULL))` -- a NULL nv->val is now skipped (preserve whatever value is already there), the same defensive 'continue' posture already applied to the other two NULL checks. The pre-existing `if (nv->val != NULL && nv->val->which == Value_u_fvalue)` TRACE guard just below was simplified to `if (nv->val->which == Value_u_fvalue)` since nv->val == NULL can no longer reach that line. New regression test in src/scheduler/testsuite/test_core_events.c: test_workers_push_with_null_val_does_not_clobber_existing_resval -- pushes 2 resources (mem=100.0, r1m=5.0) for worker 'hostA', then pushes the SAME names with nv->val==NULL for both, asserts resvals.count stays 2 and both values survive unchanged; a THIRD identical NULL push re-confirms count stays 2 (would have been 4 under the pre-fix count- doubling behavior). RED confirmed live (git stash of just this file's fix, rebuilt, ran sched_core_events_test: 'Expected Non-NULL' failure at the post-second-push value check, 10/11 tests passing otherwise); GREEN confirmed after git stash pop + rebuild (11 Tests 0 Failures). +(2) src/scheduler/src/repl_info.c: repl_info_worker_one()'s resource loop bound changed from `w->resvals.count` to `w->resvals.caps`, mirroring resource.c:228's `int num_res = (int)w->resvals.caps;` convention exactly; the loop's existing avail_v==NULL/rname==NULL skip is unchanged and already handles the now-larger, sparser index range safely. New regression test in src/scheduler/testsuite/test_repl_breakpoints_inspection.c: test_info_worker_shows_all_populated_resources_when_count_is_stale -- worker with 2 populated resvals slots (idx0 'mem', idx1 'r1m') but resvals.count forcibly set to 1 (the exact stale/sparse shape bug (1)'s prior failure mode produced), asserts `info worker` output contains BOTH resource lines. RED confirmed live (git stash of just this file's fix, rebuilt sched_repl_breakpoints_inspection_test: 28/29 passing, new test failing with the exact expected message 'a populated slot at an index >= the (stale) .count must still display'); GREEN confirmed after git stash pop + rebuild (29 Tests 0 Failures). +End-to-end reproduction against the ORIGINAL /tmp/scheduler-replay-20260825-090251.replay, driving `info worker pc13v` before any steps, then 30x `step 5`, then `info worker pc13v` again: BEFORE the fix, the final `info worker pc13v` printed only 11 of 20 resource lines (mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp survive via RESVALS_DELTA refresh; slots/type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks silently vanish) -- a live-reproduced instance of the reported regression class. AFTER both fixes, the same sequence's final `info worker pc13v` shows all 20 resource lines identical in shape to the very first (pre-step) dump, with only the naturally-updated dynamic values (mem/r1m/r15m/etc.) differing in magnitude as expected from real RESVALS_DELTA updates over 30 rounds -- status=online, zero missing lines. +Full scheduler suite (xce-build.sh scheduler, ASAN profile): 53/54 tests OK, 1 failure -- test_recordstart_symlink_escape_replies_error (sched_admin_request_test), the exact pre-existing bug-1560 sandbox artifact (Expected 'invalid record file name' Was 'permission denied accessing record file'), confirmed unrelated (no files this bug touches -- core_events.c/repl_info.c/zarray.c -- are anywhere near record-file-path validation). Zero new failures; both new tests (sched_core_events_test 21/54, sched_repl_breakpoints_inspection_test 52/54) pass in the full suite run. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1568 diff --git a/.wolf/bugs/bug-1689.md b/.wolf/bugs/bug-1689.md new file mode 100644 index 00000000..f3af78df --- /dev/null +++ b/.wolf/bugs/bug-1689.md @@ -0,0 +1,42 @@ +--- +id: "bug-1689" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T18:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c (apply_job_finish_transition ~L2292", "record_job_finish_fact ~L2371", "restore_job_finish_state new)", "src/controller/persist.c (apply_committed_entry's shared job-fact group ~L3890-4080", "apply_wal_entry's WAL_JOB_FINISH case", "persist_submit_job_finish_fact new", "decode_jobexitinfo new)", "src/include/persist.h", "src/include/wal_types.h", "src/controller/testsuite/test_wal_job_finish_fact.c (new)"] +tags: ["controller", "job.c", "persist.c", "wal", "wal-job-finish", "job-exit-info", "finish-time", "run-time", "restart-recovery", "data-loss", "apply-seam", "native-trailing-payload"] +related: ["bug-1678", "bug-1687", "bug-1561"] +legacy: true +--- + +# bug-1689: xce-audit-invariants finding: WAL_JOB_FINISH silently drops job fields on a controller restart… + +## Description +xce-audit-invariants finding: WAL_JOB_FINISH silently drops job fields on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs) and bug-1687 (WAL_JOB_START/job.pidinfo+start_time), but for the worker-confirmed-finish fact. apply_job_finish_transition() (job.c) sets job->job.exit_info->exitcode and job->job.finish_time/run_time, but the WAL_JOB_FINISH fact written by record_job_finish_fact() only carried wal_jobstate_t{jobid, state} -- exit_info was never in the log at all (and, worse, apply_job_finish_transition() itself is effectively dead code on the real production apply path -- both apply_committed_entry()'s and apply_wal_entry()'s WAL_JOB_FINISH handling route through apply_job_update_transition() (state-only) instead, per raft-apply-decoupling's deferred-apply design, so exit_info was NEVER populated through the real commit-apply pipeline, restart or not, prior to this fix), and finish_time was re-fabricated with time(NULL) at every apply_committed_entry() apply (and, on apply_wal_entry()'s standby catch-up seam, never touched at all -- an even larger gap than WAL_JOB_START's pre-1687 state) instead of carrying the true worker-reported receipt time. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +record_job_finish_fact() (job.c, on_job_finish()'s only caller) called persist_submit_job_fact(&c->persist, WAL_JOB_FINISH, job->job.jobid, state) -- a thin wrapper whose payload type wal_jobstate_t is exactly {jobid, state, _pad}, discarding finish->exit_code entirely even though it was available at the call site (the worker's real FINISH report). Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/FINISH/.../DISPATCH fall-through group (persist.c) for primary boot replay AND live commit-advance apply, and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) for standby catch-up -- never restored exit_info. apply_committed_entry() additionally fabricated finish_time = time(NULL) at apply time (rather than carrying the receipt-time value) and derived run_time from that fabricated finish_time; apply_wal_entry()'s shared group touched neither finish_time nor run_time at all for WAL_JOB_FINISH (a strictly worse gap than WAL_JOB_START's pre-bug-1687 apply_wal_entry behavior, which at least existed as a documented no-op). Confirmed via CodeGraph source read that apply_job_finish_transition() -- the richer core that DOES set exit_info/finish_time/run_time correctly -- is only ever invoked from record_job_finish_fact()'s duplicate-terminal-state peek (a no-op call whose own guard prevents any mutation), i.e. it is effectively dead code on the real production apply path today; the raft-apply-decoupling redesign defers all real mutation to apply_committed_entry()/apply_wal_entry(), which use the leaner apply_job_update_transition() (state-only) instead. job_exitinfo_t (xce/types.h) is a single int (`{int exitcode;}`) with NO existing Cap'n Proto mapping (confirmed -- no JobExitInfo struct in controller.capnp), unlike bug-1687's JobPIDInfo, so no capnp wrapper was invented for it. + +## Fix +FIXED (2026-08-25). Mirrors bug-1687's WAL_JOB_START fix pattern exactly, but uses a small NATIVE (non-capnp) trailing payload instead of Cap'n Proto, since job_exitinfo_t has no existing capnp mapping and no variable-length data (unlike JobPIDInfo). src/controller/persist.c gains decode_jobexitinfo() (a native memcpy decode, mirroring decode_pidinfo()/decode_joballoc()'s doc-comment style but without capnp) and persist_submit_job_finish_fact(persist_t*, jobid, state, finish_time, exit_code) -- WAL_JOB_FINISH's payload is now a wal_jobstate_t header + a fixed uint64_t finish_time (receipt-time clock, captured ONCE by record_job_finish_fact() rather than re-derived independently at each apply seam) + a fixed int32_t exit_code, declared in src/include/persist.h. src/controller/job.c's record_job_finish_fact() now calls persist_submit_job_finish_fact(..., time(NULL), finish->exit_code) instead of persist_submit_job_fact(...). Both apply seams updated: apply_wal_entry() (standby catch-up) splits WAL_JOB_FINISH out of its shared group into its own case (mirroring bug-1687's WAL_JOB_START split), decoding trailing finish_time/exit_code and calling new restore_job_finish_state() (job.c, restore_job_start_state()'s sibling -- finish_time==0 means 'not carried, leave finish_time/run_time untouched', matching this seam's pre-fix behavior exactly; exit_info==NULL means 'not carried, leave untouched'; job_exitinfo_t has no nested pointers so it is released with plain FREEUP, not a capnp free_*_ptr helper). apply_committed_entry() stays in its existing shared fall-through group but extends its wal_type==WAL_JOB_FINISH branch: a payload with trailing bytes (len >= sizeof(wal_jobstate_t)+sizeof(uint64_t)) decodes and adopts the wire finish_time and exit_code directly; a payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_FINISH entry written before this fix) falls through unchanged to the old fabricated-time(NULL)/exit_info-untouched behavior. run_time is NEVER carried on the wire in either case -- it is always re-derived as finish_time - start_time at apply time (matching apply_job_finish_transition()'s own formula), since start_time is already correctly recovered via bug-1687's own fix. Full backward compatibility verified by 2 dedicated old-format tests plus every pre-existing WAL_JOB_FINISH test in test_apply_committed_entry.c/test_apply_reapply_idempotency.c/test_apply_pump.c/test_commit_barrier_reply.c/test_commit_barrier_rollout.c (all still using the old bare-wal_jobstate_t shape) passing unchanged. + +New regression tests: src/controller/testsuite/test_wal_job_finish_fact.c (6 tests, mirroring test_wal_job_start_fact.c's structure exactly, wired into src/controller/meson.build and testsuite/controller_test.c): (1) apply_committed_entry sets exit_info+finish_time (+re-derived run_time) from a new-format payload; (2) apply_committed_entry backward-compat with an old-format payload -- exit_info stays NULL, finish_time still fabricated via time(NULL); (3) apply_wal_entry sets exit_info+finish_time+run_time, fires no hook; (4) apply_wal_entry backward-compat -- exit_info/finish_time/run_time all left untouched (sentinel values survive), unlike apply_committed_entry's fabrication; (5) direct-payload restart-replay (WAL_JOB_SUBMIT + WAL_JOB_START + new-format WAL_JOB_FINISH via persist_wal_async + persist_recover()) recovers the exact wire finish_time, exit_code, and a correctly re-derived run_time; (6) test_wjf_real_on_job_finish_survives_restart -- drives the REAL production on_job_finish() path (record_job_finish_fact()'s only caller) with a real xce_jobfinish_t (nonzero exit_code=137), through a real WAL-write + persist_recover() restart cycle, asserting exit_info, finish_time (bounded to the real wall-clock window), and run_time all survive the restart intact -- the audit's exact confirmed root cause and the user's exact acceptance criterion. + +RED->GREEN independently verified live (not just narrated): git-stashed job.c+persist.c only (`git stash push -- src/controller/job.c src/controller/persist.c`, keeping the new test file, persist.h, wal_types.h unchanged), rebuilt via the synchronous xce-build.sh script, ran `./controller_unit_test -t test_wal_job_finish_fact` -- 6 Tests 4 Failures 0 Ignored (the 2 backward-compat tests correctly PASS unchanged; all 4 new-behavior tests correctly FAIL with exact expected mismatches, e.g. 'Expected 1700000050 Was 1787651049' for the fabricated-vs-wire finish_time on apply_committed_entry, 'Expected 1700000150 Was 0' for apply_wal_entry never touching finish_time pre-fix, and 'must set job.exit_info... not just state' for the real on_job_finish restart test). Restored via `git stash pop`, rebuilt, re-ran -- 6 Tests 0 Failures 0 Ignored, reproduced 3 consecutive times. Full controller_unit_test suite re-run after the fix: 1152 Tests, 1-2 Failures (flaky between runs) -- both failure signatures ('Expected 2 Was 3' / test_conf_trim_keeps_only_n_versions, and a garbage-uint64 mismatch / test_conf_get_saved_tag_no_config in test_config_repl.c) are the exact, previously-logged bug-1561 overlayfs-fdatasync sandbox artifact, confirmed identical across 2 consecutive re-runs of the unchanged binary (proving the flakiness is a sandbox I/O nondeterminism, not caused by this fix) -- zero new failures. Checked test_commit_barrier_reply.c's cbrp_count_jobstate_entries() (the exact helper bug-1687 had to fix for a payload-size-growth-induced hang) for the same risk: it already uses the generic `hdr.payload_len >= sizeof(wal_jobstate_t)` comparison bug-1687 fixed (not a WAL_JOB_START-specific special case), so WAL_JOB_FINISH's payload growth is already safely handled by that existing fix -- confirmed empirically by the full suite completing in ~38s with zero hangs, matching prior-session timing exactly. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1678, bug-1687, bug-1561 diff --git a/.wolf/bugs/bug-1690.md b/.wolf/bugs/bug-1690.md new file mode 100644 index 00000000..83b80307 --- /dev/null +++ b/.wolf/bugs/bug-1690.md @@ -0,0 +1,44 @@ +--- +id: "bug-1690" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T11:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/schema/controller.capnp (new HostRunusage/JobRunusage structs)", "src/controller/persist.c (capnp_encode_jobrunusage/decode_jobrunusage ~L327-388", "persist_job_hook_fun's HOOK_DETAIL_JOB_RUSAGE_UPDATE case ~L7440-7500", "apply_committed_entry's WAL_JOB_RUSAGE branch ~L4116-4144", "apply_wal_entry's WAL_JOB_RUSAGE case ~L3712-3740)", "src/controller/job.c (merge_job_runusage new ~L893-1000", "restore_job_rusage_state new)", "src/include/wal_types.h (WAL_JOB_RUSAGE doc comment)", "src/controller/testsuite/test_wal_job_rusage_fact.c (new", "8 tests)"] +tags: ["controller", "job.c", "persist.c", "wal", "wal-job-rusage", "job-run-usage", "restart-recovery", "data-loss", "apply-seam", "capnp-schema-change", "heap-corruption-hazard", "xstring-rtti-pointer", "merge-not-replace"] +related: ["bug-1678", "bug-1687", "bug-1689", "bug-1561"] +legacy: true +--- + +# bug-1690: xce-audit-invariants finding: WAL_JOB_RUSAGE silently drops job.run_usage (client-reported… + +## Description +xce-audit-invariants finding: WAL_JOB_RUSAGE silently drops job.run_usage (client-reported per-host CPU/mem usage) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time). update_job_rusages() (job.c) sets job->job.run_usage (xce_job_runusage_t*, an accumulated per-host struct) via update_host_runusage() (misc.c), but the WAL_JOB_RUSAGE fact persist_job_hook_fun (persist.c) wrote only carried wal_jobstate_t{jobid, state} -- run_usage was never in the log at all, so a restart replay left it permanently NULL even though a client actively reported CPU/mem usage. Unlike the three precedents, xce_job_runusage_t had NO existing Cap'n Proto mapping at all (confirmed: grep for xce_job_runusage_t in controller.capnp matched nothing; a similar-looking but unrelated type, xce_job_rusages_t/JobRusages/SchedJobRusages -- the CLIENT's wire update-request shape {jobid, pids[], pgids[], usages[]} -- already existed and is NOT interchangeable), so this fix required a genuine schema addition, not a reuse. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +persist_job_hook_fun()'s HOOK_DETAIL_JOB_RUSAGE_UPDATE case (persist.c) built only wal_jobstate_t{jobid, state} for the WAL_JOB_RUSAGE fact, discarding the job's run_usage entirely even though it was available (looked up via the same jobmgr lookup, under the same write lock update_job_rusages() already held). Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/FINISH/.../DISPATCH fall-through group (persist.c) and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) -- never restored run_usage, matching the exact same class of gap bug-1678/1687/1689 already fixed for allocs/pidinfo+start_time/exit_info+finish_time. Confirmed via CodeGraph source read of update_job_rusages() (job.c) and persist_job_hook_fun()'s actual HOOK_DETAIL_JOB_RUSAGE_UPDATE arm (not a direct persist_submit_job_*_fact() call site like DISPATCH/START/FINISH have -- RUSAGE is purely hook-driven from the raw xce_job_rusages_t* payload update_job_rusages() fires, a genuine divergence from the audit's initial reconstructed precedent that was verified against the real code before implementing, per this task's explicit instruction). + +## Fix +FIXED (2026-08-25). Added a genuine new Cap'n Proto schema pair to controller.capnp -- struct HostRunusage $C.mapname("xce_host_runusage_t"){hostName, cpuTime, memUsage} and struct JobRunusage $C.mapname("xce_job_runusage_t"){maxMem, avgMem, usedMem, cpuTime, peakCpuTime, peakCpuDuration, memEfficiency, avgCpuEfficiency, peakCpuEfficiency, hostRunusage: List(HostRunusage) $C.maplistcount("num_hosts")} -- following JobPendReason's exact List(NestedStruct)+maplistcount precedent already in the schema, field types verified against types.h's real xce_job_runusage_t/xce_host_runusage_t definitions via CodeGraph before writing. The meson custom_target (src/lib/meson.build) regenerated lib/controller.capnp.c/.h automatically on rebuild -- verified via nm on the built controller_unit_test binary showing the full genuinely-codegen'd symbol set (new_JobRunusage, read/write_JobRunusage, encode/decode_JobRunusage, free_JobRunusage_ptr, get/set per-field accessors, and the equivalent HostRunusage set), never hand-written. persist.c gains capnp_encode_jobrunusage()/decode_jobrunusage() (mirroring capnp_encode_joballoc()/decode_joballoc() exactly). persist_job_hook_fun()'s HOOK_DETAIL_JOB_RUSAGE_UPDATE case now also encodes j->job.run_usage (when non-NULL) as trailing capnp bytes after the wal_jobstate_t header -- inlined into that hook (not a new persist_submit_job_rusage_fact(), unlike bug-1678/1687/1689's direct-call-site pattern, since RUSAGE is genuinely hook-driven). Both apply seams updated: apply_wal_entry() splits WAL_JOB_RUSAGE out of its shared state-only group into its own case, decoding trailing bytes and calling new restore_job_rusage_state() (job.c). apply_committed_entry()'s WAL_JOB_RUSAGE handling stays in its shared job-fact group but -- critically, UNLIKE START/FINISH/DISPATCH -- decodes/adopts run_usage UNCONDITIONALLY, before the JOB_TRANSITION_APPLIED gate: a rusage report almost never changes job.state (the job stays RUN across every repeat report), so gating adoption on a forward state transition the way the other three fixes do would make it dead code for the realistic, common case (every report after the first hits JOB_TRANSITION_NOOP_DUPLICATE and the gated block never runs). This divergence was caught by actually reading apply_job_update_transition()'s guard logic rather than blindly copying the precedent pattern. A payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_RUSAGE entry written before this fix, or one fired while job.run_usage was still NULL at write time) decodes no run_usage on both seams, matching pre-fix behavior exactly -- full backward compatibility. + +CRITICAL SAFETY FINDING beyond the precedent's pattern, caught via investigation (not assumed): a naive free-and-replace (mirroring bug-1678/1687/1689's free_X_ptr()-then-adopt exactly) would be a genuine heap-corruption hazard here, specific to RUSAGE. job.run_usage can be the job's LIVE structure -- update_job_rusages()/update_host_runusage() (job.c/misc.c) mutate it synchronously and in place BEFORE the WAL fact is even appended, and host_name is sourced from makeString() (src/lib/xstring.c), an RTTI-managed, refcounted, header-before-data pointer (confirmed via librtti's actual source in the nix store: RTTI_DATA_TO_OBJ subtracts RTTI_OBJ_HEADER_SIZE from the data pointer to reach the real allocation start) -- NOT a plain malloc() allocation, unlike JobAlloc/JobPIDInfo's host/exdata fields (verified to be genuinely capnp-wire-decoded end to end via dispatch_a_job_()'s own alloc parameter, confirming why bug-1678/1687 never hit this). persist_apply_pump_run() (the live online-apply pump, confirmed via its own doc comment to run on the primary itself, not just boot replay) re-applies every committed WAL_JOB_RUSAGE fact via apply_committed_entry() moments after the SAME live mutation already happened -- so calling the capnp-generated free_JobRunusage_ptr() (written for plain malloc'd Text fields) on that live structure would free() a non-malloc'd, RTTI-managed pointer: guaranteed ASAN abort / heap corruption on the very first live rusage report for a new host. Fixed by making adoption a MERGE (new merge_job_runusage(), job.c) instead of a free-and-replace: an already-present host (matched by host_name string comparison, mirroring update_host_runusage()'s own find-existing-or-append semantics) only has its scalar cpu_time/mem_usage overwritten in place -- its host_name pointer, possibly xstring-backed, is NEVER touched or freed; only a genuinely new host's own freshly-decoded (always plain-malloc'd) host_name is adopted by ownership transfer. Applied identically in both restore_job_rusage_state() (apply_wal_entry seam) and apply_committed_entry's own branch, since a demoted-primary-turned-standby could in principle retain live-mutated run_usage data too. Noted (not fixed, genuinely out of scope -- a different file/component/bug class): job_destroy() (src/lib/jobj.c)'s own existing manual FREEUP-based walk over run_usage has this exact same latent host_name/xstring mismatch, independent of and pre-dating this fix. + +Regression tests: src/controller/testsuite/test_wal_job_rusage_fact.c (8 tests, mirroring test_wal_job_start_fact.c's structure): (1) apply_committed_entry adopts run_usage when state does NOT change (JOB_TRANSITION_NOOP_DUPLICATE) -- the suite's most important test, proving the unconditional-decode design choice; (2) apply_committed_entry adopts run_usage on a genuine forward transition, hook still fires once; (3) apply_committed_entry backward-compat with an old-format payload; (4) apply_committed_entry merges without freeing a REAL makeString()'d host_name -- builds a live-flavor run_usage by hand, applies an update for that same host plus a new host, and asserts the host_name POINTER IDENTITY survives (proving it was never freed) alongside correct scalar/new-host merging, with careful test-only cleanup (detach + dereference()) so the separate, pre-existing job_destroy() FREEUP issue can't fire inside the test's own teardown; (5) apply_wal_entry adopts run_usage, no hook, absolute state overwrite; (6) apply_wal_entry backward-compat; (7) direct-payload restart-replay (WAL_JOB_SUBMIT + WAL_JOB_RUSAGE via persist_wal_async + persist_recover()); (8) drives the REAL, unmodified persist_job_hook_fun() via the established 'variadic trampoline' pattern (test_commit_barrier_rollout.c/test_persist_hooks.c) through a real WAL-write + persist_recover() restart cycle -- NOT via update_job_rusages() itself, because this same controller_unit_test binary already links test_commit_barrier_reply.c's own non-static stub of update_host_runusage() (a no-op, since misc.c is outside controller_unit_test_src), so calling update_job_rusages() here would silently hit that stub and populate nothing -- documented in the test file's own header comment. + +RED->GREEN independently verified live (not narrated): `git stash push -- src/controller/job.c src/controller/persist.c` (keeping the schema change, the new test file, and meson.build/controller_test.c wiring), full clean rebuild via `nix develop ./src --command bash src/tools/xce-build.sh controller asan build-asan` (no --incremental, since this is the schema's first-ever compile) -- 1160 Tests 8 Failures: exactly the 6 new RUSAGE tests (5 with the exact expected 'must set job.run_usage' mismatch, 1 with 'Expected 500 Was 100' for the merge-safety test) plus the 2 pre-existing bug-1561 test_config_repl.c failures; the 2 backward-compat tests correctly passed unchanged. `git stash pop`, incremental rebuild -- 1160 Tests 2 Failures (only the pre-existing bug-1561 pair, exact signature match 'Expected 2 Was 3' / 'Expected 0 Was '), all 8 new tests pass, zero new failures, zero ASAN aborts/heap-corruption findings (explicitly grepped the full test log for AddressSanitizer/heap-buffer-overflow/invalid-free/SEGV/Aborted -- none found). Verified the generated capnp code was genuinely regenerated, not hand-written: `nm` on the built controller_unit_test binary shows the complete real codegen symbol set for both new structs (new_JobRunusage, read/write_JobRunusage, encode/decode_JobRunusage, free_JobRunusage_ptr, per-field get/set accessors, and the equivalent HostRunusage set), with generated-file mtimes strictly after the schema edit's mtime. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1678, bug-1687, bug-1689, bug-1561 diff --git a/.wolf/bugs/bug-1691.md b/.wolf/bugs/bug-1691.md new file mode 100644 index 00000000..cb1745f7 --- /dev/null +++ b/.wolf/bugs/bug-1691.md @@ -0,0 +1,46 @@ +--- +id: "bug-1691" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T12:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c:2999 (on_job_ckpt)", "src/controller/job.c:1046 (restore_job_ckpt_state", "new)", "src/controller/persist.c:354 (decode_ckptinfo", "src/controller/persist.c:7579 (persist_job_hook_fun's HOOK_DETAIL_JOB_CKPT case", "src/controller/persist.c:3757 (apply_wal_entry's WAL_JOB_CKPT case", "split out)", "src/controller/persist.c:4147 (apply_committed_entry's WAL_JOB_CKPT unconditional-adopt block)", "src/include/wal_types.h (WAL_JOB_CKPT doc comment)", "src/controller/testsuite/test_wal_job_ckpt_fact.c (new)"] +tags: ["controller", "job.c", "persist.c", "wal", "wal-job-ckpt", "job-ckptinfo", "restart-recovery", "data-loss", "apply-seam", "native-trailing-payload", "hook-driven-fact", "xstring-release-verb", "test-fixture-hook-registration-gap"] +related: ["bug-1678", "bug-1687", "bug-1689", "bug-1690", "bug-1561"] +legacy: true +--- + +# bug-1691: xce-audit-invariants finding: WAL_JOB_CKPT silently drops job.ckptinfo (the… + +## Description +xce-audit-invariants finding: WAL_JOB_CKPT silently drops job.ckptinfo (the worker/scheduler-reported checkpoint info string) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), and bug-1690 (WAL_JOB_RUSAGE/job.run_usage). on_job_ckpt() (job.c) sets job->job.ckptinfo via makeString(), but the WAL_JOB_CKPT fact -- purely hook-driven via persist_job_hook_fun's HOOK_DETAIL_JOB_CKPT arm (on_job_ckpt() itself never calls a persist_submit_job_*_fact() function directly, unlike DISPATCH/START/FINISH) -- only ever carried the shared wal_jobstate_t{jobid, state} payload. Neither apply seam restored ckptinfo -- it stayed NULL after restart. job->job.ckptinfo (xce/types.h) is a plain NUL-terminated char*, NOT the richer xce_ckptinfo_t/CheckpointInfo capnp struct used elsewhere for a different purpose (that struct is on_job_ckpt()'s OWN incoming-report parameter type). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +persist_job_hook_fun()'s job-fact switch (persist.c) routed HOOK_DETAIL_JOB_CKPT through its generic `default:` arm (shared with STOP/RESUME/UPDATE/PREASON), which captures only {jobid, state} from the job_t* -- j->job.ckptinfo, freshly set by on_job_ckpt() moments earlier under the same write lock this AFTER_EVENT fires inside, was never read or encoded. Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/FINISH/.../DISPATCH fall-through group (primary boot replay AND live commit-advance apply) and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) for standby catch-up -- never restored ckptinfo, matching the exact same class of gap bug-1678/1687/1689/1690 already fixed for allocs/pidinfo+start_time/exit_info+finish_time/run_usage. Confirmed via CodeGraph source read that on_job_ckpt() is hook-driven (fires xce_run_hooks only, no direct persist_submit_job_*_fact() call site) -- a genuine divergence from DISPATCH/START/FINISH's direct-call-site pattern, closer to bug-1690's RUSAGE precedent (also purely hook-driven) -- verified against the real code before implementing, per this task's explicit instruction. Also confirmed on_job_ckpt() never changes job.state (only job.ckptinfo), so a naive copy of the START/FINISH/DISPATCH pattern (gating adoption on a forward JOB_TRANSITION_APPLIED) would have made the fix dead code on every real invocation, mirroring bug-1690's exact same discovery for WAL_JOB_RUSAGE. + +## Fix +FIXED (2026-08-25). This is the simplest of the five field-parity fixes -- job.ckptinfo is a plain string, so NO capnp struct was needed at all (unlike bug-1678/1687/1690's capnp wrappers, though it does share bug-1689's 'native, non-capnp trailer' style). Mirrors WAL_QUEUE_CTRL's own established [header][name NUL-terminated] wire shape (persist_queue_hook_fun / apply_wal_entry's WAL_QUEUE_CTRL case, read as the reference implementation before designing this fix) exactly: the ckptinfo string is appended as a NUL-terminated string directly after the wal_jobstate_t header, bounded by the WAL entry's own payload_len (no separate length prefix). persist.c gains decode_ckptinfo() (a native memcpy-with-NUL-bound-check decode, mirroring decode_jobexitinfo()'s bug-1689 native-trailer style) and a new HOOK_DETAIL_JOB_CKPT case in persist_job_hook_fun() -- split out of the generic `default:` arm -- that encodes j->job.ckptinfo (when non-NULL) as trailing bytes appended after the wal_jobstate_t header, mirroring the rusage_buf/rusage_len construction bug-1690 already added to this same function for WAL_JOB_RUSAGE (both are hook-driven, unlike DISPATCH/START/FINISH's direct persist_submit_job_*_fact() call sites). src/include/wal_types.h's WAL_JOB_CKPT enum value gains a doc comment documenting the new payload shape and backward-compat contract, mirroring every other WAL_JOB_* entry's existing convention. Both apply seams updated: apply_wal_entry() (standby catch-up) splits WAL_JOB_CKPT out of its shared state-only group into its own case (mirroring bug-1690's WAL_JOB_RUSAGE split), decoding trailing ckptinfo bytes via decode_ckptinfo() and calling new restore_job_ckpt_state() (job.c, restore_job_rusage_state()'s sibling). apply_committed_entry() stays in its existing shared job-fact group but -- mirroring bug-1690's WAL_JOB_RUSAGE design exactly, for the identical reason -- decodes/adopts ckptinfo UNCONDITIONALLY, before the JOB_TRANSITION_APPLIED gate: a checkpoint report never changes job.state, so gating adoption on a forward state transition would make it dead code for the realistic, only-ever-occurring case (every real WAL_JOB_CKPT fact hits JOB_TRANSITION_NOOP_DUPLICATE, since js.state is captured verbatim from the job's own unchanged current state). A payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_CKPT entry written before this fix, or one fired while job.ckptinfo was still NULL at write time) decodes no ckptinfo on both seams, matching pre-fix behavior exactly -- full backward compatibility. + +Adoption is a plain free-and-replace (dereference() the prior xstring value, adopt a fresh makeString()'d copy), NOT a merge -- this DIVERGES from bug-1690's WAL_JOB_RUSAGE fix, which required a merge specifically because a naive free-and-replace via the WRONG release function (the capnp-generated free_JobRunusage_ptr(), written for plain malloc'd fields) would free a non-malloc'd, RTTI-managed xstring pointer. job.ckptinfo IS xstring-backed (makeString(), xstring.c) exactly like RUSAGE's host_name was, and CAN be the job's own live value when the primary's own live apply pump re-applies a fact moments after on_job_ckpt() already set it -- but dereference() (the correct RTTI release verb, used here instead of free()/FREEUP) is safe regardless of whether the pointer is 'old' or coincidentally 'the current live one', since it decrements a refcount rather than raw free()ing a non-owned pointer. This distinction was verified by reading xstring.c's actual makeString()/dereference() implementation before choosing the release strategy, not assumed from the RUSAGE precedent. + +Regression tests: new src/controller/testsuite/test_wal_job_ckpt_fact.c (7 tests, mirroring test_wal_job_rusage_fact.c's 'unconditional decode independent of transition result' structure plus test_wal_job_finish_fact.c's simpler real-production-call pattern, wired into src/controller/meson.build and testsuite/controller_test.c): (1) apply_committed_entry sets ckptinfo when state does NOT change (JOB_TRANSITION_NOOP_DUPLICATE) -- this suite's most important test, proving the unconditional-decode design choice, hook fires 0 times; (2) apply_committed_entry sets ckptinfo on a genuine forward transition, hook still fires once; (3) apply_committed_entry backward-compat with an old-format payload; (4) apply_wal_entry sets ckptinfo, fires no hook; (5) apply_wal_entry backward-compat; (6) direct-payload restart-replay (WAL_JOB_SUBMIT + new-format WAL_JOB_CKPT via persist_wal_async + persist_recover()); (7) test_wjc_real_on_job_ckpt_survives_restart -- drives the REAL production on_job_ckpt() path (cluster.c's NotifyData_data_ckptinfo dispatch's only callee) with a real xce_ckptinfo_t, through a real WAL-write + persist_recover() restart cycle, asserting ckptinfo survives the restart intact -- the audit's exact confirmed root cause. Test 7 required passing a real cluster_t* (not NULL) as persist_init()'s 3rd argument -- unlike test_wjf_real_on_job_finish_survives_restart's identical-looking fixture (a direct-call-site fix needing no hook registration), on_job_ckpt() is hook-driven and depends entirely on persist_start() registering persist_job_hook_fun on cluster.hooksys (gated on `p->cluster != NULL`) to reach the WAL at all -- caught via a genuine RED-signal debugging pass (the real-path test initially failed even with the fix applied, isolating to this exact fixture gap, not a fix defect). + +RED->GREEN independently verified live (not just narrated): `git stash push -- src/controller/job.c src/controller/persist.c` (keeping the new test file, wal_types.h, meson.build/controller_test.c wiring), incremental rebuild via the synchronous xce-build.sh script, ran the full controller_unit_test suite -- 1167 Tests 7 Failures (the 2 pre-existing bug-1561 test_config_repl.c failures plus all 5 new-behavior WAL_JOB_CKPT tests failing with the exact expected 'must set job.ckptinfo' mismatch; the 2 backward-compat tests correctly PASS unchanged). `git stash pop`, incremental rebuild, ran the isolated new test group directly against the binary (`ASAN_OPTIONS=detect_leaks=0 ./controller_unit_test -t test_wal_job_ckpt_fact`, needed because LeakSanitizer's atexit-based leak check bypasses libc's normal exit() stdio-flush path on this binary, silently discarding all buffered PASS/FAIL output when leaks are present -- a pre-existing, suite-wide fixture characteristic unrelated to this fix, confirmed identical on the RUSAGE suite: 78 leaked allocations in isolation, same fixture-leak pattern, not a regression) -- initially 6/7 passed, with test_wjc_real_on_job_ckpt_survives_restart failing; root-caused to the persist_init() NULL-vs-&c fixture gap above, fixed the test, reran -- 7 Tests 0 Failures. Full controller_unit_test suite re-run via xce-build.sh after all fixes: 1167 Tests, 2 Failures -- exact bug-1561 signature match ('Expected 2 Was 3' / test_conf_trim_keeps_only_n_versions, 'Expected 0 Was ' / test_conf_get_saved_tag_no_config in test_config_repl.c), zero new failures, zero ASAN heap-corruption findings (grepped the full test log for heap-buffer-overflow/invalid-free/use-after-free/SEGV/Aborted/double-free -- none found). + +ENVIRONMENT NOTE: this session's build/test runs were repeatedly disrupted by the /tmp filesystem filling to 100% (0 bytes free), traced to ~15,437 orphaned mkdtemp() scratch directories (xce_ckpt_*/xce_hook_test_*/xce_e2e_wal_*/etc., ~658GB total) accumulated across many prior sessions' test runs whose cleanup never ran (each interrupted mid-flight by the same ENOSPC condition in a compounding cycle). Cleared via `rm -rf /tmp/xce_*` (verified via /proc/*/fd that no live process held any of them open) before the final, valid GREEN run -- unrelated to this fix's correctness, but a genuine environment hazard worth flagging for other sessions sharing this machine. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1678, bug-1687, bug-1689, bug-1690, bug-1561 diff --git a/.wolf/bugs/bug-1692.md b/.wolf/bugs/bug-1692.md new file mode 100644 index 00000000..b406187e --- /dev/null +++ b/.wolf/bugs/bug-1692.md @@ -0,0 +1,50 @@ +--- +id: "bug-1692" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T13:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c:206 (capnp_encode_jobpreason", "new)", "src/controller/persist.c:290 (decode_jobpreason", "src/controller/persist.c:78 (persist_job_hook_fun local vars pend_buf/pend_len/pend_present)", "src/controller/persist.c (persist_job_hook_fun's new HOOK_DETAIL_SETPREASON/HOOK_DETAIL_CLRPREASON case", "split out of default arm)", "src/controller/persist.c (persist_job_hook_fun's shared-tail pend_present append block)", "src/controller/persist.c (apply_wal_entry's WAL_JOB_PREASON case", "split out of the WAL_JOB_FAIL/STOP/RESUME/UPDATE shared group)", "src/controller/persist.c (apply_committed_entry's WAL_JOB_PREASON unconditional-adopt block", "added after the WAL_JOB_CKPT block", "before the JOB_TRANSITION_APPLIED gate)", "src/controller/job.c (restore_job_preason_state", "new", "added after restore_job_ckpt_state)", "src/include/wal_types.h (WAL_JOB_PREASON=10 doc comment", "src/controller/testsuite/test_wal_job_preason_fact.c (new", "11 tests)", "src/controller/meson.build", "src/controller/testsuite/controller_test.c (test wiring)"] +tags: ["controller", "job.c", "persist.c", "wal", "wal-job-preason", "job-pend-reasons", "restart-recovery", "data-loss", "apply-seam", "capnp-schema-reuse", "hook-driven-fact", "clear-semantics", "presence-byte-wire-shape", "test-quality-correction"] +related: ["bug-1678", "bug-1687", "bug-1689", "bug-1690", "bug-1691", "bug-1561"] +legacy: true +--- + +# bug-1692: xce-audit-invariants finding: WAL_JOB_PREASON silently drops job.pend_reasons (the… + +## Description +xce-audit-invariants finding: WAL_JOB_PREASON silently drops job.pend_reasons (the scheduler-reported host/resource-based pending-job explanation) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), bug-1690 (WAL_JOB_RUSAGE/job.run_usage), and bug-1691 (WAL_JOB_CKPT/job.ckptinfo). set_preason()/do_clr_preason() (job.c) set/clear job->pend_reasons (xce_job_preason_t*: main_reason plus xce_host_reason_t**/xce_res_reason_t** nested arrays), but the WAL_JOB_PREASON fact -- purely hook-driven via persist_job_hook_fun's `default:` arm (shared with STOP/RESUME/UPDATE, same class as bug-1690/1691's hook-driven precedent, NOT a direct persist_submit_job_*_fact() call site like DISPATCH/START/FINISH) -- only ever carried the shared wal_jobstate_t{jobid, state} payload. Neither apply seam restored pend_reasons after a restart. This was the LAST of the five field-parity gaps from the original xce-audit-invariants pass -- closes that invariant class. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +persist_job_hook_fun()'s job-fact switch (persist.c) routed HOOK_DETAIL_SETPREASON/HOOK_DETAIL_CLRPREASON through its generic `default:` arm (shared with STOP/RESUME/UPDATE), which captures only {jobid, state} from the job_t* -- j->job.pend_reasons, freshly set/cleared by set_preason()/do_clr_preason() moments earlier under the same write lock this AFTER_EVENT fires inside, was never read or encoded. Both apply seams -- apply_committed_entry()'s shared WAL_JOB_START/FINISH/.../DISPATCH fall-through group (primary boot replay AND live commit-advance apply) and apply_wal_entry()'s identical shared group -> restore_job_state() (job.c, state-only) for standby catch-up -- never restored pend_reasons, matching the exact same class of gap bug-1678/1687/1689/1690/1691 already fixed for allocs/pidinfo+start_time/exit_info+finish_time/run_usage/ckptinfo. Confirmed via CodeGraph source read that set_preason()/do_clr_preason() are hook-driven (fire xce_run_hooks only, no direct persist_submit_job_*_fact() call site) -- closer to bug-1690/1691's RUSAGE/CKPT precedent than DISPATCH/START/FINISH's direct-call-site pattern -- verified against the real code before implementing. Also confirmed set_preason()/do_clr_preason() never change job.state (only job.pend_reasons), so a naive copy of the START/FINISH/DISPATCH pattern (gating adoption on a forward JOB_TRANSITION_APPLIED) would have made the fix dead code on every real invocation, mirroring bug-1690/1691's exact same discovery. UNIQUE to this bug (verified, not assumed from precedent): job.pend_reasons has a genuine CLEAR operation (do_clr_preason/HOOK_DETAIL_CLRPREASON) that legitimately resets it to NULL and must survive replay just as durably as the SET side -- bug-1690/1691's own wire convention ('trailing bytes present or absent' == 'field has a new value or is old-format, leave untouched') cannot distinguish that legitimate NULL from a pre-fix payload, so a blind copy of that convention would have silently reintroduced a stale-pend_reasons-after-clear bug on restart. + +## Fix +FIXED (2026-08-25). controller.capnp ALREADY had a matching struct -- `struct JobPendReason $C.mapname("xce_job_preason_t")` with `mainReason`/`hostReasons: List(HostBasedReason) $C.maplistcount("nhost_reasons")`/`resReasons: List(ResourceBasedReason) $C.maplistcount("nres_reasons")`, plus sibling `HostBasedReason $C.mapname("xce_host_reason_t")` and `ResourceBasedReason $C.mapname("xce_res_reason_t")` -- verified field-for-field against types.h's real struct layouts, and ALREADY wired into the full Job struct's own encode/decode/free (Job.pendreason field, used by the existing checkpoint/snapshot path) -- confirmed via grep that encode_JobPendReason/decode_JobPendReason/free_JobPendReason/free_JobPendReason_ptr/new_JobPendReason/read_JobPendReason/write_JobPendReason all already exist in the generated controller.capnp.c/.h. NO schema change needed (like bug-1678/1687's JobAlloc/JobPIDInfo reuse, unlike bug-1690's WAL_JOB_RUSAGE which needed a brand-new schema struct). + +SAFETY CHECK (bug-1690's RUSAGE lesson, explicitly re-verified rather than assumed): read erase_job_preason() (src/lib/preason.c) -- releases every field via FREEUP() (plain free()), never dereference()/RTTI. Read add_host_preason()/add_res_preason() (same file) -- build host/resource reason entries via strdup()/calloc() throughout, never makeString(). Read the generated decode_HostBasedReason/decode_ResourceBasedReason (controller.capnp.c) -- use STRING_DUP (plain strdup) + calloc(), and free_HostBasedReason/free_ResourceBasedReason/free_JobPendReason/free_JobPendReason_ptr all use plain free(). Also confirmed jobj.c's job_init() ALREADY does a plain MOVE_PTR(dst->job.pend_reasons, src->pend_reasons) when constructing a job from a decoded Job (no merge, no makeString) and job_destroy() releases it via erase_job_preason() (FREEUP-only) -- an existing, working precedent for exactly this adopt pattern elsewhere in the codebase. CONCLUSION: unlike WAL_JOB_RUSAGE's host_name (xstring/RTTI-backed, required a merge to avoid corrupting the codebase's own live in-place-mutated structure), job.pend_reasons and its entire nested tree are plain calloc()/strdup()-backed with NO RTTI involvement -- a straightforward free-old-then-adopt-new via the capnp-generated free_JobPendReason_ptr() is safe, matching bug-1678/1687's JobAlloc/JobPIDInfo pattern, NOT bug-1690's merge-required pattern. set_preason() itself corroborates this: it does erase_job_preason()+FREEUP() then a plain MOVE_PTR of the caller's already-built xce_job_preason_t*, never mutating the SAME live structure in place the way update_host_runusage() does for run_usage -- so there is no live-object-reapplied-via-apply-pump hazard analogous to RUSAGE's here either. + +persist.c gains capnp_encode_jobpreason()/decode_jobpreason() (mirroring capnp_encode_joballoc()/decode_joballoc() exactly). persist_job_hook_fun() gains a new HOOK_DETAIL_SETPREASON/HOOK_DETAIL_CLRPREASON case (split out of the generic default arm, mirroring bug-1691's HOOK_DETAIL_JOB_CKPT split) that encodes j->job.pend_reasons when non-NULL. + +KEY DESIGN DIVERGENCE from bug-1690/1691's wire convention (the CLEAR-semantics gap identified above): the payload appends an explicit PRESENCE BYTE after the wal_jobstate_t header whenever the fact is SETPREASON or CLRPREASON (not just when a value exists to encode) -- 0 = pend_reasons was genuinely cleared (adopt NULL on replay), 1 = trailing capnp JobPendReason bytes follow (adopt the decoded value). A payload of exactly sizeof(wal_jobstate_t) (no presence byte at all -- any entry written before this fix) leaves pend_reasons untouched on both apply seams, preserving full backward compatibility. Both apply seams updated: apply_wal_entry() splits WAL_JOB_PREASON out of its shared state-only group into its own case, decoding the presence byte (and trailing capnp bytes when present=1) via decode_jobpreason() and calling new restore_job_preason_state() (job.c) -- careful to treat a present-but-failed-decode as 'leave untouched', never as an implicit clear. apply_committed_entry() stays in its shared job-fact group but -- mirroring bug-1690/1691's unconditional-adopt design for the identical reason (set_preason()/do_clr_preason() never change job.state, so gating on JOB_TRANSITION_APPLIED would make it dead code) -- decodes/adopts pend_reasons UNCONDITIONALLY, before the transition gate, using the same presence-byte + decode-failure-safe logic. + +Regression tests: new src/controller/testsuite/test_wal_job_preason_fact.c (11 tests, mirroring test_wal_job_ckpt_fact.c's 'unconditional decode independent of transition result' structure, extended with PREASON-specific clear-semantics coverage no CKPT/RUSAGE precedent needed): (1) apply_committed_entry sets pend_reasons (BOTH a host-based AND a resource-based reason, exercising both nested list types) when state does NOT change -- most important test; (2) apply_committed_entry sets pend_reasons on a genuine forward transition; (3) apply_committed_entry backward-compat with an old bare payload; (4) apply_committed_entry CLEARS an existing pend_reasons on a presence-byte=0 payload -- the test that would have caught a naive copy of bug-1690/1691's convention; (5) apply_committed_entry leaves pend_reasons untouched when present=1 but zero trailing bytes follow (decode_jobpreason never even called) -- NOTE: an earlier draft of this test tried to exercise 'corrupt capnp bytes' instead and got a genuine RED signal for the wrong reason (Expected 4 Was 0) -- capnp-c's read path does not validate content, so garbage-but-nonempty bytes decode into a zero-valued struct rather than returning NULL (matching decode_joballoc/decode_jobrunusage's identical pre-existing leniency); corrected to the actually-reachable zero-trailing-bytes case; (6-8) the apply_wal_entry mirrors of (1)/(3)/(4); (9) direct-payload restart replay (WAL_JOB_SUBMIT + WAL_JOB_PREASON with host+resource reasons via persist_wal_async + persist_recover()); (10) test_wjp_real_do_set_preason_survives_restart -- drives the REAL, unmodified do_set_preason() (set_preason() itself is file-static to job.c; do_set_preason() is job.c's only exported entrypoint, matching the actual OPCode_setpreason wire path) through a real WAL-write + persist_recover() restart cycle, requiring a minimal zeroed (fd=-1, no iomgr) endpoint_t since do_set_preason() calls xce_reply() before processing -- mirrors test_commit_barrier_reply.c's established fixture pattern, packets drained via endpoint_deq()/free_packet() to avoid a reachable-leak false positive; (11) test_wjp_real_clear_survives_restart -- PREASON-specific, no CKPT/RUSAGE analog: drives real do_set_preason() THEN real do_clr_preason() before restarting, proving the CLEAR itself durably survives replay (not just the earlier SET) -- the audit's exact confirmed correctness requirement unique to this fact type. + +RED->GREEN independently verified live (not just narrated): `git stash push -- src/controller/job.c src/controller/persist.c` (keeping the new test file, wal_types.h doc comment, meson.build/controller_test.c wiring), incremental rebuild via the synchronous xce-build.sh script (`nix develop ./src --command bash src/tools/xce-build.sh controller asan build-asan --incremental`) -- 1178 Tests 9 Failures: the 2 pre-existing bug-1561 test_config_repl.c failures plus exactly 7 of the 11 new tests failing with the exact expected 'must set job.pend_reasons' / 'must reset pend_reasons to NULL' mismatches (tests 1,2,4,6,8,9,10 by suite order); the other 4 new tests (backward-compat x2, the decode-edge-case test, and the clear-survives-restart test) correctly PASS unchanged under the reverted code -- verified this is expected, not a test-quality defect: those 4 scenarios are either genuinely unaffected by the fix (backward-compat) or, for the two restart-replay tests, happen to trivially satisfy their own assertion under BOTH old and new code (old code never encodes pend_reasons at all regardless of set-vs-clear, so a 'must be NULL after clear' check passes vacuously pre-fix too -- acceptable, since the SET-side restart test (10) and the direct-payload test (9) already independently prove the encode/decode mechanism itself, and test (11)'s value is in proving the CLEAR case specifically once mechanism 10 is proven, not in isolating it alone). `git stash pop`, incremental rebuild -- 1178 Tests 2 Failures (only the pre-existing bug-1561 pair, exact signature match 'Expected 2 Was 3' / 'Expected 0 Was 11954236611186853435'), all 11 new tests pass, zero new failures, zero ASAN heap-corruption findings (grepped the full test log for AddressSanitizer/heap-buffer-overflow/invalid-free/double-free/use-after-free/SEGV/Aborted -- none found). Verified no new compiler warnings in persist.c/job.c/the new test file (isolated rebuild + warning grep; the handful of pre-existing format-truncation/comment warnings in persist.c are all at unrelated line numbers, predating this change). + +This was the LAST of the five field-parity gaps identified by the original xce-audit-invariants pass (bug-1678/1687/1689/1690/1691/1692) -- that invariant class (APPLY_CLEAN's entity-count+cursor-position check not proving per-field completeness, persistence.md) is now closed for job.allocs/pidinfo+start_time/exit_info+finish_time/run_usage/ckptinfo/pend_reasons. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1678, bug-1687, bug-1689, bug-1690, bug-1691, bug-1561 diff --git a/.wolf/bugs/bug-1693.md b/.wolf/bugs/bug-1693.md new file mode 100644 index 00000000..2fb6641c --- /dev/null +++ b/.wolf/bugs/bug-1693.md @@ -0,0 +1,38 @@ +--- +id: "bug-1693" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-08-25T14:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/cluster.c", "src/controller/sched.c", "src/controller/job.c", "src/controller/worker.c", "src/controller/include/cluster.h"] +tags: ["controller", "decision-log-fidelity", "cluster.c", "sched.c", "job.c", "worker.c", "OPCode_schedAdmin", "on_fetchfile", "handle_scheduler_leave", "handle_worker_leave", "fail-loud", "condvar-signal", "reply-wait-timeout", "xce-audit-invariants", "ha-async-io"] +related: ["bug-646", "bug-648", "bug-650"] +legacy: true +--- + +# bug-1693: Invariant audit (Decision Log fidelity dimension) found two violations of the recorded Decision… + +## Description +Invariant audit (Decision Log fidelity dimension) found two violations of the recorded Decision Log entry "No client reply-wait timeout as a fix for a hung request -- fail loud at the real transport/source instead of masking with a deadline backstop": (1) OPCode_schedAdmin's forward-and-block-for-reply (cluster.c) blocks up to SCHEDADMIN_TIMEOUT_MS=5000ms via pthread_cond_timedwait, but handle_scheduler_leave() (sched.c) -- the actual scheduler-disconnect handler -- only ever nulled c->sched.ep; it never signalled the pending waiter, so a scheduler disconnect mid-wait could only ever be observed at the full 5s deadline. (2) on_fetchfile's identical proxy pattern (job.c, 10s pthread_cond_timedwait) had the same gap: handle_worker_leave() (worker.c) only ever nulled w->ep, never woke a pending fetchfile waiter proxied to that worker. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Both sites already had a working signal function used on the SUCCESS path (schedadmin_signal() in cluster.c, fetchfile_signal() in job.c, both wake the pending condvar wait when a real reply arrives) -- but the corresponding DISCONNECT handlers (handle_scheduler_leave/handle_worker_leave) were never wired to call them. cluster.c's own OPCode_schedAdmin comment explicitly documents schedAdmin as mirroring on_fetchfile's proxy pattern, confirming this was one shared, repeated design mistake copied into both sites, not two independent bugs. The timeout backstop was therefore the SOLE failure-detection mechanism for a mid-wait disconnect at both sites, exactly what the Decision Log entry (itself the outcome of bug-646/648/650's original fail-loud fix) forbids. + +## Fix +Kept the timeout as a defensive backstop (per the Decision Log's exact wording: the rule is about not relying SOLELY on a timeout, not removing timeouts) and added an immediate-wake path on disconnect at both sites, mirroring bug-646/648/650's original fail-loud precedent. schedAdmin site: new int schedadmin_signal_disconnect(void) (cluster.c) -- a thin wrapper calling the existing static schedadmin_signal(-ECONNRESET, NULL) -- declared extern in cluster.h; handle_scheduler_leave() (sched.c) now calls it BEFORE acquiring c->sched.lock (calling it after would deadlock, since the OPCode_schedAdmin dispatch case holds c->sched.lock for its entire forward+wait duration; schedadmin_signal_disconnect() itself never touches c->sched.lock, only g_schedadmin_mu + the pending slot's own mu, so no new lock-ordering hazard). cluster.c's post-wait dispatch gained a new got_status==-ECONNRESET branch relaying a structured SchedAdminReply{error="scheduler disconnected"}, distinguishable from both the immediate "scheduler not connected" rejection and the deadline's "request timeout". fetchfile site: fetchfile_pending_t (job.c) gained a worker_t* worker field (set in on_fetchfile() while still holding the worker's read lock, before fetchfile_register()); new void fetchfile_signal_worker_disconnect(worker_t*) (job.c, declared extern in cluster.h) walks g_fetchfile_list under g_fetchfile_mu and signals every pending entry matching the disconnected worker with status=-ECONNRESET (unlike fetchfile_signal, which is keyed by a single jobid, more than one fetchfile request can be in flight against the same worker); handle_worker_leave() (worker.c) calls it right after nulling w->ep (no lock-ordering constraint there, since on_fetchfile's wait never holds any worker_t lock). Hidden fan-out cost discovered only at link time (matches this project's established pattern for this class of change): controller_unit_test does not link the real sched.c at all (test_config_repl.c already provides its own no-op handle_scheduler_leave() stub, to avoid an unrelated on_scheduler_join()/notify() link collision with other files in that binary) -- so the primary schedAdmin regression test drives schedadmin_signal_disconnect() directly (the real, unmodified, unstubbed production function in cluster.c) rather than through handle_scheduler_leave(), and a SEPARATE small spy-based test was added in test_sched_worker_change.c (which DOES link the real sched.c) proving handle_scheduler_leave() itself calls schedadmin_signal_disconnect() exactly once, before clearing c->sched.ep. Also had to add a one-line link-satisfying schedadmin_signal_disconnect() stub to test_job_array_sched_push.c and test_sched_worker_change.c (the two standalone binaries that link sched.c without cluster.c), mirroring their existing check_worker_() stub precedent. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-646, bug-648, bug-650 diff --git a/.wolf/bugs/bug-1694.md b/.wolf/bugs/bug-1694.md new file mode 100644 index 00000000..60fd3e82 --- /dev/null +++ b/.wolf/bugs/bug-1694.md @@ -0,0 +1,38 @@ +--- +id: "bug-1694" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T00:00:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c", "src/include/persist.h", "src/controller/testsuite/test_persist_checkpoint_offload.c", "src/controller/testsuite/controller_test.c", "src/controller/meson.build"] +tags: ["async-io", "thread-role", "delaymgr", "checkpoint", "persist.c", "slowlane", "apply_pump_lock", "vtask", "xce-audit-invariants", "raft-drive-async"] +related: ["bug-1429"] +legacy: true +--- + +# bug-1694: xce-audit-invariants finding (dimension 1, async I/O thread-role): checkpoint_task_fn… + +## Description +xce-audit-invariants finding (dimension 1, async I/O thread-role): checkpoint_task_fn (src/controller/persist.c) was the last delaymgr-triggered path still calling its full blocking body (persist_write_checkpoint_marker's pwrite+fdatasync, write_snapshot()'s unbounded open/write walk over every job/queue/host frame held under apply_pump_lock, trim_wal_segments, rotate_snapshots' unlink) directly on the delaymgr timer thread -- the same single thread that drives heartbeats and election timeouts cluster-wide (ha-async-io.md thread-role contract: delaymgr must only trigger work, never block). Triggered from the WAL-append hot path via schedule_checkpoint()/delay_task() whenever wal_record_count crosses max_wal_records. Every sibling long-blocking-I/O path (snapshot send, config-push, reconnect) had already been offloaded by raft-drive-async; checkpoint was the one documented remaining gap. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +checkpoint_task_fn called persist_checkpoint() inline with no offload mechanism at all -- unlike every other delaymgr-triggered blocking path in the codebase. Investigated whether the obvious fix (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK), the pattern used by persist_apply_pump_kick/commit.c's dispatch_continuation) was safe here: it is NOT -- persist_apply_pump_run() (which ALSO acquires apply_pump_lock, across its entire per-entry walk) is itself dispatched onto that SAME vtaskmgr N-thread pool via persist_apply_pump_kick()'s vtask_enq_classified(..., VTASK_LIVE_BLOCK). Putting checkpoint's genuinely unbounded write_snapshot() (held under apply_pump_lock) on the identical N-thread pool would risk a worker thread parked on apply_pump_lock for the whole snapshot-write duration, starving the pool's liveness-critical capacity (commit continuations, apply-pump drains) -- exactly the failure mode ha-async-io.md's own table entry warns about ("vtaskmgr must NOT do: long/unbounded blocking I/O (snapshot transfer, config-push) -- starves the pool"). The correct destination, per the codebase's OWN established distinction (re-verified by reading all 4 reference sites plus grepping every offload flag), is the SLOW LANE (slowlane_submit) -- the same mechanism snapshot-send (ha_snapshot_do_send -> snapshot_lane_fn, g_drive_async_snapshot_offload, default ON) and config-push (configpush_lane_fn, g_drive_async_configpush_offload, default ON) already use, and NOT the vtask pool. Confirms the task brief's suspicion that checkpoint was harder than a simple vtask wrap -- it's a slow-lane fix, not a vtask-pool fix. + +## Fix +FIXED. checkpoint_task_fn is now trigger-only: it releases the checkpoint_in_progress CAS slot, then submits the real work (via a new run_checkpoint_body()/checkpoint_lane_fn()) to the cluster's slow lane (slowlane_submit(&persist->cluster->slowlane, key=1, checkpoint_lane_fn, persist)), mirroring snapshot_lane_fn/configpush_lane_fn/catchup_lane_fn/demotion_lane_fn exactly. New feature flag g_persist_checkpoint_offload (persist_set_checkpoint_offload_for_test/persist_checkpoint_offload_enabled, persist.h/persist.c) defaults ON (1) -- unlike the still-experimental catchup/demotion offloads (default OFF pending an unrelated real test-fixture hang, bug-1429), this reuses an ALREADY-PROVEN-SAFE call: persist_checkpoint() already runs on this same slow lane in production today via ha_snapshot_do_send()->snapshot_lane_fn (snapshot offload also default ON), so apply_pump_lock's acquisition from a slow-lane thread is exercised in production every day, not new surface. persist_checkpoint()'s own atomic_exchange guard (checkpoint_in_progress) remains the single-flight barrier preventing double-run with the snapshot-send path, exactly as it already does today. Verified no existing controller_unit_test fixture depends on the delaymgr trigger path completing synchronously (test_ha_core_safety.c deliberately pins max_wal_records=1u<<30 specifically to prevent ever exercising it in a live-cluster fixture) -- ruling out the catchup/demotion-class 'uninitialized slow lane + a synchronous downstream waiter hangs' hazard, since checkpoint's trigger is pure fire-and-forget. Inline-fallback + WARN on genuine slowlane_submit failure only (not -EAGAIN/-EEXIST, both non-failure per slowlane's own documented contract) -- a scheduled checkpoint is never silently dropped. New regression suite test_persist_checkpoint_offload.c (3 tests, mirrors test_ha_liveness.c's reconnect/configpush offload gate-probe pattern): test_checkpoint_offload_trigger_returns_promptly (GREEN: trigger returns <200ms while the real body is observably parked on a slow-lane worker thread, not the caller's), test_checkpoint_inline_blocks_delaymgr_thread_RED (flag OFF reproduces the pre-fix inline-blocking behavior on demand, permanent regression guard), test_checkpoint_no_cluster_runs_inline (defensive NULL-cluster fallback). RED-phase independently confirmed by temporarily forcing checkpoint_task_fn's pre-fix always-inline behavior (a one-line `if (1) ||` probe) and rebuilding: the GREEN test then hung indefinitely (process had to be kill -9'd, PID confirmed via ps as the stuck controller_unit_test binary) -- proving the test genuinely detects the violation, not a tautology. Probe reverted (git diff --stat confirmed clean re-application of the real fix) and the full suite re-run GREEN: 1184 Tests, identical 2 pre-existing failures both runs (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was , test_config_repl.c, both matching bug-1561's well-documented overlayfs fdatasync-EIO artifact signature exactly -- 0 NEW failures. Live HA smoke test (real checkpoint trigger against a running deployed cluster under load) was scoped out given budget already spent on the lock-order investigation, which was this fix's primary risk -- flagged explicitly in the final report rather than silently skipped, as a reasonable follow-up soak. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1429 diff --git a/.wolf/bugs/bug-1695.md b/.wolf/bugs/bug-1695.md new file mode 100644 index 00000000..d0351c7e --- /dev/null +++ b/.wolf/bugs/bug-1695.md @@ -0,0 +1,53 @@ +--- +id: "bug-1695" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T15:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/myapp.c (myapp_run's bootstrap check", "call site only); src/controller/startup_validation.c (myapp_leader_already_known", "the extracted+fixed decision function -- placed here", "not myapp.c", "so it links into controller_unit_test); src/controller/testsuite/test_ha_startup.c (7 new regression tests)"] +tags: ["controller", "ha", "raft", "election", "election-safety", "bootstrap", "myapp", "live-deployment", "found-via-soak", "skip_election", "term_at_boot", "live-soak-verified", "bug-1487-adjacent"] +related: ["bug-1477", "bug-1684", "bug-1487"] +legacy: true +--- + +# bug-1695: Found via the mandatory live HA chaotic soak (xce-audit-invariants controller campaign, Step 4)… + +## Description +Found via the mandatory live HA chaotic soak (xce-audit-invariants controller campaign, Step 4) -- NOT caused by any of the 9 fixes that campaign made (bug-1684/1687/1689/1690/1691/1692/1693/1694), all of which are in src/controller/{ha,persist,job,cluster,sched,worker}.c; this bug is in myapp.c, untouched by any of them. 10-round --chaotic soak: every single round after the first shows 'NO new primary within window (possible election stall)'; ctrl-1's own log shows, on EVERY restart after the first boot, 'myapp_run: leader already known after startup delay (voted_for=ctrl-1, voted_term=1, raft_term=1, primary_fd=-1) -- skipping own election' -- primary_fd=-1 proves there is no live connection to any primary, yet the node skips holding its own election anyway. ctrl-2/ctrl-3 are simultaneously stuck in a permanent ~3s retry loop ('cluster_init_phase2: cluster_config not loaded yet', rc=-22, the bug-1477 self-rescheduling retry firing forever without ever succeeding). The cluster never elects a new, live, functioning primary after the very first election of the cluster's lifetime. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +myapp_run()'s post-stagger 'is a leader already known' bootstrap check (myapp.c ~361-384) has two independent fallback conditions that can each unilaterally set leader_known=1: `c.ha.raft.term > 0` and `c.ha.primary_ctrl_fd >= 0`. The comment's stated intent (lines 345-353) is to catch a narrow race -- 'iomgr was NOT running during the nap ... but the winner's ctrlhello has already established the control channel' -- i.e. genuine NEW raft activity processed during THIS boot's startup-stagger nanosleep. But `ha_restore_term()` (bug-1684's neighbor, unmodified by that fix) restores ha->term from ha-term.bin BEFORE this check ever runs, as part of normal boot -- so `raft.term > 0` is trivially true on EVERY restart after the cluster's first-ever election in its entire lifetime, regardless of whether any NEW raft traffic was processed this boot at all. The check cannot distinguish 'term advanced because I just received a real, fresh vote/heartbeat during this nap' (genuine liveness evidence) from 'term is nonzero because I loaded an old value off disk at startup' (tells us nothing about whether ANY leader is currently reachable). Consequence: after the cluster's first election, EVERY node, on EVERY subsequent restart, unconditionally takes the goto skip_election fast path and never calls ha_start_election_async again -- so no node ever re-asserts leadership through a real raft election, and the cluster permanently loses the ability to elect a NEW primary after any restart, however far in the past the original election was. This is a related-but-distinct gap from bug-1477 (which fixed a stranded node never RETRYING cluster_init_phase2 after one failed attempt -- that retry loop is confirmed still firing correctly here, every ~3s, exactly as bug-1477 designed -- but the retry can never succeed because the underlying cause, no election is ever held to produce a real live primary to push config from, was never addressed). + +## Fix +FIXED (2026-08-25). Root cause confirmed exactly as hypothesized: myapp_run()'s raft.term>0 fallback (myapp.c ~372) is trivially true on every restart after the cluster's first-ever election, since ha_restore_term() (called from cluster_init_phase1, strictly before ha_init_raft()/the startup-stagger nap) restores a persisted, monotonically-nondecreasing term from ha-term.bin, and Raft terms never reset. + +Fix: extracted the inline check into a new, unit-testable function myapp_leader_already_known(ha, my_name, term_at_boot) -- placed in startup_validation.c (NOT myapp.c) purely so it links into controller_unit_test, which meson's controller_unit_test_src deliberately excludes myapp.c/main.c from (myapp_run is an untestable integration entry point requiring a live network socket) but already builds startup_validation.c for the sibling validate_* helpers -- myapp.c now just forward-declares and calls it, mirroring the pre-existing validate_startup_config() precedent exactly. myapp_run() captures `term_at_boot = c.ha.raft.term` immediately after ha_init_raft() seeds ha->raft.term from the just-restored ha->term -- strictly BEFORE the startup-stagger nap -- and the fallback check now requires `ha->raft.term > term_at_boot` (a genuine ADVANCE during THIS boot's nap, i.e. real vote-request/heartbeat traffic actually processed by iomgr via raft_on_vote_request's/raft_on_append_entries_request's UpdateTerm steps) instead of the old bare `> 0`. The voted_for_name/vft==cur check and the primary_ctrl_fd>=0 check are UNTOUCHED, per the task's scope boundary. + +Verification of the 5 required questions: +1. Ordering confirmed via direct source read: ha_restore_term() runs inside cluster_init_phase1 (myapp_run Step 1, ~line 67), strictly before ha_init_raft() (~line 327) and the stagger nanosleep (~line 333-343) -- on every boot including restarts, confirmed via a live log line 'HA: restored term=0 from .../ha-term.bin' printed inside cluster_init_phase1, always preceding 'HA: raft initialized (...term=N...)' from ha_init_raft. +2. No other path can legitimately advance ha->raft.term between the snapshot and the check in a way the fix would wrongly treat as stale: the only writer after ha_init_raft() is raft_cb_persist, fired only from raft_on_vote_request / raft_on_append_entries_request / raft_start_election -- all driven by genuine incoming or self-initiated raft traffic processed by iomgr during the nap, exactly the signal the check intends to detect (confirmed by reading raft.c's UpdateTerm steps in full). +3. First-ever-boot preserved: ha_restore_term() leaves term at 0 with no ha-term.bin, so term_at_boot=0, identical operative comparison to the old `raft.term > 0` check on a true first boot -- covered by 2 new regression tests. +4. No new safety-relevant failure mode: the 'spurious own election while a leader is genuinely live but unobserved' race this fix re-exposes on every restart is the IDENTICAL race the cluster's first-ever boot already ran through before this fix existed (term_at_boot was always 0 pre-first-election) -- not a new class of risk, and Election Safety is unaffected (rests entirely on votedFor + majority inside the untouched raft library, never on this advisory bootstrap check). +5. voted_for_name/vft==cur deliberately left untouched per scope, but flagged in the code comment as NOT obviously immune to the same class of question (a restored (voted_for_term,term) pair is exactly as internally consistent as a live one). Investigated empirically via the live soak below rather than touched speculatively: in both the baseline and fixed runs, this path never fires for ctrl-2/ctrl-3 in the --chaotic scenario at all, because the driver only kills/restarts the current primary -- ctrl-2/ctrl-3 run continuously and never re-enter myapp_run(), so their eventual takeover goes entirely through the separate, pre-existing, untouched liveness-watchdog/PreVote/RDHUP-failover subsystem (ha_on_primary_ctrl_fd_rdhup / ha_arm_heartbeat_timeout_vtask), not through this bootstrap check. No compounding defect was exercised or found; flagged as a residual, unverified theoretical concern for a scenario this soak doesn't cover (a STANDBY node itself restarting after remembering a now-permanently-dead remote leader). + +TDD: RED confirmed by extracting the function with the OLD `raft.term > 0` logic first (marked with a RED/GREEN comment), adding 7 new tests to test_ha_startup.c (test_bug1695_stale_restored_term_does_not_skip_election is the core regression), and rebuilding -- 2 of the 7 genuinely FAILED ('Expected 0 Was 1'), exactly as predicted (the self-vote test also failed under the old logic, since raft.term>0 trivially short-circuited it too -- an incidental but correct extra confirmation of the bug's reach). Applied the one-line semantic fix (`> 0` -> `> term_at_boot`); rebuilt -- all 7 new tests GREEN, full controller suite at 1191 tests with exactly the 2 pre-existing bug-1561 failures (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config -- identical Expected/Was values to bug-1561's own documented signature, 0 new failures). + +MANDATORY live soak (6-round --chaotic, fresh ASAN binaries) -- run TWICE for a rigorous A/B: once with the fix, once with myapp.c/startup_validation.c/test_ha_startup.c git-stashed back to the pre-fix original (same binaries otherwise, same scratch-dir mechanics). BASELINE (unfixed): term frozen at 4 for the ENTIRE 6-round run (never advances even once after initial cold-boot settling); 'NO new primary within window' on 6/6 rounds; ctrl-2/ctrl-3's logs show ZERO CTRL_PRIMARY/PreVote-round activity anywhere in their entire lifetime -- a real, direct confirmation of the reported permanent-stall symptom. FIXED: term genuinely progresses 1->2->3->4->7 across the run; ctrl-2 (a DIFFERENT node from the original ctrl-1) achieved a real, log-verified CTRL_PRIMARY transition via a won PreVote round + real election at term=3 -- something that never happens even once anywhere in the baseline's logs. This is direct, unambiguous proof the core bug-1695 symptom (permanent inability to elect a NEW primary after any restart, once the cluster has ever held its first election) is fixed. committed_divergence (walcompare, the authoritative WAL-level check) stayed 0 in BOTH runs. + +Residual, honestly-reported finding: the FIXED run's chaotic verdict was `committed_job_loss=6, result=FAIL` (baseline was `committed_job_loss=0, result=PASS` -- matching this task's own prediction that the baseline passes trivially by never actually re-electing anyone). Root-cause investigation (waldump on all 3 nodes' raw WAL files): the SAME single jobid ('0-15-1787671840-0') was reported missing on every affected round, and does NOT appear in ANY of the 3 nodes' on-disk WALs at all -- it traces to the cluster's very first cold-boot settling window (timestamp ~1787671840, well before round 1), during which ctrl-1's FIRST-ever election (with peers still config-less, config_tag=0) produced ambiguous vote-eligibility/config-bootstrap behavior unrelated to this fix (confirmed via ctrl-1's own log: its very first boot conceded to CTRL_STANDBY, never wrote that job to its own WAL). This strongly resembles a pre-existing, already-documented false-positive class (bug-1487: 'reachable primary reports job missing... post-election own-term commit-barrier lag while a peer is still catching up' -- verify_classify's own in-code comment names this exact class), newly and more frequently exposed because this fix, for the first time, lets the chaotic driver's overlapping-kill design exercise GENUINE repeated elections at all (the baseline's frozen-forever cluster never gave bug-1487's window a chance to occur more than once, if ever). NOT independently re-fixed in this session (out of this fix's myapp.c-only scope, and .claude/skills/xce-test-ha/run-ha-soak.sh is test-harness tooling, not controller code) -- flagged here as a genuine, needs-owner follow-up: either extend bug-1487's verify_classify retry margin further, or investigate whether the initial cold-boot config-bootstrap vote-eligibility path can genuinely lose a pre-config-push submission. Confidence in the CORE fix remains high (direct log proof of a new node winning a real election; walcompare clean in both runs); confidence in the chaotic verdict's exact job-loss root cause is medium (waldump-verified absence + a highly plausible, precedented explanation, but not traced to a specific line the way the core fix was). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1477, bug-1684, bug-1487 diff --git a/.wolf/bugs/bug-1696.md b/.wolf/bugs/bug-1696.md new file mode 100644 index 00000000..6448b5c7 --- /dev/null +++ b/.wolf/bugs/bug-1696.md @@ -0,0 +1,38 @@ +--- +id: "bug-1696" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T16:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c"] +tags: ["controller", "ha", "raft", "election", "wal-truncation", "leader-completeness", "data-safety", "bug-1060-regression-class", "boot-ordering-race", "persist_recover", "cluster_init_phase2", "found-via-soak", "critical", "bug-1695-adjacent", "live-soak-verified", "asan-verified"] +related: ["bug-1695", "bug-1060", "bug-1445", "bug-828", "bug-1487"] +legacy: true +--- + +# bug-1696: Chaotic HA soak (post-bug-1695-fix) reports committed_job_loss on every round; waldump on ctrl-1's… + +## Description +Chaotic HA soak (post-bug-1695-fix) reports committed_job_loss on every round; waldump on ctrl-1's live data dir shows the ENTIRE on-disk WAL truncated to a single NOOP entry (entries: 1 submits: 0) despite 21+ jobs previously verified committed; no snapshot exists; log shows ha_compute_safe_committed_lsn=0 immediately before raft_cb_become_leader promotes ctrl-1 to CTRL_PRIMARY at term=2 on its SECOND boot. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +raft_cb_become_leader() computes the promotion WAL-truncation floor (line ~1658-1662: trunc_floor = ha_promotion_truncate_floor(safe_lsn, persist_durable_floor(&c->persist)); persist_truncate_wal_above(&c->persist, trunc_floor)) and evaluates the whole promotion-eligibility gate (durable_floor/truncated/apply_verdict, ~line 1699-1823) BEFORE cluster_init_phase2() -- the ONLY call that invokes persist_recover() for this process's first-ever boot -- which does not run until is_first_boot handling at line ~1873-1899, further down in the SAME function. persist_durable_floor()/persist_recovery_truncated()/persist_apply_verdict() all read fields persist_recover() is responsible for populating from the real on-disk WAL; before that call, a freshly persist_init()'d struct reports durable_lsn=0 (bug-1445's comment at line 1901 already flagged this exact 'gate evaluated blind before recovery' hazard for ONE flag -- persist_apply_resource_fenced -- with a narrow re-check-after-phase2 patch, but the far more consequential durable_floor/truncation-floor computation was never covered by that fix). With durable_floor=0 (bogus, pre-recovery) and safe_lsn=0 (every voter, including self, ALSO advertised its LSN via persist_durable_floor() before its own recovery ran -- myapp.c:324-329, ha_candidate_advertise_lsn), ha_promotion_truncate_floor(0,0)=0 and persist_truncate_wal_above(p, 0) physically ftruncates/unlinks the real on-disk WAL segments down to LSN 0/1 -- destroying committed job data that persist_recover(), had it run FIRST, would have correctly reported as durable and protected via the bug-1060 floor guard. This scenario (a node RESTARTS with real durable data on disk, then immediately WINS its own promotion on that same boot, before its own recovery has run) was unreachable before bug-1695's fix -- previously a restarted node's stale-but-nonzero persisted term made it always skip its own election and defer to a peer, so it never hit is_first_boot-during-become_leader with real data at stake. bug-1695 correctly restored real re-election on restart and, in doing so, newly exposed this pre-existing boot-ordering flaw. A genuine Leader Completeness / bug-1060-guard-defeat, not a test-harness artifact: confirmed via direct waldump on-disk evidence plus source-level trace of the read-before-recover ordering. + +## Fix +FIXED. Reordered raft_cb_become_leader() (ha.c) so cluster_init_phase2()/persist_recover() (is_first_boot path) runs BEFORE the promotion-eligibility gate and the promotion-truncation-floor computation, so persist_durable_floor()/persist_recovery_truncated()/persist_apply_verdict() always reflect this boot's REAL recovered WAL state before any eligibility/truncation decision reads them. Split the pure vote-derived safe_committed_lsn computation (reads only ha->elect_ctx, safe pre-recovery) from the truncation-floor computation (needs post-recovery persist state) -- the former stays early, the latter moved to run after phase2. Original RED test hypothesis (physical WAL destruction via truncate) was disproven by source tracing (persist_truncate_wal_above's own no-op guard plus persist_init's current_lsn=1 reset make that specific mechanism impossible); the REAL, provable defect was the promotion-eligibility gate wrongly judging a genuinely corrupt/divergent node ELIGIBLE because it read persist_recovery_truncated()/persist_apply_verdict() before recovery ran. New regression test test_first_boot_promotion_refuses_when_own_first_recovery_finds_corruption (CRC-corrupts a real on-disk WAL entry, restarts, promotes) confirmed RED against unfixed code (wrongly became CTRL_PRIMARY despite RECOVER_TRUNCATED) and GREEN against the fix (correctly refused). Full controller unit suite: 1193 tests, 2 pre-existing bug-1561 failures only, 0 new. Live proof: full 10-round --chaotic soak under ASAN (build-asan-1696repro) PASSED (VERDICT mode=chaotic rounds=10 jobs=33 committed_divergence=0 committed_job_loss=0 result=PASS) and full 10-round --sequential soak PASSED (PASS: 10 rounds, 33 jobs submitted, zero job loss; all 3 nodes led (ctrl-1,ctrl-2,ctrl-3)) -- both legs of the audit skill's Step 4 mandate satisfied. Note: the fix's own verification agent died mid-A/B-soak leaving the fix orphaned in a git stash for a period before being discovered and restored; a subsequent independent full soak (this entry) is what actually closes it out. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1695, bug-1060, bug-1445, bug-828, bug-1487 diff --git a/.wolf/bugs/bug-1697.md b/.wolf/bugs/bug-1697.md new file mode 100644 index 00000000..4e974b10 --- /dev/null +++ b/.wolf/bugs/bug-1697.md @@ -0,0 +1,38 @@ +--- +id: "bug-1697" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-25T20:22:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c", "src/controller/persist.c"] +tags: ["controller", "job-lifecycle", "wal-dispatch", "wal-job-start", "start-time", "pidinfo", "field-parity", "apply-committed-entry", "transition-noop-collision", "found-live-no-restart", "bug-1679-adjacent", "bug-1687-adjacent", "bug-1678-adjacent", "critical"] +related: ["bug-1679", "bug-1687", "bug-1678", "bug-1690", "bug-1691", "bug-1692"] +legacy: true +--- + +# bug-1697: Job start_time is always 0 for every job, live, no restart needed (observed via xce_test REPL… + +## Description +Job start_time is always 0 for every job, live, no restart needed (observed via xce_test REPL joblist during an HA soak). run_time (finish_time - start_time) is consequently also wrong for every finished job. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +dispatch_a_job_() (job.c:1686, scheduler-dispatch-commit task 3.1) synchronously sets j->job.state = JOB_STAT_RUN at DISPATCH time, before the worker ever reports back -- this is a deliberate, already-made semantic change: the controller decides RUN at dispatch, it no longer waits for a worker ack. When the worker's LATER on_job_start()/WAL_JOB_START fact commits and apply_committed_entry()'s live apply pump (persist.c, shared WAL_JOB_START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH case group starting ~persist.c:4430) processes it, apply_job_update_transition(j, JOB_STAT_RUN) (job.c:2779) sees job->job.state already == JOB_STAT_RUN (set by dispatch, moments/seconds earlier) and returns JOB_TRANSITION_NOOP_DUPLICATE (job.c:2795-2799), never JOB_TRANSITION_APPLIED. The wire-carried start_time/pidinfo decode/adopt block bug-1687 added (persist.c ~4602-4637) is gated on `result == JOB_TRANSITION_APPLIED` (persist.c:4594), so for WAL_JOB_START it can now NEVER run, for any job -- start_time and pidinfo (pids/pgids/exdata) both silently stay at their zero-initialized defaults forever. This is the exact same class of bug already fixed for WAL_JOB_RUSAGE/WAL_JOB_CKPT/WAL_JOB_PREASON in this identical function (their own comments describe it verbatim: 'gating this adoption on result==JOB_TRANSITION_APPLIED ... would make it dead code ... decode/adopt unconditionally') -- WAL_JOB_START was never given the same treatment because bug-1687 (which added the start_time/pidinfo wire fields) predates/was unaware of the dispatch-time RUN semantic collision. bug-1679's own comment (job.c:1709-1719) already independently documented and worked around the SAME root collision for a different symptom (the apiserver update-notification hook silently not firing on this same NOOP_DUPLICATE path) -- the two symptoms were never connected as the same root cause until now. + +## Fix +FIXED -- start_time moved to dispatch-owned, pidinfo adoption made unconditional, exactly the confirmed scope. (1) job.c dispatch_a_job_() (job.c:1696-1708) stamps j->job.start_time = time(NULL) synchronously alongside j->job.state = JOB_STAT_RUN, under the same write lock, before j->job.allocs/persist_submit_job_dispatch_fact(). (2) wal_types.h adds WAL_JOB_DISPATCH_HAS_START_TIME_FLAG (new _pad bit, mirroring WAL_JOB_FAIL_STAGE_FLAG/WAL_JOB_UPDATE_POSTDONE_MARK). persist_submit_job_dispatch_fact() (persist.c:7670, persist.h:1465) gained a `uint64_t start_time` parameter; payload is now [wal_jobstate_t header, _pad=FLAG][uint64_t start_time][JobAlloc bytes] -- flag-gated, not length-based, since JobAlloc's own variable length would make a length heuristic ambiguous against the old header+alloc-only shape. (3) Both apply seams' WAL_JOB_DISPATCH cases (apply_committed_entry persist.c:4716 branch; apply_wal_entry persist.c:4183 case; restore_job_dispatch_state job.c:801) decode/adopt start_time when the flag is set, else fall through to the pre-fix alloc-only decode untouched. (4) apply_committed_entry's WAL_JOB_START handling (persist.c:4627-4660): start_time touching removed entirely (dead-code branch and its time(NULL) fallback deleted); pidinfo decode/adopt moved OUT of the `result == JOB_TRANSITION_APPLIED` gate to run unconditionally, mirroring the RUSAGE/CKPT/PREASON blocks in the same function. apply_wal_entry's WAL_JOB_START case (persist.c:4105) and restore_job_start_state() (job.c:840, dropped the start_time parameter) likewise stop touching start_time; pidinfo there was already unconditional (this seam never calls apply_job_update_transition, so it never had the NOOP-gating hazard). persist_submit_job_start_fact()'s wire format is UNCHANGED (still writes start_time for on-disk layout continuity) -- only the READ side stopped consuming it. Pre-fix WAL_JOB_DISPATCH/WAL_JOB_START entries (_pad flag unset) fall back to the exact pre-fix behavior, an accepted one-time migration-boundary gap consistent with every other fix in this campaign. TDD: RED confirmed by temporarily reverting just the two defect lines (job.c's start_time stamp; persist.c's pidinfo gate) and running the full suite -- 5 genuine new failures (test_dc_success_stamps_start_time_synchronously, test_dc_success_appends_wal_job_dispatch_durably, test_wjs_apply_committed_entry_adopts_pidinfo_when_already_run_from_dispatch, test_wjs_restart_replay_recovers_dispatch_start_time_and_started_pidinfo, test_ace_job_start_sets_start_time) plus the pre-existing baseline (2 known bug-1561 test_config_repl.c failures + 1 concurrent bug-1696 test_job_array_replay_fence.c failure, both left untouched). GREEN after restoring the fix: rebuilt clean (0 compile errors) in an isolated dir (avoided the live HA soak's build-asan), full controller_unit_test run 3x direct (ASAN_OPTIONS=detect_leaks=0, matching this binary's own meson.build env) -- consistently "1198 Tests 3 Failures 0 Ignored", exactly the known baseline, 0 new failures. Live manual verification: single-node controller+scheduler+xcecd_virtual worker deployed via deploy-ha.sh (isolated port 19991, isolated workdir, no interference with the concurrent bug-1696 soak); 4 real xce_test jobsub->dispatch->finish cycles via the REPL all showed nonzero StartTime in `joblist` immediately (e.g. jobid 0-14-...-55018: StartTime 1787718938, matching SubmitTime). pidinfo's live collision path (jstart is a worker-mode-only REPL command, racy/awkward to trigger against a job a real worker is also concurrently finishing) was left to the deterministic unit tests above rather than a flaky live race -- covered exhaustively there at the exact apply_committed_entry()/on_job_start() call sites. Files: src/controller/job.c, src/controller/persist.c, src/include/persist.h, src/include/wal_types.h, src/controller/testsuite/{test_wal_job_dispatch_fact.c,test_wal_job_start_fact.c,test_dispatch_commit.c,test_apply_committed_entry.c,test_repl_apply.c (weak stub signature only)}. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1679, bug-1687, bug-1678, bug-1690, bug-1691, bug-1692 diff --git a/.wolf/bugs/bug-1698.md b/.wolf/bugs/bug-1698.md new file mode 100644 index 00000000..0152adbb --- /dev/null +++ b/.wolf/bugs/bug-1698.md @@ -0,0 +1,38 @@ +--- +id: "bug-1698" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T07:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (ha_connect_finish_peer", "reconnect_lane_fn", "ha_connect_to_peer)"] +tags: ["controller", "ha", "peer-connection", "reconnect", "ha_connect_finish_peer", "live-deployment", "found-live-no-restart", "partially-root-caused", "needs-live-repro", "needs-owner", "not-a-zombie-process-artifact"] +related: ["bug-1696", "bug-854", "bug-1487"] +legacy: true +--- + +# bug-1698: User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster, build.dbg): after… + +## Description +User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster, build.dbg): after ./scripts/start-all.sh reported 'Controller cluster healthy (primary elected)', ctrl-1's log showed it perpetually retrying ha_connect_finish_peer() against both peer[1] (localhost:9902) and peer[2] (localhost:9903) every ~3s for 8+ minutes straight (120+ attempts, INFO-level 'connecting outbound to peer[N]' logged each time -- the SUCCESS-path log line per ha_connect_finish_peer's own code, not a failure), never once completing a working peer connection. Meanwhile ctrl-2/ctrl-3 were actively, healthily replicating with each other (real AppendEntries traffic, LSN steadily advancing) -- they formed a working 2-of-3 quorum excluding ctrl-1 entirely. `xceadmin scheduler record status` failed with 'no primary available: timed out contacting localhost:9901' (ctrl-1's own client port) even though `xce_test`'s REPL joblist worked fine against the real 2-node quorum. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +PARTIALLY CONFIRMED, NOT FULLY ROOT-CAUSED. Ruled out as a test-harness/environment artifact: independently confirmed via a completely fresh, isolated 3-node deploy (~/tmp/xce-ha-fresh-repro, same build.dbg binaries, same run-ha-soak.sh topology) that the SAME symptom (impossible term jump, standby permanently failing cluster_init_phase2) reproduced ONCE due to a genuine same-session zombie-process port collision (see cerebrum.md's 2026-08-26 Do-Not-Repeat entry) -- but that specific mechanism does NOT apply to /tmp/xce-ha-deploy itself: confirmed via `ss -tlnp` that no live process holds ports 9901-9903 at investigation time, and no OTHER unrelated cluster shares those specific ports (9991-9993 was the colliding zombie's port range, unrelated). Direct on-disk evidence via `waldump` on the STOPPED cluster's data dirs CONFIRMS a real, non-artifactual divergence: ctrl-1's WAL has only 138 entries (lsn 1..138, all term=2, CRC chain intact, no gaps) while ctrl-2 has 2648 entries and ctrl-3 has 2641 entries (both showing term=3 activity in ha-term.bin, one of which -- ctrl-3 -- legitimately recorded voted_for_name='ctrl-2', a NORMAL vote-grant record, not corruption). This proves ctrl-1 was genuinely functioning (as primary, at term=2, given the job data the user's own joblist output showed) up through WAL entry 138, then something caused it to lose contact with both peers, a new election (term=3) happened among ctrl-2/ctrl-3 without it, and ctrl-1's own RECONNECT mechanism (ha_connect_finish_peer, dispatched via the SlowLane per raft-drive-async task 2.1's offload design) never once succeeded in re-establishing a working connection for the remainder of the run, despite logging its own SUCCESS path (endpoint_connect()+iomgr_monitor() both returning >=0) on every single attempt. This strongly suggests the bug is NOT in the initial connect() call itself (which appears to succeed at the socket/iomgr-registration level every time) but in whatever HIGHER-LEVEL protocol step is supposed to follow a successful low-level connect (completing a vote/ctrlhello handshake, registering the peer in standby_endpoints, or some per-peer state that should be cleared/retried differently after a genuine peer-loss) -- that step silently never completes or never gets attempted, so the SAME low-level connect()+iomgr_monitor() cycle repeats every ~3s forever without progress. Could NOT be reproduced live/interactively -- no debugger/ptrace access to the (already-stopped) real deployment's processes was available; this analysis is from static log/WAL evidence only, not a live capture of the actual failure in progress. FURTHER REPRO ATTEMPTS (2026-08-26, all failed to reproduce): (a) fresh 3-node deploy, kill+restart both standbys simultaneously while primary stays alive -- clean reconnect both times; (b) same, with continuous background jobsub load throughout -- clean reconnect (one transient ECONNREFUSED-then-retry-succeeds cycle observed, not a stuck state); (c) user-supplied exact recipe (submit a mixed array of jobs giving RUN/PEND/DONE states, then xceadmin scheduler record status) repeated 10x against a genuinely healthy full-component deploy (3x controller + worker + scheduler + xceadmin) -- succeeded cleanly every time; (d) 8x fresh-boot-only attempts (no artificial kills at all) immediately running reslist/workerlist/jobsub/joblist in sequence within the first ~5s of boot, matching the user's report that the real failure occurred within MINUTES of start-all.sh with no explicit kill -- all 8 clean, no crash, no elevated reconnect-retry counts. Total: 20+ reproduction attempts across 4 distinct scenario shapes, zero reproductions. This is a genuinely rare, timing-sensitive condition, not reliably triggerable by any tested scenario shape so far. + +## Fix +NOT YET ATTEMPTED. Needs a live reproduction with process-level tracing (gdb/strace on a debug build) or targeted new logging in ha_connect_finish_peer/reconnect_lane_fn's post-connect handshake path to see exactly which step silently fails to progress after a genuine peer-loss (RDHUP) event, as distinct from initial cold-boot connection establishment (which DOES work correctly, per the fresh-repro soak's clean PASS with normal kill/restart/reconverge cycles). Next investigation step: reproduce a genuine mid-run peer-loss (not a driver-initiated kill, which already reconverges fine per the fresh-repro soak's PASS) -- e.g. a transient network blip or a peer restart landing in an unlucky window -- under gdb/strace, and trace ha_connect_finish_peer's full call chain including whatever runs after endpoint_connect()/iomgr_monitor() succeed (the ctrlhello/vote-channel handshake) to find where it silently stalls. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1696, bug-854, bug-1487 diff --git a/.wolf/bugs/bug-1699.md b/.wolf/bugs/bug-1699.md new file mode 100644 index 00000000..6d0083db --- /dev/null +++ b/.wolf/bugs/bug-1699.md @@ -0,0 +1,38 @@ +--- +id: "bug-1699" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-08-26T07:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c or src/controller/cluster.c (exact function not identified -- crash occurred somewhere in the ctrlstatus request-handling path shortly after raft_cb_become_follower/first_boot=1)"] +tags: ["controller", "ha", "segfault", "sigsegv", "unreproduced", "rare-race", "found-via-soak", "needs-live-repro", "needs-owner", "core-dump-inaccessible"] +related: ["bug-1696", "bug-1698"] +legacy: true +--- + +# bug-1699: ctrl-1 segfaulted (SIGSEGV, core dumped -- confirmed via bash's own accurate job-control message… + +## Description +ctrl-1 segfaulted (SIGSEGV, core dumped -- confirmed via bash's own accurate job-control message, not a misread) within ~1.5s of a genuinely fresh boot, immediately after processing its first client 'ctrlstatus' probe request, during a fresh 10-round --chaotic run-ha-soak.sh --chaotic invocation (build.dbg, no ASAN) launched to verify bug-1696. Driver reported 'FAIL: no primary elected within ~90s' as a consequence (ctrl-1 never came back; ctrl-2 was subsequently Killed by cleanup). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT ROOT-CAUSED -- occurred exactly ONCE and could not be reproduced despite two dedicated repro attempts: (1) re-running the identical build.dbg chaotic soak under run-ha-soak.sh's built-in GDB_WRAP=ctrl-1 debug seam caught only a benign SIGPIPE (gdb's own default stop-on-SIGPIPE behavior firing on a normal peer-disconnect-during-TLS-write event that the real code already ignores via signal(SIGPIPE, SIG_IGN) at netio.c:1250 -- confirmed this was a gdb artifact, not the real crash, since production code never sees this signal); (2) a manual gdb session with `handle SIGPIPE nostop noprint pass` set before `run` (correctly letting SIGPIPE pass through), driving 15+ rapid ctrlstatus probes against a freshly-booted ctrl-1 via xce_test's REPL (mirroring the exact probe pattern run-ha-soak.sh's own wait-primary/discover_primary functions use), did NOT reproduce any crash. (3) A full, from-scratch 10-round --chaotic soak (33 jobs, 10 kill/elect cycles) run under a fresh ASAN rebuild (src/build-asan-1696repro) completed with a clean PASS (committed_divergence=0 committed_job_loss=0) and no SIGSEGV/ASAN report at all. This strongly suggests either a genuinely rare, narrow-window race (possibly specific to cold dynamic-linker/first-request timing that a debug or ASAN build's different memory layout/timing perturbs away) or an environment-specific fluke from that one specific boot -- NOT a deterministic, reliably-triggerable defect. No core file was accessible for postmortem gdb analysis (systemd-coredump is configured as the core_pattern handler but coredumpctl is not installed/reachable in this sandbox). FURTHER NOTE (2026-08-26): did not recur across the same 20+ reproduction attempts made while investigating bug-1698 (fresh boots, standby kill/restart cycles, load, exact user recipe replay) -- still zero occurrences since the single original sighting. FURTHER NOTE (2026-08-26, cont.): USER-SUPPLIED core dumps from the real deployment (4 core.xce_controller.*.zst files today) confirm this is a real, recurring heap corruption (glibc malloc_printerr: "unaligned tcache chunk detected", detected inside endpoint_get()/calloc for a 312-byte endpoint_t -- the abort site is downstream of the actual corruption, not the cause). Confirmed the crash DID occur at least once in this session too (attempt 7 of an earlier 8-attempt loop) but was MISSED by an earlier flawed stderr-grep detection method (setsid nohup ... & disown fully detaches a process from the shells job table, so bash never prints a Aborted/Segmentation fault line for it -- a real methodology gap, not evidence of rarity). Redone with PROPER job-control-based exit-code detection: 15 attempts under a fresh ASAN build (build-asan-1696repro) = 0 reproductions; 15 more attempts with all 3 nodes wrapped in a properly-SIGPIPE-configured gdb session against build.dbg (where it previously reproduced) = 0 reproductions. Total 30 further attempts, zero catches. Hypothesis: ASANs different allocator/timing may suppress the race entirely (consistent with 0/15 under ASAN); build.dbg without a live gdb/gdb overhead may still need many more attempts or a specific concurrent-connection pattern not yet isolated. bug-1698s stuck-reconnect symptom and this heap corruption are plausibly the SAME root cause (both point at endpoint_t lifecycle -- corrupted/UAFd endpoint state could equally explain a connection that never completes ITS handshake). Next step if revisited: run MANY more attempts under build.dbg with proper exit-code detection (not ASAN, which appears to suppress it), or request the user run the real deployment under a debug build with core dumps enabled and share the NEXT core file for backtrace analysis the same way this session did. + +## Fix +NOT ATTEMPTED -- cannot fix an unreproduced crash responsibly. If this recurs, capture a live gdb session (with SIGPIPE explicitly passed through, per this investigation's corrected GDB_WRAP-equivalent technique) or install/access a working coredumpctl-equivalent BEFORE the crashing process is cleaned up, to get a real backtrace with symbols. Also worth checking core_pattern/ulimit -c in the actual execution environment ahead of time so a future occurrence leaves a usable core file automatically. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1696, bug-1698 diff --git a/.wolf/bugs/bug-1700.md b/.wolf/bugs/bug-1700.md new file mode 100644 index 00000000..c4b788e3 --- /dev/null +++ b/.wolf/bugs/bug-1700.md @@ -0,0 +1,42 @@ +--- +id: "bug-1700" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T16:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/cluster.c (myreq_handler", "~line 406-414)"] +tags: ["controller", "xceadmin", "redirect", "primary-discovery", "OPCode_schedAdmin", "OPCode_upgradeConfig-adjacent", "usability", "found-live", "root-caused", "fix-scoped", "low-risk", "code-fix-landed", "live-verified", "unblocked-by-bug-1701", "see-bug-1701"] +related: ["bug-1698", "bug-854", "bug-1701"] +legacy: true +--- + +# bug-1700: User-reported: `xceadmin scheduler record status` (and by the same mechanism, `record… + +## Description +User-reported: `xceadmin scheduler record status` (and by the same mechanism, `record start`/`record stop`) fails with `Error: controller returned status -10014` whenever the `--controller`/default address happens to point at a controller that is currently a STANDBY, not the primary. User's expectation: 'xceadmin/libxce should find the correct primary to talk to it' instead of surfacing the raw rejection. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +FULLY ROOT-CAUSED, gap is server-side only. `-10014` is `E_XCE_STANDBY` (errnox.h, XCE_ERRNO_BASE+13) -- ha_dispatch_gate() (ha.c:4745-4775) correctly rejects OPCode_schedAdmin on a CTRL_STANDBY node (it is absent from that role's opcode allow-list, by design -- schedAdmin is a primary-only forward-to-scheduler operation). The gate's generic caller in myreq_handler (cluster.c:406-414) sends the rejection reply with `reply.status = rc; reply.data_which = RequestReply_data_nodata;` and NEVER calls fill_redirect_reply() (join.c:27-31, the exact same one-line helper OPCode_join's handler already uses successfully) -- so redirectHost/redirectPort are left at their zero-value defaults on EVERY dispatch-gate rejection, for EVERY opcode, not just schedAdmin (OPCode_upgradeConfig has the identical, already-documented gap -- see test_e2e_live_ha_push.py's module docstring, confirmed 2026-07-30-era). Critically, this is NOT a client-side deficiency: xceadmin's redirect-following client logic is ALREADY fully implemented, generic/opcode-agnostic, and well-tested (`xceadmin.discovery.follow_redirects()`, with a full test suite covering 1-hop/3-hop/too-many-hops/timeout cases; `xceadmin.scheduler._send_admin_request()` already calls it for every record_start/stop/status call). `_redirect_target()` (discovery.py:80-89) correctly requires BOTH redirectHost and redirectPort to be non-empty/non-zero to treat a reply as a redirect -- given the gate-rejection reply supplies neither, the client correctly concludes there is nothing to follow and surfaces the bare rejection status as-is. The client is not 'giving up' -- it is working exactly per its documented contract given the reply it receives; the whole path already works end-to-end against a properly-behaving server (test_cli_scheduler_record_start_follows_redirect_to_primary already exercises the client side against a mock that DOES send a redirect). + +## Fix +PARTIALLY FIXED (2026-08-26), live verification FAILS -- see bug-1701 for the blocking root cause, do not close this bug until that is resolved. IMPLEMENTED, unit-tested, and CORRECT exactly as scoped: myreq_handler's generic ha_dispatch_gate rejection path (cluster.c ~410-433) now calls fill_redirect_reply(&reply, &c->ha) when, and only when, rc == -XCE_ERR_STANDBY -- confirmed via ha_dispatch_gate's full body (ha.c:4573-4781) that -XCE_ERR_STANDBY (CTRL_STANDBY's default-case rejection, ha.c:4774) and -XCE_ERR_NOT_READY (CTRL_INIT/CTRL_ELECTING, and CTRL_PRIMARY's config-serve-fence / upgrade-materialization-fence / no-write-quorum branches) are the ONLY two nonzero codes the gate can return -- no third code path exists. -XCE_ERR_NOT_READY is deliberately excluded (no settled primary to redirect to in those states). RED/GREEN proof: added test_schedadmin_standby_with_known_primary_redirects (FAILED pre-fix with 'Expected 10.0.1.7 Was ''', PASSED post-fix) and test_schedadmin_standby_no_known_primary_no_redirect (regression companion, passes in both states) to test_sched_admin_dispatch.c, driving the real myreq_handler via myreq_handler_for_test with OPCode_schedAdmin + CTRL_STANDBY role. Full controller_unit_test: 1200 Tests, 2 Failures both times (pre- and post-fix) -- exactly the pre-existing bug-1561 overlayfs pair (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), zero new failures. Encode-path NULL-safety CONFIRMED (not assumed): ha_state_t.primary_host is a fixed char[XCE_MAXNAME_LEN] buffer embedded in ha_state_t (ha.h:161), never a pointer, so its address is never NULL; encode_RequestReply (controller.capnp.c:13777-13786) gates strictly on `s->redirect_host != NULL` and falls back to an explicit empty string/0 otherwise; decode_RequestReply (controller.capnp.c:13841) strdup()s the wire text unconditionally (so a client-side redirect_host is always a real, non-NULL, possibly-empty C string, never NULL) -- an unknown primary therefore safely round-trips as redirectHost='' / redirectPort=0, which xceadmin's discovery.py _redirect_target() (host and port both required truthy) already correctly treats as 'no redirect, use this reply as-is'. No NULL-pointer/crash risk anywhere in this path, confirmed by code inspection of every hop, not assumed. LIVE VERIFICATION FAILS, however: deployed a fresh 3-node cluster twice (deploy-ha.sh, controller+worker+scheduler via scripts/start-all.sh) -- once on a binary built from this fix (git stash of cluster.c to produce a clean before-binary, git stash pop to restore it for the after-binary; xce_test ctrlstatus confirmed ctrl-1 elected primary, ctrl-2/ctrl-3 standby, in both deploys). `xceadmin scheduler record status --controller localhost:9992` (a standby) against BOTH the pre-fix and the post-fix binary produced the IDENTICAL result: 'Error: controller returned status -10014', exit 1, no 'redirected: ...' line on stderr either time -- i.e. this fix has ZERO observable effect on the user-reported symptom in a real deployment. Root cause of the no-op: ha_state_t.primary_host/primary_port (ha.h:161-162) are NEVER WRITTEN by any production code path on a standby -- confirmed by grepping every .c file in src/ for an assignment site (none exists outside test fixtures, which set it manually) and by ctrlstatus.c's own pre-existing doc comment (line 38/95: 'the vestigial, never-written ha->primary_host') -- so fill_redirect_reply() always sees an empty string / 0 on a real standby, in every state, permanently, not just during the 'hasn't learned yet' transient window this bug's own original root-cause writeup anticipated and asked to be verified. This is independently corroborated by the ALREADY-CLOSED bug-854 (this bug's own related_bugs entry), whose root_cause section states the identical fact ('primary_host is vestigial/never-written') and whose fix explicitly says 'Deliberately NOT adding server-side primaryName source -- it is only a latency optimization; the client rotation fix alone resolves the hang' for the DIFFERENT ctrlstatus/client-rotation code path. New implication surfaced by this session (see bug-1701): since fill_redirect_reply() is fed by the same permanently-empty field for EVERY caller, OPCode_join's pre-existing, supposedly-successful use of the exact same helper (join.c:277) is, by the same logic, equally non-functional against a live standby in production today -- this was not previously flagged anywhere in the bug log. Per this bug's own instructions, NOT papering over this with a workaround guessed in this session -- stopping for review. The gate-rejection code-path change above is correct, tested, and worth keeping (it is necessary infrastructure and fixes the OPCode_upgradeConfig-adjacent gap's code shape identically to join.c's precedent), but closing the user-visible bug additionally requires a second, larger, currently-unscoped change: something must actually populate primary_host/primary_port on a standby from live heartbeat/AppendEntries/ctrlhello traffic (touches ha.c, which has other work in flight this session was told not to touch), OR xceadmin needs a bug-854-style client-side master-ring rotation fallback for schedAdmin/upgradeConfig (mirroring xcecli_ctrlstatus_primary's fix) so an empty-redirect standby reply drives a retry against the next configured master instead of surfacing the bare rejection. See bug-1701 for the dedicated tracking of the underlying data-gap finding. + +UPDATE (2026-08-26): bug-1701 is now FIXED (local peers_cfg lookup in handle_ctrlhello, src/controller/config_repl.c:2170-2246 -- see bug-1701's own entry for the full fix writeup) and LIVE-VERIFIED end to end, closing the gap that made this bug's fix a live no-op. Re-ran the exact failing scenario from this bug's own live-verification section: fresh 3-node cluster, `xceadmin scheduler record status --controller ` now prints `redirected: -> ` on stderr and the real recording status on stdout, exit 0 -- confirmed against both standbys in the cluster and again after a re-election changed which node was primary. This bug's own code change (myreq_handler's fill_redirect_reply() call on -XCE_ERR_STANDBY, cluster.c ~410-433) required NO further modification -- it was correct all along; it only needed a real, populated ha->primary_host/primary_port to redirect to, which bug-1701 now supplies. Status: FIXED, LIVE-VERIFIED. + +Original PENDING fix text (superseded by the above, kept for history): PENDING. Minimal, single-call-site fix: in myreq_handler's generic ha_dispatch_gate rejection path (cluster.c:406-414), when the rejection reason is specifically -XCE_ERR_STANDBY (i.e. this node knows or may know who the primary is) call fill_redirect_reply(&reply, &c->ha) before xce_reply(), exactly mirroring OPCode_join's existing handle_worker_join()-adjacent usage (join.c:277). This fixes OPCode_schedAdmin AND the already-documented OPCode_upgradeConfig gap simultaneously, with ZERO xceadmin/client-side changes needed -- the existing follow_redirects() mechanism will start working immediately once the server populates the fields. Must verify: (a) ha_state_t.primary_host/primary_port are populated/valid on a CTRL_STANDBY node that has learned of a live primary (the common case) vs. a node that genuinely does not know any primary yet (mid-election, or has never heard from one) -- fill_redirect_reply() unconditionally aliases these fields with no NULL/empty guard, so confirm what a standby's primary_host looks like before any primary is ever discovered (likely NULL/empty, in which case the reply's redirectHost would encode as empty/absent and the client's existing `if host and port` guard already handles this correctly as 'no redirect' -- but verify, don't assume); (b) add a regression test proving a schedAdmin request to a standby-with-known-primary now receives a populated redirect reply (mirroring test_join_redirect.c's existing pattern, `test_redirect_reply_carries_primary_address`) and a live xceadmin CLI test (extending test_e2e_live_ha_push.py's pattern) proving `xceadmin scheduler record status` against ANY of the 3 nodes now transparently reaches the primary and succeeds. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1698, bug-854, bug-1701 diff --git a/.wolf/bugs/bug-1701.md b/.wolf/bugs/bug-1701.md new file mode 100644 index 00000000..c46bc1aa --- /dev/null +++ b/.wolf/bugs/bug-1701.md @@ -0,0 +1,38 @@ +--- +id: "bug-1701" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.h:161-162 (ha_state_t.primary_host/primary_port)", "src/controller/join.c:27-31 (fill_redirect_reply)", "src/controller/ctrlstatus.c:38 (pre-existing doc comment already naming this)"] +tags: ["controller", "ha", "primary-discovery", "redirect", "primary_host", "config_repl.c", "handle_ctrlhello", "join.c", "fill_redirect_reply", "found-live", "root-caused", "live-verified", "cross-cutting"] +related: ["bug-1700", "bug-854"] +legacy: true +--- + +# bug-1701: Discovered while implementing/live-verifying bug-1700's fix (myreq_handler's generic dispatch-gate… + +## Description +Discovered while implementing/live-verifying bug-1700's fix (myreq_handler's generic dispatch-gate rejection now calls fill_redirect_reply() on -XCE_ERR_STANDBY, cluster.c ~410-433): the fix has ZERO effect in a real deployment. `xceadmin scheduler record status --controller ` against a live 3-node cluster still fails with 'Error: controller returned status -10014' identically before and after the fix, with no 'redirected: ...' hop ever printed. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_state_t.primary_host/primary_port are NEVER WRITTEN by any production code path on a standby -- confirmed by grepping every assignment site across the entire src/ tree (`grep -rn primary_host src/ --include=*.c --include=*.h`): the only writers are test fixtures (test_join_redirect.c, test_p1_integration.c, test_p3_hardening.c, test_ha_redirect.c, test_ha_role_lifecycle.c, this bug's own new test_sched_admin_dispatch.c tests) manually poking the field for isolated unit tests; no heartbeat/AppendEntries/ctrlhello handler in ha.c ever copies the leader's identity into it on the receiving standby. ctrlstatus.c's own field-sourcing doc comment (lines 37-38, 93-101) already names this explicitly ('the vestigial, never-written ha->primary_host') and documents that ctrlstatus_fill() deliberately derives primaryName from role + voted_for_name instead, NEVER from primary_host, for exactly this reason -- so the gap was known and worked around for ctrlstatus, but not for the redirect-reply mechanism (fill_redirect_reply/redirect_host/redirect_port), which has no equivalent workaround and is the ONLY consumer of this permanently-empty field. bug-854 (2026-06-21, CLOSED/FIXED) independently documented the identical fact while root-causing a related but distinct bug (positional client-side primary discovery for ctrlstatus) and explicitly chose NOT to add a server-side primaryName/primary_host source, fixing that bug entirely client-side (xcecli_ctrlstatus_primary master-ring rotation) instead -- a reasonable call for THAT bug, but it leaves fill_redirect_reply() as a piece of server-side machinery that looks functional (it has a real caller, a real unit test proving its own field-copying logic, and a real capnp wire encode/decode path) yet is fed a field that is permanently empty in every real deployment. Direct consequence: OPCode_join's existing use of fill_redirect_reply() (join.c:277, the very precedent bug-1700's fix was told to mirror) is, by the same reasoning, ALSO non-functional against a live standby in production today -- a worker/scheduler/controller join redirect to the primary silently never happens; nothing in the existing test suite (test_join_redirect.c) catches this because those tests call fill_redirect_reply() directly with a manually-populated ha_state_t, never through a live standby that only knows what production heartbeat/election code actually wrote into it. + +## Fix +FIXED (2026-08-26) via option (a): a pure LOCAL lookup in handle_ctrlhello() (src/controller/config_repl.c:2170-2246, inside the existing primary_ctrl_fd==-1 idempotency-guarded block, ~2185-2233), NOT a new wire-protocol field -- every node already statically knows every candidate's host:port from boot (c->myconf->peers_cfg, parsed once from --peers). At the exact moment a standby first establishes primary_ctrl_fd (the same instant it 'discovers' its primary), it now matches req->from->name (populated for EVERY incoming request at src/lib/request.c:379, `req->from = ep->peer;` -- the same name= identity used in --peers name=host:port) against c->myconf->peers_cfg.candidates[] by name (loop pattern mirrors the existing voted_for_name-by-index lookup in ha_init_raft, ha.c:7822-7829) and, on a match, copies that candidate's hostname/port into ha->primary_host/primary_port. No match (real config mismatch) or a NULL req->from/name leaves primary_host/primary_port at their previous value (never wiped) and logs a WARN identifying the unmatched sender name. Locking: ha.h's `lock` field doc explicitly says 'Protects primary_host/port + voted_for_term/name'; the existing writers of voted_for_name (the other field under the same lock, ha.c:2304-2332 and ha.c:5559-5593) take c->ha.lock around the write, so the new write here does too (pthread_mutex_lock/unlock around the strncpy+port assignment only -- the name-matching loop itself runs outside the lock, matching the narrow critical-section style of the precedent). Types verified by direct read, not assumed: ha_state_t.primary_host is `char primary_host[XCE_MAXNAME_LEN]` (ha.h:161, XCE_MAXNAME_LEN=256, config.h:24) and primary_port is `uint32_t` (ha.h:162) -- NOT matching the task brief's assumption that primary_port's type already lined up with xce_candidate_t.port (`int`, cluster.h:22); the int->uint32_t assignment is a safe implicit widening for any real port value (0-65535), cast added for clarity, no risky truncation. No clear-on-primary-loss needed: ha_on_primary_ctrl_fd_rdhup (ha.c, read-only verified, not modified) resets primary_ctrl_fd back to -1 on genuine primary loss, so the next real ctrlhello (from the same or a new primary) re-satisfies the same idempotency guard and naturally re-runs this lookup -- confirmed by reading ha.c:8594-8680, no explicit clear required. RED/GREEN proof: added test_handle_ctrlhello_populates_primary_host_from_known_peer (FAILED pre-fix: "Expected '10.0.1.7' Was ''"; PASSED post-fix) and test_handle_ctrlhello_unknown_peer_leaves_primary_host_unchanged (edge case: unmatched sender name -- no crash, primary_host/primary_port stay at fresh-node empty/zero, primary_ctrl_fd bookkeeping still proceeds) to src/controller/testsuite/test_ha_connection_reuse.c (same file as the pre-existing test_handle_ctrlhello_sets_primary_ctrl_fd, which continues to pass unchanged, proving the idempotency behavior for primary_ctrl_fd itself is untouched). Full controller_unit_test: 1202 Tests, 2 Failures both pre- and post-fix -- exactly the pre-existing bug-1561 overlayfs pair (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), zero new failures. LIVE END-TO-END VERIFIED -- the exact scenario bug-1700 failed on: deployed two independent fresh 3-node clusters (deploy-ha.sh) on git-stash-isolated before/after controller binaries (stashed only config_repl.c + its test file, leaving other in-flight ha.c/job.c/persist.c work untouched). BEFORE (pre-fix binary): `xceadmin scheduler record status --controller localhost:29992` (a standby) -> `Error: controller returned status -10014`, exit 1, no redirect line -- reproduces bug-1700's original live-verification failure exactly. AFTER (post-fix binary, fresh deploy + also re-verified after clearing controller data to force a different node to win election): `xceadmin scheduler record status --controller ` -> stderr prints `redirected: localhost:19991 -> localhost:19992` (confirmed via separate stdout/stderr capture: the redirect line is genuinely on stderr, the status body on stdout, matching xceadmin.discovery.follow_redirects()'s documented behavior), stdout prints the real recording status (`recording: no`, etc.), exit 0 -- verified against BOTH standbys in the cluster and re-verified after a re-election flipped which node was primary (ctrl-2 elected instead of ctrl-1), proving the fix is dynamic/general, not coincidentally tied to one node. This unblocks bug-1700's fix, which was otherwise correct and complete but fed by this permanently-empty field -- see bug-1700's own entry for its updated live-verified confirmation. OPCode_join's pre-existing fill_redirect_reply() usage (join.c:277) is also now unblocked by the same fix (not separately re-verified live in this session, but shares the identical previously-broken data source). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1700, bug-854 diff --git a/.wolf/bugs/bug-1702.md b/.wolf/bugs/bug-1702.md new file mode 100644 index 00000000..681d9b24 --- /dev/null +++ b/.wolf/bugs/bug-1702.md @@ -0,0 +1,38 @@ +--- +id: "bug-1702" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test jobrun -m host strncpy into 256B stack buffer unbounded (src/harness/test.c)" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:2130-2136 (cmd_jobrun)"] +tags: ["controller", "testsuite", "xce_test", "cmd_jobrun", "buffer-overflow", "memory-safety", "cli-ux-audit"] +related: ["bug-1703"] +legacy: true +--- + +# bug-1702: cmd_jobrun's `-m :` parsing copies the host-name substring into a fixed 256-byte… + +## Description +cmd_jobrun's `-m :` parsing copies the host-name substring into a fixed 256-byte stack buffer with an unbounded strncpy (length = bytes before ':' in the user-supplied argument, never clamped to sizeof(buf)). A -m argument with a hostname longer than 256 bytes before the ':' overflows the stack buffer. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +`strncpy(buf, optarg, (int)(p - optarg))` uses the attacker/user-controlled substring length directly with no `MIN(..., sizeof(buf)-1)` clamp, unlike sibling code elsewhere in the file that does bound similar copies. + +## Fix +NOT YET FIXED. Recommended: clamp the copy length to sizeof(buf)-1 (or reject a host-name segment longer than the buffer with a usage error) before strncpy. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1703 diff --git a/.wolf/bugs/bug-1703.md b/.wolf/bugs/bug-1703.md new file mode 100644 index 00000000..891185b8 --- /dev/null +++ b/.wolf/bugs/bug-1703.md @@ -0,0 +1,38 @@ +--- +id: "bug-1703" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test jobsub -n min strncpy into 64B stack buffer unbounded (src/harness/test.c)" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:2246-2254 (cmd_jobsub)"] +tags: ["controller", "testsuite", "xce_test", "cmd_jobsub", "buffer-overflow", "memory-safety", "cli-ux-audit"] +related: ["bug-1702"] +legacy: true +--- + +# bug-1703: cmd_jobsub's `-n ,` parsing copies the min-value substring into a fixed 64-byte stack… + +## Description +cmd_jobsub's `-n ,` parsing copies the min-value substring into a fixed 64-byte stack buffer with an unbounded strncpy (length = bytes before ',' in the user-supplied argument, never clamped to sizeof(buf)). A -n argument with 64+ characters before the comma overflows the stack buffer. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same pattern as bug-1702: `strncpy(buf, optarg, (int)(p - optarg))` with no bound check against sizeof(buf). + +## Fix +NOT YET FIXED. Recommended: clamp the copy length to sizeof(buf)-1 (or reject) before strncpy, same fix shape as bug-1702. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1702 diff --git a/.wolf/bugs/bug-1704.md b/.wolf/bugs/bug-1704.md new file mode 100644 index 00000000..be748c30 --- /dev/null +++ b/.wolf/bugs/bug-1704.md @@ -0,0 +1,38 @@ +--- +id: "bug-1704" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:2663 (declaration)", ":2668-2688 (getopt loop)", ":2695 (queuectrl.action = action)"] +tags: ["controller", "testsuite", "xce_test", "cmd_qctrl", "uninitialized-variable", "undefined-behavior", "cli-ux-audit"] +related: [] +legacy: true +--- + +# bug-1704: cmd_qctrl declares `int action;` with no initializer and only assigns it inside the -a/-i/-o/-c… + +## Description +cmd_qctrl declares `int action;` with no initializer and only assigns it inside the -a/-i/-o/-c getopt cases; there is no post-loop check that one of them was given before `queuectrl.action = action` is sent over the wire. Running `qctrl -q normal` with no action flag (syntactically valid per USAGE_QCTRL, which lists all four as optional) transmits indeterminate stack memory as the queue action. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Missing initializer + missing post-loop validation, unlike the very next sibling command cmd_hostctrl which initializes action=HOST_DO_NONE and explicitly rejects that sentinel. + +## Fix +FIXED (2026-09-09, xce-test-cli-ux task 3.1, flagspec migration). cmd_qctrl now initializes action = QUEUE_DO_NONE (xce/types.h pre-existing sentinel, same enum family as HOST_DO_NONE) before the getopt_long() loop, and rejects the invocation post-loop if action is still QUEUE_DO_NONE after parsing -- mirroring cmd_hostctrl's exact HOST_DO_NONE pattern. Landed as part of migrating cmd_qctrl (and 16 other flag-accepting commands) from getopt() to the shared flagspec_t + getopt_long() mechanism (flagspec.c/.h, tasks 1.1/1.2). Regression-locked by test_qctrl_missing_action_flag_rejected and test_qctrl_flag_forms_short_and_long_both_accepted in src/controller/testsuite/test_test_cli.c. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1705.md b/.wolf/bugs/bug-1705.md new file mode 100644 index 00000000..95b122cb --- /dev/null +++ b/.wolf/bugs/bug-1705.md @@ -0,0 +1,38 @@ +--- +id: "bug-1705" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test jobsub -x exdata overwritten by build_job_exdata and leaked" +component: "controller" +severity: "leak" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:2243-2245 (-x parsing)", ":2181-2195 (build_job_envs unconditional overwrite)", ":2336 (unconditional call site)", "USAGE_JOBSUB at :484-485"] +tags: ["controller", "testsuite", "xce_test", "cmd_jobsub", "dead-flag", "silent-data-loss", "memory-leak", "cli-ux-audit"] +related: [] +legacy: true +--- + +# bug-1705: cmd_jobsub's `-x ` flag is documented (USAGE_JOBSUB) as overriding the built-in PATH-only… + +## Description +cmd_jobsub's `-x ` flag is documented (USAGE_JOBSUB) as overriding the built-in PATH-only exdata JSON, but build_job_exdata()/build_job_envs() unconditionally overwrite submit.start_info->exdata with the built-in PATH JSON regardless of whether -x already set a value -- the exact opposite of the documented behavior. The original -x value is also leaked (never freed before being replaced). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +build_job_exdata()/build_job_envs() have no guard for an already-non-NULL exdata field before overwriting it. + +## Fix +NOT YET FIXED. Recommended: skip build_job_exdata() when submit.start_info->exdata is already non-NULL (from -x), and free the old pointer before any legitimate reassignment. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1706.md b/.wolf/bugs/bug-1706.md new file mode 100644 index 00000000..4a994628 --- /dev/null +++ b/.wolf/bugs/bug-1706.md @@ -0,0 +1,39 @@ +--- +id: "bug-1706" +status: "duplicate" +status_updated: "2026-09-24" +status_note: "same defect as bug-1574 (set_resval() has no string case)" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:3323-3364 (set_resval", "missing Value_u_svalue case)", ":3483 (vals[] never zeroed)", ":363-364/375 (USAGE_RESUPDATE's own -t string example)"] +tags: ["controller", "testsuite", "xce_test", "cmd_resupdate", "uninitialized-variable", "silent-data-corruption", "cli-ux-audit"] +related: ["bug-1707", "bug-1712"] +duplicate_of: "bug-1574" +legacy: true +--- + +# bug-1706: cmd_resupdate's own documented example (`resupdate -n mylabel -v foo -t string`) and any… + +## Description +cmd_resupdate's own documented example (`resupdate -n mylabel -v foo -t string`) and any string-typed resource (e.g. `type`, `model`) corrupts the submitted resource value: set_resval() has no `case Value_u_svalue:` in its switch, so for a string-typed resource, vals[idx].name is left as uninitialized stack memory and the wire type stays the calloc'd default (ivalue32) instead of the string the user typed. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +set_resval()'s switch only handles fvalue/ivalue32/ivalue64/bvalue; resval_resolve_type() can resolve to Value_u_svalue (type/model, or any name with -t string) but nothing handles that case. A comment at test.c:3365-3372 acknowledges the gap as intentionally deferred by a prior task. + +## Fix +NOT YET FIXED. Recommended: add a Value_u_svalue case to set_resval() that sets name/val->u.svalue correctly, and zero-init vals[] in cmd_resupdate as defense in depth. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1707, bug-1712 diff --git a/.wolf/bugs/bug-1707.md b/.wolf/bugs/bug-1707.md new file mode 100644 index 00000000..450307a6 --- /dev/null +++ b/.wolf/bugs/bug-1707.md @@ -0,0 +1,38 @@ +--- +id: "bug-1707" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test resupdate trailing -n without -v silently dropped" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:3551-3561 (trailing-pair flush", "comment admits the drop)", ":3502-3507 (mid-list warning that trailing case lacks)", ":3567 (unconditional success message)"] +tags: ["controller", "testsuite", "xce_test", "cmd_resupdate", "silent-data-loss", "cli-ux-audit"] +related: ["bug-1706"] +legacy: true +--- + +# bug-1707: cmd_resupdate: `resupdate -n slots -v 4 -n mem` silently drops the trailing `mem` resource with… + +## Description +cmd_resupdate: `resupdate -n slots -v 4 -n mem` silently drops the trailing `mem` resource with zero diagnostic (the final pending -n/-v pair is only flushed if a value was seen), yet the command still prints "resource update ok!" as if everything the user typed succeeded. A mid-list missing -v does print a warning; only the trailing one is silent. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Trailing pending -n/-v pair flush logic only runs when resval_str != NULL, with no fallback warning path like the mid-list case has. + +## Fix +NOT YET FIXED. Recommended: apply the same "no value specified, default will be used" warning (or reject outright) to the trailing pair instead of a silent drop. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1706 diff --git a/.wolf/bugs/bug-1708.md b/.wolf/bugs/bug-1708.md new file mode 100644 index 00000000..bc650bea --- /dev/null +++ b/.wolf/bugs/bug-1708.md @@ -0,0 +1,38 @@ +--- +id: "bug-1708" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test jobkill -s non-numeric -> signal 0, still reports success" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:2572-2574 (signo = atoi(optarg))", ":2601 (unconditional success message)"] +tags: ["controller", "testsuite", "xce_test", "cmd_jobkill", "silent-failure", "cli-ux-audit"] +related: [] +legacy: true +--- + +# bug-1708: cmd_jobkill's `-s ` is parsed via unchecked atoi(); a non-numeric value (e.g + +## Description +cmd_jobkill's `-s ` is parsed via unchecked atoi(); a non-numeric value (e.g. a plausible typo of the symbolic name `SIGKILL` instead of `9`) silently becomes signo=0 (POSIX null signal, delivers nothing), yet the command still prints "Job signaled" as if it succeeded. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +No validation that -s's argument is numeric/nonzero before using it and reporting success. + +## Fix +NOT YET FIXED. Recommended: reject non-numeric/unparseable -s values with a clear error instead of silently defaulting to signal 0. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1709.md b/.wolf/bugs/bug-1709.md new file mode 100644 index 00000000..3ff6ef45 --- /dev/null +++ b/.wolf/bugs/bug-1709.md @@ -0,0 +1,38 @@ +--- +id: "bug-1709" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test jobinfo returns 0 on malformed/missing jobid" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:2029-2058 (cmd_jobinfo)"] +tags: ["controller", "testsuite", "xce_test", "cmd_jobinfo", "silent-failure", "exit-code", "cli-ux-audit"] +related: [] +legacy: true +--- + +# bug-1709: cmd_jobinfo always returns 0 (success) even when its -j jobid is malformed (parse_jobid's return… + +## Description +cmd_jobinfo always returns 0 (success) even when its -j jobid is malformed (parse_jobid's return code is discarded), when no -j is given at all, or when xcecli_jobinfo fails for every -j given -- breaking the tool's own scripted-mode exit-code-verdict contract for this command specifically. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +No error/found flag tracked across the -j loop; the function unconditionally `return 0`s regardless of whether any valid lookup succeeded. + +## Fix +NOT YET FIXED. Recommended: track an error/found flag across the loop and return nonzero (with a usage message) when no valid -j was given or every lookup failed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1710.md b/.wolf/bugs/bug-1710.md new file mode 100644 index 00000000..ce8c576d --- /dev/null +++ b/.wolf/bugs/bug-1710.md @@ -0,0 +1,38 @@ +--- +id: "bug-1710" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test hello -c <=0 reports vacuous success" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:1341 (rc=0 init)", ":1351 (count=atoi(optarg)", "unchecked)", ":1367 (while count>0 loop)", ":1373 (return rc)"] +tags: ["controller", "testsuite", "xce_test", "cmd_hello", "silent-failure", "cli-ux-audit"] +related: [] +legacy: true +--- + +# bug-1710: cmd_hello with -c 0, a negative count, or a non-numeric value (all yield count<=0 from unchecked… + +## Description +cmd_hello with -c 0, a negative count, or a non-numeric value (all yield count<=0 from unchecked atoi()) sends zero hellos (the while(count>0) loop body never runs) but still returns the still-zero-initialized rc, reporting success even though nothing was sent. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +No validation that count is >=1 before entering the loop; the initial success value is never overwritten when the loop body never executes. + +## Fix +NOT YET FIXED. Recommended: reject count<=0 with an explicit error instead of falling through to a silent vacuous success. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1711.md b/.wolf/bugs/bug-1711.md new file mode 100644 index 00000000..f5b3f6af --- /dev/null +++ b/.wolf/bugs/bug-1711.md @@ -0,0 +1,39 @@ +--- +id: "bug-1711" +status: "fixed" +status_updated: "2026-09-24" +status_note: "a8154acb" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:1435 (UNKOWN)", ":1069 (matchs)", ":2939 (sequentail)", ":1401/1404 (reslist formatting)", ":1454/1473 (workerlist formatting for contrast)", ":1551-1572 (joblist raw epoch + NTasks label)", ":1486 (job_state_str)", ":1972-1992 (print_jobinfo's raw-state fallback for contrast)", ":3632/3634 (print_grouplist/print_configlist stubs)", ":3659-3666 (getconf headers)"] +tags: ["controller", "testsuite", "xce_test", "typo", "output-formatting", "cli-ux-audit"] +related: [] +fix_commits: ["a8154acb"] +legacy: true +--- + +# bug-1711: xce_test: several command-specific typos and output-formatting inconsistencies found during a… + +## Description +xce_test: several command-specific typos and output-formatting inconsistencies found during a /cli-ux command-by-command audit: worker_status_str returns the misspelled "UNKOWN" for an unrecognized worker status bit (visible in every workerlist row hitting that path); cmd_help prints the malformed "No commands matchs `%s'" on an unknown help topic; cmd_runtasks's default launch_type for a single task is misspelled "sequentail" (display/data-only, not behavior-altering -- worker only special-cases "openmpi"/"mpich"); cmd_reslist right-justifies its Name/Description text columns (inconsistent with cmd_workerlist's left-justified equivalent columns) and prints the Flags column as unlabeled %x hex (ambiguous vs decimal, unlike this file's own "0x%x"-prefixed state_raw precedent); cmd_joblist prints SubmitTime/StartTime/FinishTime as raw Unix epoch seconds instead of using this file's own ctrlstatus_format_time_ms-style formatting, and labels a hidden mintasks,maxtasks pair as a single "NTasks" column; cmd_joblist's job_state_str has no raw-state fallback for an ambiguous RUN+EXIT state the way print_jobinfo already guards against two functions away; getconf's User Groups/Host Groups/Configs sections always print their header followed by nothing (print_grouplist/print_configlist are no-op stubs), indistinguishable from a genuinely empty cluster. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Accumulated small inconsistencies/typos/unimplemented stubs found via a systematic per-command audit; none individually severe, several visible in routine tool usage. + +## Fix +NOT YET FIXED. Recommended: fix the three typos directly; left-justify reslist's text columns and label/prefix its Flags column as hex to match workerlist/state_raw conventions; format joblist's three time columns like ctrlstatus does and relabel/split the NTasks column; either implement print_grouplist/print_configlist from the already-fetched data or print "(not implemented)" instead of silent nothing. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1712.md b/.wolf/bugs/bug-1712.md new file mode 100644 index 00000000..97a297ad --- /dev/null +++ b/.wolf/bugs/bug-1712.md @@ -0,0 +1,38 @@ +--- +id: "bug-1712" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test resupdate duplicate -v gives misleading message" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T09:32:18Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c:3525-3531 (case 'v' else-branch)"] +tags: ["controller", "testsuite", "xce_test", "cmd_resupdate", "misleading-error-message", "cli-ux-audit"] +related: ["bug-1706", "bug-1707"] +legacy: true +--- + +# bug-1712: cmd_resupdate: a duplicate -v for the same pending resource (`resupdate -n slots -v 4 -v 8`)… + +## Description +cmd_resupdate: a duplicate -v for the same pending resource (`resupdate -n slots -v 4 -v 8`) prints the misleading message "ignore res value `%s' as no res name found" even though a resource name WAS found -- the real problem is a value is already pending for it, not a missing name. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The else-branch of the -v case handles two distinct situations (no preceding -n at all, vs. a duplicate -v for an already-named pending resource) with one shared, incorrectly-worded message. + +## Fix +NOT YET FIXED. Recommended: distinguish the two cases with an accurate message for the duplicate-value case, e.g. "ignoring duplicate value '%s' for res '%s' (already has a pending value)". + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1706, bug-1707 diff --git a/.wolf/bugs/bug-1713.md b/.wolf/bugs/bug-1713.md new file mode 100644 index 00000000..3eea54dd --- /dev/null +++ b/.wolf/bugs/bug-1713.md @@ -0,0 +1,38 @@ +--- +id: "bug-1713" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-08-26T17:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request)", "possibly xceadmin/discovery.py (follow_redirects)"] +tags: ["xceadmin", "scheduler-admin", "E_XCE_NOT_READY", "E_XCE_NOMASTER", "retry", "transient-state", "primary-discovery", "bug-1700-adjacent", "bug-1701-adjacent", "found-live", "root-caused", "fix-scoped", "live-verified", "push.py-sibling-gap-flagged"] +related: ["bug-1700", "bug-1701"] +legacy: true +--- + +# bug-1713: User-reported, live 3-node cluster, right after bug-1700/bug-1701's redirect fix landed and was… + +## Description +User-reported, live 3-node cluster, right after bug-1700/bug-1701's redirect fix landed and was live-tested: `xceadmin scheduler record status --controller ` now correctly redirects ('redirected: localhost:9903 -> localhost:9901'), but the SAME command against the SAME redirect target intermittently fails with `Error: controller returned status -10013` (sometimes succeeds with real recording status, sometimes fails on an immediate repeat) instead of always succeeding. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +FULLY ROOT-CAUSED. -10013 is E_XCE_NOT_READY (errnox.h: XCE_ERRNO_BASE+12, 'Controller in CTRL_INIT or CTRL_ELECTING state'). The redirect from bug-1701's fix correctly points at whichever node was primary as of its last ctrlhello, but that node can be transiently mid-(re-)election by the time the redirected request actually arrives -- expected/normal on a freshly-booted cluster (election churn during the first several seconds), or any time a real failover is in flight. ha_dispatch_gate() returns -XCE_ERR_NOT_READY for exactly this transient window, with NO redirect populated (by design -- a CTRL_INIT/CTRL_ELECTING node genuinely has no settled primary to point to yet). The sibling code E_XCE_NOMASTER (errnox.h:22) carries an explicit design-intent comment: 'Standby has no known primary -- client should retry' -- but `_send_admin_request()` (scheduler.py:119-161) treats EVERY nonzero reply.status, including both E_XCE_NOT_READY and E_XCE_NOMASTER, as an immediate hard SchedAdminError with no retry at all. This is a documented client-side gap, not a server bug: the server-side behavior (refuse with a specific, distinguishable transient-state code) is correct and intentional; the client simply never implements the retry the errno's own comment already promises. + +## Fix +FIXED and LIVE-VERIFIED (2026-08-26). `_send_admin_request()` (src/xceadmin/src/xceadmin/scheduler.py:119-231) now wraps its whole request/redirect/decode cycle in a bounded retry loop, re-running `follow_redirects()` FROM SCRATCH each attempt (not a bare resend to the same host:port) -- a fresh attempt can discover a newly-settled or different primary once an election concludes. Retries EXACTLY `reply.status in (_E_XCE_NOMASTER, _E_XCE_NOT_READY)` (module-level constants at scheduler.py:80-111, values -10011/-10013 -- NEGATIVE, matching the real wire value confirmed via join.c:284 `rc = -E_XCE_NOMASTER` / commit.c:259 and controller.capnp:608's signed `Int32 status` field). `E_XCE_STANDBY` (-10014) is deliberately excluded -- untouched, still resolved purely by `follow_redirects()`'s existing redirect-following (bug-1701); a standby reply that DOES carry a redirect never even reaches this status check (follow_redirects consumes it internally), and one that does NOT carry a redirect is a genuine non-transient rejection that must keep failing immediately. No shared Python-side mirror of errnox.h's enum exists anywhere in xcecommon/xceadmin (confirmed by grepping the whole package) -- followed the one existing precedent instead of inventing a new convention: test_e2e_live_ha_push.py's own local `_E_XCE_STANDBY = 10014` constant with an errnox.h-citing comment. Retry budget: `_MAX_TRANSIENT_ATTEMPTS=4` total attempts (1 initial + 3 retries), `_TRANSIENT_RETRY_DELAY_S=1.5` fixed delay between attempts (`time.sleep`) -- sized at the same scale as this project's existing bounded-retry shapes (follow_redirects' own `_MAX_REDIRECT_HOPS=3`; the HA soak driver's verify_classify retry loop) since election-churn settling is single-digit-seconds, not a long outage; no exponential backoff needed at this scale. Retry-budget exhaustion still raises the existing `SchedAdminError` (no new exception type needed) but with a message appending '(still transient after N attempts over ~Xs -- giving up)' for clear distinguishability from an immediate non-retried hard failure. RED/GREEN proof (src/xceadmin/tests/test_scheduler.py): added `_SequentialMockController` (a finite, exact-connection-count multi-TLS-connection mock server, since the retry loop reconnects from scratch each attempt -- distinct from the file's existing one-shot `_MockController`) plus 5 new tests -- `test_record_status_retries_on_transient_status_then_succeeds` (parametrized NOT_READY/NOMASTER, 2 failures then success -> eventual success, FAILED pre-fix at collection time since the referenced constants didn't exist yet, and confirmed behaviorally via an ad hoc script that the pre-fix code raises immediately on a single transient reply with only 1 connection attempt), `test_record_status_retry_exhaustion_raises_distinguishable_error` (always-transient mock, `scheduler.time.sleep` monkeypatched so the test does not actually wait; asserts exact attempt count + distinguishable message), `test_record_status_standby_without_redirect_not_retried` and `test_record_status_standby_with_redirect_follows_redirect_not_retried` (regression locks: E_XCE_STANDBY is never retried, with/without a populated redirect, zero sleep calls in either case) -- all 5 PASS post-fix in 0.20s total (no real sleeping). Full xceadmin suite: 274 passed, 1 failed -- the failure (`test_live_config_push_against_standby_returns_standby_rejection_not_redirect`, test_e2e_live_ha_push.py) is PRE-EXISTING and unrelated: confirmed via `git stash push -- ` that it reproduces IDENTICALLY on the unmodified codebase -- it is a live e2e test whose own docstring already anticipated going stale once bug-1700/1701's generic dispatch-gate redirect fix (already landed this session) generalizes to `OPCode_upgradeConfig`, which it now has; zero new failures caused by this fix. LIVE VERIFICATION (real 3-node cluster, deploy-ha.sh + xce.build's real controller binary, editable-install xceadmin venv pointed at scheduler.py so before/after needed no rebuild, only a file swap): a plain fresh-boot tight loop immediately after start-all.sh did NOT naturally reproduce -10013 in this sandbox (election settles with highly variable timing, sometimes near-instantly, since `ha_has_write_quorum()`'s standby_endpoints are populated synchronously inside `raft_cb_become_leader` and are typically already satisfied by the time start-all.sh's 1s-granularity health-check first observes CTRL_PRIMARY) -- but a REAL FAILOVER (kill -9 the live primary, then hammer the two survivors with zero sleep, no health-check wait) reliably reproduced the exact user-reported symptom, which this bug's own root-cause writeup already named as an equally valid trigger ('...or any time a real failover is in flight'): pre-fix, the client surfaced a bare 'Error: controller returned status -10013' to the user at t+8.60s post-kill (exit 1); post-fix, an identical failover on a fresh cluster produced ZERO -10013/-10011 failures across the equivalent ~15s post-kill window (52 requests; every non-success was either a legitimate pre-election-completion 'connection refused' to the just-killed primary's now-dead listener, or a legitimate post-gate 'scheduler not connected' business-level SchedAdminError from this sandbox having no real xce_scheduler process deployed -- neither is a transient status code, and correctly reaches the caller unretried in both runs, proving the fix does not overreach). `push.py`'s `push_config()`/`probe_config()` (src/xceadmin/src/xceadmin/push.py:92-179) have the IDENTICAL gap -- same two transient codes, same immediate-return-as-data with zero retry -- confirmed by direct code read; NOT fixed here (out of this bug's Python-scheduler-admin-path scope, and push.py's error-vs-return convention differs from scheduler.py's raise-based one, so a proper fix there is more than a trivial shared-helper change) -- flagged as a sibling finding for a future bug, not silently left undocumented. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1700, bug-1701 diff --git a/.wolf/bugs/bug-1714.md b/.wolf/bugs/bug-1714.md new file mode 100644 index 00000000..aedc30b9 --- /dev/null +++ b/.wolf/bugs/bug-1714.md @@ -0,0 +1,137 @@ +--- +id: "bug-1714" +status: "cannot-reproduce" +status_updated: "2026-09-25" +status_note: "recipe no longer reproduces on this machine: its trigger (array pushes redelivered to the scheduler) did not occur on either build, so it was already gone before 2026-09-25's fixes; ASAN run clean. Plausibly related fixes since: bug-1864 (per-endpoint message reordering in the vtask dispatcher), bug-1769 (ALTER heap overflow write). Reopen to investigating if a scheduler segfault recurs." +component: "scheduler" +severity: "crash" +timestamp: "2026-08-26T18:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 8 +files: ["src/scheduler/ (exact crash site unknown -- no core dump", "no backtrace available for either occurrence)"] +tags: ["scheduler", "segfault", "sigsegv", "unreproduced", "no-core-dump", "coredump-capture-gap", "found-live", "needs-live-repro", "needs-owner", "kernel-confirmed", "reproduced-post-fix", "asan-confirmed", "uaf-mechanism-confirmed", "malloc-reuse-confirmed", "watchpoint-captured", "side-bugs-fixed", "dataset-schedjobs-ruled-out", "redelivery-storm-found", "live-repro-not-captured-2026-08-30", "large-quarantine-asan-tried-no-repro-2026-08-30", "redelivery-storm-also-not-reproduced-2026-08-30", "reconnect-storm-confirmed", "live-repro-2026-09-04", "crash-reproduced", "e-xce-not-ready-correlated", "amplification-mechanism-found", "stale-reader-still-unknown", "resreq-select-ast-ruled-out-2026-09-04", "dataq-path-ruled-out", "controller-dual-push-race-ruled-out", "redelivery-storm-fully-explained"] +related: ["bug-1699", "bug-1731"] +repro_attempts: "0/2 runs, 2026-09-25: documented recipe (3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' = 2400 array elements each) on b63bb581 build.dbg and on d87b78dc-era ASAN build -- no crash, scheduler alive throughout, 0 redelivered array pushes (0 'sched_add_job failed' duplicates; the original repro saw up to 18x redelivery, the recipe's precondition)" +legacy: true +gitea_issue: 265 +gitea_synced_status: "cannot-reproduce" +--- + +# bug-1714: User-reported live deployment: xce_scheduler process died and restarted unexpectedly (PID changed… + +## Description +User-reported live deployment: xce_scheduler process died and restarted unexpectedly (PID changed from 3964918 to 4137745 in scheduler.log with a ~79s gap, no manual restart by the user) during a live 3-node cluster test session; xceadmin briefly reported 'Error: scheduler disconnected' (the already-correct bug-1693 disconnect-detection path in cluster.c, working as designed -- not itself a bug) then recovered on the next attempt once the scheduler rejoined. User confirmed via `sudo dmesg -T` a REAL kernel-level segfault: `xce_scheduler[2815660]: segfault at 30 ip 0000601569e1fb35 sp 00007db7db1f6e00 error 4 in xce_scheduler[23b35,601569e01000+de000] likely on CPU 5 (core 16, socket 0)` -- though this specific dmesg entry is dated Aug 25 19:21:39, a DIFFERENT, earlier occurrence than today's specific restart (Aug 26 ~18:00-18:02), not necessarily the direct cause of THIS restart. No core dump exists for xce_scheduler anywhere in /var/lib/systemd/coredump/ (only xce_controller and python entries present, despite the scheduler segfault being kernel-confirmed) -- a separate, notable gap in this environment's crash-capture coverage for the scheduler process specifically. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +- Attempts: 0/2 runs, 2026-09-25: documented recipe (3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' = 2400 array elements each) on b63bb581 build.dbg and on d87b78dc-era ASAN build -- no crash, scheduler alive throughout, 0 redelivered array pushes (0 'sched_add_job failed' duplicates; the original repro saw up to 18x redelivery, the recipe's precondition) + +## Root cause +NOT ROOT-CAUSED -- confirmed real (kernel dmesg segfault line, error code 4 = user-mode read fault, not a random/unrelated event) but no core dump or backtrace exists for any xce_scheduler crash observed so far, including the Aug 25 19:21:39 kernel-confirmed occurrence and today's (Aug 26, ~18:00-18:02) restart (PID 3964918 -> 4137745), which may or may not be the SAME underlying bug -- the two occurrences have not been correlated beyond both being unexplained scheduler deaths. Notable secondary finding: systemd-coredump is confirmed working for xce_controller (4 real core.xce_controller.*.zst files captured and used earlier this session, see bug-1699) but has captured ZERO xce_scheduler crashes despite at least one kernel-confirmed scheduler segfault -- needs investigation on the user's real machine (check core_pattern/ulimit -c/systemd-coredump size or process-name filtering specifically for the scheduler binary; also worth checking whether the scheduler's own launch wrapper -- start-scheduler.sh, a bare unwrapped invocation with no core-dump-enabling ulimit call -- inherits a different ulimit -c than however the controller happens to get its unlimited setting). FOLLOW-UP (2026-08-26, disambiguating today's restart from the Aug-25 segfault): checked dmesg for today's exact restart window (17:45-18:15) -- zero xce_scheduler entries (the ONLY dmesg segfault hit remains the single Aug 25 19:21:39 occurrence). journalctl for the same window -- zero xce_scheduler entries at all. dmesg confirms OOM killer was never actually invoked against any process in this window (only periodic OOM-killer enable/disable toggling, unrelated). ulimit -c is unlimited and core_pattern correctly routes to systemd-coredump (confirmed working for xce_controller, 4 real cores captured this session). The combination of (a) unlimited core-dump capacity + correct core_pattern, (b) zero kernel fault-signal record, (c) zero OOM record, (d) zero journalctl record for today's specific window strongly indicates today's PID change (3964918 -> 4137745) was NOT a crash of any kind -- a genuine fault signal (SIGSEGV/SIGABRT/SIGBUS/SIGILL/SIGFPE) is unconditionally kernel-logged regardless of which process triggers it, and none was. The scheduler is launched as a bare backgrounded process (start-scheduler.sh, no systemd unit), so an ordinary kill/pkill sent by an unrelated concurrent process on this shared machine would leave NO trace in dmesg or journalctl -- most likely explanation for today's specific restart, DECOUPLED from this bug going forward. This buglog entry now tracks ONLY the confirmed, kernel-verified Aug 25 19:21:39 xce_scheduler segfault (still unreproduced, still no core available, still worth investigating on recurrence) -- not today's restart, which is environmental noise, not a code bug. + +UPDATE (2026-08-29, bug-1731 live gdb-wrapped repro session): CRASH CAPTURED LIVE, first-ever backtrace for this crash family. Root cause of the earlier no-core-dump gap circumvented by launching xce_scheduler wrapped in `gdb -batch -ex run` FROM PROCESS START (not attach-after-the-fact) during a heavy burst-array workload (6 concurrent `jobsub -a 50` array submissions, 300 near-instant /bin/true elements, on a live 3-node HA cluster, debug build profile). Reproduced at T+150s into the burst. Full backtrace: +SIGSEGV in sched_get_project(core, name=0x41d0000000000000) at scheduler/src/core.c:412, called from core_build_job() at scheduler/src/core_events.c:221, called from xce_core_push_events() at scheduler/src/core_events.c:725 (crashed on the 3rd of a 6-event batch), called from xce_sched_driver_translate_pending_events() (driver.c:555) <- xce_sched_driver_run_normal_mode_round() (driver.c:749) <- main() (main.c:465). Registers: rax=rdx=rsi=0x41d0000000000000 (corrupted `name` arg present in 3 registers), rdi=0x555555695da0 (valid `core` arg). Single-thread fault in the main scheduling-round thread (all 9 other threads idle in pthread_cond_wait/epoll_wait) -- not a cross-thread race at the fault site itself, though the corruption's origin is not yet located. +core_build_job() (core_events.c:221) calls sched_get_project(core, j->submit->project_name). The two preceding calls in the same function (sched_get_user, sched_get_queue, reading earlier fields of the same job_submit_data_t*) succeeded, so j->submit itself is a valid pointer -- but its project_name field held 0x41d0000000000000, not NULL and not a plausible string pointer. clone_job_submit_data() (handler.c:102-150) calloc()s the destination and only strdup()s project_name when the source is non-NULL, so a legitimately-absent project name would be NULL (which sched_get_project's own guard at core.c:412 handles safely) -- this crash requires the field to have been overwritten with garbage AFTER allocation. Points to memory corruption or a stale/reused pointer in the array-job event-translation/batching path under heavy concurrent load, not a missing-null-check. NOT YET further isolated -- the corruption's write site is still unknown. Fault class (SIGSEGV, invalid-pointer read via strlen()) matches the original kernel dmesg description (`error 4` = user-mode read fault) closely enough that this is plausibly the same crash family, now finally attributable to an exact function/line/call-chain instead of an unattributed dmesg line. + +UPDATE (2026-08-29, post-fix live-repro session -- bug-1730/1731/1733 fixes now all landed and included): two-phase reproduction attempt, both phases succeeded. + +PHASE 1 (ASAN, fresh clean builds of controller/scheduler/worker/xceadmin including commits 42eeaa1e[bug-1733 vtaskmgr UAF fix], b1323f8d[bug-1730], ebe691c3+67009815[bug-1731]): deployed fresh 3-node HA cluster (/tmp/xce-deploy-bug1714-asan2), launched xce_scheduler directly under ASAN (ASAN_OPTIONS=detect_leaks=0:halt_on_error=1:abort_on_error=1), ran a 10x50-element concurrent array-job burst (500 elements, wave 1 of a planned two-wave/1000-element burst). CRASH REPRODUCED WITHIN WAVE 1 ITSELF -- before all 10 clients' submissions even completed, after only 2 of 10 array pushes had been expanded by the scheduler (100 of the intended 500 elements). This is FASTER than the original bug-1731-session capture (T+150s) and definitively proves the bug is STILL PRESENT after all three recent fixes, not masked or perturbed by ASAN instrumentation. Verbatim ASAN report: + AddressSanitizer:DEADLYSIGNAL + ==562488==ERROR: AddressSanitizer: SEGV on unknown address (pc 0x64b0be11076e ...) + ==562488==The signal is caused by a READ memory access. + #0 sched_get_project ../scheduler/src/core.c:412 + #1 core_build_job ../scheduler/src/core_events.c:221 + #2 xce_core_push_events ../scheduler/src/core_events.c:725 + #3 xce_sched_driver_translate_pending_events ../scheduler/src/driver.c:555 + #4 xce_sched_driver_run_normal_mode_round ../scheduler/src/driver.c:749 + #5 main ../scheduler/src/main.c:465 + Register values: rax=rdx approx corrupted; rax = 0x41d0000000000000 (still exactly 2^30=1073741824.0 as an IEEE-754 double, matching the original capture bit-for-bit) -- IDENTICAL crash signature, file, line, and call chain to the original bug-1731-session capture. SUMMARY: AddressSanitizer: SEGV core.c:412 in sched_get_project. + Diagnostically important: ASAN's own heap-buffer-overflow and use-after-free shadow-memory detectors did NOT fire -- only the raw SIGSEGV/DEADLYSIGNAL handler caught this, exactly as a non-ASAN build would. This RULES OUT a classic redzone-violating heap overflow into an adjacent allocation (ASAN would have caught that at the WRITE) and argues for either (a) a wrong-offset/type-confusion write within a still-valid allocation, or (b) a use-after-free where the stale read happens only after the freed chunk has already left ASAN's quarantine and been legitimately reallocated for a new, unrelated live object -- which Phase 2 then confirmed directly. + +PHASE 2 (gdb scripted watchpoint, debug -O0 -g3 profile, same fix set, fresh cluster /tmp/xce-deploy-bug1714-dbg2): launched xce_scheduler under `gdb -batch -x scheduler-watchpoint.gdb`, breaking on clone_job_submit_data() (handler.c:102, called once per array element from make_array_element), sampling every 31st call and arming a hardware watchpoint on the returned job_submit_data_t's ->project_name field (via `finish` + cast of $rax to the known x86-64 System V pointer-return-value register), capped at 3 simultaneous watchpoints to stay within HW watchpoint budget. Ran the same burst pattern. ALL THREE armed watchpoints fired within the first burst wave. Full verbatim capture in this session's transcript; key facts: + - All three watched job_submit_data_t->project_name fields transitioned from a plausible-looking heap string pointer (e.g. 'Old value = 0x7fffe000bc60 "0\275"') to exactly NULL (0x0) -- i.e., we caught a real WRITE, but a NULL-out, not the resource-double corruption pattern itself. + - Root cause of that write, nailed by backtrace: the write instruction was __memset_avx2_unaligned_erms, called from libCapnC_Runtime.so's capn_init_mem()/init_fp(), called from xce_decode() (lib/request.c:103) -- i.e. Cap'n Proto's own message-decode-buffer zero-initialization for a BRAND NEW INCOMING NETWORK REQUEST, executing on a COMPLETELY DIFFERENT thread (LWP 580496, decoding its own client's array submission) than the thread that had originally cloned the watched job_submit_data_t (LWP 580499, decoding a DIFFERENT client's array submission, job_array pointer 0x7fffe0000cf0 vs 0x7fffe4002fe0). + - This is DEFINITIVE PROOF of the corruption mechanism: a job_submit_data_t's heap chunk, after being freed, gets immediately recycled by glibc malloc to serve an UNRELATED incoming request's Cap'n Proto decode buffer on a different thread. Any stale pointer still referencing the old (freed) job_submit_data_t will, from that point on, read whatever the CURRENT occupant of that memory happens to contain -- first zeros (as caught here), but on a LATER reuse cycle for something that stores a raw IEEE-754 double (e.g. an xce_alloc_t's `value` field, src/include/xce/types.h:270), it would read exactly the 2^30/1GiB bit pattern seen in both the original and this session's crash. + - IMPORTANT CAVEAT / what remains UNRESOLVED: the free() itself, in this specific capture, was almost certainly LEGITIMATE, not a bug by itself. Verified by static reading of driver.c's xce_sched_driver_translate_pending_events(): within one drain/translate call, xce_core_push_events(core, events, n) (driver.c:555, which runs every core_build_job() call for the whole batch) unconditionally executes BEFORE the cleanup loop that calls release_translated_payload()->free_Job_ptr()->free_JobSubmitData_ptr() (driver.c:585-588) -- ownership/free ordering within a single batch is correct by inspection. sched_on_job_array() (handler.c) also does not push a client's array to the scheduler until its ENTIRE 50-element clone loop completes, so the specific object we watched (armed at clone call #31, i.e. mid-loop) could only have been freed AFTER its owning thread's full loop finished, was submitted as one combined XCE_SCHED_OP_UPDATE_JOB request, and was drained+processed+freed by an EARLIER, already-completed scheduling round -- i.e. a normal, successful lifecycle, not an early/premature free caught in the act. The actual bug is therefore NOT in this ownership-transfer/free ordering itself, but in some OTHER, not-yet-identified code path that reads a job_submit_data_t (or the containing xce_job_t) a SECOND time, AFTER it has already been correctly processed and freed once. Did not have time this session to isolate that second/stale reader. Two concrete leads for the next session: (1) whether a redelivered/retried array push can race sched_on_job_array()'s per-element `sched_get_job(&(sched->core), jobid) != NULL` dedup check (handler.c ~212) against the window before a just-submitted job is actually added to sched->jobs (which only happens later, inside core_build_job()/sched_add_job(), not at submit time) -- if so, two independent xce_job_t/job_submit_data_t clones could exist for the same jobid, and it's worth checking whether anything then conflates or double-references them; (2) core_build_schedobj() (core_events.c:174-197, called from core_build_job() at core_events.c:253 with the ORIGINAL `j` still valid) does `obj->project = makeString(j->submit->project_name)` -- a SECOND independent read of j->submit->project_name beyond the sched_get_project() call at core_events.c:221 that actually crashes; worth double-checking core_build_schedobj/its 3 tag-dispatch helpers for any raw pointer-aliasing (vs. deep copy) into the resulting long-lived xce_sched_job_t/xce_sched_obj_t that could keep referencing freed submit-owned memory after `j` itself is gone. + +Both clusters deployed this session (/tmp/xce-deploy-bug1714-asan2, /tmp/xce-deploy-bug1714-dbg2) were torn down cleanly after their captures; the crash/watchpoint logs remain on disk at /tmp/xce-deploy-bug1714-asan2/logs/scheduler_asan.log and /tmp/xce-deploy-bug1714-dbg2/logs/scheduler_gdb_watchpoint.log for reference (not committed, local /tmp only -- copy out if this environment gets reset before a follow-up session). + +UPDATE (2026-08-30, Part 1/Part 2 investigation session): PART 1 (static-only, XceData_data_schedjobs dataset->rows aliasing) -- RULED OUT, not a bug. Traced the full decode chain for the schedjobs/jobs table (src/lib/xcectx.c:139 xce_decode() -> src/lib/request.c:1786 extract_data() -> src/lib/controller.capnp.c decode_Job_list()/decode_Job_ptr() (line 6348/6369) -> decode_JobSubmitData_ptr() (line 12427) -> decode_JobSubmitData() (line 12223)). Confirmed every xce_job_t (calloc, decode_Job_ptr/decode_Job_list) and every job_submit_data_t (calloc, decode_JobSubmitData_ptr) is an independent heap allocation, and every job_submit_data_t string field uses STRING_DUP (= strdup, controller.capnp.h:9) -- structurally identical to clone_job_submit_data()/decode_pidinfo(). extract_data() (request.c:1786-1815) only memcpy's the already-decoded xce_job_t* POINTER VALUES into dataset->rows (then memsets the source array to prevent double-free) -- it never aliases into the capnp segment/request buffer, and this happens entirely INSIDE xce_decode(), before capn_free(&c) at request.c:281 destroys the wire buffer. So sched_handle_dataset()'s XceData_data_schedjobs case handing dataset->rows to sched_on_jobs() for deferred driver-thread processing is a correct, safe zero-copy ownership transfer of already-independent objects, not a second UAF class. The XceData_data_jobArray case's contrasting comment ('we never alias... unlike the list-shaped cases above') is about a different concern (N-way sharing of one spec across array elements), not capnp-buffer lifetime -- both cases are equally safe on that specific axis. + +PART 2 (live pointer-lifecycle tracking, main task) -- did NOT capture a fresh crash this session despite extensive effort (see below), but surfaced a significant, previously-undocumented, independently-reproducible finding that materially narrows the investigation and directly answers this entry's own Question 3 lead about a redelivery/retry mechanism. Full method: added temporary mutex-guarded fprintf/fflush diagnostic logging (address, jobid, tid, timestamp) at three sites -- make_array_element() (handler.c, ALLOC), every free_Job_ptr()/free_JobSubmitData_ptr() call site in driver.c's release_translated_payload() (FREE), and core_build_job() (core_events.c) immediately before the historically-crashing sched_get_project() line (READ). The shared logging implementation was placed in src/lib/utils.c (a genuine, non-generated libxce.a source file) rather than controller.capnp.c, after discovering mid-session that src/lib/controller.capnp.c is NOT actually compiled -- meson's custom_target regenerates lib/controller.capnp.c/.h from src/schema/controller.capnp fresh into the BUILD directory on every build, silently discarding any manual source-tree edit to that file (confirmed empirically: an initial edit there vanished from the compiled object with no error). Built (debug profile), deployed multiple fresh 3-node HA clusters, launched xce_scheduler under `gdb -batch -ex run -ex "thread apply all bt full"`, and drove very heavy concurrent `jobsub -a 50` array-job bursts -- far exceeding the original recipe's intensity: 6-15 concurrent submissions per wave, 20+ waves across several redeploys (both -v and -vvv controller verbosity), well over 100 total concurrent array submissions / 5000+ array elements pushed. The scheduler did NOT crash in any of these attempts (longest single continuous run: ~16 minutes / 5 waves without a restart), a notably higher bar than the original session's 150s/6-concurrent capture -- see 'What changed / open question' below. + +KEY FINDING (from the diagnostic log, cross-referenced against the scheduler's own pre-existing Bug-A WARN logging): every array-job jobid gets independently, repeatedly RE-CONSTRUCTED and pushed to the scheduler via the ORDINARY (non-array) XceData_data_schedjobs/JOBS_PUSH bulk path -- confirmed by cross-referencing FREE events against ALLOC events by jobid: every array-path jobid showed MORE free events than array-path (make_array_element) allocs, each with a DISTINCT heap address (never the same address freed twice -- ruling out a literal double-free), e.g. jobid 4658596961282883584 was array-allocated once but freed 18 times from 18 distinct addresses over a 16-minute run. Each of these extra frees is preceded by a matching WARN in scheduler.log: 'xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid ... (rc=-17), discarding duplicate/orphaned job' -- i.e. Bug A's fix (this same buglog entry, prior session) is correctly catching and safely discarding every one of these redelivered duplicates before they reach the picker. CRITICALLY, this was reproduced in a run with ZERO client-side submission failures/retries (confirmed: grep for 'jobsubmit failed'/'retry' across all burst client output files in the clean final run returned zero matches, yet the SAME redelivery pattern still occurred, up to 18x per jobid) -- this RULES OUT the client-retry-on-timeout hypothesis (XCECLI_FAILOVER_CALL's retry loop, client_internal.h) as the trigger; the redelivery is NOT a symptom of client-observed transport failure, it happens under clean, fully-acknowledged submissions too. Static reading of the two most likely controller-side mechanisms did not find an obvious culprit: (1) persist_apply_pump_run() (persist.c:5521) advances last_applied_lsn strictly one entry at a time under apply_pump_lock and only ever processes next=last_applied_lsn+1, so a genuine re-walk of an already-applied LSN range looks structurally impossible by construction; (2) apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case (persist.c:4391-4461) explicitly fires HOOK_DETAIL_JOB_NEW exactly once per successful apply, with an EEXIST short-circuit for an idempotent whole-record re-apply. Neither explains a genuinely repeated, single-job-shaped (l->count==1, routed through xcesched_push_joblist() per sched.c:326-327's dispatch, not xcesched_push_jobarray()) HOOK_DETAIL_JOB_NEW fire for jobids that were originally created via the array path. bug-1419's in-flight submission registry (job.c:71-140, tracks [base_jobid, base_jobid+array_size) ranges) is the most likely place a gap could exist for this specific pattern but was not dynamically traced this session -- NEXT STEP for a follow-up session: either a live gdb breakpoint on sched_job_new_fun()/xcesched_push_joblist() (sched.c) capturing the call's l->count and full backtrace, or controller-side instrumentation logging out_new_jobs.count at every HOOK_DETAIL_JOB_NEW fire site, to determine whether this is a genuine second independent WAL commit for an overlapping jobid range (a real gap in bug-1419's registry or jobid_range_exists()) or some other apply-side effect not yet identified by static reading. + +(a) vs (b) QUESTION -- NOT SETTLED this session (no fresh crash captured to inspect). The redelivery-storm finding above establishes a plausible AMPLIFYING mechanism (far more job_submit_data_t alloc/free churn under array-job load than previously suspected, all safely discarded by Bug A but still consuming and freeing large volumes of same-shaped heap chunks under concurrent capnp-decode pressure on other threads) but does not by itself prove a stale-pointer read, since every redelivered object observed in this session's log was a freshly-decoded, valid object at its own READ time (not a stale/freed one) -- the two prior sessions' watchpoint-proven UAF mechanism (freed job_submit_data_t memory reused by malloc for an unrelated concurrent capnp decode buffer) still requires a genuinely stale SECOND reference somewhere, which this session did not catch live. + +What changed / open question: this session's burst intensity substantially exceeded the recipe that reliably crashed within 150s-2.5min in the two prior sessions (same debug build profile, same gdb wrapper, same jobsub -a 50 command), yet produced zero crashes across ~20+ waves / 100+ concurrent submissions / 5000+ array elements over multiple redeploys. Possible explanations, none confirmed: (1) the two prior sessions' side-bug fixes (bug-1730's core_events.c fix, bug-1731's HOOK_DETAIL_JOB_FAIL fix, and this entry's own Bug A/Bug B, all landed 2026-08-29 per git log -- 'fix(scheduler): don't hand an orphaned duplicate job to the picker' at commit 38437e57 among them) may have shifted timing enough to reduce (not eliminate) the race window; (2) this session's burst driver script (piped `xce_test --norc` REPL commands, one process per concurrent submission) may differ subtly in timing/pacing from whatever exact mechanics the original crash-capturing sessions used; (3) simple bad luck across a large but still finite number of attempts for a genuinely rare race. All temporary diagnostic instrumentation (handler.c, driver.c, core_events.c, src/lib/utils.c) was fully reverted before this session ended -- confirmed via `git diff` returning clean/identical-to-HEAD for all four tracked files, plus a manual re-check of the untracked/regenerated src/lib/controller.capnp.c (git-invisible, confirmed no leftover bug1714_diag_log references anywhere under src/). Full scheduler ASAN test suite re-run after revert: 54/55 pass, the one failure (test_recordstart_symlink_escape_replies_error) is the already-documented pre-existing sandbox filesystem-permissions flake, unrelated to this session. All deployed clusters (3x) torn down; zero xce_controller/xce_scheduler/xcecd_virtual/gdb processes left running. NEXT STEP for a follow-up session: (a) pursue the redelivery-storm's exact trigger per the NEXT STEP above (this is now the most concrete, evidence-backed lead); (b) if reproduction is attempted again, consider a substantially longer single continuous run (30+ min) rather than many shorter redeploys, since this session's longest unbroken run was only ~16 minutes and the original captures may have benefited from cumulative state a fresh redeploy resets each time. + +UPDATE (2026-08-30, later same day, large-ASAN-quarantine live-repro session): NEW APPROACH TRIED, NOT YET DOCUMENTED IN THIS ENTRY -- launched the ALREADY-BUILT ASAN xce_scheduler (src/xce.build/scheduler/xce_scheduler, built 2026-08-30 11:36, already including every fix through commit 22aa16bf/38437e57 -- no rebuild needed since ASAN_OPTIONS is a runtime env var, not compile-time) with a deliberately oversized quarantine/redzone (ASAN_OPTIONS=quarantine_size_mb=256:redzone=256:max_redzone=2048:detect_leaks=0:abort_on_error=1:halt_on_error=1) specifically to test whether the documented "freed block reused too fast for ASAN's default quarantine" explanation for plain-ASAN's prior silence (see the 2026-08-29 post-fix update above) would resolve with a much larger quarantine. Deployed a controller-only + scheduler 3-node HA cluster (NO worker -- confirmed by static reading of the crash call chain, in this same entry, that sched_get_project()/core_build_job()/xce_core_push_events() all fire at JOBS_PUSH submission-ingestion time, before any picker/allocator/worker involvement, so a worker is not required to trigger this specific crash; xcecd_virtual was also unavailable to build this session, see environmental note below, making this a load-bearing simplification, not just a shortcut). Drove a sustained, unthrottled burst-submission loop (8 concurrent `client`+`jobsub -u root -q normal -c hostname -a 50` xce_test invocations per wave, back-to-back waves, ~1-8s apart) for ~8 minutes continuous wall clock, reaching 64 waves = 25,600 array elements pushed (verified via a final controller `joblist` count of 25,756 PEND jobs, consistent with no jobs lost at the controller level) -- substantially higher volume and, without a worker providing any execution-time backpressure, likely higher submission RATE than any prior session's recipe (the original crash-capturing sessions used 6 concurrent x 50 = 300 elements in 150s; this session pushed 25,600 in ~480s, roughly an order of magnitude more elements per unit time). RESULT: the scheduler did NOT crash and produced ZERO AddressSanitizer reports of any kind (grep -ci AddressSanitizer scheduler.log == 0) across the full run. ADDITIONAL NEGATIVE FINDING: the 2026-08-30-morning session's "redelivery-storm" (array-job jobids redelivered via the ordinary JOBS_PUSH path, discarded by Bug A with a "discarding duplicate/orphaned job" WARN, observed up to 18x per jobid that session) did NOT reproduce AT ALL this run either -- zero occurrences of that WARN string anywhere in 94,794 scheduler.log lines, despite comparable-or-higher load. This is a second, independent "previously-reliable symptom now absent" data point in addition to bug-1714's own crash, on the same day, suggesting SOMETHING about either the exact submission-driver mechanics (this session used direct `client`+`jobsub` xce_test invocations; the morning session used a different piped-REPL driver script whose exact mechanics were not preserved) or genuine environment/timing variance materially changes whether either symptom fires -- not just bad luck on the crash alone. CAVEAT (important, do not skip): a controller build from an EARLIER point in this same work session (commit 38437e57 onward, built 2026-08-30 03:21) and a scheduler build from 11:36 the same day were reused as-is (not rebuilt) for this attempt -- both already post-date every fix this entry and its siblings (bug-1730/1731/1733) describe, so this is a like-for-like continuation of the "post-fix" investigation, not a regression to older code. + +ENVIRONMENTAL INCIDENT (important for future sessions on this sandbox): this session began by finding the root filesystem at 0 bytes available (936.8G/936.8G used), which blocked ALL work (builds, deploys, even log writes) until resolved. `nix-collect-garbage` (no -d) was run to reclaim space, which worked (freed ~790GB, root cause was almost certainly months of accumulated `nix develop` closures across many past sessions' worth of ASAN toolchains, never garbage-collected) but had a serious side effect: it evicted this flake's own cached bootstrap-toolchain closure (gnum4/bison/gmp/mpfr/isl/libmpc/autoconf/a full multi-stage GCC bootstrap), forcing EVERY subsequent `nix develop ./src` invocation on this box -- including a DIFFERENT, concurrently-running agent's own controller build for an unrelated bug (bug-1735) -- into a full from-source toolchain rebuild. This cost roughly 15-20 minutes of wall clock before ANY build (mine or the other agent's) could proceed, and materially ate into this session's time budget. LESSON for next time: if disk space must be reclaimed on this box again, prefer `nix-store --gc --print-dead` / a bounded `nix-collect-garbage --max-freed ` first to gauge what will be evicted, since this flake's own dev-shell closure is apparently NOT protected by a persistent gcroot between sessions and is fully re-buildable-from-scratch (not just re-fetchable) when evicted -- a full unbounded `nix-collect-garbage` is disproportionately expensive here. Once resolved, this session worked around not having a fresh full build by (a) reusing the ALREADY-BUILT src/xce.build/{controller,scheduler} binaries directly (their /nix/store runtime deps were, fortunately, NOT evicted -- `ldd` still resolved cleanly) with the new ASAN_OPTIONS applied purely at launch time, and (b) building ONLY the missing `xceadmin` component fresh (into a separate dir, src/xce.build.bug1714xa) since a cluster.bin config cannot be produced without it -- this minimal-rebuild strategy is worth reusing directly if this situation recurs, rather than a full `xce-build.sh all`. + +bug-1734 was NOT reproduced this session: its repro recipe needs a worker to actually run jobs to DONE/EXIT so the scheduler-vs-controller view can be compared; no `xcecd_virtual` binary exists in any build dir on this box currently (checked src/xce.build, src/xce.build.validate, src/build-asan, src/build-asan-1696repro -- only worker *test* binaries exist, no `xcecd_virtual` itself), and this session's bounded time budget (already consumed by the disk/nix incident above) did not allow building `worker` fresh on top of the already-spent time. See bug-1734's own entry for a same-day cross-reference note. + +## Fix +STILL NOT FIXED -- but no longer blocked on evidence collection; see root_cause UPDATE above for the first captured backtrace. NEXT STEP: find what writes garbage into job_submit_data_t->project_name after clone_job_submit_data() allocates it -- likely somewhere in the array-job batching/event-translation path (xce_core_push_events / xce_sched_driver_translate_pending_events) given the crash hit specifically while processing event index 2 of a 6-event batch under heavy concurrent array-job load. Reproduction recipe confirmed reliable enough to use again: debug-profile build, xce_scheduler launched via `gdb -batch -ex run -ex "thread apply all bt full"` from process start (script preserved at /tmp/xce-deploy-bug1731/scripts/start-scheduler-gdb.sh), 6 concurrent 50-element array submissions against a live 3-node HA cluster -- crashed within 150s this run. + +UPDATE (2026-08-29): STILL NOT FIXED. This session's two-phase live-repro (ASAN + gdb watchpoint, see root_cause UPDATE) upgrades the diagnosis from 'read-site backtrace only' to 'corruption mechanism proven' (freed job_submit_data_t memory gets recycled by malloc for unrelated Cap'n Proto decode buffers on other threads; any surviving stale pointer to the old object then reads garbage matching whatever now occupies that memory) but the actual SOURCE of the stale pointer/second-reader is still not isolated. NEXT STEP, sharpened from this session's two leads: (1) audit sched_on_job_array()'s per-element dedup check (handler.c ~212) for a race window against a redelivered/retried array push (job not yet added to sched->jobs at submit time, only later inside core_build_job()/sched_add_job()); (2) audit core_build_schedobj() and its 3 tag-dispatch helpers (core_events.c:110-197) for any raw pointer stored into the long-lived xce_sched_obj_t/xce_sched_job_t that aliases (rather than deep-copies via makeString()) memory owned by the short-lived original xce_job_t/job_submit_data_t, which release_translated_payload() frees shortly after core_build_job() returns. + +UPDATE (2026-08-29, two side-bugs found during this investigation FIXED -- bug-1714's OWN root cause remains UNFIXED, see this session's root_cause UPDATE above and the still-open leads there): while chasing this bug's stale-pointer UAF, static code reading (not live repro) surfaced two SEPARATE, independently-confirmed defects, real bugs on their own merits, not proven to BE this bug's own root cause. Both now fixed with TDD (RED->GREEN) and full regression evidence. + +Bug A -- xce_core_push_events()'s JOBS_PUSH case (src/scheduler/src/core_events.c, the XCE_CORE_EVENT_JOBS_PUSH switch case) never checked sched_add_job()'s return value. sched_add_job() (core.c:219) is a thin wrapper around key_add(core->jobs, jobid, job), which returns a negative errno (duplicate-jobid class failure) if the jobid already exists -- previously silently discarded, so a freshly-built xce_sched_job_t that FAILED to register in core->jobs was still handed to picker->picker_iface->on_job_push() anyway: an orphaned job object the picker now tracks but that sched_get_job()/any other core->jobs-keyed lookup can never find again. FIX: check sched_add_job()'s return value; on failure, skip the on_job_push() call, log a WARN (matching this case's existing WARN convention for the job==NULL branch), and free the orphaned job at the same granularity core_build_job()'s own early-failure paths already use (FREEUP(job->resreq); FREEUP(job->schedobj); FREEUP(job);) -- job->user/queue/project deliberately left alone (kvmgr-owned get-or-create lookups, not owned by this job instance); job->allocs/accounts deliberately left alone too (out of scope -- xce_sched_job_t has no complete destructor anywhere in this codebase; kill_sched_job() in schedobj.c only frees job->rusages, a genuine pre-existing gap this fix does not attempt to close). TDD: RED -- added test_jobs_push_duplicate_jobid_does_not_call_on_job_push_or_leak to src/scheduler/testsuite/test_core_events.c (pushes one job successfully, baseline on_job_push count 1, then pushes a second JOBS_PUSH batch with the SAME jobid); on unfixed code this failed with 'Expected 1 Was 2' (on_job_push fired a second time for the orphaned duplicate), confirming the bug. GREEN -- after the fix, the same test passes: on_job_push count stays at 1, and sched_get_job() for that jobid still returns the ORIGINAL job object, not the duplicate. + +Bug B -- sched_on_job_array()'s per-element dedup check (src/scheduler/src/handler.c ~line 179-226) has a TOCTOU race window: it runs on the network request-handling thread and checks sched_get_job() against core->jobs, but a job is only actually inserted into core->jobs later, when the separate DRIVER thread drains the pending-update queue and calls xce_core_push_events() -> JOBS_PUSH -> sched_add_job(). A redelivered (e.g. network-retried) copy of the same array push arriving on the request thread during that window is not caught by this check, so a second, independent set of xce_job_t/job_submit_data_t clones for the same jobids gets constructed and enqueued too. Per this session's analysis, eliminating the race itself (e.g. a new synchronized pending-jobids set) was deliberately NOT attempted -- out of scope, bigger architectural change, own risk -- because it is not actually necessary: once Bug A is fixed, this race is SAFE rather than dangerous, since the redelivered duplicate is now correctly and safely rejected at the authoritative uniqueness boundary (sched_add_job()'s key_add()), freed, and never reaches the picker. FIX: documentation only, no logic change -- added a comment on the dedup check explaining the race window, that it is a best-effort optimization (not a correctness boundary), and that sched_add_job()'s key_add() (now correctly handled per Bug A) is the actual authoritative dedup boundary a redelivered duplicate ultimately hits. No test needed (no behavior change); covered indirectly by Bug A's test, which models exactly the duplicate-jobid scenario this race can produce. + +Full regression (nix develop ./src --command bash src/tools/xce-build.sh scheduler asan --incremental): 54/55 scheduler test suites pass. The one remaining failure, test_recordstart_symlink_escape_replies_error in test_admin_request.c (expected 'invalid record file name', got 'permission denied accessing record file'), is the already-known, pre-existing bug-1561-adjacent sandbox filesystem-permissions flake, unrelated to and unaffected by this change -- present identically before these fixes were applied (also seen in the RED run). IMPORTANT: these two fixes are scoped, standalone correctness fixes for real, independently-confirmed defects -- they do NOT resolve bug-1714's own root cause (the stale-pointer UAF into a freed job_submit_data_t, see root_cause above). That remains open; the two still-open leads listed in this entry's prior fix UPDATE (2026-08-29) are what a future session should pursue next, now narrowed since lead (1) [sched_on_job_array()'s dedup race] has been read closely enough to conclude it is not itself hazardous post-Bug-A -- lead (2) [core_build_schedobj()'s makeString()-vs-alias handling] remains the live suspect. + +UPDATE (2026-08-30): STILL NOT FIXED -- no fix attempted this session (per this entry's own 'do not attempt a speculative fix without a confirmed root cause' guidance). This session's Part 1 static audit ruled out the XceData_data_schedjobs dataset->rows aliasing as a second UAF class (see root_cause UPDATE). Part 2's live pointer-lifecycle instrumentation did not capture a fresh crash despite ~20+ burst waves substantially exceeding the historically-successful recipe's intensity, but did surface a new, real, independently-reproducible finding: array-job jobids get redelivered to the scheduler via the ordinary single-job XceData_data_schedjobs/JOBS_PUSH path many times (up to 18x observed) after their original array-path submission, confirmed unrelated to client-side retries (reproduced with zero client submission failures). Bug A (this same entry, prior session) correctly discards every one of these before they reach the picker, so this redelivery is not itself a proven crash cause, but it is a strong candidate AMPLIFYING factor for the previously-proven malloc-reuse race (more job_submit_data_t churn = more opportunities to hit the race) and a concrete, scoped lead for a follow-up session (see root_cause UPDATE's 'NEXT STEP'). All temporary diagnostic instrumentation was fully reverted (verified via git diff + full re-run of the scheduler ASAN test suite: 54/55, same pre-existing flake as before). No code changes remain in the working tree from this session. + +UPDATE (2026-08-30, later same day): STILL NOT FIXED, STILL NOT REPRODUCED. Tried the large-ASAN-quarantine idea this entry's own root_cause section flagged as untried (quarantine_size_mb=256:redzone=256:max_redzone=2048, launched via already-built binaries, no rebuild) against a controller+scheduler-only cluster (no worker -- confirmed unnecessary for this specific crash's call chain, which fires entirely at JOBS_PUSH submission-ingestion time). 64 waves / 25,600 array elements over ~8 minutes, an order of magnitude more throughput than the original 150s/300-element capture: zero crash, zero ASAN reports of any kind, and (new, separate negative finding) zero recurrence of the morning session's reliably-observed redelivery-storm WARN either. Ruling nothing new out beyond what's already documented -- this is a THIRD consecutive session-day attempt (after the 08-29 post-fix session and the 08-30-morning session) where heavy live load did not reproduce a symptom that DID reproduce reliably in the two original capturing sessions (bug-1731's live-repro session and this entry's own 08-29 post-fix Phase-1/Phase-2 session), now across three different techniques (plain heavy load, gdb-wrapped heavy load with diagnostic instrumentation, and now large-quarantine ASAN). NEXT STEP suggestion, sharpened by three failed-to-reproduce sessions in a row: the two ORIGINAL successful captures both happened on 2026-08-29, before commit 38437e57 (Bug A) and its sibling Bug B documentation landed the same day, and EVERY session since (including this one) has been AFTER those fixes landed. While Bug A/B were carefully reasoned to be independent of bug-1714's own root cause (see this entry's history), the fact that reproduction rate appears to have dropped sharply coincident with their landing -- both the crash itself AND the previously-reliable redelivery-storm symptom now failing to reproduce under comparable or heavier load -- is a pattern worth taking seriously rather than dismissing as noise. Concrete suggestion for a follow-up session: (a) try reverting JUST commit 38437e57 (Bug A's fix) in a throwaway branch/build and re-run this exact large-quarantine-ASAN recipe against pre-Bug-A code, to directly test whether Bug A's fix (or the TOCTOU race it interacts with) was itself part of what made the race window practically hittable, rather than assuming independence holds; (b) if pre-Bug-A code DOES still crash under this recipe while post-Bug-A does not, that would suggest Bug A's fix (checking sched_add_job()'s return and discarding the duplicate instead of silently handing it to the picker) also incidentally narrows or closes bug-1714's own race window as a side effect worth understanding precisely rather than just accepting as an unexplained improvement. Both this session's build artifacts (src/xce.build.bug1714xa, a from-scratch xceadmin build with wheel+venv) were left in place (not cleaned up, unlike the temp deploy dir and processes) since they represent real, reusable build work with no live-process footprint. + +UPDATE (2026-09-04, live-repro session -- CRASH REPRODUCED, major new causal-chain finding, still no confident fix): Built an isolated debug profile (controller/scheduler/worker/xceadmin) into src/xce.build.bug1714repro specifically to avoid a concurrently-running interactive session also building/testing in this same worktree (tmux 'scheduler-1') -- deployed a fresh 3-node cluster on ports 9971-9973 (/tmp/xce-deploy-1714), launched xce_scheduler under `gdb -batch -ex run -ex "thread apply all bt full"` from process start, then drove the same recipe as the two original successful captures: 6 concurrent `jobsub -a 50` array submissions (300 elements) via xce_test against the live cluster. + +CRASH REPRODUCED, ~12s after the burst started (faster than either prior successful capture). Identical signature to every previous occurrence, byte-for-byte: SIGSEGV in sched_get_project (core.c:412), name=0x41d0000000000000 (same corrupted-double bit pattern), called from core_build_job (core_events.c:222) <- xce_core_push_events (core_events.c:754, this time on event index 16 of a 67-event batch, vs the original capture's index 2 of 6) <- xce_sched_driver_translate_pending_events (driver.c:555) <- xce_sched_driver_run_normal_mode_round (driver.c:749) <- main (main.c:465). All 9 other threads idle in a blocking syscall at fault time (same as every prior capture) -- confirms again this is a stale READ of already-corrupted memory, not a live cross-thread write race at the fault site itself. + +MAJOR NEW FINDING (not identified or looked for in any of the 6 prior sessions): the scheduler's connection to the primary controller GENUINELY, REPEATEDLY DROPS AND RECONNECTS under this exact burst load -- confirmed independently on BOTH sides of the connection, not a hypothesis: + - SCHEDULER side: this session's build included the bug-1760 partial fix (sched_check_pending_rejoin() called every round from main.c, logs on success/failure). During the ~12s burst window, it logged 'pending rejoin succeeded, cluster membership restored' SEVEN times (03:38:03-03:38:07), each preceded by scheduler.log's own `xcecli_retry_next: xcecli failover: attempt N -> localhost:99xx` lines showing the client library actively rotating through all 3 candidates. + - CONTROLLER side (ctrl-1, the primary throughout -- confirmed via `RAFT[p0] term=2 event=...` staying at term=2 the whole burst, i.e. this is NOT a controller-cluster election/failover, ctrl-1 never lost leadership): sched.c's handle_scheduler_leave() ('scheduler leave') fires repeatedly, each one immediately followed by a fresh `server sock 4 accepted 13` (a brand-new TCP accept, confirmed via a different priv-data/endpoint pointer each time) -- i.e. genuinely new connections, not the same one persisting. + - LIKELY TRIGGER, evidenced but not yet proven with byte-level certainty: ctrl-1's own persist_submit_job_array_async() logged 'lsn=6/7/8/9 appended but commit_submit_async registration failed (rc=-10013)' (E_XCE_NOT_READY -- see bug-1713's own decoding of this same errno) at 03:37:56.346, ~300ms before the scheduler's first observed failover attempt (03:37:56.632) -- i.e. the PRIMARY CONTROLLER itself was under registration/commit backpressure from the burst of 5 near-simultaneous 50-element array submissions at that exact moment. The most likely mechanism: ctrl-1, busy with WAL/commit backpressure, delays some response the scheduler's client library is waiting for past its own retry/timeout threshold; XCECLI_FAILOVER_CALL's client-side timeout-and-rotate logic (a deliberate, correct behavior in isolation) then treats this as 'candidate unresponsive' and fails over -- to OTHER, non-primary candidates, which correctly redirect it back, so it rejoins ctrl-1 again shortly after. This is a legitimate committed-backpressure symptom being misread as a connection failure, not memory corruption in itself. + +WHY THIS MATTERS FOR THIS BUG: every one of these reconnects fires on_scheduler_join() (sched.c:742) controller-side, which unconditionally re-sends a FULL xcesched_push_jobs() resync of every committed job via the ordinary (non-array) XceData_data_schedjobs wire path -- the EXACT mechanism the 2026-08-30 session's 'redelivery-storm' finding observed (array-job jobids redelivered via the single-job JOBS_PUSH path up to 18x per jobid) but could not explain the source of. This session's finding directly explains it: 7 reconnects in 12s here would scale to easily 18+ over a longer/heavier session. Each full resync constructs an entirely fresh set of job_submit_data_t/xce_job_t clones (decode_JobSubmitData_ptr(), independent heap allocations per PART 1's own prior static-analysis finding) for EVERY committed job, all safely discarded by Bug A's sched_add_job() dedup on the scheduler side -- but this churns a much higher volume of same-shaped job_submit_data_t alloc/free cycles under concurrent capnp-decode pressure than array-job load alone would produce, which is exactly the amplifying condition the 2026-08-29 watchpoint session proved is necessary for the malloc-reuse UAF (a freed job_submit_data_t chunk reused by an unrelated concurrent decode on another thread) to actually get hit. + +WHAT STILL IS NOT FOUND: the actual second, stale reader -- i.e. which specific code path retains a pointer to a job_submit_data_t (or its containing xce_job_t) PAST its own correct one-time processing+free, and reads it again later. This session did not have time to also capture a fresh gdb watchpoint trace (PHASE 2 style, 2026-08-29 precedent) to nail that down given how much of the session went into establishing the reconnect-storm causal chain above; the batch gdb run (-ex run -ex 'thread apply all bt full' -ex quit) auto-quit after the crash, so no live interactive follow-up was possible against this specific capture. + +NO FIX ATTEMPTED this session (per this entry's own established 'do not attempt a speculative fix without a confirmed root cause' guidance, followed again here) -- the reconnect-storm mechanism is a strong, well-evidenced AMPLIFYING factor, not yet the root cause of the UAF itself. + +NEXT STEPS, sharpened and now much more concrete than before: +(1) The reconnect-storm itself may be its own separate, worthwhile fix regardless of bug-1714: investigate why ctrl-1's commit/registration backpressure under a burst of concurrent array submissions is severe enough to trip the scheduler client's failover timeout even though ctrl-1 never actually loses leadership -- possibly a client-side timeout tuned too aggressively for this load pattern, or a controller-side responsiveness gap under WAL/commit pressure worth its own investigation (cross-reference bug-1713's E_XCE_NOT_READY history). Fixing or tuning this would likely reduce (not necessarily eliminate) bug-1714's own reproduction rate as a side effect, the same way Bug A/B's landing seemed to reduce it, per the 2026-08-30 session's own observation. +(2) To find the actual stale-pointer read site: repeat this session's exact recipe (now confirmed reliable again, ~12s to crash) but with a scripted gdb watchpoint pass (PHASE 2 precedent, 2026-08-29 session) armed BEFORE the burst starts, on several job_submit_data_t->project_name fields sampled from make_array_element() calls, to catch the actual write-after-free this time under the now-better-understood load conditions. +(3) Given the crash this session hit event index 16 of a 67-event batch (previous captures: index 2 of 6, and index unspecified in the 2026-08-29 session) -- worth checking whether crash position scales with reconnect-storm-driven batch size (more concurrent JOBS_PUSH full-resync traffic -> larger translate_pending_events batches -> more job_submit_data_t churn per batch -> proportionally higher crash odds within a batch), which would further support the amplification theory quantitatively. + +Cleanup: isolated build dir (src/xce.build.bug1714repro) and deploy dir (/tmp/xce-deploy-1714) left in place (reusable, no live-process footprint) -- all started processes (3x xce_controller, 1x xcecd_virtual, xce_scheduler + its gdb wrapper) confirmed killed via ps aux before this session ended. + +UPDATE (2026-09-04, static-only session, ~30 min): Pursued a new, previously-unchecked lead -- traced full ownership of j->submit->resreq (the raw wire string) through parse_resreq() -> parse_resreq_primary() (resreq.c:307) -> parse_section() (splits select[]/rusage[]/span[]/same[]/affinity[] substrings, MOVE_PTR'd into a local xce_resreq_data_t) -> parse_select() (resreq.c:205, builds the actual xce_select_t AST attached to job->resreq long-term) -> the select-expression PEG parser itself (xce_select.peg/xce_select.c). Hypothesis: if any hop stores a RAW pointer into the short-lived job_submit_data_t/its derived substrings instead of an owned copy, job->resreq (long-lived, read on every subsequent scheduling round) would hold a dangling pointer once job_submit_data_t is freed post-push -- a different, more delayed UAF trigger than the push-time-only theories every prior session chased. + +RULED OUT, with exact citations. (1) Every SEL_STR string-literal node in the select AST is makeString()-interned (xstring-refcounted, heap-independent) at the exact grammar action that parses it, BEFORE being stored -- xce_select.peg:161 ("char* s = makeString(buf);") and xce_select.peg:261 ("char* s = makeString((char*)$0);" -- $0 is the PEG grammar's own captured-token metavariable, not a shell variable), confirmed by makeSelectNode()'s own SEL_STR handling (xce_select.c) explicitly calling dereference()/reference() on op1, i.e. treating it as an xstring, not a raw pointer. (2) Every attribute/resource-name reference (e.g. the `type` in `type == "x86_64"`) resolves via resolve_resname() (xce_select.peg:17-55), which does key_find(auxil->resmgr, (void*)name, ...) -- the raw parsed token is used ONLY as a transient lookup key; the node stores the LOOKUP RESULT, a record owned by resmgr itself (long-lived, cluster-wide, unconditionally outlives any single job), never the job's own substring. (3) parse_resreq_primary() (resreq.c:307-349) only frees the intermediate section-split strings (kill_resreq_data(&data), line 346) AFTER parse_select()/parse_rusage() have already synchronously consumed them within the same function call -- no cross-call aliasing window exists there either. Net: job->resreq's entire AST is fully self-contained/independently-owned by the time core_build_job() returns; this lead does not explain bug-1714's crash and should not be re-chased. + +No fix attempted (none needed -- hypothesis ruled out, not confirmed). No live repro attempted this session (static tracing alone was sufficient to close this specific lead; the actual stale-second-reader for j->submit itself, as opposed to any of its parsed derivatives, remains unfound -- see this entry's existing NEXT STEP notes, still the sharpest open leads). + +UPDATE (2026-09-04, later same session): a second static lead was checked and RULED OUT -- whether the SAME xce_job_t* object (not just a duplicate jobid via independent clones, already covered by Bug A) could be drained/processed twice via the pend_updates dataq double-buffer swap (driver.c's xce_sched_driver_translate_pending_events(), handler.c's enq_pend_update()/sched_on_jobs(), lib/dataq.c). Traced dataq_enq()/dataq_deq() (mylock.c-style single mutex per queue, dataq.c) and the pend_updates[0]/[1] swap-under-core->lock pattern line by line: dataq_deq_head() unconditionally unlinks the returned item (b->next = NULL, q->head advanced) before returning it, so a given enqueued data_item_t can structurally never be dequeued twice without an intervening fresh dataq_enq() of that same pointer -- and no such re-enqueue site exists (sched_on_jobs()/sched_handle_admin_request() both calloc() a fresh xce_sched_update_request_t per call, enq_pend_update() is their only caller). The swap itself (pend_updates[0] <-> [1]) and the concurrent producer-side enq_pend_update() both hold core->lock, so there is no window for the drain loop to observe a queue the producer is still mutating. This path is clean; does not explain bug-1714. + +Running tally of candidates now RULED OUT across all sessions (do not re-chase without new evidence): (1) XceData_data_schedjobs dataset->rows aliasing (2026-08-30). (2) core_build_schedobj()'s user/queue/project handling -- confirmed makeString() deep copies (2026-09-04). (3) resreq/select-expression string ownership through parse_resreq() -- confirmed fully independent AST, makeString()-interned literals, resmgr-owned resname lookups (2026-09-04). (4) pend_updates dataq double-enqueue/double-drain (this update). Sharpest remaining unexplored angle: the controller-side HOOK_DETAIL_JOB_NEW/xcesched_push_joblist() live trace (2026-08-30's own NEXT STEP, never actually run live) and the connection-flap-driven resync amplification mechanism (found 2026-09-04, this session's earlier fork) -- both point at the CONTROLLER side re-pushing job data the scheduler already has, not at anything in the scheduler's own translate/build/free pipeline (now thoroughly checked and clean). A future session should pursue the controller side next, not re-audit scheduler.c/core_events.c/driver.c string or pointer ownership again. + +UPDATE (2026-09-04, controller-side dual-push-path check): a third static lead was checked and RULED OUT before committing to another live gdb session (the main session's own codegraph read already showed the key evidence; independently re-confirmed by re-reading src/controller/sched.c directly) -- whether the controller's two independent job-push paths could race and deliver the same jobid via two concurrent wire messages: (1) on_scheduler_join() (sched.c:742-752), which unconditionally calls xcesched_push_jobs() (sched.c:214, a FULL resync of every currently-committed job via XceData_data_schedjobs/collect_jobs_committed()) on EVERY scheduler join or rejoin -- including a reconnect-triggered rejoin after a connection flap; and (2) sched_job_new_fun() (sched.c:305-337), the XCE_CONTROLLER_HOOK_JOB hook installed exactly once per controller process lifetime (on_scheduler_join()'s own static hook_installed guard), which fires incrementally on every new job commit via xcesched_push_joblist()/xcesched_push_jobarray(). RULED OUT: both functions hold c->sched.lock for their ENTIRE body (mylock_lock(&c->sched.lock) wraps all of on_scheduler_join()'s pushes including xcesched_push_jobs(); the same lock wraps sched_job_new_fun()'s push call too) -- a single mutex held for the full duration of each makes concurrent/interleaved execution of the two paths structurally impossible. No live capture could show otherwise; this is provable by code reading alone. + +This DOES fully explain (without needing a race) the still-unexplained 'redelivery storm' the 2026-08-30 session found (array-job jobids redelivered via the ordinary single-job XceData_data_schedjobs path, up to 18x per jobid, safely discarded by Bug A's sched_add_job() dedup): since the two paths are strictly SEQUENTIAL rather than concurrent, an already-committed job is pushed once via the incremental commit-hook, then pushed AGAIN in full every single time the scheduler's connection flaps and rejoins -- on_scheduler_join() re-pushes the ENTIRE committed job set on every join, not just the first. Combined with this session's earlier finding that the scheduler's connection genuinely flaps under controller-side commit-path backpressure (E_XCE_NOT_READY) during heavy array-job load, a burst with N flaps produces N redundant full resyncs of every already-known job -- sufficient on its own to explain 'up to 18x per jobid' with zero race involved. Expected (if wasteful) behavior, not a crash cause. + +Running tally of candidates now RULED OUT across all sessions (do not re-chase without new evidence): (1) XceData_data_schedjobs dataset->rows aliasing. (2) core_build_schedobj()'s user/queue/project handling. (3) resreq/select-expression string ownership through parse_resreq(). (4) pend_updates dataq double-enqueue/double-drain. (5) controller-side dual job-push-path race (this update) -- serialized by c->sched.lock, and its sequential nature fully explains the redelivery-storm volume without a race. All scheduler-side memory-ownership candidates AND the controller-side push-path race are now exhausted. The actual stale-second-reader for j->submit remains unfound after 9 sessions total. No further cheap/static leads are known at this time -- a future session likely needs either (a) a fresh live gdb-watchpoint campaign specifically instrumenting core->jobs's OWN lifecycle (does anything else hold a raw xce_sched_job_t*/job_submit_data_t* reference outside core->jobs and outside the short window core_build_job() itself runs in? e.g. snapshot/recording code paths, REPL inspection commands, or a picker plugin caching a job reference across sessions) which has NOT been checked by any prior session, or (b) accepting this as a rare, low-frequency-under-extreme-load production risk to monitor rather than a bug worth further dedicated live-debugging investment at this time -- a decision for the project owner, not something to keep re-attempting speculatively. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1699, bug-1731 +- Gitea: #265 diff --git a/.wolf/bugs/bug-1715.md b/.wolf/bugs/bug-1715.md new file mode 100644 index 00000000..adf6e747 --- /dev/null +++ b/.wolf/bugs/bug-1715.md @@ -0,0 +1,38 @@ +--- +id: "bug-1715" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "crash" +timestamp: "2026-08-26T18:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request)"] +tags: ["xceadmin", "scheduler-admin", "scheduler-disconnected", "retry", "transient-state", "bug-1713-adjacent", "bug-1693-adjacent", "found-live", "root-caused", "fix-scoped", "live-verified"] +related: ["bug-1713", "bug-1693"] +legacy: true +--- + +# bug-1715: User-reported, live 3-node cluster: `xceadmin scheduler record status --controller localhost:9901`… + +## Description +User-reported, live 3-node cluster: `xceadmin scheduler record status --controller localhost:9901` failed once with `Error: scheduler disconnected`, then succeeded on the very next identical invocation moments later. Root-caused to a full cluster redeploy (user confirmed: 'This is a full rebuilt and re-deployed cluster') restarting all 5 processes (3 controllers + worker + scheduler) together -- PIDs across ctrl-1/ctrl-2/ctrl-3/scheduler/worker logs are all tightly clustered (4137667/4137684/4137696/4137745/4138046), confirming a coordinated restart, not an isolated scheduler crash. The request happened to land in the brief, normal window between the old scheduler disconnecting and the new one reconnecting. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +FULLY ROOT-CAUSED, same family as bug-1713 but NOT covered by that fix. 'scheduler disconnected' is delivered via a DIFFERENT signaling path than bug-1713's two numeric status codes: cluster.c's bug-1693 disconnect-detection mechanism (cluster.c:1052-1067) sends `reply.status = 0` (success at the RPC level) with `schedadminreplymsg{ok=false, error="scheduler disconnected"}` -- a normal-status reply carrying a domain-level failure, not a nonzero `reply.status` the way E_XCE_NOT_READY/E_XCE_NOMASTER are. `_send_admin_request()` (scheduler.py:145-154) treats `admin_reply.ok == False` as an unconditional hard SchedAdminError for EVERY reason string, with no distinction between genuinely transient conditions ('scheduler disconnected', 'scheduler not connected', 'request timeout' -- all connection/timing related, expected to self-resolve within seconds, especially during a redeploy/restart cycle) and genuinely permanent domain errors ('not recording', 'already recording', 'invalid record file name' -- real error states that must NOT be retried, e.g. calling record-stop on an already-stopped recorder is a correct, meaningful rejection, not a transient hiccup). + +## Fix +FIXED and LIVE-VERIFIED (2026-08-26). Extended bug-1713's retry loop in `_send_admin_request()` (src/xceadmin/src/xceadmin/scheduler.py:158-249) to a SECOND, differently-signaled transient family: after decoding `admin_reply = reply.data.schedadminreplymsg` and finding `admin_reply.ok == False`, the code now checks `admin_reply.error in _TRANSIENT_ADMIN_REASONS` (new module-level frozenset, scheduler.py:114-160) and, if so and `attempt < _MAX_TRANSIENT_ATTEMPTS`, sleeps `_TRANSIENT_RETRY_DELAY_S` and `continue`s the SAME `for attempt in range(1, _MAX_TRANSIENT_ATTEMPTS+1)` loop bug-1713 already added -- no second retry loop, same budget (4 attempts, 1.5s fixed delay), same re-run-`follow_redirects()`-from-scratch shape (a fresh attempt can discover a newly-reconnected scheduler, exactly like a fresh attempt can discover a newly-settled primary in bug-1713's case). On exhaustion, raises the same `SchedAdminError` shape with the reason string annotated '(still transient after N attempts over ~Xs -- giving up)', mirroring bug-1713's own distinguishable-message convention exactly. AUTHORITATIVE error-string list (grepped every literal `sched_admin_reply_t`/`SchedAdminReply` `.error = "..."` assignment across src/controller/ and src/scheduler/, then read each site's surrounding code, not just the string text): transient (all 4 now in `_TRANSIENT_ADMIN_REASONS`) -- "scheduler disconnected" (cluster.c:1062, bug-1693's disconnect-detection path), "scheduler not connected" (cluster.c:958, `c->sched.ep==NULL`), "request timeout" (cluster.c:1078, SCHEDADMIN_TIMEOUT_MS wait expired), and "server busy, try again" (cluster.c:937, `vtaskmgr_blocking_acquire()` admission-reservation exhaustion -- a 4th transient reason NOT in this bug's original three-string hypothesis, added after reading the admission-gate's own 'try again' design comment: resource contention that self-resolves as soon as another admitted vtask op completes, not a domain rejection). Permanent (left untouched, still raise on the very first attempt): "already recording"/"not recording" (driver.c:390/426, rc==-EALREADY) and validation_error_text()'s six filename/path-rejection strings (driver.c:298-316: "invalid record file name", "record directory not found", "record directory path is not a directory", "permission denied accessing record file", "a non-file entry with this name already exists", "record file path too long"), plus a dynamic (not a fixed literal, so it could never join an exact-match retry set anyway) "cannot write to %s: %s" I/O-failure message (driver.c:398-404). RED/GREEN proof (src/xceadmin/tests/test_scheduler.py): confirmed RED by temporarily reverting only the two bug-1715 edits back to bug-1713's exact shipped state (not via git stash, since bug-1713's own fix was itself still an uncommitted working-tree change sharing the same file -- a bare `git stash` would have reverted bug-1713 too) -- the new parametrized `test_record_status_retries_on_transient_admin_reason_then_succeeds`/`..._retry_exhaustion_on_transient_admin_reason_...` tests failed at COLLECTION time (`AttributeError: module 'xceadmin.scheduler' has no attribute '_TRANSIENT_ADMIN_REASONS'`), and a standalone ad hoc script against that same pre-fix code confirmed BEHAVIORAL red too: a real TLS mock returning `ok=false, error='scheduler disconnected'` once raised `SchedAdminError` immediately after exactly 1 connection, no retry. Reapplied the fix -> GREEN: all 41 tests in test_scheduler.py pass, including 4 new `_TRANSIENT_ADMIN_REASONS`-parametrized retry-then-succeed tests, 4 new retry-exhaustion tests (one per confirmed-transient reason, each asserting exact connection_count == _MAX_TRANSIENT_ATTEMPTS and a distinguishable exhaustion message), and a new `test_record_stop_not_recording_never_retried` non-regression test (asserts connection_count==1, zero sleep calls, for the permanent 'not recording' reason). The pre-existing `test_record_start_error_codes_raise_sched_admin_error` parametrize list was edited (not left untouched): "scheduler not connected"/"request timeout" were removed (they are now confirmed-transient and retried, not raised immediately -- their coverage moved to the new retry tests) and replaced with two more confirmed-permanent validation_error_text() strings ("record directory not found", "permission denied accessing record file") alongside the original "invalid record file name", for stronger permanent-error coverage. bug-1713's own 5 tests (transient-status retry/exhaustion, both E_XCE_STANDBY regression locks) were run UNMODIFIED and all still pass -- no shared-loop refactor was needed since bug-1713's `for attempt in range(...)` loop and its `reply.status != 0` branch were reused as-is; only a second `if`/retry check was added inside the existing `if not admin_reply.ok:` branch. Full xceadmin suite: 283 passed, 1 failed -- the failure (`test_live_config_push_against_standby_returns_standby_rejection_not_redirect`) is the SAME pre-existing, already-flagged-stale failure named in bug-1713's own report (test_e2e_live_ha_push.py, stale since bug-1700/1701's redirect-gap fix generalized to OPCode_upgradeConfig) -- zero new failures caused by this fix (284 total tests vs bug-1713's reported 275, consistent with the 9 new tests added here). LIVE VERIFICATION (real 3-node cluster, deploy-ha.sh + start-all.sh, build.dbg's real xce_controller/xce_scheduler/xcecd_virtual binaries -- no C rebuild needed since this fix is Python-only and none of the C-side cluster.c/driver.c code changed): hammered `xceadmin.scheduler.record_status('localhost:9991')` (the elected primary) in a tight zero-delay loop against the fixed client for 20s while concurrently `kill -9`-ing the live xce_scheduler process mid-run and restarting it via `start-scheduler.sh` (a genuine scheduler-process-death-and-restart, not a mock, not a controller failover). Result: 153 of 154 requests during the window succeeded, most transparently absorbing the scheduler-down period with zero visible failure to the caller; ONE request genuinely hit the real 'scheduler not connected' condition, went through the full retry budget (logged exactly as designed: 'scheduler not connected (still transient after 4 attempts over ~4.5s -- giving up)'), and still surfaced a SchedAdminError because this particular sandbox's scheduler restart (relaunched via a fresh `nix develop` subshell each time, which carries its own multi-second evaluation overhead on top of the scheduler's own re-join handshake) took longer to fully rejoin than the shared 4-attempt/1.5s (~4.5s total) budget -- the very next request after that succeeded again. This is honestly reported as PARTIAL absorption, not a zero-failure live result like bug-1713's controller-failover scenario got: the retry mechanism is proven to engage correctly on a genuine, non-mocked signal end-to-end (real controller, real scheduler restart, real wire reply, real client-side retry/give-up message), and it converted what would have been 100% visible failure for the whole outage window (pre-fix) into 1 failure out of 154 requests (post-fix) -- but this sandbox's nix-develop-relaunch overhead for the scheduler restart is likely slower than a real production supervisor restart would be, so the single exhaustion here should not be read as proof the shared 4.5s budget is too short in general. Kept the identical bug-1713 budget as instructed (no concrete evidence specific to THIS class of condition that it's wrong -- the one exhaustion observed is plausibly a sandbox-restart-mechanism artifact, not a signal about the real-world scheduler-restart timescale) rather than inventing a different cadence from one ambiguous data point. `push.py`'s `push_config()`/`probe_config()` gap flagged by bug-1713 as a sibling finding remains unfixed and out of this bug's scope (unchanged from bug-1713's report). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1713, bug-1693 diff --git a/.wolf/bugs/bug-1716.md b/.wolf/bugs/bug-1716.md new file mode 100644 index 00000000..0a3650ed --- /dev/null +++ b/.wolf/bugs/bug-1716.md @@ -0,0 +1,38 @@ +--- +id: "bug-1716" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T14:25:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/rcfile.c"] +tags: ["rcfile", "fopen", "EISDIR", "directory", "getline", "xce-test-rc-file", "controller-testsuite", "stderr-warning"] +related: [] +legacy: true +--- + +# bug-1716: test_rcfile_load_directory_path_warns_and_returns_0:FAIL: Expected TRUE Was FALSE (stderr capture… + +## Description +test_rcfile_load_directory_path_warns_and_returns_0:FAIL: Expected TRUE Was FALSE (stderr capture was empty -- no warning printed) when rcfile_load() opened a directory path + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +On this platform (glibc/Linux), fopen(path, "r") SUCCEEDS for a directory path -- it does not return NULL/set errno the way the design doc's error-handling section (and a naive reading of 'the file could not be opened for another reason') implied. The open failure only surfaces on the FIRST READ: getline() returns -1 with errno==EISDIR and ferror(f) set (feof(f) stays 0). rcfile_load()'s original implementation only checked `if (!f)` right after fopen() and had no post-loop check on the stream's error state, so a directory path silently produced rc=0 with zero lines processed and NO warning ever printed -- functionally 'safe' (no crash, no runner call) but silently violating the design's documented 'present but unreadable: print one stderr warning' contract for exactly this case. + +## Fix +Added a check after the getline() loop in rcfile_load(): `if (rc == 0 && ferror(f)) { fprintf(stderr, ...); }` -- prints the same warning message used for the fopen()-failure branch, keyed off ferror(f) rather than assuming the failure is only detectable at fopen() time. Confirmed live via a small standalone C repro (fopen("/tmp","r") then fread/getline) that fopen() returns a valid FILE* for a directory and the read call is what fails with EISDIR + ferror set. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1717.md b/.wolf/bugs/bug-1717.md new file mode 100644 index 00000000..472f0aba --- /dev/null +++ b/.wolf/bugs/bug-1717.md @@ -0,0 +1,38 @@ +--- +id: "bug-1717" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-08-26T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/commit.c"] +tags: ["flaky", "asan", "use-after-free", "controller_unit_test", "commit.c", "persist.c", "fsync_worker", "test_apply_pump.c", "teardown-race", "unreproduced"] +related: [] +legacy: true +--- + +# bug-1717: ASAN heap-use-after-free abort in controller_unit_test: commit_resolve_pending (commit.c:807)… + +## Description +ASAN heap-use-after-free abort in controller_unit_test: commit_resolve_pending (commit.c:807) racing fsync_worker (persist.c:867) against tap_teardown (test_apply_pump.c:243) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Background fsync_worker thread appears to still be resolving a pending commit via commit_resolve_pending() while a test teardown (tap_teardown in test_apply_pump.c) is freeing/tearing down state concurrently -- a shutdown-ordering race between the async fsync/commit-resolution thread and per-test teardown. Observed once during xce-test-rc-file task 1.2 review (ad-hoc standalone controller_unit_test run); did NOT reproduce on a second standalone run or in the canonical xce-build.sh run (both completed cleanly with 1213 Tests 2 Failures 0 Ignored, the 2 failures being the unrelated tracked bug-1561). Not caused by and unrelated to rcfile.c/test_rcfile.c (xce-test-rc-file spec) -- flagged as a pre-existing flake surfaced incidentally during that task-local review. + +## Fix +Not yet root-caused or fixed -- logged for tracking since it did not reproduce reliably enough to investigate further in that review pass. Next occurrence should capture the full ASAN report (not just the top frames) and check whether test_apply_pump.c's teardown waits for fsync_worker to fully quiesce before freeing shared state. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1718.md b/.wolf/bugs/bug-1718.md new file mode 100644 index 00000000..554d3b0c --- /dev/null +++ b/.wolf/bugs/bug-1718.md @@ -0,0 +1,38 @@ +--- +id: "bug-1718" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T16:23:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_rcfile_integration.sh"] +tags: ["bash", "python", "pty", "heredoc", "stdin", "xce-test-rc-file", "task-3.1"] +related: [] +legacy: true +--- + +# bug-1718: A python3 pty-driver embedded via a heredoc on a command that ALSO receives piped stdin silently… + +## Description +A python3 pty-driver embedded via a heredoc on a command that ALSO receives piped stdin silently discards the piped input + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +First draft of run_interactive_pty() did: printf '%s\n' "$script" | python3 - "$XCE_TEST" ... <"$PTY_DRIVER" < does not consume stdin to obtain its own source. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1719.md b/.wolf/bugs/bug-1719.md new file mode 100644 index 00000000..cadd186a --- /dev/null +++ b/.wolf/bugs/bug-1719.md @@ -0,0 +1,38 @@ +--- +id: "bug-1719" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-26T17:14:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/xce-build.sh"] +tags: ["xce-build.sh", "incremental", "build-dir-clobber", "scheduler", "controller", "xce-test-rc-file", "task-3.3"] +related: [] +legacy: true +--- + +# bug-1719: resource-value-verification skill's deployment failed: 'controller binary missing' immediately… + +## Description +resource-value-verification skill's deployment failed: 'controller binary missing' immediately after a scheduler-only xce-build.sh run that had just succeeded + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +During xce-test-rc-file task 3.3 (compatibility verification, no product-code change), ran `nix develop ./src --command bash src/tools/xce-build.sh scheduler` (no --incremental) to fill a gap where xce.build/scheduler/xce_scheduler had never been linked. xce-build.sh's documented default behavior (see its own top-of-file comment) does a full `rm -rf "$BUILD_DIR"` + fresh `meson setup` on every invocation unless --incremental is passed, then `meson compile` only rebuilds the ONE requested component -- so the plain `scheduler` invocation silently wiped xce.build/controller/xce_controller and xce.build/controller/xce_test (built moments earlier by task 3.1/3.2's controller build, containing the rc-file feature under test) even though I never touched controller source. A prior attempt in the same session had additionally passed an explicit build_dir arg (`src/xce.build`) which -- combined with xce-build.sh's own internal cd into src/ -- produced a bogus nested `src/src/xce.build` directory instead of reusing the real one (harmless, fully gitignored via meson's own auto-generated per-builddir `.gitignore: *`, but wasted a build cycle and caused brief confusion investigating why `git status`/`git check-ignore` showed nothing for it). + +## Fix +Rebuilt controller incrementally (`xce-build.sh controller asan xce.build --incremental`) to restore xce_controller/xce_test alongside the already-built scheduler, without wiping it back out. Confirmed both controller_unit_test (1213 Tests 0 Failures -- even bug-1561's usually-flaky 2 tests passed clean this run) and the scheduler suite (53/54, only the pre-existing tracked bug-1560 symlink-escape failure) came back healthy. General lesson for any task that needs a SECOND component built into an ALREADY-built xce.build dir: always pass --incremental, and never pass an explicit build_dir positional unless it already matches xce-build.sh's own default relative-path resolution (omit the 3rd arg and let it default to 'xce.build' resolved from src/tools/xce-build.sh's own cwd, don't hand it a repo-root-relative 'src/xce.build' path). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1720.md b/.wolf/bugs/bug-1720.md new file mode 100644 index 00000000..f571e464 --- /dev/null +++ b/.wolf/bugs/bug-1720.md @@ -0,0 +1,38 @@ +--- +id: "bug-1720" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-08-26T17:17:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["environment (stale orphaned process", "not a code defect)"] +tags: ["xce-test-resources", "apiserver", "port-conflict", "orphaned-process", "sandbox-hygiene", "xce-test-rc-file", "task-3.3"] +related: [] +legacy: true +--- + +# bug-1720: xce-test-resources run-resource-verify.sh: apiserver layer UNREACHABLE for all 9 injected… + +## Description +xce-test-resources run-resource-verify.sh: apiserver layer UNREACHABLE for all 9 injected resources; apiserver.log showed 'API Server fails to start: listen tcp :8080: bind: address already in use' + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +A leftover /tmp/apiserver process (pid 1458981, launched via /tmp/xce-deploy-bug1701/scripts/start-apiserver.sh) from an unrelated, already-concluded prior session's bug-1701 investigation was still running and bound to :8080 -- 7.5 hours old (started 09:44, found at 17:18) with its own deploy directory (/tmp/xce-deploy-bug1701) already deleted, i.e. genuinely abandoned rather than an active concurrent session. This is a general sandbox hygiene gap: xce-test-resources' own trap-based cleanup only covers processes IT started: nothing in this sandbox periodically reaps orphaned daemons left behind by an unrelated script/session that itself never got to run its own cleanup (e.g. was killed externally, or its subagent's context ended before the trap fired). + +## Fix +Identified via `lsof -i :8080` / `ss -ltnp`, confirmed genuinely orphaned (dead deploy dir, long uptime, no relation to this session's own work), then `kill -9` on both the wrapper script and the apiserver binary PID. Re-ran run-resource-verify.sh immediately after and got a clean PASS (9 resources, 3 layers, zero mismatches/unreachable/rejected). General lesson: an apiserver-layer UNREACHABLE result in this skill should prompt checking `lsof -i :8080` / `pgrep -af apiserver` for a stale bind-conflict BEFORE suspecting a propagation regression -- the skill's own FAILURE/UNREACHABLE line doesn't distinguish 'never started due to port conflict' from 'started but never logged the trace line', both surface identically as UNREACHABLE. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1721.md b/.wolf/bugs/bug-1721.md new file mode 100644 index 00000000..852f19f0 --- /dev/null +++ b/.wolf/bugs/bug-1721.md @@ -0,0 +1,38 @@ +--- +id: "bug-1721" +status: "open" +status_updated: "2026-09-24" +status_note: "clean_failover_preservation.sh default ha/data dir no longer exists" +component: "controller" +severity: "unknown" +timestamp: "2026-08-26T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/clean_failover_preservation.sh"] +tags: ["test-harness", "stale-fixture", "clean_failover_preservation.sh", "ha-term-safety", "pre-existing", "admin-config-tool-regression"] +related: [] +legacy: true +--- + +# bug-1721: clean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite) + +## Description +clean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The script hardcodes its DATADIR precondition default to $REPO_ROOT/ha/data (a pre-generated cluster.bin + TLS cert template directory it expects to already exist), but ha/data/ was deliberately deleted from the repo by an unrelated commit (ee8473bd, admin-config-tool: drop ha/data/ entirely, match real client TLS). The script was never updated afterward, so its own documented precondition check now fails unconditionally in any fresh checkout/worktree that has not manually recreated that directory out-of-band. Discovered while adding --norc to the script (xce-test-rc-file spec follow-up) and independently verifying the edit did not break anything -- confirmed via git history that the failure reproduces identically on the pre-edit script too, so it is unrelated to that change. + +## Fix +Not fixed -- out of scope for the xce-test-rc-file spec. Needs either: (a) DATADIR default updated to point at a still-valid TLS/config template location consistent with admin-config-tool's TLS changes, or (b) the script updated to self-generate its own template via deploy-ha.sh/xce-deploy-ha before the precondition check, matching how run-ha-soak.sh and run-resource-verify.sh already self-deploy their own scratch clusters instead of depending on a fixed pre-existing directory. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1722.md b/.wolf/bugs/bug-1722.md new file mode 100644 index 00000000..1d3b6934 --- /dev/null +++ b/.wolf/bugs/bug-1722.md @@ -0,0 +1,38 @@ +--- +id: "bug-1722" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-08-27T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/xcealloc.c"] +tags: ["heap-buffer-overflow", "asan", "xcealloc", "multi-host-job", "make_job_account", "make_host_allocs", "pre-existing"] +related: ["bug-1621"] +legacy: true +--- + +# bug-1722: AddressSanitizer heap-buffer-overflow in make_job_account() (src/lib/xcealloc.c:188), WRITE of… + +## Description +AddressSanitizer heap-buffer-overflow in make_job_account() (src/lib/xcealloc.c:188), WRITE of size 8 -- found while verifying viability of a proposed scheduler snapshot-allocation-capture fix, not user-reported. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +make_job_account() allocates account->accounts via calloc(1, sizeof(xce_host_account_t*)) -- room for exactly 1 host pointer -- but its own loop writes account->accounts[i] for i in [0, jalloc->nhosts), overflowing the heap allocation for any job spanning more than one host. Already reachable from live production dispatch (core_events.c:262, schedobj.c:207, scheduler.c:1444) for any multi-host job, not introduced by this session. Sibling function make_host_allocs() (xcealloc.c:134) has the identical bug shape (calloc(1, ...) vs a loop writing hallocs[i] up to nhosts-1), though every current call site happens to pass nhosts=1 so it has never actually overflowed in production -- still a live footgun for any future 2+-host caller of a public, exported helper. + +## Fix +FIXED, TDD RED->GREEN. New test_xcealloc.c (src/lib/testsuite/, wired into libxce_test_src in meson.build + test.c) drives a 2-host xce_job_alloc_t through make_job_account(), reproducing the exact ASAN heap-buffer-overflow at xcealloc.c:188 pre-fix. Fix: both calloc(1, ...) call sites changed to calloc((size_t)nhosts, ...) / calloc((size_t)jalloc->nhosts, ...) -- xcealloc.c:134 (make_host_allocs) and xcealloc.c:180-181 (make_job_account). Also added a covering test for make_host_allocs multi-host correctness. Verified: full libxce test suite 303/303 (was 301, +2 new), scheduler suite 53/54 (1 pre-existing unrelated bug-1560 flake, test_recordstart_symlink_escape_replies_error), controller suite 1211/1213 (2 pre-existing unrelated bug-1561 overlayfs flakes, test_conf_trim_keeps_only_n_versions/test_conf_get_saved_tag_no_config) -- zero regressions, zero new failures. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1621 diff --git a/.wolf/bugs/bug-1723.md b/.wolf/bugs/bug-1723.md new file mode 100644 index 00000000..0c79917d --- /dev/null +++ b/.wolf/bugs/bug-1723.md @@ -0,0 +1,38 @@ +--- +id: "bug-1723" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-08-27T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/testsuite/test_xcealloc.c"] +tags: ["memory-leak", "asan", "leaksanitizer", "xcealloc", "test-fixture-teardown", "scheduler-snapshot-job-allocation", "pre-existing", "out-of-scope"] +related: ["bug-1722", "bug-1621", "bug-1437"] +legacy: true +--- + +# bug-1723: LeakSanitizer (xcelib_test's ASAN default detect_leaks=1, per bug-1437's established precedent… + +## Description +LeakSanitizer (xcelib_test's ASAN default detect_leaks=1, per bug-1437's established precedent that this is a separate pass/fail axis from Unity PASS/FAIL) aborts the whole xcelib_test binary at exit with SIGABRT even on a completely clean run (all Unity tests, including the new scheduler-snapshot-job-allocation task 1.1 dup_job_alloc() suite, report 0 Failures) -- 456 byte(s) leaked in 29 allocation(s), confirmed present identically at HEAD before task 1.1's changes (baseline A/B comparison: stashed task 1.1's 3 changed files, rebuilt, reran standalone binary -- same 456 bytes/29 allocations, same two leak sources, same SIGABRT). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +test_make_job_account_multi_host_no_overflow() (added in bug-1722's fix) calls free_job_account(account) but never frees its own directly-built fixture tree (jalloc/hallocs/h0/h1/a0/a1 -- make_job_alloc()'s calloc at xcealloc.c:156 and the two make_alloc()-internal strdup()s it roots). test_make_host_allocs_multi_host_no_overflow() (same commit) frees nothing at all (make_host_allocs()'s calloc at xcealloc.c:134 and its two hosts/allocs). Neither test calls free_JobAlloc_ptr() or any other teardown on its own locally-constructed xce_job_alloc_t/xce_host_alloc_t** tree. Pre-existing since bug-1722's commit (00207503), not introduced by task 1.1 -- confirmed via git-stash A/B rebuild showing byte-for-byte identical leak signature with task 1.1's 3 files reverted. + +## Fix +FIXED (2026-09-08, follow-up session). Applied exactly the fix this entry already specified: added free_JobAlloc_ptr(&jalloc); at the end of test_make_job_account_multi_host_no_overflow() (frees jalloc/hallocs/h0/h1/a0/a1's whole tree, after free_job_account(account) already frees the account side) and free_HostAlloc_list(2, hallocs); at the end of test_make_host_allocs_multi_host_no_overflow() (which previously freed nothing at all). This reduced the leak from 456 bytes/29 allocations to 12 bytes/2 allocations -- the residual was a DIFFERENT, deeper pre-existing production bug (free_host_account() never freeing acct->host), already independently logged as bug-1621 and fixed in the same pass; see that entry. Verified: standalone xcelib_test binary now 1/1 Ok with zero LeakSanitizer reports (was: SIGABRT on every run, even with all Unity tests passing); full xce-build.sh all asan --incremental: 94/94 Ok (controller/scheduler/lib/worker/apiserver C suites + all Go test packages), zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1722, bug-1621, bug-1437 diff --git a/.wolf/bugs/bug-1724.md b/.wolf/bugs/bug-1724.md new file mode 100644 index 00000000..b73e6e74 --- /dev/null +++ b/.wolf/bugs/bug-1724.md @@ -0,0 +1,38 @@ +--- +id: "bug-1724" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-08-28T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/meson-build.sh"] +tags: ["xceadmin", "meson-build.sh", "repo-mirror", "live-e2e", "test-discovery", "symlink", "build-infra", "found-live", "root-caused", "live-verified"] +related: [] +legacy: true +--- + +# bug-1724: 3 xceadmin live E2E tests… + +## Description +3 xceadmin live E2E tests (test_e2e_live_push.py::test_live_config_push_against_real_single_node_controller, test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path, test_e2e_live_ha_push.py::test_live_config_push_against_standby_returns_standby_rejection_not_redirect) always SKIPPED under the meson-driven build path (`/xce-build xceadmin`, `nix build .#xcecore`) with 'no xce_controller binary found under xce.build/build.release/build.dbg', regardless of whether a real xce_controller/xceadmin binary actually existed anywhere in the checkout. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +tests/_live_e2e_common.py's CONTROLLER_CANDIDATES/XCEADMIN_CANDIDATES/DEPLOY_HA_SH resolve paths relative to the test file's own on-disk location (Path(__file__).resolve().parents[2]). That works when pytest runs directly from the real checkout, but meson-build.sh copies the whole xceadmin test tree into an isolated out-of-tree repo-mirror before invoking pytest (to keep the meson build pure), and only ever symlinked ha/, src/configs, src/schema into that mirror -- never the sibling xce.build/build.release/build.dbg controller-binary directories, nor src/tools/ (deploy-ha.sh). So the mirror's own SRC_DIR (repo-mirror/src/) never had a controller subdir, a deploy-ha.sh, or a matching xceadmin binary at the expected relative path, no matter what was actually built in the real repo -- the skip was structural, not a signal about the real build state. + +## Fix +Two changes to meson-build.sh: (1) symlink each profile's `controller` subdirectory (xce.build/controller, build.release/controller, build.dbg/controller -- only the controller subdir, never the whole profile-root, since the profile-root can be an ancestor of the mirror itself when xceadmin is built in that same profile, which would create a self-referential symlink loop) into the mirror so CONTROLLER_CANDIDATES can find a real xce_controller wherever it was built; (2) symlink src/tools (checked-in, not a build-output dir, so a plain safe symlink like configs/schema) so DEPLOY_HA_SH resolves. For xceadmin's OWN binary, which genuinely is self-referential (the mirror lives inside that very build's own xceadmin output), routed around it via the existing XCEADMIN_BIN env-var override (resolve_binary_candidates in _live_e2e_common.py), set to $VENV_DIR/bin/xceadmin -- the binary this very meson-build.sh invocation just pip-installed -- before invoking pytest. Verified end-to-end via a real `xce-build.sh xceadmin debug build.dbg --incremental` run: all 3 tests now execute for real against a live controller/cluster instead of skipping. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1725.md b/.wolf/bugs/bug-1725.md new file mode 100644 index 00000000..df8d3057 --- /dev/null +++ b/.wolf/bugs/bug-1725.md @@ -0,0 +1,38 @@ +--- +id: "bug-1725" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-08-28T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/tests/test_e2e_live_ha_push.py"] +tags: ["xceadmin", "config-push", "redirect", "OPCode_upgradeConfig", "stale-test", "design-doc-sync", "roadmap-sync", "race-condition", "config_activate_committed", "found-live", "root-caused", "live-verified"] +related: ["bug-1724", "bug-1700", "bug-1701", "bug-1468"] +legacy: true +--- + +# bug-1725: After bug-1724's fix made… + +## Description +After bug-1724's fix made test_live_config_push_against_standby_returns_standby_rejection_not_redirect actually execute (previously always skipped), it FAILED for real: pushing config to a standby now redirects to the primary and succeeds, contradicting the test's own hard-coded 'standby flatly rejects, no redirect' assertions. Separately, the rewritten success-path test's reused _assert_primary_committed() helper intermittently failed its 4th assertion ('config activated from committed marker tag=...' not found in the primary's log immediately after the push CLI returned success). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +(1) The test's assumption was stale, not wrong test infra: bug-1700/1701 (fixed 2026-08-26) closed the exact standby-redirect gap this test was written to regression-lock, at the generic ha_dispatch_gate() rejection level in cluster.c -- the test's own docstring had explicitly predicted this exact failure as the correct signal to replace it with a success-path assertion. design.md ('Boundary Commitments'/'Revalidation Triggers') and roadmap.md still described the gap as open. (2) config_activate_committed() on the primary runs off the WAL apply-pump dispatch (persist.c ~5111, the bug-1469 fix), asynchronously relative to the majority-ack-gated client reply (commitIndex vs lastApplied are distinct steps) -- the CLI can legitimately return success before the primary's own local apply of that marker has logged its activation line, a latent race in the existing single-shot log read shared by the direct-to-primary test too. + +## Fix +Renamed and rewrote the test as test_live_config_push_against_standby_redirects_and_succeeds, asserting the redirect line IS present and the push succeeds (same success bar as the direct-to-primary test: positive ackedCount, exit 0, primary commit evidence). Updated the module docstring, design.md (Boundary Commitments, Revalidation Triggers, sequence-diagram note, redirect-implementation rationale, Testing Strategy), and roadmap.md's corresponding item to describe the gap as closed instead of open -- also corrected an adjacent, independently-verified stale claim (bug-1468, fixed 2026-07-25, already resolved but never reflected in design.md) that no controller role could reach the upgradeConfig handler at all. For the activation race, changed _assert_primary_committed() from a single-shot log_text argument to a Path-based helper that polls (up to 5s, 0.2s interval) for the activation line instead of assuming synchronous ordering, and updated both call sites. Verified end-to-end: full xce-build.sh xceadmin run now passes 288/288 with zero skips/failures. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1724, bug-1700, bug-1701, bug-1468 diff --git a/.wolf/bugs/bug-1726.md b/.wolf/bugs/bug-1726.md new file mode 100644 index 00000000..e54a48a2 --- /dev/null +++ b/.wolf/bugs/bug-1726.md @@ -0,0 +1,39 @@ +--- +id: "bug-1726" +status: "cannot-reproduce" +status_updated: "2026-09-24" +status_note: "single occurrence, no surviving logs" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-08-28T08:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/tests/test_e2e_live_ha_push.py (live_ha_cluster fixture", "_wait_for_primary)"] +tags: ["flaky-test", "ha", "raft-election", "live-ha-push", "unresolved", "unreproduced", "low-priority", "distinct-from-bug-1485", "found-live"] +related: ["bug-1485", "bug-1486"] +repro_attempts: "5 immediate retries, 0 reproduced" +legacy: true +--- + +# bug-1726: test_e2e_live_ha_push.py::test_live_config_push_against_standby_redirects_and_succeeds ERRORed at… + +## Description +test_e2e_live_ha_push.py::test_live_config_push_against_standby_redirects_and_succeeds ERRORed at live_ha_cluster fixture setup (single occurrence, user-run via `nix develop ./src` + direct pytest against the built build.dbg mirror, not via any Claude-driven build): _wait_for_primary(nodes, 30.0) raised TimeoutError -- none of the 3 freshly-spawned xce_controller processes ever printed the CTRL_PRIMARY election-complete marker within the 30s timeout (normal election completes ~10.2s per this file's own docstring, keyed to HA_DEFAULT_HEARTBEAT_TIMEOUT_MS=10000). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +- Attempts: 5 immediate retries, 0 reproduced + +## Root cause +NOT ISOLATED -- deliberately distinct from bug-1485 (do not merge into it): bug-1485's signature is election COMPLETING (all 3 markers print) then the freshly-promoted primary going /unreachable shortly after; this occurrence is election never visibly settling AT ALL within 3x the normal budget, a different symptom with a different plausible cause. XCE_KEEP_DEPLOY_DIR was not set for this run, so the 3 nodes' stdout/stderr logs from the actual failure were deleted by the fixture's own cleanup before this could be forensically investigated -- no diagnostic evidence survives from this specific occurrence. Immediate reruns (5/5, by Claude directly after the report, same build.dbg binaries, same test, quiet system: load avg ~0.6-1.5/22 cores, no concurrent xce_controller/meson/nix processes found) all passed cleanly in ~14-15s each, well inside the 30s budget -- consistent with (but not proof of) transient CPU/IO contention from another concurrent session on this shared multi-agent sandbox (the established pattern documented at length in bug-1486 and this same file's bug-1485 investigation) delaying all 3 nodes' heartbeat timers past the window, rather than a deterministic protocol liveness bug. Not ruled out: a genuine, rare election-liveness edge case (e.g. a PreVote/RequestVote split-vote condition) that happens not to leave a marker before some other transient condition times it out. + +## Fix +NOT FIXED -- single occurrence, unreproduced on 5 immediate retries, no surviving log evidence to root-cause further. Logged per this project's own established convention (bug-1485) of tracking an inconclusive live-cluster-election flake rather than silently dismissing it. Next step if this recurs: re-run with XCE_KEEP_DEPLOY_DIR=1 (already supported by the fixture, added for exactly this class of investigation per bug-1485's own remediation) to preserve deploy_dir/logs/*.log and *.stderr.log for real forensic inspection instead of losing them to the fixture's own cleanup. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1485, bug-1486 diff --git a/.wolf/bugs/bug-1727.md b/.wolf/bugs/bug-1727.md new file mode 100644 index 00000000..9046df6e --- /dev/null +++ b/.wolf/bugs/bug-1727.md @@ -0,0 +1,44 @@ +--- +id: "bug-1727" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-30T22:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/scheduler/src/repl_info.c (repl_info_worker_one", "~line 281-390)"] +tags: ["confirmed-live", "display-bug", "found-live", "not-caused-by-task-4.6", "pre-existing", "repl", "resInUse", "resource-accounting", "resvals", "scheduler"] +related: ["bug-1568", "bug-1590"] +legacy: true +--- + +# bug-1727: User-reported (live REPL, real cluster-captured replay… + +## Description +User-reported (live REPL, real cluster-captured replay /tmp/scheduler-replay-20260828-085356.replay, 40 sessions, 60 jobs): after `xce_scheduler -i -s ` + `step`, `info worker all` shows `resource slots: value=44.00 used=22.00` for worker pc13v, but the worker's real installed capacity is 22 slots (confirmed by the same worker's own `resource maxcpus: value=22.00` line) and all 22 running jobs are 1-slot-each, so the correct display is `value=22.00 used=22.00` (fully packed, 0 free) -- `used=22.00` is correct, `value=44.00` is double the true installed capacity. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT FIXED -- recorded for future investigation, but well-narrowed: repl_info_worker_one()'s own header comment (repl_info.c:281-291) asserts w->resvals holds 'currently-AVAILABLE' capacity that shrinks as jobs are charged, and that 'value=' should be printed as avail+inUse ('a conservation invariant this codebase's own resource accounting already relies on'), implemented at repl_info.c:376-390's Value_u_fvalue branch: `avail_v->u.fvalue + used` printed as value=. Traced the actual charge path (sched_account_jobs -> sched_charge_account, resource.c:586 -> sched_update_account, resource.c:559-583 -> sched_update_res_accounts, resource.c:401-470) and confirmed it ONLY ever writes to w->resInUse (`sched_update_res_accounts(core, &(w->resInUse), 0, haccount->naccount, haccount->accounts, charge)`, resource.c:580-581) -- it never touches w->resvals. Exhaustively grepped every `zarray_put(&(w->resvals), ...)`/`zarray_put(&(worker->resvals), ...)` call site in the scheduler (core_events.c:500, core_events.c:848, snapshot_load.c:333, schedobj.c:311): all four are WORKERS_PUSH / worker-update / snapshot-restore INGESTION paths only -- i.e. w->resvals is populated purely from what the worker daemon self-reports (its own configured/installed capacity, e.g. 'slots: 22'), and is architecturally NEVER decremented when a job is dispatched or charged. So for a live worker's CONSUMABLE resource like 'slots', w->resvals already IS the total installed capacity (not a live 'remaining free' counter), making repl_info.c's own documented 'avail+used = value, a conservation invariant' assumption appear to be FALSE in practice -- the comment's claimed invariant does not match how w->resvals is actually populated for this resource type. This makes `value = avail + used` double-count: true installed capacity (22) gets added to itself once resInUse also reads 22, yielding 44. CONFIRMED PRE-EXISTING, NOT caused by today's scheduler-snapshot-job-allocation task 4.6 fix (the sched_account_jobs()-at-load-time change earlier this session): this exact 44/22 pattern already appeared in the user's ORIGINAL first-ever paste this session, produced via the normal `step` -> sched_account_jobs() path (not the load-time path) -- task 4.6 only changed WHEN the display first goes wrong (now visible immediately at load, previously only after the first step/session, since sched_account_jobs() previously never ran until then). Two competing hypotheses for the real fix, not yet distinguished: (a) repl_info.c's display formula is simply wrong for how resvals is actually populated -- should print avail_v directly as value= (already the total) with used= from resInUse, no addition; or (b) w->resvals was INTENDED to be a live 'remaining free' counter (matching the comment's claimed invariant) and something is missing a `w->resvals -= charged_amount` step at dispatch/charge time that was never implemented -- but no evidence of (b) was found anywhere in resource.c's charge/uncharge functions, so (a) is the better-supported hypothesis pending closer investigation of what a live worker daemon (xcecd) actually reports for a 'slots' resval on each heartbeat (constant install size vs. something already load-adjusted). + +--- CONFIRMED + FIXED 2026-08-30 (scheduler-framework session, user's live Symptom 2 report) --- + +Hypothesis (a) from the prior investigation is CONFIRMED correct, and hypothesis (b) is definitively ruled out. Live-reproduced end to end: built a fresh scheduler+controller+worker (schema change for a separate bug bundled in the same build, unrelated to this fix), deployed a real 3-node HA cluster + xcecd_virtual worker (22 real cores, matching the user's original pc13v report), submitted a 50-element array job (`sleep 60 && hostname`, overlapping runtime) against the 22-slot worker, captured a scheduler recording mid-run (`step 100`, 19 jobs genuinely RUN), and inspected `info worker worker-1` on the UNPATCHED binary first: `resource slots: value=41.00 used=19.00` -- NOT even a clean 2x (44), because avail_v (=22, the worker's real installed capacity, confirmed never decremented -- grepped every zarray_put(&w->resvals,...) call site: all four (core_events.c:500/848, snapshot_load.c:333, schedobj.c:311) are WORKERS_PUSH/RESVALS_DELTA/snapshot-restore INGESTION only) plus used_v (=19, however many jobs happen to be running at inspection time) are ADDED together by repl_info_worker_one()'s own display formula -- so the displayed 'value' grows in lockstep with 'used' as more jobs run, not a fixed doubling. The ORIGINAL user report's exact 22+22=44 was simply the specific case of a fully-packed worker (used == the worker's own real capacity), which is why it read as a clean 2x. Cross-checked against the actual admission-control formula that this exact w->resvals field feeds -- resource.c's unsatisfy_(): `total[i] < inuse[i] + uncomitted[i] + wanted[i]` -- which treats get_resvals(&w->resvals,...) as `total` directly, already inclusive of whatever is `inuse`. This is airtight proof w->resvals[idx] is the STATIC TOTAL, not a live 'remaining available' counter that would need `used` added back to reconstruct the total (hypothesis (b), now ruled out -- no such 'w->resvals -= charged_amount' step exists anywhere, and none should, since the admission check already subtracts inuse separately). The real scheduling/admission decisions are UNAFFECTED by this bug -- they read w->resvals and w->resInUse directly, not through this display formula -- so job placement correctness was never at risk; this is a pure `info worker`/`info resource` REPL display defect. IMPORTANT DISAMBIGUATION: this display bug is entirely SEPARATE from Symptom 1 (phantom RUN jobs exceeding real worker capacity, logged as bug-1739) -- that symptom is a genuine allocation-bookkeeping gap in bug-1734's JOBS_PUSH resync-merge path, unrelated to this printf-time double-count. + +Rebuilt with the SAME binary against the SAME live-captured replay file: `resource slots: value=22.00 used=19.00` -- matches `resource maxcpus: value=22.00` exactly, confirming the fix. + +## Fix +FIXED (2026-08-30). src/scheduler/src/repl_info.c: repl_info_worker_one()'s per-worker CONSUMABLE-resource branch and repl_info_resource_one()'s identical GLOBAL/SHARED-resource branch both changed from printing `avail + used` as `value=` to printing `avail` alone (used= is unchanged, still read from w->resInUse / core->sharedInUse) -- avail_v already IS the total capacity, so adding `used` on top double-counted it. Corrected the header comment above repl_info_worker_one() (previously asserted a false 'avail+used is a conservation invariant' claim) to document the real semantics, with a citation to resource.c's unsatisfy_() as the proof. TDD: updated the 2 existing tests that had baked in the wrong expected value (test_repl_breakpoints_inspection.c: test_info_worker_reports_consumable_value_and_used, expected string changed from 'value=7.00 used=2.00' to 'value=5.00 used=2.00' for avail=5/used=2; test_info_resource_reports_consumable_value_and_used, 'value=11.00 used=3.00' to 'value=8.00 used=3.00' for avail=8/used=3) -- both now correctly assert the raw total, not the double-counted sum. The two tests in test_sim_e2e_original_motivation.c asserting 'value=32.00 used=0.00' / 'value=15.00 used=0.00' needed no change (used=0 makes the old and new formulas identical). Full scheduler + controller suite: 61/61 Ok after the fix, zero regressions. Live re-verified against the real captured replay: value now matches maxcpus exactly at every inspection point (mid-run and post-completion). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1568, bug-1590 diff --git a/.wolf/bugs/bug-1728.md b/.wolf/bugs/bug-1728.md new file mode 100644 index 00000000..54b25240 --- /dev/null +++ b/.wolf/bugs/bug-1728.md @@ -0,0 +1,39 @@ +--- +id: "bug-1728" +status: "fixed" +status_updated: "2026-09-27" +status_note: "33c4f28c: failed join barrier unbinds w->ep (was dangling); idx -1 teardown now WARN" +component: "controller" +severity: "unknown" +timestamp: "2026-08-28T10:25:14Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/cluster.c (handle_endpoint_error", "~line 52-77); src/controller/worker.c or join.c (handle_worker_join", "commit-barrier caller -- not yet located)"] +tags: ["controller", "ha", "logging-severity", "false-error", "worker-join", "commit-barrier", "found-live", "session-sibling-of-bug-1727"] +related: ["bug-1727"] +fix_commits: ["33c4f28c"] +legacy: true +--- + +# bug-1728: Controller logs `ERROR [handle_endpoint_error] incorrect worker idx -1` in all 3 nodes of a live… + +## Description +Controller logs `ERROR [handle_endpoint_error] incorrect worker idx -1` in all 3 nodes of a live HA cluster (ctrl-1.log, ctrl-2.log, ctrl-3.log), always immediately after the same two-line sequence: `WARN [commit_submit] commit: WAIT deadline expired without commit (lsn=X committed_lsn=Y writable_lsn=1 timeout_ms=500)` then `WARN [handle_worker_join] handle_worker_join commit barrier failed for pc13v, rc -110`. All three occurrences in the sampled log correlate exactly with an in-progress PreVote/failover window (liveness poll timeout -> primary_ctrl_fd RDHUP -> PreVote begin, immediately preceding each ERROR). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT FIXED -- recorded for future investigation. Read handle_endpoint_error() (cluster.c:52-119): on any endpoint teardown/error, it inspects priv->role; for HostRole_worker it branches on priv->u.idx -- if >=0, calls handle_worker_leave(c, idx) (the normal, silent teardown path); if <0 (the sentinel for 'never finished joining'), it logs ERROR('incorrect worker idx %d', priv->u.idx) instead (cluster.c:75). Working hypothesis (not yet confirmed by reading handle_worker_join() itself): a worker's endpoint gets `priv->u.idx` assigned only once its join fully completes; the commit barrier inside handle_worker_join() is what actually blocks that completion, and it just failed (rc -110 = -ETIMEDOUT) because quorum was mid-PreVote-failover and couldn't ack the WAL entry within the fixed 500ms commit_submit deadline (already WARNed one line above, correctly). So the worker's join never finishes, its endpoint's idx sentinel never advances off -1, and when that half-joined endpoint then errors out (e.g. the join's own failure path tearing the connection down, or the worker retrying), handle_endpoint_error() sees role==worker && idx==-1 and logs it as a hard, unexplained ERROR -- even though it is the direct, already-WARNed consequence of the commit-barrier timeout two lines above, not a new/distinct fault. Same shape as bug-1727's session sibling finding for scheduler sched_init_resources() and the worker kill-job ERROR-logging issue: an expected, self-healing HA-transition condition (worker retries the join once quorum stabilizes -- confirmed later in the same logs by 'HA quorum restored') surfacing as an unexplained hard ERROR instead of being correlated with the WARN that actually explains it. NOT YET CONFIRMED: have not read handle_worker_join()'s own body (location not pinned down -- likely worker.c or join.c) to verify priv->u.idx really stays -1 specifically because of THIS commit-barrier failure path (vs. some other reason), nor traced exactly what tears the endpoint down afterward to trigger handle_endpoint_error(). + +## Fix +NOT FIXED -- recorded for future investigation only, no code change made. Next steps: (1) locate and read handle_worker_join()'s full body to confirm priv->u.idx is left at -1 specifically on the commit-barrier-timeout path (rc -110) and identify what closes/errors the endpoint afterward; (2) if confirmed, either (a) downgrade cluster.c:75's ERROR to WARN with a message that explicitly references the join-in-progress/commit-barrier-timeout context (avoids the unexplained-hard-ERROR read), or (b) have the commit-barrier-timeout failure path in handle_worker_join() itself suppress/mark the endpoint so handle_endpoint_error() can recognize 'this was a known, already-logged join failure' and skip the redundant ERROR entirely; (3) verify against a live 3-node HA soak (xce-test-ha) that downgrading doesn't hide a genuinely-different idx<0 cause (e.g. a real accounting bug elsewhere assigning idx incorrectly) -- keep ERROR (or WARN-with-detail) for any idx<-1 or any case NOT immediately preceded by a commit-barrier-timeout WARN for the same worker, so a genuinely new failure mode doesn't get silently swallowed by too broad a downgrade. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1727 diff --git a/.wolf/bugs/bug-1729.md b/.wolf/bugs/bug-1729.md new file mode 100644 index 00000000..e1446d8a --- /dev/null +++ b/.wolf/bugs/bug-1729.md @@ -0,0 +1,38 @@ +--- +id: "bug-1729" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "worker" +severity: "unknown" +timestamp: "2026-08-28T10:25:14Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/uv_ipc.c (ipc_worker_send_status", "JOB_EXITED branch)"] +tags: ["worker", "logging-severity", "false-error", "job-kill", "tdd", "found-live", "session-sibling-of-bug-1727", "session-sibling-of-bug-1728"] +related: ["bug-1727", "bug-1728"] +legacy: true +--- + +# bug-1729: xcecd_worker.log logs `[ERROR] Job exited with code 137: Killed` for an ordinary… + +## Description +xcecd_worker.log logs `[ERROR] Job exited with code 137: Killed` for an ordinary, deliberate job kill/cancel (exit code 137 = 128+SIGKILL) -- user-reported as misleading, since 'kill job' is a normal admin operation, not a failure. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED: process_exit_callback() (xcecd_worker.c:467-502) already treats a signal-terminated job exit as informational -- `XCECD_WORKER_LOG(XCE_LOG_INFO, "Child %d killed by signal %d\n", pid, term_signal)` at line 497 -- immediately before calling ipc_worker_send_status(&job_runtime_info, "Killed"). But ipc_worker_send_status() (uv_ipc.c, JOB_EXITED branch) re-derived its own severity purely from job_status, hardcoded to XCE_LOG_ERROR for every JOB_EXITED regardless of cause -- so the SAME kill event logged INFO two lines earlier and ERROR two lines later, in the same call chain. There is no JOB_KILLED status distinct from JOB_EXITED, so a deliberate xcecli kill (worker_signal_callback(), xcecd_worker.c:1118, itself logs INFO: 'Terminating job %s with signal %d') produced an ERROR-level status log downstream regardless of intent. worker_signal_callback() only ever sends SIGTERM or SIGKILL for a normal cancel/force-kill (xcecd_worker.c:1116-1125) -- exit_code is always exactly 128+term_signal (xcecd_worker.c:493), so exit_code==143 (SIGTERM) or 137 (SIGKILL) reliably identifies a deliberate worker-initiated termination, distinct from a genuine crash signal (SIGSEGV=139, SIGBUS=135, SIGABRT=134, etc.). + +## Fix +FIXED and unit-tested (TDD red->green, negative-control verified: temporarily forced job_exit_log_level() to always return XCE_LOG_ERROR, confirmed 2/4 new tests failed as expected, then restored the real fix and confirmed all 4 pass). Extracted the severity decision into a small pure function, `static LogLevel job_exit_log_level(int exit_code)` (uv_ipc.c, just above ipc_worker_send_status()): returns XCE_LOG_INFO for exit_code == 128+SIGTERM or 128+SIGKILL (matches process_exit_callback()'s own INFO judgment for the identical event), XCE_LOG_ERROR otherwise (preserves ERROR for genuine crash signals and any other/unrecognized exit code). Wired into the JOB_EXITED branch: `XCECD_WORKER_LOG(job_exit_log_level(job_info->exit_code), "Job %s exited with code %d: %s", ...)`. Added #include for SIGTERM/SIGKILL/SIGSEGV/etc. New test worker/testsuite/test_uv_ipc_job_exit_severity.c (4 cases: deliberate SIGTERM, deliberate SIGKILL, genuine crash signals SIGSEGV/SIGABRT/SIGBUS, unrecognized/plain exit codes) white-box #includes uv_ipc.c directly (existing suite convention) and exercises only the pure helper -- no libuv/IPC machinery needed. Two link-only stub functions added (find_worker_by_pid, xcecd_job_remote_start) to satisfy the linker for code paths this test never calls, same stubbing idiom test_resource_sync.c already establishes. Full worker test suite (5 binaries, 25 tests total) green after the fix; real xcecd_worker binary rebuilds cleanly. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1727, bug-1728 diff --git a/.wolf/bugs/bug-1730.md b/.wolf/bugs/bug-1730.md new file mode 100644 index 00000000..12e0f808 --- /dev/null +++ b/.wolf/bugs/bug-1730.md @@ -0,0 +1,42 @@ +--- +id: "bug-1730" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-28T13:39:36Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_UPDATE case)", "src/controller/sched.c (sched_job_exit_fun/sched_job_start_fun/sched_job_fail_fun)", "src/controller/job.c (HOOK_DETAIL_JOB_EXIT/START/FAIL fire sites)"] +tags: ["scheduler", "controller", "job-state", "dead-code", "hook-detail-mismatch", "resource-leak", "accounts-allocs", "raft-apply-decoupling-regression", "tdd", "found-live", "session-sibling-of-bug-1731"] +related: ["bug-765", "bug-1679", "bug-1731"] +legacy: true +--- + +# bug-1730: User-reported (live 3-node HA cluster, real workload): a finished job's array elements sometimes… + +## Description +User-reported (live 3-node HA cluster, real workload): a finished job's array elements sometimes remain visible as RUN forever in the scheduler's own state (confirmed both live and via a scheduler session recording's .replay snapshot -- xce_scheduler -i -s showing several 0-12-1787907177-0[N] elements as RUN when the controller's joblist/xcecli already shows them DONE). User's own log correlation: scheduler-side handler.c:414 'sched_handle_sched_update_job updated' fires once per job (the dispatch-time notification) but a second fire for the finish notification never arrives for the affected jobs, while the worker log independently proves those jobs genuinely completed (uv_ipc.c:1042 'Job completed successfully', milliseconds after dispatch -- these are near-instant /bin/true-style array elements). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +TWO CONFIRMED FINDINGS from this investigation, one FIXED here, one still OPEN (see bug-1731, its own entry, for the still-open half -- kept separate since the fixed half does not explain the original reported symptom by itself, see below). + +FINDING 1 (FIXED by this entry): sched_job_exit_fun()/sched_job_start_fun()/sched_job_fail_fun() (controller/sched.c) -- the controller-side notify handlers wired to route a job exit/start/fail specifically to the scheduler's accounts/allocs-cleaning sched_job_state_change() (via NotifyData_data_schedExitJob etc. -> XCE_SCHED_OP_JSTATE_CHANGE) -- are PERMANENTLY DEAD CODE: each only acts on `situation == HOOK_AFTER_EVENT` (default: break for HOOK_BEFORE_EVENT), but HOOK_DETAIL_JOB_EXIT/JOB_START/JOB_FAIL are ONLY EVER fired as HOOK_BEFORE_EVENT anywhere in the controller (job.c:2515/2640/2744/2749/2783, all inside the durable-append-only record_job_*_fact() receipt-time functions -- confirmed via exhaustive grep, zero HOOK_AFTER_EVENT fires of these three details exist). The actual state mutation + notification for a job finish/start/fail happens later, at deferred-apply time (apply_committed_entry()'s shared WAL_JOB_START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH 'group' case, persist.c:4464), which fires a DIFFERENT, generic detail -- HOOK_DETAIL_JOB_UPDATE -- at HOOK_AFTER_EVENT, routing instead through sched_job_update_fun() -> NotifyData_data_schedUpdateJob -> scheduler's sched_handle_sched_update_job() (handler.c:404) -> XCE_SCHED_OP_UPDATE_JOB_FIELDS -> (in the currently-live core_events.c-based driver path) XCE_CORE_EVENT_JOB_UPDATE (core_events.c:933). That handler correctly sets job->state, but -- unlike its sibling XCE_CORE_EVENT_JOB_STATE_CHANGE case right above it in the same file, and unlike scheduler.c's own (also now-dead) sched_job_state_change() -- it never released job->accounts/job->allocs on a terminal transition, so every job finishing via this (the only live) path leaked its resource-allocation/accounting objects for the rest of the scheduler process's life. Almost certainly a regression from the raft-apply-decoupling refactor, which collapsed START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH's deferred-apply hook fire into one generic HOOK_DETAIL_JOB_UPDATE without ever rewiring (or retiring) the scheduler-notification handlers that were built for the old, type-specific hook details. + +Confirmed this FINDING alone does NOT explain the user's original reported symptom (job->state itself staying RUN): job->state is set correctly by XCE_CORE_EVENT_JOB_UPDATE's existing code regardless of this leak -- the leak only means job->accounts/allocs are never released, a distinct, confirmed-real defect. Extensive live investigation of the actual state-staying-wrong symptom is tracked separately in bug-1731 (open). + +## Fix +FIXED and TDD-verified (RED->GREEN) for FINDING 1 only. src/scheduler/src/core_events.c's XCE_CORE_EVENT_JOB_UPDATE case: added the identical `if (!JOB_IS_RUN(updated->state)) { free job->accounts via free_job_account(), job->accounts = NULL; free job->allocs via free_JobAlloc_ptr() }` block XCE_CORE_EVENT_JOB_STATE_CHANGE's case already has, right after `job->state = updated->state;`. New test test_job_update_to_terminal_state_frees_accounts_and_allocs (scheduler/testsuite/test_core_events.c): builds a real RUNNING job with a genuine allocation via production make_alloc()/make_host_alloc()/make_job_alloc()/make_job_account() (src/lib/xcealloc.c), pushes a JOB_UPDATE transitioning it to DONE, asserts job->accounts/job->allocs are both NULL afterward. RED confirmed (FAIL: Expected NULL) before the fix; GREEN after. Full scheduler test suite: 52/52 with the new test, only the pre-existing, unrelated symlink-escape-error-message mismatch (sched_admin_request_test) failing before and after -- zero regressions. Real xce_scheduler binary rebuilds cleanly. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-765, bug-1679, bug-1731 diff --git a/.wolf/bugs/bug-1731.md b/.wolf/bugs/bug-1731.md new file mode 100644 index 00000000..b862bbcc --- /dev/null +++ b/.wolf/bugs/bug-1731.md @@ -0,0 +1,73 @@ +--- +id: "bug-1731" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-08-28T13:39:36Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/persist.c (apply_committed_entry", "WAL_JOB_FINISH/START/DISPATCH shared case)", "src/controller/sched.c (sched_job_update_fun)"] +tags: ["scheduler", "controller", "job-state", "notification-loss", "unresolved", "unreproduced", "found-live", "possible-crash", "possible-stall", "session-sibling-of-bug-1730"] +related: ["bug-1714", "bug-1730"] +legacy: true +--- + +# bug-1731: STILL OPEN half of the bug-1730 investigation: the user's originally-reported symptom itself -- a… + +## Description +STILL OPEN half of the bug-1730 investigation: the user's originally-reported symptom itself -- a finished job's job->state staying stuck as RUN in the scheduler's own view (live and in a .replay snapshot) despite the controller and the worker both having genuinely completed it -- was NOT conclusively reproduced live despite extensive targeted testing, so its exact trigger mechanism remains unconfirmed. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT ISOLATED. Investigation trail: traced the full controller-to-scheduler-to-replay-recording pipeline for a job finish end-to-end via static code reading (every hop -- live apply, hook fire, record-encode in recording_events.c, replay-decode in sim_loop.c, replay-apply -- checked individually and found correctly wired for the ordinary case) and found no defect that would explain a PERMANENT loss by itself. Added temporary diagnostic WARN logging at apply_committed_entry()'s two previously-100%-silent early-return points (job not found in jobmgr; apply_job_update_transition() returning REJECTED_BACKWARD/NOOP_DUPLICATE) and reproduced a burst-array-job workload (matching the user's original scenario -- a 30+ element array of near-instant jobs) against both a single-node and a real 3-node HA deploy of this exact worktree's build. Findings from live reproduction: (a) WAL_JOB_FINISH consistently showed result=APPLIED (hook fires) in every observed instance -- did not catch a live silent-drop for FINISH specifically; (b) DID catch WAL_JOB_START correctly no-op'ing as a harmless duplicate when WAL_JOB_DISPATCH already set RUN moments earlier in the same batch (expected, not a bug); (c) a genuinely concerning, SEPARATE, NOT-YET-DIAGNOSED finding: under a very heavy burst (4 concurrent 50-element arrays on a live 3-node cluster), the xce_scheduler process itself DIED entirely mid-run with zero error/fatal/assert output in its own log -- no backtrace captured yet (a subsequent gdb-attached rerun of a similarly-sized burst did not reproduce the crash within the session's remaining time budget); (d) a separate apparent scheduling-throughput stall under a large PEND backlog (200+ near-instant jobs against a 22-slot worker progressing far slower than expected) -- not distinguished from ordinary fork/exec overhead of spawning hundreds of real OS processes on one worker, so not confirmed as a genuine scheduler defect either. Any of (c)/(d) could plausibly be entangled with, or be a completely independent symptom alongside, the original state-staying-wrong report -- not established either way. + +UPDATE (2026-08-29, live gdb-wrapped burst repro): two concrete new findings, neither yet closing this bug's original symptom (job state itself stuck at RUN). +(1) LIVE-CAPTURED NOTIFY-DROP, SELF-HEALING IN THE OBSERVED CASE: the DEBUG3 diagnostic trace added earlier this session (sched_job_update_fun, sched.c) fired for real under this burst -- e.g. `DIAG sched_job_update_fun: jobid=4618064451492872238 state=0x2 -- c->sched.ep is NULL, notify SKIPPED` during a ~2s scheduler reconnect blip (handle_scheduler_leave then a rejoin then a second, fatal leave at the crash). Across the run: 173 RUN notifies attempted, 178 DONE notifies delivered, 53 SKIPPED (dropped because c->sched.ep was NULL at fire time). The one traced job above still received its terminal DONE notify ~300ms later and self-healed -- but this is a live, reproducible demonstration of the exact drop mechanism this bug has theorized: if a job's TERMINAL notify (not just an intermediate RUN one) were ever dropped in the same way, with no subsequent notify to self-heal it, the scheduler's view would stay stuck at whatever state it last received -- matching the original reported symptom exactly. Not yet caught happening to a terminal notify specifically, so still not a full reproduction, but the mechanism is now demonstrated live rather than theoretical. +(2) THE SCHEDULER CRASH (open finding (c) above) WAS FINALLY CAPTURED WITH A FULL BACKTRACE -- see bug-1714 for the complete crash detail (SIGSEGV in sched_get_project via a corrupted job_submit_data_t->project_name pointer, core_events.c:221). Tracked as bug-1714 rather than duplicated here since it's a distinct symptom (crash vs. stuck-state) that may or may not share a root cause with this bug's own open half -- both occur under the same heavy concurrent-array-batching conditions, so they remain plausibly entangled, not yet proven either way. +(3) NO THROUGHPUT STALL observed this run (open finding (d) above) -- DIAG counts climbed steadily and continuously right up to the crash; of 300 submitted array elements only 178 reached DONE before the scheduler died at T+150s, but that is a crash artifact (elements cut off mid-flight), not evidence of an independent stall. Finding (d) remains unconfirmed either way, not further investigated this run. + +## Fix +FIXED (2026-08-29). Root cause (confirmed, not the same theory this entry's prior root_cause investigation was chasing -- see note at the end): apply_committed_entry() (src/controller/persist.c, the shared WAL_JOB_START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH case) fired exactly ONE generic HOOK_DETAIL_JOB_UPDATE at HOOK_AFTER_EVENT for every job-state fact -- correct and still relied on by sched_job_update_fun/apiserver_job_update_fun -- but NEVER the type-specific HOOK_DETAIL_JOB_START/HOOK_DETAIL_JOB_EXIT, even though live handler pairs for both (sched_job_start_fun/sched_job_exit_fun in sched.c, apiserver_job_start_fun/apiserver_job_exit_fun in apiserver.c, all wired via xce_add_hook) have existed the whole time, subscribed at HOOK_AFTER_EVENT. Those handlers were permanently dead code: job.c only ever fires HOOK_DETAIL_JOB_START/JOB_EXIT/JOB_FAIL at HOOK_BEFORE_EVENT (receipt-time, for WAL-append purposes), never AFTER_EVENT -- see persist.h's apply_committed_entry doc comment, 'Deviation from a literal per-type hook fan-out', for the original documented rationale (the old wal_jobstate_t {jobid, state} wire format could not supply the richer job_pidinfo_t/xce_jobfinish_t payloads those handlers require). That blocker is now stale for START and FINISH specifically: bug-1697 (WAL_JOB_START) and bug-1689 (WAL_JOB_FINISH) already enriched this exact wire format with trailing job_pidinfo_t / job_exitinfo_t+finish_time bytes, decoded every apply into j->job.pidinfo / j->job.exit_info -- just never forwarded to the dead handlers. On the apiserver side this meant DefaultHandler.OnJobFinished (src/frontend/apiserver/backend/handler.go) -- itself correct and fully wired via handleNotifyData's NotifyData_data_jobFinishInfo case (clib.go) -- was permanently starved of data: a job could finish for real (worker and controller both correct) and the apiserver's store.Jobs entry would simply never transition off its pre-finish status. + +FIX (src/controller/persist.c, apply_committed_entry's shared job-state-fact case): (1) WAL_JOB_FINISH -- inside the existing `result == JOB_TRANSITION_APPLIED` block (the same guard HOOK_DETAIL_JOB_UPDATE already used), also fire HOOK_DETAIL_JOB_EXIT with a stack-local xce_jobfinish_t{jobid=js->jobid, exit_code=(j->job.exit_info ? j->job.exit_info->exitcode : 0)} -- this is the PRIMARY fix. (2) WAL_JOB_START -- a new call-scoped `bool pidinfo_just_decoded` flag, set ONLY where the pre-existing pidinfo decode-and-adopt block (bug-1697) actually decodes a NEW job_pidinfo_t this call (deliberately NOT `j->job.pidinfo != NULL`, which could already be true from an earlier apply and would misfire on an unrelated later transition for the same job); also belt-and-suspenders stamps `decoded->jobid = js->jobid` explicitly right where it is adopted. HOOK_DETAIL_JOB_START now fires whenever pidinfo_just_decoded is true, INDEPENDENT of `result` -- required because bug-1697 already established that every real job's WAL_JOB_START fact applies as a confirmed NOOP_DUPLICATE in production (dispatch_a_job_() sets state=RUN synchronously before WAL_JOB_START ever applies), so gating on result==APPLIED the way JOB_UPDATE is gated would make the new fire dead code forever, live, exactly the class of bug being fixed. The early-return gate right after unlock(j) was widened from a flat `if (result != JOB_TRANSITION_APPLIED) return 0;` to `if (result != JOB_TRANSITION_APPLIED && !pidinfo_just_decoded) return 0;` so a genuinely-just-decoded START pidinfo still reaches the single t_apply_seam_notifying-guarded hook-fire bracket even on a NOOP_DUPLICATE transition; all three fires (new JOB_START, new JOB_EXIT, pre-existing JOB_UPDATE) stay inside that ONE bracket, not duplicated or narrowed. NULL-safety: sched_job_start_fun/sched_job_exit_fun (sched.c) dereference info->jobid unconditionally with no NULL guard (confirmed via codegraph read of both) -- pidinfo_just_decoded's call-scoped design and the belt-and-suspenders jobid stamp exist specifically to make firing with a NULL/stale payload structurally impossible. Confirmed via codegraph/grep that persist_job_hook_fun (the third XCE_CONTROLLER_HOOK_JOB subscriber, registered via ha_register_persist_hooks) does have live HOOK_DETAIL_JOB_START/EXIT cases (unlike the doc comment's characterization of it as 'HOOK_BEFORE_EVENT only for these details') -- but it early-returns immediately on `t_apply_seam_notifying` (Hazard-1 guard, already true for every fire inside apply_committed_entry's bracket) before ever reaching those cases, so no unintended WAL re-append happens; this nuance is a buglog correction to the doc-comment's own framing, not a defect. + +TDD evidence: added test_wjf_apply_committed_entry_fires_job_exit_hook_exactly_once and test_wjf_apply_committed_entry_reapply_finish_fires_no_hook (test_wal_job_finish_fact.c), test_wjs_apply_committed_entry_fires_job_start_hook_with_pidinfo and test_wjs_apply_committed_entry_old_format_no_job_start_hook (test_wal_job_start_fact.c), using a new detail-aware spy hook per file (distinguishes HOOK_DETAIL_JOB_EXIT/JOB_START from JOB_UPDATE and captures the fired payload, unlike the existing blunt 'any AFTER_EVENT' counters already in those files). RED confirmed on unfixed code (persist.c edits temporarily reverted, test files kept): exactly the 2 new primary-path assertions failed (test_wjs_..._fires_job_start_hook_with_pidinfo: Expected 1 Was 0; test_wjf_..._fires_job_exit_hook_exactly_once: Expected 1 Was 0), plus the pre-existing bug-1561 test_config_repl.c flake pair -- 4 failures total, matching expectations exactly. The two negative-guard tests (old-format-no-pidinfo, reapply-fires-no-hook) already passed RED (correctly -- pre-fix code never fires these hooks at all, so 'must not fire' trivially holds), confirming they are genuine regression guards, not accidentally-tautological. + +GREEN evidence: after restoring the fix, four PRE-EXISTING tests broke because their assertions hard-coded the old 'exactly one notification' invariant that this fix deliberately changes (more hooks now fire, by design) -- all four updated in place with corrected expected counts plus an explanatory bug-1731 comment (not weakened, not deleted): test_wjs_apply_committed_entry_sets_pidinfo_but_not_start_time (1 -> 2: JOB_UPDATE + new JOB_START both fire on this genuine PEND->RUN transition with pidinfo), test_wjs_apply_committed_entry_adopts_pidinfo_when_already_run_from_dispatch (0 -> 1: this IS bug-1697's exact NOOP_DUPLICATE production scenario -- the new JOB_START fire is the whole point of the fix here, so asserting 0 would mean re-breaking the bug), test_wjf_apply_committed_entry_sets_finish_time_and_exit_info (1 -> 2: JOB_UPDATE + new JOB_EXIT), test_tari_finish_then_stale_start_preserves_terminal_state_via_dispatch in test_apply_reapply_idempotency.c (2 -> 3 after FINISH, stays 3 after the stale-START no-op -- confirms idempotency: the stale START carries no pidinfo trailing bytes, so pidinfo_just_decoded stays false and the already-guard-rejected transition fires nothing new). Full controller regression after all fixes: 1217 tests, 2 failures -- exactly the known, pre-existing, unrelated bug-1561 flake pair (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config) and nothing else. scheduler/worker/xceadmin builds also completed clean (scheduler build surfaced one unrelated, pre-existing test failure -- test_recordstart_symlink_escape_replies_error, in record-file-path validation, zero relation to job-state hooks or any file this fix touched -- not investigated further, out of scope). + +LIVE VERIFICATION (single-node cluster, src/tools/deploy-ha.sh to /tmp/xce-deploy-1731, controller+worker+scheduler+apiserver all built from this fix and started via start-all.sh; two real jobs submitted via xce_test jobsub -u root -q normal -c /bin/true, both ran to completion): + Scheduler side -- scheduler.log shows, for jobid 0-7-1787994329-448775 (and identically for a second job, 0-14-1787994109-0): `event = schedStartJob` immediately followed by `sched_job_state_change 0-7-1787994329-448775 state 2` (handler.c:388, inside sched_job_state_change, handler.c:381 -- state 2 = JOB_STAT_RUN, the NEW HOOK_DETAIL_JOB_START fire), then later `event = schedExitJob` followed by `sched_job_state_change 0-7-1787994329-448775 state 4` (state 4 = JOB_STAT_EXIT, the NEW HOOK_DETAIL_JOB_EXIT fire) -- with `sched_handle_sched_update_job ... updated` (the pre-existing JOB_UPDATE path) still firing on both sides of it, confirming the new fires are additive, not a replacement. Final `joblist` output confirms the job's state in the scheduler-driven view reached DONE, not stuck at RUN. + Apiserver side (the part specifically asked to be verified) -- added permanent Info-level logging to DefaultHandler.OnJobFinished (handler.go, mirroring this file's existing zap-field logging convention) since it had none before; rebuilt /tmp/apiserver, restarted it against the live cluster, submitted a fresh job. apiserver.log recorded: `OnJobFinished: job status updated {\"jobid\": 33400045333567751, \"exit_code\": 0, \"job_status\": 8}` -- job_status 8 = models.JOBSTAT[\"DONE\"], exit_code 0 matching /bin/true's real exit code, jobid matching the submitted job -- direct proof OnJobFinished, previously never invoked in production, now fires correctly end-to-end. All 4 processes (controller, worker, scheduler, apiserver) remained healthy throughout with zero ASAN/crash output in any log. + +Kept out of scope, as directed: WAL_JOB_FAIL's own dead HOOK_DETAIL_JOB_FAIL handler pair (apiserver_job_fail_fun -> jobFailInfo -> OnJobFinished too) was NOT touched -- it lives in a separate, more complex `case` (dispatches on js->_pad, apply_job_fail_transition vs. apply_job_update_transition) a bit further down persist.c's apply_committed_entry, and is a distinct follow-up fix, not covered here. + +Files touched: src/controller/persist.c (the fix), src/controller/testsuite/test_wal_job_finish_fact.c and test_wal_job_start_fact.c (new regression tests + corrected pre-existing assertions), src/controller/testsuite/test_apply_reapply_idempotency.c (corrected one pre-existing assertion), src/frontend/apiserver/backend/handler.go (permanent OnJobFinished logging, kept -- not reverted, judged a reasonable permanent addition matching this file's existing logging conventions). + +Correction to this entry's own prior root_cause note: the earlier investigation (2026-08-28/29 updates above) was chasing a DIFFERENT theory -- a transient sched.ep==NULL notify-drop window in sched_job_update_fun -- as the suspected mechanism for the original stuck-at-RUN symptom, and left bug-1714 (a scheduler crash found during that chase) as a possibly-entangled open thread. This fix does not address that notify-drop theory or bug-1714 at all; it closes a DIFFERENT, independently confirmed defect (the dead JOB_START/JOB_EXIT hook pair) that was diagnosed and fixed in this session. Both could in principle contribute to real-world stuck-job reports; this entry now tracks the JOB_START/JOB_EXIT hook fix as FIXED, with bug-1714's scheduler-crash thread and any residual sched.ep notify-drop risk left open elsewhere (bug-1714's own entry) rather than reopening speculation here. + +FOLLOW-UP (2026-08-29, same session): the WAL_JOB_FAIL HOOK_DETAIL_JOB_FAIL gap explicitly left out of scope above ("Kept out of scope, as directed") is now ALSO fixed. HOOK_DETAIL_JOB_FAIL was fired only at HOOK_BEFORE_EVENT (job.c's mark_job_fail and record_job_fail_fact, at receipt time, for WAL-append purposes) -- never at HOOK_AFTER_EVENT -- leaving two already-written, already-correct handlers permanently dead: sched_job_fail_fun (sched.c, routes to the scheduler via NotifyData_data_schedFailJob) and apiserver_job_fail_fun (apiserver.c, routes to the Go apiserver's NotifyData_data_jobFailInfo case in clib.go -> DefaultHandler.OnJobFinished). A job that FAILED (as opposed to finished cleanly, which this bug's primary fix already covers via WAL_JOB_FINISH/HOOK_DETAIL_JOB_EXIT) never updated the apiserver's job store. + +FIX (src/controller/persist.c, apply_committed_entry's `case WAL_JOB_FAIL:` block): hoisted the case-local `job_failinfo_t failinfo` to case scope (it needs to survive past `unlock(j)`) with a new `bool have_failinfo` flag, set true only in the `js->_pad & WAL_JOB_FAIL_STAGE_FLAG` branch (the only branch that constructs a real job_failinfo_t -- the legacy `_pad == 0` absolute-overwrite branch, job_ctrl's synchronous mark_job_fail path, has no fail-specific data to report and must NOT fabricate one, exactly mirroring how this bug's own WAL_JOB_START fix only fires HOOK_DETAIL_JOB_START when pidinfo was genuinely decoded). Inside the existing `t_apply_seam_notifying`-guarded bracket, added `if (have_failinfo) xce_run_hooks(..., HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_FAIL, c, &failinfo, j);` immediately before the pre-existing HOOK_DETAIL_JOB_UPDATE fire -- both fires stay inside the SAME bracket, additive not replacing. Confirmed via reading apply_job_fail_transition() (job.c) that unlike WAL_JOB_START's dispatch-time-optimization no-op'ing, this transition has no internal guard at all -- it unconditionally mutates and returns JOB_TRANSITION_APPLIED for every stage -- so the existing `if (result != JOB_TRANSITION_APPLIED) return 0;` early-return gate did NOT need widening the way WAL_JOB_START's did; a rejectable/no-op outcome for this case can only come from the legacy _pad==0 branch's apply_job_update_transition() guard (job_state_is_backward / duplicate-state checks), which was already correctly gated before this follow-up. + +TDD evidence: added 3 tests to test_apply_committed_entry.c -- test_ace_job_fail_stage_flag_fires_faildetail_and_update (seeds job 605 RUN|EXIT, applies WAL_JOB_FAIL with _pad=WAL_JOB_FAIL_STAGE_FLAG|JOB_EXEC_STAGE_POST_EXEC, asserts JOB_STAT_PERR merged AND both hooks fire with the correct jobid in the captured job_failinfo_t), test_ace_job_fail_legacy_pad_zero_fires_update_only (regression guard: _pad==0 branch fires JOB_UPDATE only, never fabricates JOB_FAIL), test_ace_job_fail_guard_rejected_fires_no_hook (a backward transition via the legacy branch, mirroring test_ace_job_state_transition_rejects_backward_suppresses_notify, fires NEITHER hook). New detail-aware spy hook tace_faildetail_spy_hook added (modeled on the file's existing tace_array_spy_hook), with va_arg order (cluster_t*, job_failinfo_t*, job_t*) matching job.c's own HOOK_BEFORE_EVENT fire sites for this detail; g_tace_last_fail_jobid reset to -1 in tace_setup() alongside the existing g_tace_hook_fires reset. + +RED confirmed on unfixed code (persist.c fix temporarily git-stashed, test file kept): exactly test_ace_job_fail_stage_flag_fires_faildetail_and_update failed (Expected 2 Was 1 -- HOOK_DETAIL_JOB_FAIL did not fire), plus the pre-existing bug-1561 test_config_repl.c flake pair (2 failures) -- 3 failures total, matching expectations (the other two new tests are regression guards that correctly already passed pre-fix, since the unfixed code never fires HOOK_DETAIL_JOB_FAIL at all). + +GREEN confirmed after restoring the fix: all 3 new tests pass. Full controller regression: 1220 tests, 2 failures -- exactly the known, pre-existing, unrelated bug-1561 flake pair (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config) and nothing else. + +LIVE VERIFICATION: attempted against a fresh single-node deploy (controller+worker(xcecd_virtual)+scheduler+apiserver, src/tools/deploy-ha.sh to /tmp/xce-deploy, all built from this fix). Confirmed via code reading (xcecd_job_finished, worker/src/controller_connector.c) that a worker-side nonzero exit code routes through xcecd_notify_job_fail (WAL_JOB_FAIL with STAGE_FLAG), never WAL_JOB_FINISH -- so a genuinely-failed job is the right live trigger. Submitted a `sleep 60` job (chosen so the virtual worker's simulated run-time, parsed from the literal command string in VIRTUAL_CLUSTER builds, stays RUN long enough to signal) and sent jobkill -s 9 while it was RUNNING; the controller/worker IPC path correctly delivered the signal and set worker->killed_signal=9 (confirmed in xcecd-worker-1.log). However this sandbox has no D-Bus system bus (no systemd manager reachable), and xcecd_dbus_manager.c's dbus_workers_update() -- unchanged by this fix, in src/worker/, not src/controller/ -- calls get_systemd_job_exit_code() every poll even under VIRTUAL_CLUSTER; that call fails to connect to D-Bus and returns -1, which routes into the `exit_code < 0` (dbus-unreachable) branch of the VIRTUAL_CLUSTER dispatch -- a branch which, on inspection, only checks check_worker_sleep_time_elapsed() and never checks worker->killed_signal (that check exists only in the sibling `exit_code == 0` branch, unreachable in this sandbox since dbus never succeeds). Net effect: in THIS sandboxed environment specifically, xcecd_virtual can only ever report a job as succeeding (exit_code=0) once its simulated sleep time elapses, or hang indefinitely if killed before then -- a genuine nonzero-exit/killed-job WAL_JOB_FAIL could not be produced live here. This is a pre-existing environment/worker-code limitation unrelated to and untouched by this fix (no file outside the fix's own scope was modified to work around it, per this task's constraints) -- an honestly-reported limitation, not a gap in the fix itself. The unit-test coverage above (RED->GREEN on the exact apply-time dispatch code) is the primary proof of correctness; the same live-verification shape this bug's primary fix already used successfully for WAL_JOB_FINISH/WAL_JOB_START (a real dbus-independent xce_test jobsub) does not exist for a genuine WAL_JOB_FAIL trigger in this environment. + +Files touched by this follow-up: src/controller/persist.c (the fix), src/controller/testsuite/test_apply_committed_entry.c (3 new tests + 1 new spy hook + 1 new reset line). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1714, bug-1730 diff --git a/.wolf/bugs/bug-1732.md b/.wolf/bugs/bug-1732.md new file mode 100644 index 00000000..6d8213b6 --- /dev/null +++ b/.wolf/bugs/bug-1732.md @@ -0,0 +1,38 @@ +--- +id: "bug-1732" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-08-29T05:21:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/tools/xce-build.sh (worker component case)"] +tags: ["build-tooling", "xce-build.sh", "worker", "xcecd_virtual", "low-priority"] +related: [] +legacy: true +--- + +# bug-1732: src/tools/xce-build.sh's `worker` component case never builds `xcecd_virtual` (only `xcecd… + +## Description +src/tools/xce-build.sh's `worker` component case never builds `xcecd_virtual` (only `xcecd xcecd_worker xrun controller_connector_test`), even though deploy-ha.sh requires xcecd_virtual whenever no real systemd manager is present (the common case in sandboxes) -- `/xce-build worker` alone leaves a deploy unable to auto-detect/generate start-worker.sh for the virtual worker. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The `worker)` case's explicit meson target list omits `xcecd_virtual`. Confirmed while dispatching a debug-profile build for a bug-1731 live repro: `/xce-build worker debug --incremental` completed successfully but did not produce `xcecd_virtual` in the build dir, requiring a manual `meson compile -C xcecd_virtual` before deploy-ha.sh's own worker-binary auto-detection (Step 3, per the xce-deploy-ha skill) would find it and generate start-worker.sh instead of soft-skipping it. + +## Fix +FIXED. Added `xcecd_virtual` to the `worker)` case's meson compile target list in src/tools/xce-build.sh (alongside the existing xcecd xcecd_worker xrun controller_connector_test), with a bug-1732-referencing comment. Verified: `xce-build.sh worker asan` now builds xcecd_virtual with a fresh mtime, and the full worker test suite (5/5, including test_uv_ipc_job_exit_severity) still passes with zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1733.md b/.wolf/bugs/bug-1733.md new file mode 100644 index 00000000..e2c517a8 --- /dev/null +++ b/.wolf/bugs/bug-1733.md @@ -0,0 +1,46 @@ +--- +id: "bug-1733" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-08-29T06:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/vtaskmgr.c (vtask_enq", "line 517)", "src/lib/dataq.c (dataq_enq)"] +tags: ["controller", "vtaskmgr", "use-after-free", "asan-confirmed", "concurrency", "high-severity", "live-crash", "found-via-bug-1731-repro"] +related: ["bug-1298", "bug-1415", "bug-1714", "bug-1731"] +legacy: true +--- + +# bug-1733: AddressSanitizer: heap-use-after-free WRITE of size 8 at vtask_enq (src/lib/vtaskmgr.c:517)… + +## Description +AddressSanitizer: heap-use-after-free WRITE of size 8 at vtask_enq (src/lib/vtaskmgr.c:517) -- crashed 2 of 3 controllers in a live 3-node HA cluster under a heavy burst-array-job workload (6 concurrent `jobsub -a 50` submissions, 300 near-instant job elements, driving high-frequency WAL-replication-ACK traffic). Found while chasing bug-1714/bug-1731's scheduler crash under ASAN -- the scheduler itself did not crash in this run, but two controllers did, independently, ~7 and ~14 minutes into the run, with byte-for-byte identical ASAN stacks (different PIDs/addresses). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +vtask_enq() (vtaskmgr.c:497-526) calls dataq_enq(&(vtaskmgr->taskq), &(task->link)) at line 514, which PUBLISHES the task onto the shared taskq (under q->lock, released on return) -- immediately making it visible to any idle vcpu worker thread. Only AFTER that publish does line 517 write `task->owner = vtaskmgr;`. A concurrent vcpu thread can dequeue the just-published task, run it to completion (run_task -> the task's own callback, here peer_ctrl_vtask_entry), and vcpu_entry (vtaskmgr.c:106-109) can then call vtask_fini(t)+free(t) -- all BEFORE the producer thread reaches line 517 -- so the producer's write lands on already-freed memory. ASAN's two captured stacks confirm exactly this: thread T6 allocates+enqueues the task (persist_apply_pump_kick -> vtask_alloc/vtask_enq, called via the replication-ACK path repl_ack_decode_and_dispatch -> repl_on_ack -> commit_on_ack -> commit_resolve_pending -> persist_apply_pump_kick), while thread T5 has ALREADY freed the same 48-byte vtask_t region inside vcpu_entry by the time T6's own `task->owner = vtaskmgr` write executes. This is a publish-before-init race -- the task is made visible to concurrent consumers before its own initialization (the `owner` field) completes. Notably, the code immediately above (lines 508-512, `endpoint_ref(ep)`) already gets this right, per an existing bug-1298 comment explicitly calling out 'acquire BEFORE dataq_enq publishes the task to the dispatcher' -- line 517's `owner` assignment simply doesn't follow that same established discipline. Reproduces reliably under this project's burst-array live-HA workload (both controller instances that received the assignment crashed on it during a single ~30-minute run); did not reproduce in the test suite's pool-less fixtures (matches the same class of gap bug-1415 documents for a different vtaskmgr race -- production n_vcpus>1 pool concurrency needed to manifest, not exercised by tests that never call commit_set_pool). + +## Fix +FIXED and TDD-verified (RED attempted, GREEN confirmed). Root cause and fix exactly as diagnosed: vtask_enq() (src/lib/vtaskmgr.c) now sets `task->owner = vtaskmgr;` BEFORE calling dataq_enq(), mirroring the existing bug-1298 endpoint_ref-before-publish discipline immediately above it in the same function; the post-enqueue success block now only calls vtaskmgr_wakeup(vtaskmgr) (the owner write no longer straddles the publish that makes the task visible to concurrent vcpu workers). + +New stress test test_vtask_enq_race_stress (src/lib/testsuite/test_vtask_enq_race.c, wired into libxce_test_src in src/lib/meson.build and testsuite/test.c's extern/TEST() list as test_vtask_enq_race, right after test_vtask_admission): a real vtaskmgr_t pool (8 vcpus) with 16 producer threads each calling vtask_enq_classified(mgr, task, VTASK_LIVENESS) in a tight loop (8000 iterations each, 128,000 tasks total) with true no-op entry callbacks (atomic increment only, matching test_vtask_admission.c's live_task_entry) -- maximizing the odds a consumer vcpu dequeues, runs, and frees a task before the producer's own next instruction lands, exactly the bug-1733 race window. + +RED ATTEMPT: this is a genuine data race confined to the two-line window between dataq_enq()'s publish and the owner write, which cannot be forced deterministically from a test without modifying production code (unlike test_vtask_admission.c's fully deterministic cbuf-gated design). Extensive tuning was attempted against the UNMODIFIED vtaskmgr.c before applying the fix -- over a dozen configurations across 60+ manual runs: the default stress shape (6 vcpus/10 producers/4000 iters), bumped concurrency (16v/24p/6000iters, 32v/48p/20000iters), taskset-pinned to 4 cores and to a single core, background CPU-load saturation (22-44 concurrent `yes` processes churning all 22 host cores to force involuntary scheduler preemption right after dataq_enq() returns), and a few-producers/many-idle-vcpus variant (4p/16v/100000iters) meant to keep the dispatcher in a tight dequeue-polling loop. None reproduced the ASAN heap-use-after-free within these bounded attempts, including the official `nix develop ./src --command bash src/tools/xce-build.sh lib asan` run against the unmodified file (the new test itself PASSED cleanly there; the run's only SIGABRT came from a pre-existing, unrelated LeakSanitizer report in test_xcealloc.c -- 456 bytes/29 allocations via strdup in make_alloc/make_host_alloc -- confirmed present byte-for-byte identically both before and after this fix, so not a regression from this change). This is consistent with this entry's own root-cause note that the race needed real multi-node HA production load (WAL-replication-ACK bursts under `jobsub -a 50` burst-array-job load) to manifest, and that it also did not reproduce in prior pool-less test fixtures (same class of gap bug-1415 documents). The original ASAN report captured in this entry (heap-use-after-free WRITE of size 8 at vtask_enq ../lib/vtaskmgr.c:517, freed inside vcpu_entry, matching stacks on 2 of 3 live controllers) remains the authoritative confirmation the bug is real; a stress test that does not happen to hit a hard-to-trigger race in bounded iterations is a weaker regression guard than one that does, but per this fix's own instructions is not a reason to withhold the fix. + +GREEN: after the fix, the new stress test passes cleanly and repeatably -- 8 consecutive standalone runs, zero ASAN reports -- at both the isolated and full-suite (`xce-build.sh lib asan --incremental`) level. + +Full controller-suite regression (`xce-build.sh controller asan --incremental`, the real consumer of vtask_enq_classified via persist_apply_pump_kick/commit.c): 1213 tests, 2 failures -- both the pre-existing, already-logged bug-1561 timing-collision flake in test_config_repl.c (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), unrelated to this change. The specific real vtask_enq_classified callers this fix touches indirectly -- test_ha_dispatch_integration.c, test_ha_vep_teardown.c, test_apply_pump.c, test_apply_pump_integration.c, test_commit_admission_pool.c, test_commit_resolve.c -- all pass with zero failures. No regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1298, bug-1415, bug-1714, bug-1731 diff --git a/.wolf/bugs/bug-1734.md b/.wolf/bugs/bug-1734.md new file mode 100644 index 00000000..5140dd17 --- /dev/null +++ b/.wolf/bugs/bug-1734.md @@ -0,0 +1,53 @@ +--- +id: "bug-1734" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "security" +timestamp: "2026-08-30T10:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/scheduler/src/handler.c (sched_on_job_array's dedup-race", "already documented as a known TOCTOU race by commit 38437e57)", "src/scheduler/src/core_events.c (xce_core_push_events", "WARN log site + the JOBS_PUSH duplicate-discard path from commit 38437e57)", "src/scheduler/src/driver.c / src/controller/sched.c / src/controller/persist.c (the job-update notify pipeline this session's bug-1731/bug-1714 investigations already covers)"] +tags: ["scheduler", "controller", "job-state-sync", "state-desync", "found-live", "array-job", "root-caused", "reconnect-storm", "notify-drop-on-disconnect", "resync-data-loss", "session-sibling-of-bug-1714", "session-sibling-of-bug-1731", "session-sibling-of-bug-1733", "human-readable-jobid-logging-gap-still-open", "xcecli-failover-under-load-needs-followup"] +related: ["bug-1714", "bug-1731", "bug-1733"] +fix_commits: ["38437e57"] +legacy: true +--- + +# bug-1734: User-reported live deployment: scheduler's view of a job array's state (`[sim] jobs` /… + +## Description +User-reported live deployment: scheduler's view of a job array's state (`[sim] jobs` / xce_scheduler -i -s) diverges sharply from the controller's own joblist for the SAME array, well after every element genuinely finished. Array 0-4-1788084587-0 (50 elements, command `hostname`): controller's `joblist` shows ALL 50 elements as DONE with real, monotonically-increasing StartTime/FinishTime timestamps (1788084587..1788084602) -- the array is genuinely, fully complete. The scheduler's own concurrent view of the identical array shows a scrambled mix: [0-4]=RUN, [5-16]=DONE, [17]=EXIT, [18-19]=DONE, [20]=EXIT, [21]=DONE, [22-24]=EXIT, [25]=DONE, [26-29]=EXIT, [30-46]=RUN, [47-49]=PEND. Two things about this pattern are themselves suspicious, not just 'the scheduler is behind': (a) elements [30-46] show RUN and [47-49] show PEND, i.e. the scheduler believes 20 elements haven't even STARTED yet, even though the controller's timestamps show the whole array finished by 1788084602 -- this is not a small lag, the scheduler's view is drastically behind reality for over a third of the array; (b) EXIT appears interspersed with DONE for a uniform array of a single reliably-successful command (`hostname`, not `/bin/false` or anything failure-prone) -- elements 17/20/22-24/26-29 show EXIT while their immediate neighbors show DONE, which is not what a simple 'scheduler hasn't caught up yet' lag would produce (a lagging-but-otherwise-correct view should show a clean PEND/RUN prefix behind a DONE suffix, not DONE and EXIT interleaved for what should be uniform outcomes). CORRELATED evidence from the SAME run: xce_scheduler's own log shows 10 consecutive WARN lines at the very same timeframe -- `xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid (rc=-17), discarding duplicate/orphaned job` for jobid 4631575359520768000 through 4631575359520768009 -- which decode (JobID bit layout: tag/cluster/apiserver/timestamp/array_idx) to EXACTLY array elements 0-4-1788084587-0[0] through [9] of this SAME array. This WARN is emitted by the bug-1714-adjacent fix landed earlier this session (commit 38437e57, 'fix(scheduler): don't hand an orphaned duplicate job to the picker') -- confirms live, in this exact reproduction, that this array's first 10 elements were pushed to the scheduler TWICE (the TOCTOU dedup race in sched_on_job_array()/handler.c that commit documented as theoretically possible is firing for real here), and the second copy is now being correctly discarded rather than corrupting state or crashing (that part of the fix is working as designed). User also separately requested (logged here as part of the same report, lower priority, not a correctness bug): xce_core_push_events's WARN line above prints the raw numeric jobid (`4631575359520768000`) instead of this project's human-readable fmt_jobid() form (`0-4-1788084587-0[0]`) that every other job-related log line in this codebase uses -- a logging-usability gap in that specific WARN, worth a one-line fix. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT YET ROOT-CAUSED -- logged for follow-up investigation per user request, not investigated live this session (a separate, still-running background investigation was mid-flight on bug-1714's own stale-reader when this was reported). Working hypothesis, NOT confirmed: this looks like more than one mechanism compounding. (1) The already-known, already-partially-mitigated sched_on_job_array() dedup race (see bug-1714's own root_cause history and commit 38437e57's message) is CONFIRMED firing live here (the WARN evidence decodes to exactly this array's first 10 elements) -- but discarding a genuine duplicate PUSH should be harmless on its own (both copies start from identical fresh JOB_STAT_PEND state; sched_add_job()'s key_add() correctly keeps the first copy), so this alone should not explain elements staying stuck at RUN/PEND when the controller confirms DONE. (2) The DONE/EXIT interleaving for a uniform, reliably-successful command (hostname) suggests either an ordering bug in how terminal-state notifications get applied (an older notification overwriting a newer one, or vice versa, for different elements) -- which would tie into bug-1731's own notify-drop/redelivery investigation and its live-captured evidence of scheduler-endpoint-NULL notify drops -- or a distinct bug in how JOB_STATE_CHANGE vs JOB_UPDATE detail routing interacts for array elements specifically (both hook-detail paths were touched this session by bug-1731's fixes; worth checking whether an array element's terminal notification can race between the two). (3) The ~20 elements the scheduler shows as never having started (RUN/PEND for [30-49]) despite controller-confirmed completion by 1788084602 is the most severe and least explained part of this report -- if genuinely not just a stale UI snapshot, this suggests SOME elements' notifications are being lost entirely, not just delayed, distinct from anything bug-1731's fix addressed (bug-1731 fixed the CONTROLLER never firing the right hook; this could be a SEPARATE loss further downstream, e.g. in delivery to the scheduler, or in how the scheduler's REPL/inspect view is rendered from its own internal state). NOT RULED OUT: this could also be nothing more than a snapshot-timing artifact if `[sim] jobs`/`joblist` were captured at genuinely different moments and the scheduler was still catching up -- but the user's own framing ('still have some issues') and the DONE/EXIT interleaving for a uniform command argue against pure timing lag being the whole explanation. Needs live reproduction with synchronized capture (query both views at the same instant, ideally with sequence numbers/timestamps on the query itself) to distinguish 'genuinely stuck' from 'was mid-catch-up when sampled'. + +ADDITIONAL CONFIRMATION (same report, user follow-up): xce_test itself (the submitting client) also reports every job in this array finished successfully -- i.e. all three independent vantage points agree the work genuinely completed (worker execution, controller's own joblist/WAL state, and the submitting client's own completion tracking). This rules out worker-side execution failure and controller-side completion-tracking failure as explanations, narrowing this specifically to the scheduler-side notification delivery/application path (or, less likely, the scheduler's own REPL/inspect rendering of otherwise-correct internal state) -- consistent with, but not identical to, bug-1731's already-fixed dead-hook gap and its still-open notify-drop-on-disconnect finding. + +UPDATE (2026-08-30, later same session, FULLY ROOT-CAUSED via live reproduction): Built controller+scheduler+worker+xceadmin fresh into src/xce.build.bug1734 (asan), deployed a real 3-node HA cluster (localhost:19991-19993) + xcecd_virtual worker + xce_scheduler (-vvv), submitted a 50-element `hostname` array job via xce_test, and captured synchronized evidence: controller `joblist` (ground truth, all 50 DONE with monotonic timestamps) vs the SAME scheduler process's own live session recording (`xceadmin scheduler record start/stop`) replayed via `xce_scheduler -i -s ` + `continue` (runs the ENTIRE recorded session set to completion, ruling out lag) + `info jobs`. REPRODUCED EXACTLY: final replayed state showed 23 of 50 elements wrong (3 EXIT, 13 RUN, 7 PEND) despite `continue` reaching 'replay complete' -- confirms GENUINE PERMANENT DESYNC, not lag. + +Full causal chain (all three links independently confirmed in the SAME live run's logs): (1) The scheduler's own OUTBOUND xcecli connection to the primary controller repeatedly failed over and reconnected under the array's load -- scheduler.log shows `xcecli failover: attempt N -> localhost:1999X` bouncing across all 3 candidates, 8 extra times in an ~8s window (exact trigger for THIS layer not further isolated -- plausibly -vvv DEBUG3 log volume + concurrent WAL/replication load causing a client-side request timeout to fire spuriously; flagged as a follow-up, see below). (2) Every one of those 8 reconnects made ctrl-1.log log `handle_scheduler_join` again, which runs on_scheduler_join() (controller/sched.c) -- a FULL resync (xcesched_push_configs + xcesched_push_workers + xcesched_push_jobs) on every join, no incrementality. xcesched_push_jobs()'s XceData_data_schedjobs wire path re-sends EVERY currently-committed job (not just the array), hitting the scheduler's ordinary (non-array) JOBS_PUSH handling with zero pre-existence filtering. Confirmed live: 700 `sched_add_job failed ... discarding duplicate` WARNs across the session (100 distinct jobids x up to 8 resyncs each), vs only ONE `job array push received and expanded` INFO line -- proving these duplicates come from the ordinary resync path, NOT bug-1714's documented sched_on_job_array() TOCTOU race (that race is real but was a red herring for THIS bug's data-loss mechanism -- it only ever double-submits freshly-PEND array elements, which key_add() correctly and harmlessly rejects, exactly as bug-1714/commit 38437e57 already established). (3) During the gap between disconnect and each reconnect, c->sched.ep is NULL, and any real job-finish notify due in that window is SILENTLY SKIPPED -- ctrl-1.log's pre-existing bug-1731-era DEBUG3 diagnostic `DIAG sched_job_update_fun: jobid=... state=... -- c->sched.ep is NULL, notify SKIPPED` fired 36 times in this run, for jobids matching the array. The very next full resync (link 2) is that job's ONLY remaining chance to reach the scheduler -- but XCE_CORE_EVENT_JOBS_PUSH's handling of an already-tracked jobid (core_events.c) discarded the incoming (correct, newer) job object UNCONDITIONALLY on sched_add_job()->key_add() EEXIST, NEVER updating the existing tracked xce_sched_job_t's state -- so a job whose terminal state only ever reached the scheduler via this resync path was permanently frozen at whatever PEND/RUN state it had before, exactly matching the reported DONE/EXIT-interleaved-with-stuck-RUN/PEND symptom. CONFIDENCE: HIGH -- root cause reproduced live, instrumented with the project's own existing diagnostics (no new printf needed), and closed with a RED->GREEN regression test that fails against the pre-fix code and passes after. + +NOT FIXED / follow-up (deliberately out of scope for this session, logged for a future bug): WHY the scheduler's own outbound xcecli connection to the primary keeps failing over under load (causal link 1 above) is not isolated to a specific timeout value or code site -- worth its own investigation (possibly a too-aggressive client-side request timeout in the shared xcecli/xce_request layer under DEBUG3 logging + concurrent WAL replication load). Reducing that flapping would reduce how often on_scheduler_join()'s full resync fires at all, but this session's fix (below) makes the resync SAFE regardless of how often it fires, which is the more load-bearing property (a resync will always be needed on genuine scheduler restart/rejoin, not just spurious flapping). + +## Fix +FIXED (2026-08-30). Root cause: src/scheduler/src/core_events.c's XCE_CORE_EVENT_JOBS_PUSH case treated sched_add_job()'s -EEXIST (job already tracked) as always a harmless no-op duplicate (per bug-1714/commit 38437e57's Bug A fix) and unconditionally freed the incoming xce_job_t without ever touching the existing tracked job. That assumption held for the array-push redelivery bug-1714 covers (both copies start fresh JOB_STAT_PEND) but not for on_scheduler_join()'s full-resync redelivery (controller/sched.c's xcesched_push_jobs(), fired on every scheduler (re)join), which can legitimately carry a job's CURRENT state -- including a terminal state the scheduler never saw because the matching real-time notify was dropped while c->sched.ep was NULL during the disconnect gap (bug-1731's still-open 'notify SKIPPED' diagnostic). Discarding that redelivery unconditionally silently and PERMANENTLY froze the scheduler's own view of the job at its pre-disconnect state, exactly reproducing this entry's reported controller-vs-scheduler divergence live. + +FIX (src/scheduler/src/core_events.c, XCE_CORE_EVENT_JOBS_PUSH case): on sched_add_job() EEXIST, look up the existing tracked xce_sched_job_t via sched_get_job() and, if its state differs from the redelivered copy's state, apply the SAME terminal-transition accounts/allocs cleanup XCE_CORE_EVENT_JOB_STATE_CHANGE/JOB_UPDATE already perform, update existing->state in place, and fire picker->picker_iface->on_job_state_change() so the picker's own bookkeeping stays consistent -- instead of silently dropping the newer information. The redelivered xce_job_t clone itself is still freed afterward (it was never registered in core->jobs and never will be) -- only its STATE is salvaged into the object future sched_get_job() lookups actually resolve to. Deliberately scoped to state only (not queue/resreq/other fields) -- documented as an explicit scope decision in the fix's own comment, same convention persistence.md's field-completeness guidance uses elsewhere. + +TDD: new regression test test_jobs_push_duplicate_jobid_with_newer_state_updates_existing_job (src/scheduler/testsuite/test_core_events.c) -- pushes jobid 56 as JOB_STAT_PEND, then redelivers the SAME jobid via a second JOBS_PUSH batch carrying JOB_STAT_DONE (simulating on_scheduler_join()'s resync racing a dropped terminal notify), asserts sched_get_job() still resolves to the SAME xce_sched_job_t (no orphaned duplicate, no key_add() churn) AND that its state is now JOB_STAT_DONE, not frozen at JOB_STAT_PEND. RED confirmed first (`Expected 8 Was 1` -- JOB_STAT_DONE vs JOB_STAT_PEND); GREEN after the fix. Full scheduler suite: 55/55 passing after the fix, zero regressions (bug-1714's own test_jobs_push_duplicate_jobid_does_not_call_on_job_push_or_leak still passes unchanged -- the two duplicate-handling paths, state-differs vs state-identical, are both exercised and both correct). + +LIVE RE-VERIFICATION: rebuilt xce_scheduler with the fix, restarted it against the SAME live 3-node cluster (still under the same reconnect-storm conditions -- 700 duplicate-discard WARNs still fired, confirming the fix does not suppress or depend on suppressing the underlying reconnect flapping), submitted a second 50-element array job, captured a fresh session recording, and replayed it via `xce_scheduler -i -s ` + `continue` + `info jobs`: all 100 jobs across both arrays (the pre-fix array replayed through the fixed binary this time, plus the new array) now show DONE, matching the controller's joblist exactly -- zero divergence, confirming the fix closes the reported gap under the exact live conditions that originally exposed it. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1714, bug-1731, bug-1733 diff --git a/.wolf/bugs/bug-1735.md b/.wolf/bugs/bug-1735.md new file mode 100644 index 00000000..43ca3bc5 --- /dev/null +++ b/.wolf/bugs/bug-1735.md @@ -0,0 +1,53 @@ +--- +id: "bug-1735" +status: "fixed" +status_updated: "2026-09-24" +status_note: "d8a37fa2" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-30T10:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/xcecd_cgroups.c (collect_job_rusages", "line ~1418-1436 -- the worker-side cgroup-based measurement and resname assignment)", "src/controller/misc.c (update_host_runusage", "line ~128-138 -- the controller-side parser)", "src/controller/job.c (update_job_rusages", "line ~3200 -- pre-existing 'FIXME update other fields of job rusage later' comment)", "src/scheduler/include/sched/types.h:115 (xce_sched_job_t.rusages -- confirmed the scheduler stores/forwards the RAW xce_job_rusages_t wire report unmodified", "does not itself re-parse or aggregate it)", "src/scheduler/src/recording_events.c:142 (wire.body.job_rusage = ev->u.job_rusage.rusages -- confirms the .replay file records whatever raw resname/value pairs it received", "unchanged)"] +tags: ["worker", "controller", "resource-usage", "rusage", "memory-tracking", "resname-mismatch", "cgroups", "xcecd_cgroups", "found-via-user-report", "static-analysis-only", "unreproduced-in-sandbox", "no-dbus-systemd-in-sandbox", "needs-real-deployment-verification", "needs-owner", "session-sibling-of-bug-1734", "priority-high", "split-2026-08-30", "core-usage-metrics-only", "verified-2026-09-03"] +related: ["bug-1734", "bug-1736"] +fix_commits: ["d8a37fa2"] +legacy: true +--- + +# bug-1735: affects data the scheduler needs for resource-usage-aware scheduling decisions (see priority note… + +## Description +PRIORITY: HIGH -- affects data the scheduler needs for resource-usage-aware scheduling decisions (see priority note below). Split 2026-08-30 from a broader initial report into this (core usage: memory + CPU time) and bug-1736 (display-only statistics: swap, task count, peak/efficiency percentages) at user request, since the two have very different urgency and the broader one should not block this one. + +User-reported: job memory usage always shows 0 in the scheduler's simulation record file (.replay), despite the worker daemon presumably measuring real resource usage. User asked to check whether the worker even reports this data at all -- CONFIRMED: it does, but the value never survives to the controller due to a plain resource-name string mismatch, plus a second, independent gap in how the controller aggregates whatever it does receive. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +TWO CONFIRMED, INDEPENDENT GAPS -- either alone would produce 'memory usage always 0'; both are real. Neither was live-verified this session (no real cgroup-backed xcecd deployment was exercised -- see the environment-limitation note below), but both are confirmed by direct code reading with exact citations, and there is NO translation/mapping layer anywhere between the two mismatched naming conventions (grepped the whole worker/controller/lib/scheduler source tree for both string sets -- only the two sites below reference them at all). + +FINDING 1 (the primary, most likely explanation): resource-name string mismatch between producer and consumer. collect_job_rusages() (xcecd_cgroups.c, the REAL, cgroup/systemd-backed worker's rusage collector -- queries a job's own 'xcecd-worker-.service' systemd unit via sd_bus_open_system()) reports exactly 4 resnames: 'cpu_time_sec', 'memory_rss', 'memory_swap', 'tasks_current'. update_host_runusage() (misc.c, the controller-side parser that receives this over the wire) only recognizes the LITERAL strings 'cputime' and 'mem' -- neither of which matches ANY of the 4 names the worker actually sends. Every rusage report from the real worker therefore falls through both `if (strcmp(...))` checks doing nothing, silently, for every single field -- host_runusage[j]->cpu_time and ->mem_usage stay at their calloc-zeroed default FOREVER, regardless of what the worker's cgroup query actually measured. No error, no log, no indication anything was dropped -- this is why it reads as 'always 0' rather than an obvious failure. + +FINDING 2 (independent, compounding): even if Finding 1 were fixed and host_runusage[]'s PER-HOST fields started populating correctly, the TOP-LEVEL AGGREGATE fields of xce_job_runusage_t (max_mem, avg_mem, used_mem, cpu_time, peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency) are NEVER computed from the per-host breakdown anywhere -- update_job_rusages() (job.c) calls update_host_runusage() (which only touches host_runusage[]) then has an explicit, still-open '/* FIXME update other fields of job rusage later */' comment immediately after, with no code implementing that FIXME anywhere in this codebase (grepped). Any downstream consumer that reads the AGGREGATE fields (rather than iterating host_runusage[] itself) would see 0 regardless of Finding 1's fix. + +SCHEDULER-SIDE STRUCTURE NOTE (relevant context, not itself a bug): the scheduler does NOT receive or store the aggregated xce_job_runusage_t at all -- xce_sched_job_t.rusages (sched/types.h:115) is typed xce_job_rusages_t*, the SAME flat wire-report struct the worker originally sent (jobid/npids/pids/count/usages[] of raw {resname,value} pairs) -- forwarded/stored as-is, and recording_events.c's capnp encode for the .replay file writes that same raw struct unchanged. So the .replay file the user is inspecting genuinely contains whatever resname/value pairs the worker sent, byte for byte -- if Finding 1 is fixed (or the worker already sent matching names), the RAW data WOULD flow all the way into the recording faithfully. Grepped src/scheduler/src/repl*.c for any resname-aware rendering of memory usage in the interactive REPL/inspect commands and found NONE -- could not determine within this session's time budget whether the specific tool/view the user used to inspect the .replay file (unclear from the report whether it's the live REPL's own display, a separate dump/analysis tool, or something else) has ITS OWN resname expectation that might differ yet again from both 'mem' and 'memory_rss' -- flagged as an open question for whoever picks this up, not confirmed either way. + +ENVIRONMENT LIMITATION NOTE: this session's own live burst-testing (bug-1714 investigation) exclusively used xcecd_virtual (the no-root virtual worker), never the real, cgroup/systemd-backed xcecd binary collect_job_rusages() lives in -- this sandbox has no D-Bus/systemd (already confirmed in this session's bug-1731 follow-up work: get_systemd_job_exit_code() always returns -1 here), so sd_bus_open_system() at collect_job_rusages()'s own first real step would itself fail immediately in this environment (returns NULL before ever reaching the resname assignment code) -- this bug could not have been, and was not, live-reproduced in this sandbox; it is a pure static-code finding, applicable specifically to the user's own real deployment (which does have real cgroups/systemd, matching the user's own 'live deployment' framing throughout this investigation). + +PRIORITY NOTE (2026-08-30, user request): this half of the original finding -- memory (memory_rss/'mem') and CPU time (cpu_time_sec/'cputime') specifically -- is the HIGH-PRIORITY half. These are exactly the two metrics xce_host_runusage_t already has dedicated fields for (cpu_time, mem_usage) and the two the controller's parser already has matching (if misspelled) if-branches for -- clearly the originally-intended 'core' usage metrics, as opposed to the swap/task-count/peak/efficiency fields split out to bug-1736, which have no controller-side field or computation logic at all today. IMPORTANT CAVEAT, checked directly this session: grepped the full scheduler source tree for any consumer of rusage/usage data in scheduling/picker/allocation logic and found NONE -- xce_sched_job_t.rusages (sched/types.h:115) is stored and forwarded into recordings but not read by any picker plugin or allocation code today. So this fix corrects the DATA PIPELINE (worker measurement -> controller aggregation -> scheduler storage/recording), the necessary prerequisite for any future usage-aware scheduling policy (e.g. backfill sizing, over-commit detection, preemption on real memory pressure) -- it does not itself wire usage data into a scheduling decision, since no such consumer exists yet. Marked high priority on that basis (unblocks correct data for future scheduling work, and is also simply a correctness bug in its own right for any operator-facing usage reporting), not because it changes scheduling behavior today. + +## Fix +NOT FIXED -- logged for follow-up, scope narrowed 2026-08-30 to memory + CPU time only (see bug-1736 for the split-off lower-priority statistics). RECOMMENDED FIX, Finding 1 (resname mismatch): align 'cpu_time_sec'/'memory_rss' (worker, xcecd_cgroups.c) with 'cputime'/'mem' (controller, misc.c) -- pick a direction (see the two options in the original finding, not repeated here) and apply consistently. RECOMMENDED FIX, Finding 2 (aggregation): implement update_job_rusages()'s (job.c) still-open FIXME specifically for max_mem/avg_mem/used_mem/cpu_time -- derive from host_runusage[]'s per-host entries each time a report merges in (max/avg/sum per field semantics). Both need TDD (unit tests, not just a live check) plus live verification against a REAL cgroup-backed xcecd deployment -- this session's sandbox has no D-Bus/systemd and cannot provide that verification itself (see the original environment-limitation note). + +UPDATE (2026-09-03, verification session): CONFIRMED ALREADY FIXED. Both recommended fixes were implemented and committed 2026-08-30 (commit d8a37fa27 for this entry's resname-mismatch Finding 1; commit 5c2c6664e for bug-1736's sibling swap/task-count fields) but this entry was never marked fixed. update_host_runusage() (misc.c) now matches the worker's real resnames ('cpu_time_sec'/'memory_rss'), and aggregate_job_runusage() (misc.c) computes max_mem/avg_mem/used_mem/cpu_time/peak_cpu_time from the per-host breakdown, called from update_job_rusages() (job.c:3502) -- replacing the old FIXME. TDD unit tests exist (src/controller/testsuite/test_host_runusage.c, 7 tests covering resname matching, multi-host update-in-place, swap/task-count parsing, and aggregation single/multi-host/running-max-peak/null-safety). Re-verified this session: fresh /xce-build controller asan, all 6 controller test suites pass including 'host runusage test' (test_host_runusage.c). No code change was needed -- this session's action was verification + closing out the bug-log entry. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1734, bug-1736 diff --git a/.wolf/bugs/bug-1736.md b/.wolf/bugs/bug-1736.md new file mode 100644 index 00000000..58ea2bfc --- /dev/null +++ b/.wolf/bugs/bug-1736.md @@ -0,0 +1,47 @@ +--- +id: "bug-1736" +status: "partial" +status_updated: "2026-09-24" +status_note: "swap/tasks/peak_cpu_time fixed 5c2c6664; peak_cpu_duration/efficiency still not computed" +component: "controller" +severity: "unknown" +timestamp: "2026-08-30T10:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/xcecd_cgroups.c (collect_job_rusages -- already sends memory_swap/tasks_current", "no controller-side field to receive them)", "src/include/xce/types.h (xce_host_runusage_t -- would need new fields for swap/task-count; xce_job_runusage_t -- the peak/efficiency fields with no producer)", "src/controller/misc.c (update_host_runusage -- would need new parse branches)", "src/controller/job.c (update_job_rusages -- same open FIXME as bug-1735", "this is the other half of what it would need to cover)"] +tags: ["worker", "controller", "resource-usage", "rusage", "display-only", "reporting-only", "priority-low", "split-2026-08-30", "no-scheduling-impact", "static-analysis-only", "unreproduced-in-sandbox", "needs-design-not-just-fix", "peak-cpu-time-fixed", "swap-taskcount-fixed", "efficiency-fields-remain-scope-decision"] +related: ["bug-1735"] +fix_commits: ["5c2c6664"] +legacy: true +--- + +# bug-1736: display/reporting only, not consumed by any scheduling logic + +## Description +PRIORITY: LOW -- display/reporting only, not consumed by any scheduling logic. Split 2026-08-30 from bug-1735 at user request: the 'statistics-only' half of that finding -- resource usage fields with no scheduling relevance, purely for operator-facing job accounting/reporting (bjobs -l/bhist-style views), as opposed to bug-1735's memory+CPU-time core, which is a prerequisite for future scheduling decisions. Worker (collect_job_rusages(), xcecd_cgroups.c) already measures and sends 'memory_swap' and 'tasks_current' per job, but the controller has NO field to receive them at all (xce_host_runusage_t only has cpu_time/mem_usage -- no swap or task-count field exists) -- these two are silently dropped entirely, not just misnamed. Separately, xce_job_runusage_t's peak/efficiency fields (peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency) have no computation logic anywhere in the codebase (grepped) -- nothing populates them, ever, regardless of bug-1735's fix. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +See bug-1735's root_cause for the full original investigation (resname-mismatch mechanism, environment-limitation note re: this session's sandbox having no real cgroup-backed worker to live-verify against) -- this entry covers only the subset of that finding split out as lower-priority. Unlike bug-1735's memory/cpu_time (which have existing-but-mismatched fields), memory_swap/tasks_current have NO receiving field in xce_host_runusage_t at all -- the worker's measurement is a complete dead end on the controller side, not a naming mismatch. The peak/efficiency fields (xce_job_runusage_t's own top-level fields) are simply never computed by anything, anywhere -- confirmed via grep, no partial implementation exists to build on. + +## Fix +PARTIALLY FIXED (2026-08-30). Of the two gaps this entry covers: + +(1) memory_swap/tasks_current -- FIXED. Added mem_swap (double)/tasks_current (int) fields to xce_host_runusage_t (types.h), matching capnp fields (memSwap @3/tasksCurrent @4) appended to the HostRunusage schema (controller.capnp) for WAL field-completeness, parsing in update_host_runusage() (misc.c) for the worker's 'memory_swap'/'tasks_current' resnames, and per-host copy in merge_job_runusage()'s (job.c) existing-host branch (the new fields would otherwise have been silently dropped on WAL replay/catchup for a host already seen). + +(2) peak_cpu_time -- FIXED. aggregate_job_runusage() (misc.c) now tracks it as the running MAX of the per-call cpu_time sum across every report a job has ever received (not just the latest), guarding against a later report reducing it (e.g. an out-of-order merge/replay). + +STILL NOT FIXED, DELIBERATELY (documented scope decision, not an oversight -- see aggregate_job_runusage()'s own doc comment in misc.c): peak_cpu_duration and the three efficiency fields (mem_efficiency/avg_cpu_efficiency/peak_cpu_efficiency). These need a 'used vs. requested' ratio against a job's resource request (a resreq expression tree, not a plain core/slot count) and/or an unspecified duration semantic -- genuine product/design decisions, not a formula that can be safely guessed. Left at their existing zero value rather than populated with an invented number that could mislead an operator. + +Verified: full `meson test --suite controller` clean, Ok:6 Fail:0, 1220/1220 individual controller_unit_test cases pass (zero regression from touching the shared xce_host_runusage_t struct + its capnp schema), plus 2 new regression tests in test_host_runusage.c (7/7 cases pass). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1735 diff --git a/.wolf/bugs/bug-1737.md b/.wolf/bugs/bug-1737.md new file mode 100644 index 00000000..1400baf2 --- /dev/null +++ b/.wolf/bugs/bug-1737.md @@ -0,0 +1,38 @@ +--- +id: "bug-1737" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-08-30T22:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/sched.c (sched_job_exit_fun", "renamed sched_job_finish_fun)", "src/scheduler/src/handler.c (sched_handle_notify's NotifyData_data_schedExitJob case)", "src/schema/controller.capnp (schedExitJob field)", "src/include/typemap.h (xce_notify_data_t's sched_exit_job union member)", "src/lib/helper.c (xce_notify_data_ptr)"] +tags: ["scheduler", "controller", "wal", "notify", "capnp", "job-state", "exit-code", "done-vs-exit", "confirmed-live"] +related: ["bug-1731", "bug-1734", "bug-1738"] +legacy: true +--- + +# bug-1737: User-reported (live 3-node HA cluster + worker, 50-element array job `sleep 100 && hostname`… + +## Description +User-reported (live 3-node HA cluster + worker, 50-element array job `sleep 100 && hostname`, xce_scheduler -i -s vs controller joblist): job 0-8-1788143749-0[38] shows EXIT in the scheduler's replay (`info jobs`) but DONE in the controller's joblist -- confirmed a REAL state divergence (JOB_STAT_EXIT=0x04 and JOB_STAT_DONE=0x08 are genuinely different bits, src/include/xce/types.h), not cosmetic. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code reading (both static and, indirectly, live -- the fixed behavior was verified against a real captured replay; the pre-fix defect itself was not separately re-reproduced live in this session since fixing it was done in the same pass). HOOK_DETAIL_JOB_EXIT fires for EVERY job finish, both success and failure -- job.c's record_job_finish_fact()/apply_job_finish_transition() decide DONE (exit_code==0) vs EXIT (exit_code!=0) BEFORE firing this exact hook detail either way (mirrors persist.c's own WAL_JOB_FINISH -> HOOK_DETAIL_JOB_EXIT mapping, `case HOOK_DETAIL_JOB_EXIT: return WAL_JOB_FINISH;`). sched.c's sched_job_exit_fun() (the HOOK_DETAIL_JOB_EXIT subscriber that notifies the scheduler) receives xce_jobfinish_t* info (which HAS an exit_code field) but never read it -- it unconditionally sent a bare jobid via NotifyData_data_schedExitJob (controller.capnp's schedExitJob was a plain UInt64, no exit_code at all on the wire). scheduler/src/handler.c's sched_handle_notify() then unconditionally mapped ANY schedExitJob to JOB_STAT_EXIT, with no way to ever land on JOB_STAT_DONE via this live path. Under STABLE connectivity this was normally self-correcting: the SAME WAL apply also fires HOOK_DETAIL_JOB_UPDATE moments later (persist.c:4854-4855, immediately after the HOOK_DETAIL_JOB_EXIT fire, same lock scope), which carries the job's REAL, already-final job->job.state via schedUpdateJob -> XCE_CORE_EVENT_JOB_UPDATE's `job->state = updated->state;` (core_events.c), overwriting the wrongly-forced EXIT with the correct DONE within the same notify batch. But this is TWO SEPARATE notify() calls, not atomic -- if the scheduler's connection blipped between them (the same reconnect-flakiness bug-1731/bug-1734 already documented live under this exact 50-element-array load), the correcting schedUpdateJob could be dropped (c->sched.ep NULL at fire time) while the earlier, wrong schedExitJob(EXIT) had already landed -- leaving the scheduler permanently stuck at EXIT for a job the controller correctly recorded as DONE. This exactly explains why only ONE array element (out of 50) showed the divergence: only that job's particular pair of notifies happened to straddle a reconnect gap. + +## Fix +FIXED (2026-08-30). Reused the EXISTING JobFinishInfo wire shape (jobid + code) apiserver_job_exit_fun() (apiserver.c) already sends for this identical hook, rather than inventing a new capnp struct: controller.capnp's schedExitJob field changed from `UInt64` to `JobFinishInfo`; typemap.h's xce_notify_data_t.sched_exit_job union member changed from `uint64_t` to `xce_jobfinish_t*`; sched.c's sched_job_exit_fun() (renamed sched_job_finish_fun() for accuracy -- it covers both DONE and EXIT outcomes, not just EXIT) now sends `.sched_exit_job = info` (the whole struct, mirroring apiserver_job_exit_fun's `.finishinfo = info` pattern) instead of `info->jobid`; scheduler/src/handler.c's sched_handle_notify() now reads finish->exit_code and mirrors job.c's own decision exactly: exit_code==0 -> JOB_STAT_DONE, else -> JOB_STAT_EXIT, instead of unconditionally assuming EXIT. helper.c's xce_notify_data_ptr() updated to match the new pointer type (dropped a now-redundant void* cast). TDD: added test_handle_notify_job_exit_maps_exit_code_to_done_or_exit (test_handlers.c) -- exit_code==0 -> DONE, exit_code!=0 -> EXIT, NULL payload -> -EINVAL (no crash). Full lib+controller+scheduler+worker build and test suite (both a dedicated full build via xce-build.sh and an incremental rebuild+retest after further edits): 61/61 Ok for controller+scheduler (66/67 overall including lib, the 1 failure a pre-existing unrelated ASAN-leak fixture bug already logged separately, not caused by this diff), xceadmin 288/288. Live-verified: deployed a fresh 3-node HA cluster + worker with the fixed binaries, submitted a 51-job workload (50-element array + 1 standalone job, all exiting 0), let everything run to completion -- the scheduler's replay showed DONE for every single job, matching the controller's joblist exactly, with zero EXIT/DONE divergence. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1731, bug-1734, bug-1738 diff --git a/.wolf/bugs/bug-1738.md b/.wolf/bugs/bug-1738.md new file mode 100644 index 00000000..30609a36 --- /dev/null +++ b/.wolf/bugs/bug-1738.md @@ -0,0 +1,38 @@ +--- +id: "bug-1738" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-30T22:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_STATE_CHANGE case", "and the JOBS_PUSH duplicate-jobid merge bug-1734 added)", "src/controller/sched.c (sched_job_fail_fun", "comment only)"] +tags: ["scheduler", "job-state", "terminal-state", "job-fail", "regression-guard"] +related: ["bug-1734", "bug-1737", "bug-1739"] +legacy: true +--- + +# bug-1738: Found investigating Symptom 4 (EXIT vs DONE divergence) alongside bug-1737: a SEPARATE, genuinely… + +## Description +Found investigating Symptom 4 (EXIT vs DONE divergence) alongside bug-1737: a SEPARATE, genuinely different mechanism that can ALSO clobber a correct terminal job state on the scheduler side, independent of the schedExitJob exit_code gap. Not separately live-reproduced this session (no post_exec-stage job failure was exercised against the live cluster) -- confirmed by direct code reading with an exact, unambiguous mechanism. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code reading. HOOK_DETAIL_JOB_FAIL is a narrower event than HOOK_DETAIL_JOB_EXIT/finish -- job.c's apply_job_fail_transition() only forces JOB_STAT_EXIT for a genuinely pre-exit failure (the `default:` case, no prior exit info exists yet); for JOB_EXEC_STAGE_POST_EXEC/Q_POST_EXEC (a post-exec script failing on an otherwise-already-decided job) it MERGES a JOB_STAT_PERR flag onto whatever DONE/EXIT the job's own finish already decided -- it never re-decides DONE vs EXIT after the fact. But sched.c's sched_job_fail_fun() (the HOOK_DETAIL_JOB_FAIL subscriber) always sends the SAME bare-jobid schedFailJob notify regardless of which case applied, and scheduler/src/handler.c's sched_handle_notify() unconditionally maps THAT to JOB_STAT_EXIT via XCE_CORE_EVENT_JOB_STATE_CHANGE, which (before this fix) had NO guard against overwriting an already-terminal state: `job->state = new_state;` unconditionally. Net effect: a job that finished successfully (DONE, correctly notified live) and then has an UNRELATED post-exec script fail afterward would have the scheduler's view of it forcibly clobbered from DONE back to EXIT -- a real, distinct correctness bug from bug-1737's exit_code gap. SEPARATELY, this session's own bug-1734 fix (landed earlier the same session, src/scheduler/src/core_events.c, XCE_CORE_EVENT_JOBS_PUSH's duplicate-jobid merge path) has the IDENTICAL class of gap: on a resync redelivery carrying a DIFFERENT state than the existing tracked job, it merged `existing->state = job->state` unconditionally with no ordering/terminal-permanence check -- a stale or out-of-order resync redelivery claiming EXIT for a job already correctly DONE would silently regress it. Both gaps share one missing invariant: once a job reaches a terminal state (DONE or EXIT) on the scheduler's own tracked copy, that state must be PERMANENT -- mirroring the controller's own job_state_is_terminal() doctrine (job.c: apply_job_finish_transition() no-ops a duplicate FINISH once terminal) -- and no single-job notify or resync redelivery should ever be able to change it to something DIFFERENT. + +## Fix +FIXED (2026-08-30). Added one shared static helper, job_state_regresses_terminal(current_state, new_state) (core_events.c, just above xce_core_push_events()): true iff current_state is already DONE or EXIT AND new_state is different. Applied it at BOTH vulnerable call sites: (1) XCE_CORE_EVENT_JOB_STATE_CHANGE's case -- rejects (WARN-logged, no-op) a new_state that would regress an already-terminal job, before the existing accounts/allocs-cleanup + state-assignment logic runs; (2) the JOBS_PUSH duplicate-jobid merge path (bug-1734) -- same guard, applied before deciding how to merge. Also added a clarifying comment to sched.c's sched_job_fail_fun() documenting the POST_EXEC-vs-default distinction and why this core-side guard (not a sched.c-side fix) is the right place to close the gap, since the scheduler has no JOB_STAT_PERR-equivalent concept of its own to merge instead -- 'never regress a terminal state' is simpler and sufficient. TDD: added test_job_state_change_does_not_regress_terminal_state (drives a job to DONE via JOB_STATE_CHANGE, then a second JOB_STATE_CHANGE claiming EXIT must be ignored) and test_jobs_push_duplicate_jobid_does_not_regress_terminal_state (same invariant via the JOBS_PUSH merge path) in test_core_events.c. Full scheduler+controller suite: 61/61 Ok, zero regressions -- the pre-existing bug-1734 test (PEND -> DONE merge, a legitimate forward transition) still passes unchanged, confirming the new guard only blocks a REGRESSION (terminal -> different), not a legitimate non-terminal -> terminal merge. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1734, bug-1737, bug-1739 diff --git a/.wolf/bugs/bug-1739.md b/.wolf/bugs/bug-1739.md new file mode 100644 index 00000000..3a9f53e1 --- /dev/null +++ b/.wolf/bugs/bug-1739.md @@ -0,0 +1,38 @@ +--- +id: "bug-1739" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-30T22:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge path)"] +tags: ["scheduler", "job-state", "phantom-run", "allocation", "resync"] +related: ["bug-1734", "bug-1738", "bug-1737"] +legacy: true +--- + +# bug-1739: User-reported (live 3-node HA cluster + worker, 50-element array job): `info summary` showed 30… + +## Description +User-reported (live 3-node HA cluster + worker, 50-element array job): `info summary` showed 30 elements of an array marked RUN in `info jobs`, but `info worker pc13v` only listed 22 'assigned job:' lines (matching the worker's real 22-core capacity) -- 8 elements showed job-level state RUN with NO corresponding worker/slot-assignment record at all, a 'phantom RUN' exceeding real hardware capacity. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code reading (this exact mechanism was identified and fixed before a dedicated live reproduction of the PRE-fix phantom-RUN state was attempted; the FIX itself was verified live -- a real 50-element array job run against the fixed binary showed zero phantom RUN jobs at any inspection point). This session's own bug-1734 fix (landed earlier the same session) added a merge for XCE_CORE_EVENT_JOBS_PUSH's duplicate-jobid (EEXIST) case: on a resync redelivery carrying a DIFFERENT state than the existing tracked job, if the new state was RUN, it set `existing->state = job->state` (RUN) directly -- but did NOT transfer any allocation data. core_build_job() (this same file, called earlier in the SAME event to construct the redelivered `job` clone from the wire xce_job_t) already adopts the wire payload's `allocs` field into a fresh xce_sched_job_t exactly the same way a brand-new RUN job's JOBS_PUSH would (`job->accounts = make_job_account(job->allocs)` when `j->allocs != NULL`) -- i.e. the redelivered clone DOES carry real allocation data when the controller's resync wire payload includes it (xcesched_push_jobs(), sched.c, collects the controller's live job_t structs directly, whose ->job.allocs is the real dispatch record for a running job). But bug-1734's merge branch discarded this clone entirely (FREEUP(job) at the end of the EEXIST handler, with allocs/accounts explicitly left out of scope in that fix's own comment) without ever copying job->allocs/job->accounts onto `existing` -- leaving `existing->state == RUN` with `existing->allocs == NULL`: a job the scheduler's own resource/slot bookkeeping (`info worker`'s 'assigned job:' scan, which walks core->jobs' allocations) has no record of at all, exactly the reported phenotype. + +## Fix +FIXED (2026-08-30), same edit pass as bug-1738's core_events.c guard (both touch the same JOBS_PUSH EEXIST branch). When the merge's terminal-permanence guard (bug-1738) allows the merge to proceed and the redelivered state is RUN: if the redelivered clone carries real allocation data (job->accounts != NULL && job->allocs != NULL), TRANSFER it onto `existing` (free whatever stale accounts/allocs `existing` had, then move the pointers over -- `existing->accounts = job->accounts; existing->allocs = job->allocs; job->accounts = NULL; job->allocs = NULL;` so the later FREEUP(job) doesn't double-free them) instead of leaving them untouched/discarded. If the redelivery claims RUN but carries NO allocation data (a defensive case -- an older/degraded controller payload, or a race where the dispatch record wasn't attached yet), the merge into RUN is REFUSED (WARN-logged, existing state kept unchanged) rather than manufacturing a phantom RUN from a different cause. TDD: added test_jobs_push_duplicate_jobid_run_state_transfers_allocation (RUN + real allocs -> existing->state==RUN AND existing->allocs/accounts non-NULL, transferred not discarded) and test_jobs_push_duplicate_jobid_run_state_without_allocation_is_refused (RUN + no allocs -> existing stays at its prior safe state) in test_core_events.c. Full scheduler+controller suite: 61/61 Ok, zero regressions -- bug-1734's own original test (PEND -> DONE merge, no allocation involved) still passes unchanged. LIVE RE-VERIFICATION: rebuilt, redeployed a fresh 3-node HA cluster + 22-core worker, submitted a 50-element overlapping-runtime array job (sleep 60 && hostname) plus one standalone job against the 22-slot capacity, captured a mid-run scheduler recording (19 jobs genuinely RUN, matching the worker's real capacity minus a few still-queued elements) and inspected `info worker`/`info jobs`: RUN-state job count matched 'assigned job:' count exactly at every inspection point (23 assigned = 19 RUN + 4 jobs with an in-flight allocation reservation not yet finalized to RUN, a benign timing artifact of catching mid-dispatch state, not a phantom RUN) -- no job ever showed RUN with a missing worker assignment. Could not force the SAME reconnect-storm conditions that originally exposed bug-1734's gap (this deployment's controller/scheduler connection stayed stable throughout), so the EEXIST/resync branch itself was exercised only via the unit tests above, not a live resync redelivery -- high confidence from code + unit tests, not a live end-to-end repro of the exact original trigger. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1734, bug-1738, bug-1737 diff --git a/.wolf/bugs/bug-1740.md b/.wolf/bugs/bug-1740.md new file mode 100644 index 00000000..55662e01 --- /dev/null +++ b/.wolf/bugs/bug-1740.md @@ -0,0 +1,38 @@ +--- +id: "bug-1740" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-08-31T22:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_RUSAGE handler -- live fix site)", "src/scheduler/src/schedobj.c (sched_apply_job_rusage -- dead-code path", "left in place)", "src/scheduler/src/scheduler.c (sched_update -- unreachable caller)", "src/scheduler/src/resource.c (sched_reconcile_job_resource_usage/sched_reconcile_host_resource_usage)", "src/scheduler/src/driver.c (xce_sched_driver_translate_pending_events/xce_core_push_events -- the live path that supersedes sched_update)", "src/scheduler/testsuite/test_core_events.c (new regression test) [file field reconstructed post-recovery from context in error_message/root_cause/fix -- not the verbatim original string", "see recovery note]"] +tags: ["scheduler", "resource-accounting", "rusage", "dead-code", "wiring-gap", "test-coverage-gap", "core-events", "mem", "duplicate-implementation", "found-via-self-review"] +related: [] +legacy: true +--- + +# bug-1740: The bug-1735/resource-accounting.md reservation-vs-usage reconciliation feature… + +## Description +The bug-1735/resource-accounting.md reservation-vs-usage reconciliation feature (sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage(), resource.c, committed this session) was wired into schedobj.c's sched_apply_job_rusage() -- but that function has exactly ONE caller anywhere in this codebase: scheduler.c's sched_update(), which itself has NO live caller at all (confirmed by exhaustive grep -- superseded by driver.c's xce_sched_driver_translate_pending_events()/xce_core_push_events() mechanism, per driver.c's own comments: "Replaces sched_update()'s inline handling"). The REAL, live production path for job rusage -- XCE_CORE_EVENT_JOB_RUSAGE in core_events.c's xce_core_push_events(), the SAME core-level event both the normal-mode driver (driver.c) and the sim/replay driver (sim_loop.c) translate their own rusage events into -- has its OWN separate, duplicate inline implementation of "store rusage on job" that never called sched_apply_job_rusage() or the new reconciliation logic at all. Result: the entire reconciliation feature was completely inert in both normal-mode (live cluster) and sim/replay mode -- exactly the context the user's original bug reports (phantom RUN jobs, EXIT/DONE divergence, mem always 0) were observed through -- despite passing all 11 of its own unit tests, which called the reconciliation functions directly, bypassing the real event-dispatch path entirely. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Classic 'tested the function in isolation, never proved the real event path reaches it' gap. schedobj.c's sched_apply_job_rusage() and core_events.c's XCE_CORE_EVENT_JOB_RUSAGE handler are two independent, duplicate reimplementations of the same 'store rusage on job' bookkeeping (a pre-existing architectural duplication, likely left over from the scheduler-core rewrite that introduced the core/driver split -- sched_update() and its whole XCE_SCHED_OP_* dataq-dispatch mechanism appear to be the OLD, pre-rewrite path, fully superseded by driver.c's XCE_CORE_EVENT_* translation but never deleted). The two implementations had already drifted: they call DIFFERENT picker hooks (on_job_rusage vs plugin_job_rusage_update). Adding new logic to the dead one instead of the live one was an easy mistake to make without first confirming which path is actually reachable in production -- found only by directly grepping for every call site of sched_apply_job_rusage() and sched_update(), not by reading either function's own code or comments, which gave no hint of the duplication. + +## Fix +FIXED (2026-08-31). Added a call to sched_reconcile_job_resource_usage() directly inside core_events.c's XCE_CORE_EVENT_JOB_RUSAGE handler (the real, live path), right after its existing job->rusages bookkeeping. schedobj.c's sched_apply_job_rusage() keeps its own copy of the reconciliation call too (harmless dead code, but still exercised by its own existing unit tests, so left in place rather than removed as an unrelated cleanup). Added a genuine end-to-end regression test, test_job_rusage_reconciles_mem_reservation_against_usage (test_core_events.c), that drives the REAL xce_core_push_events() with a dispatched job carrying a real 4MB mem allocation, then a JOB_RUSAGE event claiming 7MB actual usage, and asserts the 3MB overage lands on both the account and the resource pool -- this is the test that would have caught the original wiring gap immediately, since it exercises the actual event dispatch instead of calling the reconciliation function directly. Required white-box #include-ing resource.c into test_core_events.c (previously only core.c/core_events.c) plus two link-satisfying stubs (plugin_prepare_allocation/sched_ctx_get_uncomit_host, matching test_resource_accounting.c's own established pattern). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1741.md b/.wolf/bugs/bug-1741.md new file mode 100644 index 00000000..62174515 --- /dev/null +++ b/.wolf/bugs/bug-1741.md @@ -0,0 +1,39 @@ +--- +id: "bug-1741" +status: "fixed" +status_updated: "2026-09-27" +status_note: "0cad6850: OPCode_notifyAcked -- job start/finish/fail acknowledged after commit, resent until acked; FF fallback for old controllers" +component: "lib" +severity: "unknown" +timestamp: "2026-08-31T23:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/controller_connector.c (xcecd_job_finished)", "src/lib/client.c (xcecli_notify", "xcecli_notify_jobfinish)", "src/lib/client_internal.h (XCECLI_FF_CALL)", "src/controller/job.c (on_job_finish", "never invoked for the 6 lost reports)", "src/controller/cluster.c (do_notify/handle_notify)"] +tags: ["controller", "worker", "job-lifecycle", "job-finish", "notify", "xcecli_notify", "XCECLI_FF_CALL", "fire-and-forget", "burst-load", "array-job", "phantom-run-job", "stuck-job", "data-loss", "found-live", "partially-root-caused", "needs-live-repro", "needs-owner", "related-to-bug-1548", "distinct-from-bug-1731", "distinct-from-bug-1698", "possibly-same-root-cause-as-bug-1698"] +related: ["bug-1548", "bug-1731", "bug-1737", "bug-1738", "bug-1739", "bug-1698"] +fix_commits: ["0cad6850"] +legacy: true +--- + +# bug-1741: User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster + xcecd_virtual worker… + +## Description +User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster + xcecd_virtual worker, build.dbg): after submitting a 50-element array job (jobsub -a 50 -c hostname) immediately following a fresh primary election, `xce_test`'s `joblist` shows array elements [6] through [12] (7 consecutive elements) permanently stuck in state RUN (FinishTime=0) while every other element (0-5, 13-49) correctly reached DONE with real FinishTimes, including elements submitted/dispatched both before and after the stuck range and well after the cluster had fully stabilized (up to ~46s post-submit). The user confirmed via repeated `joblist` checks over several minutes that elements 6-12 never progress -- not a transient lag. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +PARTIALLY ROOT-CAUSED via direct log correlation across worker, controller, and scheduler logs from the live incident (not yet independently live-reproduced in isolation this session -- found while investigating the unrelated bug-1698 HA-hang occurrence in the same incident's data, but happened ~19s BEFORE ctrl-1's hang began and is confirmed unrelated to it: ctrl-1 was fully healthy and primary at the time these 7 finish reports were lost). Timeline reconstructed from three independent logs for elements 6-12: xcecd_worker.log shows the xcecd_virtual master daemon spawned one subprocess per array element (its own established per-job-process architecture), each connecting to the master via a local unix-socket IPC (uv_ipc.c) and reporting 'Completed' for all 7 elements within a ~600ms window (16:25:00.943 to 16:25:01.535). ctrl-1.log shows dispatch_a_job_/on_job_start logged correctly for all 7 elements, but job.c's on_job_finish() (traced via 'job finish jobid ... exit code' TRACE, its own first log line) fired for element [6] ONLY -- elements [7] through [12]'s finish reports never reached on_job_finish at all, i.e. the loss is upstream of job.c entirely, not a hook-firing gap (distinct from bug-1731's fix, which addressed apply_committed_entry() firing HOOK_DETAIL_JOB_START/EXIT -- that fix only helps once on_job_finish/WAL_JOB_FINISH already happened; here it never happens for 6 of 7 jobs). scheduler.log's later repeated 'sched_add_job failed ... rc=-17, merging state into existing tracked job' WARNs for exactly elements 6-12 (recurring at 16:25:23, 16:25:41, 16:25:46 across multiple post-failover JOBS_PUSH resyncs from the new primary ctrl-2) confirm the loss is durable, not just delayed: ctrl-2 inherited ctrl-1's own WAL/replicated state, which never recorded a finish for these 6 elements either, so every resync just re-merges the same stale RUN state -- there is no surviving source of truth anywhere in the cluster that ever learned these jobs finished. Traced the worker-side send path for the likely drop mechanism: xcecd_job_finished() (controller_connector.c:1680) -> xcecli_notify_jobfinish() -> xcecli_notify() (client.c:610, explicitly commented 'Fire-and-forget chokepoint') -> XCECLI_FF_CALL (client_internal.h:290-302), whose own doc comment states plainly: 'single-shot, NOT a retry loop... evaluate EXPR exactly ONCE... A lost send rc is surfaced as-is, never retried.' All 7 near-simultaneous finish notifications for this burst go out over the SAME single xcecli_ctx/network connection (one xcecd master daemon process per worker host, matching this codebase's one-connection-per-daemon model) via this explicitly best-effort, non-retrying path -- exactly the kind of rapid same-connection burst most likely to hit whatever failure mode XCECLI_FF_CALL's own comment already acknowledges as a known risk ('A failed reconnect is non-fatal -- the send below is simply best-effort'). This is the SAME XCECLI_FF_CALL mechanism bug-1548 already flagged for a related-but-distinct reliability gap (silently self-healing the transport across ticks, masking a needed rejoin) -- this occurrence is a second, independent real-world manifestation of the same primitive's fire-and-forget design causing silent data loss, this time for job-finish reports specifically. NOT YET CONFIRMED: the exact internal reason xce_notify()'s single-shot send fails/drops 6 of 7 calls in this specific burst (whether it's a genuine transport error being silently discarded because nothing checks xcecd_job_finished()'s return value, some reentrancy/buffer-reuse issue in xce_notify()'s own encode/send path when called back-to-back without waiting for the prior call's completion, or contention on ac->failover.lock combined with a narrow race) -- would need either a live gdb/strace-instrumented repro of this exact burst pattern, or a close read of xce_notify()'s implementation and whether xcecd_job_finished()'s callers check/log its return code at all (not yet done this session). USER-PROMPTED REFINEMENT (2026-08-31, same session): the user hypothesized this might be a stale-primary artifact (worker kept reporting to the old primary and never resent to the new one). That exact framing doesn't fit the millisecond timing (these 7 finishes/sends happened at 16:25:00.9-01.5, before any failover was detected by anyone), but chasing it surfaced an important adjacent fact from the worker daemon's OWN controller-connection log (xcecd.log, root-owned -- required sudo to read, distinct from xcecd_worker.log's per-job-subprocess-only content used for the rest of this investigation): the WORKER's client-side failover logic logged its own first sign of trouble with ctrl-1 at 16:25:12.164 ('xcecli failover: attempt 1 -> localhost:9901') -- 8 SECONDS BEFORE ctrl-1 itself logged anything wrong ('standby[0] silent for 12s ... evicting', 16:25:20 per bug-1698's own timeline), and about 11-19 seconds AFTER the lost job-finish notifies for elements 6-12 were sent (16:25:00.943-01.535). The worker successfully failed over to the new primary at 16:25:22.332 ('rejoin after transparent reconnect succeeded', following attempt 2 -> localhost:9902 at 16:25:22.165). This reframes the likely relationship between this bug and bug-1698: rather than two unrelated defects that happened to occur in the same incident, the timing is consistent with a SINGLE underlying trigger -- ctrl-1's own I/O/connection handling apparently began degrading measurably before ctrl-1 itself noticed (silently failing to process/ack the worker's job-finish sends circa 16:25:00-01, then dropping the worker's connection entirely by ~16:25:12) -- with ctrl-1's own self-diagnosis (the 12s standby-liveness timeout) simply being the slowest of the three symptoms to surface. If that shared-trigger theory holds, this bug's fix should NOT rely solely on 'the worker eventually fails over correctly' (bug-1698's territory, and confirmed here to work correctly once detected, ~10s after the fact) -- XCECLI_FF_CALL's fire-and-forget send has zero resend/replay mechanism, so a notify lost during exactly this kind of covert pre-failover degradation window is gone forever regardless of how well the subsequent failover behaves. Still not proven whether ctrl-1's degradation genuinely started as early as 16:25:00 (matching the lost notifies) or only at 16:25:12 (matching the worker's own first detected symptom) with the 16:25:00-01 notify loss being a coincidentally-timed SEPARATE glitch -- distinguishing these needs the same live gdb/strace capture bug-1698's own next-steps already call for, ideally instrumented to also watch ctrl-1's handling of inbound OPCode_notify requests specifically, not just its outbound peer reconnect attempts. + +## Fix +NOT YET FIXED -- found and root-caused-to-this-depth this session while investigating bug-1698 in the same incident's log data; not yet independently reproduced or fixed. Next steps: (1) confirm whether xcecd_job_finished()'s callers (worker_job_callback / process_exit_callback path in xcecd_worker.c) check and log its return code -- if not, that alone is worth fixing regardless of the deeper cause, so a real send failure is at least loud instead of silent; (2) read xce_notify()'s actual implementation (xceapi.c, shared with every other XCECLI_FF_CALL user) to determine whether it is safe to call back-to-back in a tight loop on the same api_ctx_t without waiting for a prior call's underlying I/O to fully complete, or whether it has an internal single-buffer/reentrancy hazard; (3) attempt a targeted live repro (a single worker host completing 5+ jobs within under a second, e.g. a small array of near-instant jobs against one host with concurrency >= the array size) with the worker daemon under gdb/strace to catch xce_notify()'s return code and underlying syscall result for the calls that get lost; (4) once root-caused, likely fix directions include making job-finish notifications durable/retried (not fire-and-forget) given their consequence (a permanently stuck job holding worker slots/resources forever is a correctness issue, not just a display one -- unlike heartbeats, a job-finish report has no self-healing follow-up message the way bug-1737's DONE-vs-EXIT divergence did), or serializing/queuing bursts of notify calls from the same worker daemon so they cannot race or overwrite each other on the shared connection. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1548, bug-1731, bug-1737, bug-1738, bug-1739, bug-1698 diff --git a/.wolf/bugs/bug-1742.md b/.wolf/bugs/bug-1742.md new file mode 100644 index 00000000..58f3dc1a --- /dev/null +++ b/.wolf/bugs/bug-1742.md @@ -0,0 +1,40 @@ +--- +id: "bug-1742" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-09-01T04:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/controllers/jobs.go (validateJobControl:2997-3072", "foundJob:3077-3089", "ControlJobByID:2515-2599)", "src/frontend/apiserver/models/jobs.go (JobInfo.JobStatus/.ArrayIdx:489", "493)", "src/frontend/apiserver/store/convert.go (StructToOrderMap:10-35)", "src/frontend/apiserver/router/routers.go (gin.Default() Recovery middleware:19)", "src/frontend/xcecli (request.go:219 HandleErrorResponse empty-body fallback)"] +tags: ["apiserver", "panic", "type-assertion", "interface-conversion", "job-control", "kill", "xcecli", "recovery-middleware", "empty-error-body", "new", "found-static-analysis", "needs-live-repro"] +related: [] +legacy: true +gitea_issue: 45 +gitea_synced_status: "fixed" +--- + +# bug-1742: Gitea issue #45: `xcecli jobs kill ` fails for every job-control action… + +## Description +Gitea issue #45: `xcecli jobs kill ` fails for every job-control action (kill/stop/resume/checkpoint) with `request to /xce/v1/jobs//control failed: []` -- an empty error body, not a normal ReturnError JSON payload. Reproduced by static analysis of the full CLI->apiserver call path (job ID conversion verified correct by hand: MakeJobID(1,0,1788186846,0,1) == 73932643848159233, matching the URL in the report, so the numeric-vs-string ID mismatch in the bug report is a red herring). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +models.JobInfo declares JobStatus and ArrayIdx as plain Go `int` (jobs.go:489,493), not `int64`. store.StructToOrderMap builds jobinfo.Data via reflection (val.Field(i).Interface()), which preserves the field's concrete runtime type -- so Data["job_status"] and Data["array_idx"] are boxed as `int`, never `int64`. validateJobControl's guard at jobs.go:3014 (`if jobStatus, ok := rawStatus.(int64); ok`) therefore ALWAYS evaluates false for every job, unconditionally routing into the `else` error branch (3052-3060). That branch immediately does an UNCHECKED single-return assertion at jobs.go:3057, `int(jobinfo.Data["array_idx"].(int64))` -- since array_idx is actually `int`, this panics with `interface conversion: interface {} is int, not int64` on every single job-control call, regardless of job status, action, or whether the job is an array member (the reporter's job happening to be an array index is incidental; the same unsafe `.(int64)` pattern is repeated at jobs.go:3024/3031/3040/3048/3066, and in foundJob at 3081-3082 for the array-index-lookup path). Gin's default Recovery() middleware (routers.go:19) catches the panic and returns an empty 500 body with no JSON, which is exactly what the xcecli client's error-fallback path (request.go:219, `HandleErrorResponse`) renders as `"%s failed: [%s]"` with an empty httpResp.String() -- producing the reported `failed: []` message. Confirmed the type mismatch directly by reading the struct declarations; confirmed the panic site's code by reading validateJobControl in full; not yet confirmed via a live repro/server-log stack trace (static analysis only this session). + +## Fix +FIXED (2026-09-01). validateJobControl (jobs.go:2997) now converts job_status and array_idx via utils.IntOrInt64() (already used elsewhere for this exact int/int64 mismatch, e.g. processJobData) instead of bare `.(int64)` assertions -- a failed conversion is now reported as a normal failedJobs entry with a descriptive msg instead of panicking. foundJob (jobs.go:3077) gets the same fix for its array-index lookup path. TDD: added TestValidateJobControl_HandlesRealJobInfoTypes (builds a real models.JobInfo and converts it via store.StructToOrderMap(), the same reflection path production uses, then asserts validateJobControl doesn't panic and returns a correctly-typed success job -- this is the test that would have caught the original bug, since it exercises the real int/int64 boundary instead of hand-built int64 map literals), TestValidateJobControl_RejectsNonKillableStatus, and TestFoundJob_MatchesRealArrayIdxType (jobs_test.go). Confirmed red (panic at jobs.go:3057, "interface conversion: interface {} is int, not int64") before the fix, green after. Full apiserver suite (store/utils/controllers/backend/backend/auth) passes with zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #45 diff --git a/.wolf/bugs/bug-1743.md b/.wolf/bugs/bug-1743.md new file mode 100644 index 00000000..01ceff99 --- /dev/null +++ b/.wolf/bugs/bug-1743.md @@ -0,0 +1,43 @@ +--- +id: "bug-1743" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "config" +severity: "unknown" +timestamp: "2026-09-01T04:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/resource.c (sched_check_host_satisfy_:328-377)", "src/scheduler/src/plugins/limit.c (build_vres_name/per-user-queue-project virtual resource accounting:183-227", "312-376)", "src/configs/resources.yaml (slots resource:4", "consumable:true)", "src/configs/limits.yaml (ug1_normal_slots example", "value:8", "lines 41-49)", "src/frontend/apiserver/models/hosts.go (HostInfo.NumRun:87)"] +tags: ["scheduler", "slots", "admission-control", "limits.yaml", "usergroup-limit", "queue-limit", "hosts-list", "NumRun", "dead-field", "visibility-gap", "feature-request", "new", "found-static-analysis", "needs-live-repro", "related-to-bug-1673", "related-to-bug-1556", "needs-reporter-confirmation"] +related: ["bug-1673", "bug-1556", "bug-1784", "bug-1785", "bug-1786", "bug-1787", "bug-1788", "bug-1789", "bug-1790"] +legacy: true +gitea_issue: 46 +gitea_synced_status: "fixed" +--- + +# bug-1743: Gitea issue #46 (还需要一个slots的概念): host test01 shows NCPUS=16, MAX_SLOT=16 via `xcecli hosts list`… + +## Description +Gitea issue #46 (还需要一个slots的概念): host test01 shows NCPUS=16, MAX_SLOT=16 via `xcecli hosts list`, but of 9 submitted single-slot `sleep` jobs on queue `normal`, only 8 reach RUN and the 9th is stuck PEND -- dispatch stops at 8/16 host capacity. Secondary ask: `hosts list` should show slots used/free, not just NCPUS/MAX_SLOT. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Two independent findings. (1) Dispatch-stops-at-8 is most likely NOT a slot-counting bug in the scheduler -- MAX_SLOT genuinely gates per-host admission in sched_check_host_satisfy_() the same way `mem` does (per .kiro/steering/resource-accounting.md, reservation==usage for virtual resources like slots), but there is a SEPARATE, independent admission gate: cluster-wide usergroup+queue slot limits enforced by the scheduler's limit plugin (limit.c), configured in limits.yaml. This repo's example limits.yaml defines `ug1_normal_slots`: users in group ug1 on queue `normal` share a pool capped at exactly 8 (limits.yaml:41-49) -- an exact match to the reporter's queue (`normal`) and the observed 8/16 cutoff. This exact scenario (a queue/usergroup slot limit as an alternate explanation to a host-capacity bug) was explicitly predicted-but-unconfirmed in bug-1673 (filed against the related earlier report Gitea #40) and appears to be that prediction now reproduced. CAVEAT: limits.yaml/users.yaml in this repo look like sample/demo config (users.yaml's stock ug1 group only contains user1/user2, not `admin`) -- confirming this requires checking the reporter's actual deployed limits.yaml/users.yaml or their job's pend-reason output, neither of which was available this session (static analysis only). (2) The feature-request half (show slot usage in hosts list) is NOT starting from scratch: models.HostInfo.NumRun (hosts.go:87) already exists in the schema and is already an output column in `xcecli hosts list` (matching the reporter's own output), but `grep -rn "\.NumRun\s*=" apiserver --include=*.go` returns zero hits anywhere in the apiserver -- the field is never assigned and is permanently stuck at Go's zero value, which is exactly why the reporter's own output shows NUM_RUN=0 while 8 jobs are actually running on that host. Same for NumSSusp/NumUSusp. + +## Fix +PARTIALLY FIXED (2026-09-01). Part 2 (hosts-list slot visibility) is FIXED: added computeHostSlotUsage() (controllers/hosts.go) which scans store.Jobs' live ExecHosts JSONB (the same {"host_slots":[{"host_name","num_slots"}],...} shape backend/convert_pure.go populates) and JobStatus bits (JOBSTAT["RUN"]/["SSUSP"]/["USUSP"]) to count occupied slots per host, per sub-state -- counting SLOTS (not just job count) so the numbers are directly comparable to max_slot, matching the reporter's actual ask ('host上需要体现用了多少slots'). processHostData() now overwrites row.Data["num_run"/"num_ssusp"/"num_ususp"] with these live counts instead of leaving HostInfo's always-zero struct fields untouched. TDD: added TestProcessHostData_ComputesSlotUsageFromJobs (host_control_test.go) covering plain RUN, USUSP, a 2-slot SSUSP job (verifies slot-counting, not job-counting), a PEND job with no exec host (must not count), and a job on a different host (must not bleed across hosts). Confirmed red (num_run=0, want 6) before the fix, green after. Full apiserver suite passes with zero regressions. + +Part 1 (dispatch-stops-at-8 root cause): FULLY RESOLVED (2026-09-07/08, follow-up session). Confirmed live (no reporter config needed after all -- the leading hypothesis was right, just broken at a much deeper layer than 'is a limit configured'): the entire usergroup/queue limit-matching machinery in the limit plugin was silently non-functional, a 6-bug cascade -- see bug-1784 (named usergroups never registered into the scheduler), bug-1785 (unconfigured scope matched no one instead of falling back to all), bug-1786 (election loop aborted after the first matched limit), bug-1787 (extra pointer indirection broke a resmgr handle lookup), bug-1788 (instantiate_limit() conflated not-found with a real error), and bug-1789/bug-1790 (even once matching worked, the limit's own consumption computation read the wrong request vector and always saw a 0 'wanted' value, so the numeric cap itself never bound). All six fixed and live-verified. Final confirmation (2026-09-08): fresh single-node deploy at current HEAD, 9 single-slot jobs submitted as user1 (a genuine ug1 member) on queue normal -- the exact repro shape from this issue -- scheduler trace shows limit1/ug1_slots/ug1_normal_slots all correctly matching and, for the first time, a real nonzero wanted value ('unsatisfy_ resource 35 total 3.000000 inuse 3.000000 uncomit 0.000000 wanted 1.000000' / 'unsatisfy_ unmet resource 35 (limit=limit1) shortfall 1.000000'), correctly PENDing jobs past the cap. Posted as a comment on gitea#46 and closed the issue (2026-09-08). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1673, bug-1556, bug-1784, bug-1785, bug-1786, bug-1787, bug-1788, bug-1789, bug-1790 +- Gitea: #46 diff --git a/.wolf/bugs/bug-1744.md b/.wolf/bugs/bug-1744.md new file mode 100644 index 00000000..8949e9d0 --- /dev/null +++ b/.wolf/bugs/bug-1744.md @@ -0,0 +1,43 @@ +--- +id: "bug-1744" +status: "fixed" +status_updated: "2026-09-27" +status_note: "b970ab75: apiserver checks rusage[] syntax at submit via libxce's grammar; select/order names still scheduler-side only" +component: "lib" +severity: "unknown" +timestamp: "2026-09-01T05:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/xcecli/cmd/jobs/submit.go (resReq flag", "no value validation", "just passed through: jobReq.Resreq = js.resReq)", "src/frontend/apiserver/controllers/jobs.go:1602-1606 (only length-checks Resreq", "no syntax/grammar validation)", "src/lib/resreq.c (parse_resreq/parse_rusage -- the real PEG-generated grammar parser)", "src/lib/rusage.peg (rusage_param grammar", "mem value must match `number <- [0-9]+('.'[0-9]*)?`", "so \"100000000a0\" is a genuine grammar violation)", "src/scheduler/src/core_events.c:236-245 (parse_resreq call site inside core_build_job()) and :751-760 (JOBS_PUSH caller: on core_build_job()==NULL", "WARNs server-side and `continue`s -- job silently dropped", "never added to core->jobs)"] +tags: ["apiserver", "xcecli", "job-submission", "resreq", "rusage", "validation-gap", "silent-failure", "phantom-pend", "new", "found-static-analysis", "needs-live-repro", "related-to-bug-1740", "not-yet-fixed", "deprioritized", "deferred"] +related: [] +fix_commits: ["b970ab75"] +legacy: true +gitea_issue: 47 +gitea_synced_status: "fixed" +--- + +# bug-1744: Gitea issue #47 (rusage是否做了？): `xcecli jobs submit -R "rusage[mem=100000000a0]"` -- a malformed… + +## Description +Gitea issue #47 (rusage是否做了？): `xcecli jobs submit -R "rusage[mem=100000000a0]"` -- a malformed resreq string (trailing non-numeric "a0" on the mem value) -- is silently ACCEPTED and returns a normal 'Job <...> is submitted' success message. Reporter's original comment: '乱写也可以提交' (garbage input can be submitted too). Maintainer (rshen) already commented on the issue: this is a front-end (apiserver+cli) validation gap by design intent. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The ONLY resreq validation in the entire submission path is a length check (apiserver/controllers/jobs.go:1602-1606, `len(req.Resreq) > 4094`) -- there is no syntax/grammar validation anywhere client-side or in the apiserver before a job is accepted and persisted. The REAL validator, a PEG-generated grammar parser (parse_resreq()/parse_rusage(), src/lib/resreq.c, built from src/lib/rusage.peg + xce_resreq.peg via meson custom_target), exists but is only ever invoked deep inside the C scheduler at core_build_job() time (core_events.c:236), i.e. asynchronously, well AFTER the apiserver has already accepted the job, returned a submission-success response to the client, and persisted it. When parse_resreq() fails on malformed syntax (confirmed via rusage.peg's `number` rule, `[0-9]+('.'[0-9]*)?` -- "100000000a0" cannot match this and leaves "a0" unconsumed, a genuine PEG parse failure), core_build_job() logs an ERROR and returns NULL (core_events.c:238-245); its caller in the XCE_CORE_EVENT_JOBS_PUSH handler (core_events.c:756-760) only logs a server-side WARN and `continue`s to the next job in the batch -- the malformed job is silently dropped from the scheduler's core->jobs registry entirely. Net effect is worse than 'garbage accepted': the job is accepted, shows the user a success message, is almost certainly persisted and shown as PEND via `jobs list` (this session did not confirm the exact persisted/listed state live), and then sits PEND FOREVER with zero indication to the user that anything is wrong -- the only trace is a server-side log line the user never sees. This is architecturally the same class of gap as bug-1740 (a real, working validator/reconciler exists but is wired into the wrong -- or in this case, too-late -- point in the pipeline) and matches the maintainer's own comment that front-end validation is the intended design, not yet implemented. + +## Fix +NOT YET FIXED -- investigated and design-confirmed this session; DEPRIORITIZED, deferred to later (explicit user decision, not urgent). Confirmed fix approach: validate resreq synchronously in the apiserver's submit handler (controllers/jobs.go:1602, alongside the existing length check), reusing libxce's existing PEG parser rather than reimplementing the grammar in Go. Specifically parse_rusage() (src/lib/resreq.c:154, declared resreq.h:301 -- `int parse_rusage(xlist_t*, char*, char**)`) needs NO resmgr/resource-manager context, unlike the fuller parse_resreq()/parse_select() (resreq.h:303,305 -- both take a `var resmgr` to validate resource NAMES in select[]/order[] clauses, which the apiserver doesn't currently construct) -- so a resreq-name-agnostic rusage[]-only syntax check is a small, low-risk CGo wrapper, not a larger resmgr-plumbing project. apiserver/backend already has the exact split pattern to follow: a `//go:build cgo` file with the real implementation (e.g. clib.go, which already #includes libxce headers and links -lxce) plus a `//go:build !cgo` stub (stub.go, returns an errNoCGO-style error) for the CGO-disabled test path. Full parse_resreq()/resmgr-backed select[]/order[] validation is explicitly out of scope for now -- would be a separate, larger follow-up if wanted. + +SCOPE DECISION (2026-09-01): will be implemented as a direct bugfix when picked up, NOT a kiro spec -- small, contained blast radius (one new CGo wrapper + stub in apiserver/backend, one call site in controllers/jobs.go, matching the size/shape of the already-shipped bug-1742/bug-1743 fixes), no HA/persistence/RBAC involvement, and no open design questions left (exact function, signature, and existing cgo/stub pattern to follow are already nailed down). Only the FULLER resmgr-backed select[]/order[] validation (out of scope here) would warrant at least a /kiro-spec-quick, since it needs new apiserver-side state (a resmgr context kept in sync with resource definitions) -- not needed for this rusage-only fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #47 diff --git a/.wolf/bugs/bug-1745.md b/.wolf/bugs/bug-1745.md new file mode 100644 index 00000000..793ed694 --- /dev/null +++ b/.wolf/bugs/bug-1745.md @@ -0,0 +1,41 @@ +--- +id: "bug-1745" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-09-01T00:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".kiro/specs/worker-rejoin-resync/design.md"] +tags: ["design-phase", "worker-rejoin-resync", "kiro-validate-design", "bug-1678", "bug-1631", "bug-1679", "scheduler-dispatch-commit", "not-a-runtime-bug"] +related: ["bug-1678", "bug-1631", "bug-1679", "bug-1514"] +legacy: true +--- + +# bug-1745: kiro-validate-design worker-rejoin-resync (design phase, spec never implemented yet): initial pass… + +## Description +kiro-validate-design worker-rejoin-resync (design phase, spec never implemented yet): initial pass found 2 critical issues in design.md, self-correction during the fix pass found 2 more. None are runtime bugs (no code exists for this spec yet) -- this entry records design-document defects caught before implementation, same category as bug-1514. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +(1) design.md's Data Models section cited bug-1678 (WAL_JOB_DISPATCH not carrying job->allocs through replay) as a still-open, tracked gap justifying record_job_reconcile_assign_fact() not populating an allocation -- bug-1678 was actually fixed 2026-08-25, before this design.md was finalized; the citation was stale, and the design missed that persist_submit_job_dispatch_fact() (the fixed primitive) already has a tested NULL-alloc path it could directly reuse instead of inventing a new WAL fact shape. +(2) design.md's dispatch-gating reuse (Requirement 5.1, XCE_WORKER_RECONCILING via dispatch_a_job_'s w->host.status != 0 check) was researched before scheduler-dispatch-commit (2026-08-24/25) added xce_classify_dispatch_fail()'s bounded-retry-then-mark_job_fail() machinery behind that exact check -- the design's own 'Revalidation Triggers' section named this exact check as a trigger but it had not actually been re-checked against current code. A RECONCILING host could be selected as a dispatch candidate by the scheduler (whose own filter only checks IS_WORKER_UNAVAIL, confirmed via grep) and then get retried-then-terminal-failed by the controller purely for being mid-reconciliation. +(3) Fixing (1) surfaced bug-1631 (job.allocs never released on dispatch_a_job_'s retryable-failure paths, deliberately left unfixed because nothing in the system re-admitted an already-RUN job to the scheduler's PEND pool) -- worker-rejoin-resync's own Requirement 4.6 (unrecognized job -> return to PEND, eligible for re-dispatch) IS such a re-admission path, and design.md's record_job_reconcile_pending_fact spec did not release job->job.allocs, reactivating bug-1631's exact deferred trigger condition. +(4) Fixing (3) by adding an eager job->job.allocs release exposed a self-introduced inconsistency: apply_job_update_transition() only mutates job->job.state, deferred until apply_committed_entry applies the committed fact (confirmed by reading record_job_finish_fact()'s identical deferred pattern) -- an eager allocs release paired with a deferred state change would produce a real, brief RUN-state-with-NULL-allocs window. Resolved by modeling both new job.c functions on dispatch_a_job_'s own eager-mutation-plus-explicit-persist_notify_job_updated_already_persisted pattern (bug-1679's precedent) instead, rather than record_job_finish_fact's deferred one. + +## Fix +All 4 corrected directly in design.md during the same validation session (not deferred to implementation): (1) Data Models' 'Documented scope decision' + Components section rewritten -- record_job_reconcile_assign_fact() now specified to call the existing, fixed persist_submit_job_dispatch_fact(alloc=NULL) directly. (2) Added a new Scheduler-side component (scheduler.c:1211's candidate filter gets one added disjunct, IS_WORKER_RECONCILING, mirroring the existing IS_WORKER_UNAVAIL check) as the primary Requirement 5.1 enforcement, with dispatch_a_job_'s existing check kept as an unmodified defensive backstop; Requirement 5.4's disconnected-worker case given its own pre-classifier branch (bypassing xce_classify_dispatch_fail entirely, not a 3rd enum value, to avoid forcing ~5 unrelated switch sites to handle a case they can't hit). (3) record_job_reconcile_pending_fact() now specified to release job->job.allocs eagerly. (4) Both new job.c functions redesigned around dispatch_a_job_'s eager-mutation + persist_notify_job_updated_already_persisted pattern, explicitly not record_job_finish_fact's deferred one, with the rationale (worker-confirmed ground truth vs. an unconfirmed local admin command) documented inline so a future editor doesn't accidentally revert to the deferred pattern. spec.json's design.approved flipped to true with a validation_note recording this trail. No code was written -- this is a design-document-only fix; implementation-time discovery may still surface further refinement, per design.md's own remaining 'must confirm at implementation time' caveats. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1678, bug-1631, bug-1679, bug-1514 diff --git a/.wolf/bugs/bug-1746.md b/.wolf/bugs/bug-1746.md new file mode 100644 index 00000000..0e6ece14 --- /dev/null +++ b/.wolf/bugs/bug-1746.md @@ -0,0 +1,38 @@ +--- +id: "bug-1746" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-09-01T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/persist.c"] +tags: ["worker-rejoin-resync", "task-2.2", "persist-c", "wal-job-dispatch", "bug-1678", "bug-1745", "null-pointer-deref", "asan", "record_job_reconcile_assign_fact"] +related: ["bug-1678", "bug-1745", "bug-1631"] +legacy: true +--- + +# bug-1746: worker-rejoin-resync task 2.2 implementation: persist_submit_job_dispatch_fact(p, jobid, state… + +## Description +worker-rejoin-resync task 2.2 implementation: persist_submit_job_dispatch_fact(p, jobid, state, ts, alloc=NULL) crashes with a NULL-pointer SEGV instead of producing the "no allocation" WAL_JOB_DISPATCH payload design.md (Data Models / Out-of-Boundary section) and bug-1745's own design-review fix claimed was already "a tested NULL-alloc path" reusable verbatim. Confirmed via AddressSanitizer: SEGV on unknown address 0x0 in encode_JobAlloc (lib/controller.capnp.c:9805), called from capnp_encode_joballoc (controller/persist.c:235), called from persist_submit_job_dispatch_fact (controller/persist.c:7856), called from the new record_job_reconcile_assign_fact (controller/job.c) -- reproduced twice, deterministically, both via the full meson controller-test suite and via an isolated -t test_worker_resync run. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +persist_submit_job_dispatch_fact() (persist.c:7844) unconditionally calls capnp_encode_joballoc(alloc, ...) regardless of whether alloc is NULL. capnp_encode_joballoc() (persist.c:225) calls encode_JobAlloc(cs, &cja, (xce_job_alloc_t*)a) directly -- NOT the NULL-safe encode_JobAlloc_ptr() wrapper that already exists in the same generated file (lib/controller.capnp.c:9388, used correctly by every OTHER encode call site) and already has an explicit NULL guard (if s==NULL, sets a capn_null pointer instead of encoding). encode_JobAlloc() itself (lib/controller.capnp.c:9322, generated capnp binding) has no NULL guard at all: its first statement dereferences its xce_job_alloc_t* parameter unconditionally (reads s->jobid). design.md (worker-rejoin-resync) and bug-1745's own design-validation fix pass both asserted this call is safe -- quoting design.md: bug-1678 already gave WAL_JOB_DISPATCH a persist_submit_job_dispatch_fact() primitive with both apply seams decoding an optional trailing JobAlloc, a NULL alloc being the documented, tested backward-compatible case -- but that claim conflates two different things: the DECODE/apply seams (apply_committed_entry, apply_wal_entry) genuinely ARE NULL-safe and tested for a payload with no trailing alloc bytes (test_wal_job_dispatch_fact.c constructs such payloads by hand, e.g. test_wjd_apply_committed_entry_transitions_pend_to_run) -- but NO existing test, and no existing production call site (dispatch_a_job_ is the only one, job.c:1731, and it always passes a freshly-calloc'd non-NULL j->job.allocs), ever calls persist_submit_job_dispatch_fact() itself with alloc=NULL. The function's own doc comment (persist.h, near line 1447) actually states the opposite precondition: alloc must be non-NULL, the real call site (dispatch_a_job_) always has one by construction. The ENCODE side was never made NULL-safe when bug-1678 fixed the DECODE side. + +## Fix +FIXED (coordinator-authorized scope expansion of worker-rejoin-resync task 2.2 to include exactly this one function in persist.c, mirroring task 1.1's typemap.h/clusterconfig.h expansion precedent -- no schema or generated-code change). Added a NULL guard as the first statement of capnp_encode_joballoc() (persist.c:225): `if (a == NULL) { *out = NULL; *out_len = 0; return 0; }`. Produces a genuinely ZERO-LENGTH buffer for NULL input -- NOT a capnp-encoded null-pointer JobAlloc (what the existing encode_JobAlloc_ptr() wrapper's NULL branch would produce, which is non-zero bytes on the wire). This distinction is load-bearing: both apply seams (apply_committed_entry persist.c:4206-4227, apply_wal_entry persist.c:4759-4781) discriminate 'is there an allocation to decode' purely by `alloc_len > 0` on the trailing bytes after the {jobid,state[,start_time]} header -- a non-zero null-pointer-encoding byte count would have been misdecoded as a real (if empty/garbage) allocation to adopt, silently producing a non-NULL-but-garbage job.allocs on replay instead of the required NULL. Verified: (1) record_job_reconcile_assign_fact()'s 2 new tests (test_reconcile_assign_applies_running_no_allocs_and_notifies, test_reconcile_assign_survives_restart_replay -- the latter exercising a real disk WAL append + persist_recover() restart-replay round trip) now PASS cleanly, no SEGV. (2) record_job_reconcile_pending_fact()'s 3 existing task-2.1 tests still PASS unchanged. (3) dispatch_a_job_()'s existing non-NULL-alloc call site (job.c:1731, the only other caller) is completely unaffected -- its 5 dispatch-commit tests in test_dispatch_commit.c (test_dc_success_commits_alloc_state_and_resets_counter, test_dc_success_stamps_start_time_synchronously, test_dc_success_notifies_job_update_hook_exactly_once, test_dc_worker_not_found_leaves_job_pend_and_alloc_uncommitted, and critically test_dc_success_appends_wal_job_dispatch_durably -- a real disk WAL + persist_recover() round trip through the non-NULL-alloc path) all still PASS unchanged, confirming the new guard is a pure early-return for the NULL case with zero effect on the non-NULL path. (4) Full controller test suite (meson test -C src/xce.build 'controller test'): 6/6 OK, zero regressions anywhere. Also confirmed no secondary UBSan issue from the resulting `memcpy(dst, NULL, 0)` in persist_submit_job_dispatch_fact() when alloclen==0 (allocbuf stays NULL) -- this build runs under UBSAN_OPTIONS=halt_on_error=1 and the full suite passed clean, so glibc's memcpy tolerates a NULL/0 call here in practice. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1678, bug-1745, bug-1631 diff --git a/.wolf/bugs/bug-1747.md b/.wolf/bugs/bug-1747.md new file mode 100644 index 00000000..65098324 --- /dev/null +++ b/.wolf/bugs/bug-1747.md @@ -0,0 +1,39 @@ +--- +id: "bug-1747" +status: "fixed" +status_updated: "2026-09-26" +status_note: "FINISH half eec2596c; FAIL half 1d6c04c2: WAL_JOB_FAIL trailer carries finish_time, record_job_fail_fact_at() for resync." +component: "controller" +severity: "unknown" +timestamp: "2026-09-01T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c"] +tags: ["worker-rejoin-resync", "task-3.2", "resync-c", "job-c", "requirement-4.2", "finish-time", "wal-job-fail", "wal-job-finish", "apply_job_fail_transition", "pre-existing", "predates-feature", "systemic", "resync_apply_report", "record_job_finish_fact_at"] +related: ["bug-1689", "bug-1678", "bug-1746", "bug-1631"] +fix_commits: ["eec2596c", "1d6c04c2"] +legacy: true +--- + +# bug-1747: systemic gap predating worker-rejoin-resync entirely, discovered while implementing task 3.2… + +## Description +Pre-existing, systemic gap predating worker-rejoin-resync entirely, discovered while implementing task 3.2 (resync_apply_report, resync.c): WAL_JOB_FAIL carries no fail-time/finish-time slot in its wire payload at ANY layer, and apply_job_fail_transition() (job.c) unconditionally stamps job.finish_time = time(NULL) at APPLY time (not even receipt time) for EVERY fail-reporting caller in this codebase -- not just reconciliation. This means Requirement 4.2 ('the Controller shall record the finish time the Worker reported ... rather than the time the Controller processed the reconciliation') is currently unsatisfiable for a reconciled finished-FAILURE outcome via any existing path. UPDATE (2026-09-01, coordinator-directed re-scope): the sibling FINISH-success half of this gap is FIXED -- see 'fix' field below; only the FAIL half remains open. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +job_failinfo_t (xce/types.h) is {jobid, stage, code, detail} -- no finish_time field, and WAL_JOB_FAIL's durable payload (record_job_fail_fact(), job.c: persist_submit_job_fact_ex(&c->persist, WAL_JOB_FAIL, jobid, 0, WAL_JOB_FAIL_STAGE_FLAG|stage)) carries no finish-time slot at all -- confirmed by direct inspection of the payload construction. apply_job_fail_transition() (job.c, the guarded core BOTH the live job_ctrl path (mark_job_fail) and the deferred apply seam (apply_committed_entry's WAL_JOB_FAIL case) invoke) unconditionally executes `job->job.finish_time = time(NULL);` with no external value ever consulted, at APPLY time -- meaning even a caller willing to plumb a real value through record_job_fail_fact's WAL payload would still need a apply_job_fail_transition() signature change to have it actually land in job.job.finish_time. This is a pre-existing, already-documented residual: job.c's own comment near apply_job_postdone_transition (task 4.1) already states 'finish_time is not part of the wal_jobstate_t wire format ... has no apply-time equivalent' for a sibling fact type, confirming this is systemic to the wal_jobstate_t-based fact family, not something worker-rejoin-resync introduced. Fixing it for real would require extending WAL_JOB_FAIL's wire payload (wal_types.h, mirroring bug-1689's own WAL_JOB_FINISH trailing-fields precedent) PLUS an apply_job_fail_transition() signature/behavior change reaching every fail-reporting call site in the system (job_ctrl's mark_job_fail, on_job_fail's ordinary worker-fail receipt path, and this feature's own reconciliation caller) -- real, broad-blast-radius work, not a narrow patch. + +## Fix +SPLIT (2026-09-01, coordinator-directed): the FINISH-success half of this gap is FIXED as part of worker-rejoin-resync task 3.2 -- added record_job_finish_fact_at(cluster_t*, job_t*, xce_jobfinish_t*, uint64_t finish_time_ms) (job.c), record_job_finish_fact()'s own worker-reported-finish-time sibling (identical guard/hook-fire/durable-append body, differing only in passing the caller-supplied finish_time -- converted ms->s -- to persist_submit_job_finish_fact() instead of time(NULL)); record_job_finish_fact() itself is UNMODIFIED, its own caller (on_job_finish, an ordinary near-instantaneous worker report) correctly keeps using receipt-time. resync_apply_report()'s finishedOk branch now calls record_job_finish_fact_at(c, job, &finish, entry->finish_time_ms). Verified via a real restart-replay test (test_apply_report_finished_success_uses_worker_reported_finish_time, test_worker_resync.c): a finished-success entry with finish_time_ms set to 'now minus 1 day' recovers job.finish_time equal to that worker-reported value after persist_recover(), not reconciliation-processing wall-clock time. This was possible narrowly for FINISH because persist_submit_job_finish_fact() already accepted an explicit finish_time parameter at the primitive level -- record_job_finish_fact()'s specific call site just never exposed it. The FAIL half (see root_cause) is NOT YET FIXED and is NOT task 3.2's (or any single narrow task's) job to fix -- it needs the broader WAL_JOB_FAIL payload extension + apply_job_fail_transition() signature change described above, deliberately deferred pending a dedicated task, matching the same 'deliberately deferred, documented, not blocking' treatment bug-1631's own sibling gap received earlier in this same feature. The reconciled job's STATE (EXIT) is always correct for a finished-failure outcome either way -- only its recorded finish_time reflects apply-processing time rather than the worker's true report. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1689, bug-1678, bug-1746, bug-1631 diff --git a/.wolf/bugs/bug-1748.md b/.wolf/bugs/bug-1748.md new file mode 100644 index 00000000..3d0d8c77 --- /dev/null +++ b/.wolf/bugs/bug-1748.md @@ -0,0 +1,38 @@ +--- +id: "bug-1748" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-09-01T02:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/resync.c"] +tags: ["worker-rejoin-resync", "task-3.2", "resync-c", "requirement-4.4", "requirement-4.8", "resync_apply_running_entry", "record_job_reconcile_assign_fact", "guard-logic", "short-circuit", "review-caught", "test-baked-in-bug"] +related: ["bug-1747", "bug-1697", "bug-1631"] +legacy: true +--- + +# bug-1748: worker-rejoin-resync task 3.2 (resync_apply_running_entry, resync.c), round-2 independent review… + +## Description +worker-rejoin-resync task 3.2 (resync_apply_running_entry, resync.c), round-2 independent review: the round-1 fix for the PEND/unassigned-job regression (bug-1747-adjacent, not itself in bug-1747) introduced a NEW Requirement 4.4 violation. The round-1 guard `if (!resync_job_is_terminal(job->job.state) && resync_job_assigned_to_host(job, w->host.name)) { no-op } else { record_job_reconcile_assign_fact(...) }` short-circuits the `&&` to false whenever job->job.state IS terminal, falling into the `else` branch and calling record_job_reconcile_assign_fact() on an ALREADY-TERMINAL (DONE/EXIT) job -- which has no backward-transition guard of its own and unconditionally sets job->job.state = JOB_STAT_RUN, clobbering a terminal job back to RUN. Confirmed live and, worse, CERTIFIED AS CORRECT by the round-1 implementation's own test: test_apply_report_all_five_outcomes_updates_job_table seeded j_extra at JOB_STAT_DONE with a comment claiming this 'satisfies Req 4.8's trigger,' then asserted j_extra->job.state == JOB_STAT_RUN afterward and the suite reported PASS -- the test baked the bug in as an expected outcome instead of catching it. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Requirement 4.8's actual trigger condition is 'no existing non-terminal record of that job assigned elsewhere' (requirements.md AC 4.8; record_job_reconcile_assign_fact's own doc comment and design.md's Service Interface Preconditions: 'job is non-terminal ... or has no existing non-terminal assignment'). Terminal-vs-non-terminal is NOT the axis that decides assign-eligibility on its own -- Requirement 4.4 ('shall not overwrite that state with a less final or older report') independently and unconditionally forbids ever assign-routing a terminal job, regardless of host. The round-1 guard conflated two orthogonal conditions (terminal-ness, and host-assignment) into a single boolean expression via `&&`, which is only correct when BOTH must jointly hold for the no-op case -- but Req 4.4's terminal exclusion needs to apply UNCONDITIONALLY, independent of the host check, not only when the host check also happens to be true. A two-way boolean cannot express three independent outcomes (terminal-always-noop; non-terminal-but-already-assigned-noop; non-terminal-and-unassigned-assign) without this exact class of short-circuit bug. + +## Fix +FIXED (2026-09-01, round-2 review, task 3.2). resync_apply_running_entry() rewritten as a genuine three-way `if`/`else if`/`else` split, not a boolean guard: (a) resync_job_is_terminal(job->job.state) -> ALWAYS a no-op, checked FIRST and independently, regardless of host (closes the Req 4.4 violation). (b) non-terminal AND job->job.allocs != NULL (already has SOME live non-terminal host assignment -- to the reporting worker's own host, or to a different one, it does not matter which) -> a no-op; also tightened from round 1's own host-specific check (resync_job_assigned_to_host(job, w->host.name)) to a host-agnostic allocs!=NULL check, since round-1's version would have ALSO wrongly routed a job assigned to a DIFFERENT host into the assign branch, resetting its start_time and re-appending a spurious WAL_JOB_DISPATCH fact for a job that was never actually lost. (c) non-terminal AND allocs == NULL (genuinely unassigned) -> record_job_reconcile_assign_fact(), Req 4.8's real trigger. Verified: (1) test_apply_report_all_five_outcomes_updates_job_table's j_extra is now seeded TERMINAL and asserted COMPLETELY UNTOUCHED (state stays DONE, no WAL append) -- the exact inversion of the round-1 assertion that had certified the bug. (2) test_apply_report_unqueried_extra_already_tracked_is_noop (branch (b), assigned to a DIFFERENT host) strengthened with a start_time-unchanged assertion (record_job_reconcile_assign_fact unconditionally stamps time(NULL), bug-1697 -- invisible to a bare state/allocs check since the job was already RUN either way) and an LSN-unchanged assertion (no spurious WAL_JOB_DISPATCH append) -- round 1's version of this test could not have caught either defect. (3) test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned (branch (c), genuinely unassigned/PEND) already covers the real assign-success case with full restart-replay rigor, added in the same round-1 fix pass. Full controller test suite (meson test -C src/xce.build 'controller test'): all green. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1747, bug-1697, bug-1631 diff --git a/.wolf/bugs/bug-1750.md b/.wolf/bugs/bug-1750.md new file mode 100644 index 00000000..6d9e0eda --- /dev/null +++ b/.wolf/bugs/bug-1750.md @@ -0,0 +1,38 @@ +--- +id: "bug-1750" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "worker" +severity: "unknown" +timestamp: "2026-09-01T09:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/controller_connector.c"] +tags: ["worker-rejoin-resync", "task-4.3", "xcecd_resync-c", "controller_connector-c", "requirement-6.1", "requirement-3.8", "deadlock", "gate-placement", "checking_connection", "needs_rejoin", "review-caught"] +related: ["bug-1749", "bug-1748"] +legacy: true +--- + +# bug-1750: worker-rejoin-resync task 4.3, round-1 review: the resync_pending gate was checked as… + +## Description +worker-rejoin-resync task 4.3, round-1 review: the resync_pending gate was checked as checking_connection()'s very first statement (an entry guard), returning early before xcecli_hello() or the needs_rejoin-triggered rejoin_call() could run. xcecd_resync_handle_query_page() sets resync_pending = true on receiving a job-state query's first page and only clears it when called again with a later page. If the worker's connection to the controller drops after a non-final page of a multi-page OPCode_jobStateQuery arrives (a query spanning more non-terminal jobs than fit in one page -- explicitly in scope per Requirement 8.1/8.3), no further page ever arrives, so xcecd_resync_handle_query_page() is never invoked again and resync_pending is stuck true forever -- permanently disabling checking_connection()'s entire body, including the worker's ONLY connection-recovery mechanism (xcecli_hello() / needs_rejoin-triggered rejoin). Directly violated Requirement 6.1 (the worker must be able to disconnect-and-rejoin during an incomplete reconciliation round) and regressed the intent of the pre-existing invariant test test_checking_connection_never_permanently_disables_worker, though none of round 1's own new tests drove the multi-page-then-disconnect sequence so it passed the full suite undetected. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +design.md directed mirroring the existing needs_rejoin boolean pattern for the new resync_pending gate, but needs_rejoin is self-healing by construction -- it gates one step INSIDE checking_connection() (the rejoin action itself), so the function's body including xcecli_hello() still runs every tick regardless of its value, giving the worker a fresh chance to detect and recover every cycle. Round 1 placed resync_pending's check as the function's entry guard instead, which is not actually the same pattern in the load-bearing respect that matters: it makes the gate's only exit condition (a later query page arriving) depend on the very connection health the gate is supposed to be resilient to, with no independent path back to a working state. + +## Fix +FIXED (2026-09-01, round-2 review, task 4.3). Two changes, defense-in-depth: (a) narrowed the gate so xcecd_resync_is_pending() is checked only inside the needs_rejoin branch, guarding solely rejoin_call() -- xcecli_hello() and the needs_rejoin check itself now run unconditionally on every checking_connection() tick, restoring the self-healing property. (b) added xcecd_resync_reset(), called from xcecd_on_reconnect() (the same signal that sets needs_rejoin, confirmed to fire only after xce_failover_rotate()/xcecli_invoke_rejoin() has already repaired a real, previously-detected transport failure -- never spuriously on a healthy connection), which clears resync_pending and discards any stalled partial multi-page accumulation (frees and zeroes the accumulator, not just the boolean), bounding the worst case to at most one reconnect cycle instead of forever. Verified: round-2 reviewer independently reintroduced the round-1 entry-guard bug, rebuilt, and confirmed a new structural regression test (test_checking_connection_hello_and_needs_rejoin_precede_resync_gate_in_source, asserting strict source-order) fails while the companion behavioral test (test_checking_connection_survives_permanently_stuck_resync_pending) does NOT catch the reintroduced bug alone -- establishing that placement-sensitive gates need a structural/ordering assertion, not just a behavioral one. Full worker suite green after the fix (xcecd_resync_test 28/28, controller_connector_test 22/22 including the pre-existing test_checking_connection_never_permanently_disables_worker invariant). One advisory, non-blocking follow-up noted but not fixed: resync_pending/query_accum are unsynchronized statics touched from two different OS threads (connector thread vs. a vtaskmgr pool thread) -- a narrow theoretical race on xcecd_resync_reset()'s free() vs. a concurrent read, mirroring needs_rejoin's own accepted unsynchronized-static risk profile but riskier since it frees a heap pointer. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1749, bug-1748 diff --git a/.wolf/bugs/bug-1751.md b/.wolf/bugs/bug-1751.md new file mode 100644 index 00000000..791c6afb --- /dev/null +++ b/.wolf/bugs/bug-1751.md @@ -0,0 +1,38 @@ +--- +id: "bug-1751" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-01T09:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/scheduler.c", "src/scheduler/src/core_session.c"] +tags: ["worker-rejoin-resync", "task-5", "scheduler-c", "core_session-c", "requirement-5.1", "requirement-5.2", "dead-code", "driver-equivalence", "candidate-filter", "sched_reset_worker", "core_reset_workers", "design-reality-mismatch", "found-live-during-implementation", "symbol-guard-test"] +related: ["bug-1745", "bug-1746", "bug-1747", "bug-1748", "bug-1750"] +legacy: true +--- + +# bug-1751: worker-rejoin-resync task 5 (scheduler candidate-filter exclusion for XCE_WORKER_RECONCILING)… + +## Description +worker-rejoin-resync task 5 (scheduler candidate-filter exclusion for XCE_WORKER_RECONCILING): design.md's Components/Architecture/Testing Strategy sections all cite scheduler.c:1211 (sched_reset_worker_(), the IS_WORKER_UNAVAIL skip inside the legacy sched_reset_workers()/sched_begin_session() path) as 'the existing candidate-host filter' to extend with the new reconciling-bit disjunct. Investigating before implementing (grep for every IS_WORKER_UNAVAIL skip-style check across src/scheduler/src/*.c) found sched_begin_session() has NO live callers anywhere in the current driver.c/sim_loop.c production paths -- both call xce_core_begin_session() (core_session.c, added by the scheduler-core refactor) directly instead, which runs core_reset_workers() (core_session.c:102), an independently-maintained mirror of sched_reset_worker_() with the IDENTICAL IS_WORKER_UNAVAIL-only check, per that function's own header comment ('mirrors scheduler.c's sched_reset_workers/sched_reset_worker_'). Extending only scheduler.c:1211 as design.md literally specifies would leave the task's own Observable Completion Criterion (a RECONCILING host is never selected as a dispatch candidate while the flag is set) factually false on both real drivers, since neither ever executes the code path being changed. bug-1688's own investigation (2026-08-26) independently confirms core_reset_workers()/xce_core_begin_session() as the actively-traced, live production path. No test in src/scheduler/testsuite exercises sched_reset_worker_()'s or sched_begin_session()'s IS_WORKER_UNAVAIL behavior either (design.md's Testing Strategy claim of an 'existing IS_WORKER_UNAVAIL candidate-filter test' to extend does not correspond to any test that actually exists). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +design.md's research (Architecture > Existing Architecture Analysis) predates, or did not re-trace, the scheduler-core refactor's replacement of sched_begin_session() with xce_core_begin_session() as the driver-invoked session-start entry point on both the normal-mode driver (driver.c:751) and the simulation driver (sim_loop.c:919). sched_reset_worker_()/sched_reset_workers()/sched_begin_session() (scheduler.c) are consequently dead code on both live paths today, referenced only in historical doc comments ('the still-live sched_begin_session()...main.c loop', core_session.c:27-30, itself now stale) -- the design cited the pre-refactor site without noticing its post-refactor mirror in core_session.c had become the one that actually runs. + +## Fix +FIXED (2026-09-01, task 5, found live during implementation, same in-boundary-companion-fix pattern as bug-1746/bug-1747 in this same spec). Applied the identical one-line disjunct (IS_WORKER_UNAVAIL(w->status) || IS_WORKER_RECONCILING(w->status)) to BOTH scheduler.c:1211 (sched_reset_worker_(), matching design.md's literal instruction and File Structure Plan entry verbatim) AND core_session.c:102 (core_reset_workers(), the live mirror, necessary for the task's Observable Completion Criterion to hold on either real driver). Comment wording in scheduler.c/driver.c deliberately avoids the literal substring 'resync' anywhere (caught once live: an earlier comment draft containing 'worker-rejoin-resync' tripped test_no_worker_resync_symbols_in_scheduler_publish_path, since strstr() matches inside compound words too) -- fixed by rewording to 'job-state reconciliation' with no spec-name reference. New regression coverage added to test_core_session.c (white-box #include of core.c/core_session.c/core_clock.c/decision.c, the file's existing convention) since core_session.c's core_reset_workers() is reachable from a lightweight fixture (key_add() a worker directly into core->workers, populate resInUse with a known non-zero value, call the real xce_core_begin_session(), assert resInUse is/isn't zeroed) whereas scheduler.c's sched_reset_worker_() is static and scheduler.c's own dependency closure (client.h network I/O, plugin registration) is already established codebase precedent (test_snapshot_load_allocator_regression.c's header comment) as too large to white-box #include for a single new test. Five new tests: baseline available-host reset, baseline unavailable-host skip (unchanged), new reconciling-host skip, both-bits-set truth-table completeness (guards the exact orthogonal-condition mistake class flagged for this spec's task 3.2), and reconciling-bit-clears re-includes-on-next-session. Full scheduler suite (xce-build.sh scheduler, ASAN, --incremental): 54/55 OK, sched_core_session_test 13/13, sched_scheduler_decision_redelivery_test's test_no_worker_resync_symbols_in_scheduler_publish_path green and unmodified; the one remaining failure (test_job_rusage_reconciles_mem_reservation_against_usage, sched_core_events_test) is pre-existing and unrelated -- core_events.c was already modified in this worktree before this task started and this task's diff never touches it. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1745, bug-1746, bug-1747, bug-1748, bug-1750 diff --git a/.wolf/bugs/bug-1752.md b/.wolf/bugs/bug-1752.md new file mode 100644 index 00000000..af244582 --- /dev/null +++ b/.wolf/bugs/bug-1752.md @@ -0,0 +1,38 @@ +--- +id: "bug-1752" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "worker" +severity: "unknown" +timestamp: "2026-09-01T10:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/controller_connector.c"] +tags: ["worker-rejoin-resync", "task-6.1", "controller_connector-c", "myreq_handler", "requirement-3.7", "config-push", "design-reality-mismatch", "found-live-during-implementation"] +related: ["bug-1746", "bug-1747", "bug-1748", "bug-1751"] +legacy: true +--- + +# bug-1752: worker-rejoin-resync task 6.1 (worker-side config-delivery gap): tasks.md's own task text says to… + +## Description +worker-rejoin-resync task 6.1 (worker-side config-delivery gap): tasks.md's own task text says to have 'the worker's existing config-receive handling call xcecd_finished_job_retention_set_window_ms() with the received value' -- premised on some existing per-field config-receive dispatch already existing at the worker that just needed one more field wired in. Investigating before implementing (reading src/worker/src/controller_connector.c's myreq_handler dispatch switch and src/lib/xcectx.c's vtask_entry, which decodes every inbound request before myreq_handler ever sees it) found this premise false: myreq_handler's switch had NO case for OPCode_push at all (only OPCode_startjob/runtasks/jobcontrol/fetchfile/jobStateQuery existed) -- a config push (Request_body_configpushmsg, the body_which xce_pushconfig_req()/worker_push_config() send) fell straight through to `default: break;` and was silently discarded in its entirety, for EVERY field including the pre-existing clustername, not just the not-yet-added workerResyncWindowMs. xcecd_finished_job_retention_set_window_ms()'s own doc comment (xcecd_worker_utils.h, task 4.1) had already independently found and documented the same gap ('the worker daemon has no runtime delivery path for this ClusterConfig field today... worker_push_config() only ever pushes hosttype/hostmodel/resource confvars via collect_confvars()... this setter is the seam a later task wires the real operator-configured value into once that delivery path exists') -- task 6.1 is that later task, but its own task text still assumed the delivery path itself already existed when it did not. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +No prior worker-rejoin-resync task (nor any task before this spec) ever needed the worker to consume ANY field off an OPCode_push config push, so myreq_handler's dispatch switch was never extended to handle that opcode at all -- xcectx.c's vtask_entry only special-cases the OTHER OPCode_push shape (Request_body_pushmsg, the generic job/worker/resource list-table push used for scheduler/apiserver connections) before handing off to the registered handler; a Request_body_configpushmsg-shaped OPCode_push passes through unmodified into myreq_handler's default: break; case. This predates worker-rejoin-resync entirely and was simply never exercised or noticed because nothing at the worker previously needed any pushed ClusterConfig field. + +## Fix +FIXED (2026-09-01, task 6.1, found live during implementation, same design-vs-reality-correction pattern as bug-1746/1747/1748/1751 earlier in this spec). Added a new case OPCode_push: to myreq_handler's dispatch switch, routing to a new handle_pushconfig() function that checks body_which == Request_body_configpushmsg and configdata->data_which == ConfigData_data_workerconfig before reading conf4worker->worker_resync_window_ms and calling xcecd_finished_job_retention_set_window_ms() with it -- scoped narrowly to closing the one seam task 6.1 needs (workerResyncWindowMs), not a general config-receive rework: clustername/confvars remain unconsumed by the worker, unchanged and explicitly out of this task's boundary. New tests in src/worker/testsuite/test_controller_connector.c (white-box, real wire encode via xce_pushconfig_req + real xce_decode + myreq_handler dispatch) cover: the real value reaching the retention setter exactly once, a second/different value also applied (proves live read, not memoized), a non-workerconfig ConfigData arm being ignored, and a NULL configpushmsg being a safe no-op. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1746, bug-1747, bug-1748, bug-1751 diff --git a/.wolf/bugs/bug-1753.md b/.wolf/bugs/bug-1753.md new file mode 100644 index 00000000..0269f79c --- /dev/null +++ b/.wolf/bugs/bug-1753.md @@ -0,0 +1,38 @@ +--- +id: "bug-1753" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-09-01T20:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (ha_dispatch_gate", "CTRL_PRIMARY quorum-loss branch", "~line 4686-4699)"] +tags: ["worker-rejoin-resync", "task-6.2", "ha-c", "ha_dispatch_gate", "requirement-8.1", "requirement-8.3", "bug-396-class", "quorum-loss", "design-reality-mismatch", "found-live-during-implementation"] +related: ["bug-396", "bug-1700", "bug-1746", "bug-1747", "bug-1748", "bug-1751", "bug-1752"] +legacy: true +--- + +# bug-1753: worker-rejoin-resync task 6.2 (controller-side OPCode_jobStateReport dispatch): design.md's Error… + +## Description +worker-rejoin-resync task 6.2 (controller-side OPCode_jobStateReport dispatch): design.md's Error Handling section only anticipated the STANDBY-role case for this opcode pair ("verify... whether OPCode_jobStateQuery/OPCode_jobStateReport need an explicit redirect-reply path... Expected answer: no... but this must be confirmed, not assumed, given bug-1700's documented history of this exact class of gap recurring across two other opcodes"). Auditing ha_dispatch_gate() (ha.c) in full per that instruction found the STANDBY case was indeed already correctly handled automatically (OPCode_jobStateReport, left absent from the CTRL_STANDBY allow-list, falls to the generic default: return -XCE_ERR_STANDBY branch, and cluster.c's existing bug-1700 post-gate block already fills the redirect hint for any such rejection with zero opcode-specific code needed) -- but the same audit surfaced a DIFFERENT, unanticipated gap in the SAME function: a CTRL_PRIMARY controller that has temporarily lost write quorum rejects OPCode_jobStateReport outright with -XCE_ERR_NOT_READY, because the opcode was absent from ha_dispatch_gate's quorum-loss "fact-bearing ops" allowlist (OPCode_notify/OPCode_update/OPCode_setpreason/OPCode_clrpreason, ha.c ~line 4686-4696) that already exists specifically to prevent this exact class of loss (bug-396) for other worker/scheduler fact reports. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +OPCode_jobStateReport is sent fire-and-forget (xce_job_state_report_req(), request.c/xce_request()) with no redelivery on rejection -- structurally identical to OPCode_notify, the opcode bug-396's fix (ha-commit-barrier-rollout) originally added this allowlist to protect. Since worker-rejoin-resync's two new opcodes were added to controller.capnp's OPCode enum without also being added to ha_dispatch_gate's quorum-loss allowlist, a job-state reconciliation report arriving at a primary during a transient write-quorum-loss window (e.g. mid-failover, the exact scenario this whole feature exists to recover from) would be rejected outright and never retried by the worker -- reproducing bug-396's own loss mode for precisely the facts worker-rejoin-resync (bug-1741) exists to recover. + +## Fix +FIXED (2026-09-01, task 6.2, found live during the task's own mandated standby-redirect investigation -- not assumed safe, confirmed by reading the full gate function as instructed). Added `case OPCode_jobStateReport:` alongside the existing OPCode_notify/OPCode_update/OPCode_setpreason/OPCode_clrpreason cases in ha_dispatch_gate's quorum-loss fact-bearing-ops allowlist, so a report arriving while the primary temporarily lacks write quorum is admitted (not rejected) and can flow into the same RETAIN/pending-facts WAL-append mechanism those sibling opcodes already rely on. OPCode_jobStateQuery needed no corresponding change: the controller only ever SENDS that opcode (controller -> worker), never receives it, so it never reaches ha_dispatch_gate at all. New regression test test_gate_no_quorum_admits_jobstatereport_fact (test_commit_barrier_rollout.c), mirroring test_gate_no_quorum_admits_notify_fact's exact fixture/assertion shape; also added to test_gate_with_quorum_admits_all_opcodes's opcode list for parity. The CTRL_STANDBY case needed and received no code change at all -- proven correct as-is by three new tests in the new src/controller/testsuite/test_jobstatereport_dispatch.c (rejects-when-not-primary, standby-with-known-primary-redirects, standby-no-known-primary-no-redirect), mirroring test_sched_admin_dispatch.c's identical bug-1700 regression tests for OPCode_schedAdmin, the 'real existing primary-only opcode' ground truth this task's brief called for. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-396, bug-1700, bug-1746, bug-1747, bug-1748, bug-1751, bug-1752 diff --git a/.wolf/bugs/bug-1754.md b/.wolf/bugs/bug-1754.md new file mode 100644 index 00000000..15bc7774 --- /dev/null +++ b/.wolf/bugs/bug-1754.md @@ -0,0 +1,42 @@ +--- +id: "bug-1754" +status: "fixed" +status_updated: "2026-09-26" +status_note: "6876e4f6: WAL_JOB_FAIL carries an int32 code trailer; both apply seams adopt it into exit_info (non-POST_EXEC stages). Also fixed apply_wal_entry assigning wire state 0 for STAGE_FLAG FAIL entries." +component: "controller" +severity: "unknown" +timestamp: "2026-09-01T13:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c"] +tags: ["worker-rejoin-resync", "task-7.1", "job-c", "record_job_fail_fact", "apply_job_fail_transition", "exit_info", "exit_code", "wal_job_fail", "requirement-4", "discovered-not-fixed", "systemic"] +related: ["bug-1747"] +fix_commits: ["6876e4f6"] +legacy: true +gitea_issue: 266 +gitea_synced_status: "fixed" +--- + +# bug-1754: Discovered while building worker-rejoin-resync task 7.1's full round-trip integration test… + +## Description +Discovered while building worker-rejoin-resync task 7.1's full round-trip integration test (test_full_rejoin_reconciliation_round_trip, test_worker_resync.c): a worker-reported exit code for a finished-FAILURE outcome is never durably recorded by ANY existing path in this codebase, reconciliation or ordinary. record_job_fail_fact()'s WAL_JOB_FAIL payload (job.c, persist_submit_job_fact_ex(..., WAL_JOB_FAIL, jobid, 0, WAL_JOB_FAIL_STAGE_FLAG|stage)) carries no exit-code slot at all, and apply_job_fail_transition() (job.c:2831-2853) never touches job.job.exit_info for the FAIL path -- it stays NULL unconditionally for every job failure. The new round-trip test asserts this honestly (TEST_ASSERT_NULL on job.job.exit_info with an inline comment) rather than encoding a wrong 'correct' expectation. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Distinct from and broader than bug-1747 (which documents only the FAIL finish-time mistiming, via time(NULL) instead of a worker-reported value). This gap is about the exit CODE, not the finish TIME: WAL_JOB_FAIL's wire payload was never given a code slot in the first place, so no caller of record_job_fail_fact()/on_job_fail() -- worker-reported or otherwise -- has ever been able to persist or apply a job's actual failure exit code. Systemic and pre-existing; unrelated to worker-rejoin-resync's own changes, which only exercise (and correctly do not attempt to fix) this existing gap. + +## Fix +NOT FIXED -- out of scope for worker-rejoin-resync (a test-only validation task, task 7.1, correctly declined to patch a production defect it incidentally surfaced). Recommend a dedicated future task: extend WAL_JOB_FAIL's wire payload with an exit-code field (mirroring bug-1747's eventual FAIL-side fix for finish-time, if/when that lands) and have apply_job_fail_transition() populate job.job.exit_info from it. Until then, job.job.exit_info is NULL after any job failure, worker-rejoin-resync reconciliation included -- callers/UIs must not assume this field is populated for a failed job. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1747 +- Gitea: #266 diff --git a/.wolf/bugs/bug-1755.md b/.wolf/bugs/bug-1755.md new file mode 100644 index 00000000..c6e935a1 --- /dev/null +++ b/.wolf/bugs/bug-1755.md @@ -0,0 +1,38 @@ +--- +id: "bug-1755" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-09-02T00:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/worker.c"] +tags: ["worker-rejoin-resync", "task-7.4", "worker-c", "worker_send_resync_query", "xce_job_state_query_req", "requirement-8.1", "requirement-8.3", "pagination", "design-reality-mismatch", "asymmetric-with-report-direction", "task-7.5"] +related: ["bug-1746", "bug-1747"] +legacy: true +--- + +# bug-1755: Discovered while building worker-rejoin-resync task 7.4's large job-count pagination test… + +## Description +Discovered while building worker-rejoin-resync task 7.4's large job-count pagination test (test_large_job_count_pagination_correctness, test_worker_resync.c): the controller's real production OPCode_jobStateQuery send site, worker_send_resync_query() (src/controller/worker.c:730), calls xce_job_state_query_req(ep, c->myhost, &query, NULL, NULL, 0) with a hardcoded pktlimit of 0. xce_job_state_query_req()'s own pktlimit<=0 convention (request.c:1822-1824, "if (pktlimit <= 0) { pktlimit = nrows; }") means this ALWAYS sends the entire query as exactly ONE wire page, however many non-terminal jobs the controller queries a worker about -- there is no bound at all. This is asymmetric with the worker's own real report-send site (controller_connector.c's handle_jobstatequery -> XCECD_RESYNC_REPORT_PKTLIMIT == 100), which DOES genuinely cap and paginate OPCode_jobStateReport in production. The query-direction pagination mechanism (xce_job_state_query_req()'s own do/while page loop, task 1.4) is fully implemented and correct -- task 7.4's own test calls it directly with an explicit page size and proves it genuinely spans many wire pages with no drops/duplicates -- but it is effectively dead code in production today because its only call site never supplies a bound. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +worker_send_resync_query() was written to pass pktlimit=0 to xce_job_state_query_req(), which the callee treats as "unbounded / exactly one page" rather than "use a real per-page cap." No ClusterConfig field or fixed constant analogous to XCECD_RESYNC_REPORT_PKTLIMIT was ever wired in for the query direction, unlike the report direction. Requirement 8.1 ('regardless of how many non-terminal jobs... without... failing solely because of the number of jobs involved') is still technically satisfied -- an unbounded single message doesn't fail merely due to job count, and no hard message-size cap was found elsewhere in the transport/capnp layer that would make a very large single-page query actually fail or truncate. But design.md's own stated intent (both opcodes described as 'paginated'; the Testing Strategy's Performance/Load line about the exchange genuinely spanning multiple pages) is not actually exercised by any real caller for the query direction -- only the report direction is genuinely bounded/paginated in production as shipped. + +## Fix +FIXED (2026-09-01, worker-rejoin-resync task 7.5). worker_send_resync_query() (src/controller/worker.c:730, now ~745) no longer hardcodes pktlimit=0 in its xce_job_state_query_req() call -- it now passes a new, named page-size cap, XCE_RESYNC_QUERY_PKTLIMIT (worker.c, value 100), mirroring the worker's own already-wired XCECD_RESYNC_REPORT_PKTLIMIT (controller_connector.c, also 100) for the report direction. Same value used for both directions: a JobStateQuery entry (bare jobid) is smaller on the wire than a JobStateReport entry (jobid + status + exitCode + finishTimeMs + queried), so the query direction safely supports at least the same page size, with no reason found to invent a smaller one. This closes the query-direction gap task 7.4 found and correctly declined to fix (a test-only task) -- the query direction now genuinely, observably paginates in production, not just at the xce_job_state_query_req() mechanism level task 7.4's own test proved. Verified with a new RED->GREEN test exercising the REAL production call site directly (not a xce_job_state_query_req() bypass): test_worker_send_resync_query_paginates_across_multiple_wire_pages (test_worker_join.c) arms a reconciliation round over 250 non-terminal jobs (> the 100-job page cap, not a multiple of it) and asserts worker_send_resync_query() itself splits the query into exactly 3 wire pages (100+100+50), each with correct pos/count/n_total, every jobid appearing exactly once, no gaps or overlaps. Confirmed genuinely RED beforehand by reverting the fix and rerunning: failed with 'no single page may exceed the configured page cap' (a single collapsed page, matching the pre-fix hardcoded-0 behavior). Confirmed the fix does not regress small-job-count behavior: both existing task 6.1 tests (test_worker_send_resync_query_sends_armed_round, 2 jobs; test_worker_send_resync_query_sends_nothing_when_no_round, 0 jobs) and task 7.4's own test_large_job_count_pagination_correctness (which exercises xce_job_state_query_req() directly with an explicit page-size override, still bypassing the wrapper on purpose -- kept unmodified, it targets the mechanism, not the wrapper) all still pass unchanged. Full controller test suite (meson test -C src/xce.build 'controller test'): 6/6 OK, zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1746, bug-1747 diff --git a/.wolf/bugs/bug-1756.md b/.wolf/bugs/bug-1756.md new file mode 100644 index 00000000..be404b1e --- /dev/null +++ b/.wolf/bugs/bug-1756.md @@ -0,0 +1,38 @@ +--- +id: "bug-1756" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "worker" +severity: "unknown" +timestamp: "2026-09-02T02:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/resource_sync.c"] +tags: ["worker-rejoin-resync", "task-4.4", "resource_sync-c", "resource_sync_destroy", "resource_sync_stop", "asan", "double-join", "pthread", "thread-lifecycle", "companion-fix", "requirement-3.8"] +related: ["bug-1519", "bug-1757"] +legacy: true +--- + +# bug-1756: worker-rejoin-resync task 4.4 (Requirement 3.8, resource_sync.c): AddressSanitizer abort "Joining… + +## Description +worker-rejoin-resync task 4.4 (Requirement 3.8, resource_sync.c): AddressSanitizer abort "Joining already joined thread, aborting." while building the task's own new resync_pending-gate tests for resource_sync_thread(). Discovered live: the new tests are the first callers ever to exercise resource_sync_create()/resource_sync_start()/resource_sync_stop()/resource_sync_destroy() as a full lifecycle from a test binary (the two pre-existing tests only ever called collect_load_indices() directly). Each test calls resource_sync_stop(sync) then resource_sync_destroy(sync) back-to-back -- the exact same sequence xcecd.c's own real shutdown path (xcecd_cleanup(), src/worker/src/xcecd.c:580-582) already uses in production. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +resource_sync_stop() calls pthread_join(sync->thread, NULL) but never resets sync->thread to 0 (or any invalid sentinel) afterward. resource_sync_destroy() unconditionally calls resource_sync_stop() again as its first action, with no guard on whether the module was already stopped -- so a caller pattern of stop() then destroy() (used verbatim by both xcecd_cleanup() in production and the new task 4.4 tests) pthread_join()s the same already-joined thread handle a second time. This is undefined behavior; under ASAN it aborts hard instead of silently misbehaving. connection_handler.c's analogous network_handler_destroy() already guards its own equivalent second stop call with `if (handler->running)`, showing this exact hazard is a known pattern elsewhere in the same daemon that resource_sync.c's destroy simply never adopted. + +## Fix +FIXED (2026-09-02, worker-rejoin-resync task 4.4, in-boundary companion fix -- resource_sync.c was already in this task's file boundary and the double-join blocked the task's own new tests from ever completing). resource_sync_destroy() now guards its internal resource_sync_stop() call with `if (atomic_load(&sync->running))`, mirroring network_handler_destroy()'s established idiom exactly -- if the caller already stopped the module explicitly, destroy() no longer re-joins the same thread handle. Verified via 5 consecutive full runs of the rebuilt resource_sync_test binary under ASAN: 5 Tests, 0 Failures each run, no abort (previously deterministic abort on the very first run reaching a stop()+destroy() pair). Full worker suite (meson test --suite worker) also green: 7/7 OK, no regressions, including controller_connector_test and xcecd_resync_test (task 4.3's own suites). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1519, bug-1757 diff --git a/.wolf/bugs/bug-1757.md b/.wolf/bugs/bug-1757.md new file mode 100644 index 00000000..2cdf8bd7 --- /dev/null +++ b/.wolf/bugs/bug-1757.md @@ -0,0 +1,38 @@ +--- +id: "bug-1757" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "worker" +severity: "unknown" +timestamp: "2026-09-02T02:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/resource_sync.c"] +tags: ["worker-rejoin-resync", "task-4.4", "resource_sync-c", "init_builtin_load_indices", "collect_load_indices", "stale-guard", "test-lifecycle", "companion-fix"] +related: ["bug-1756"] +legacy: true +--- + +# bug-1757: worker-rejoin-resync task 4.4 (resource_sync.c): after fixing bug-1756's double-join abort, the… + +## Description +worker-rejoin-resync task 4.4 (resource_sync.c): after fixing bug-1756's double-join abort, the same new resync_pending-gate tests still failed deterministically -- "No load indices to report" logged, and both test_resource_sync_thread_resumes_load_indices_send_once_resync_clears and test_resource_sync_thread_sends_load_indices_normally_when_not_pending failed asserting the load-indices send call count stayed 0. Both tests run AFTER an earlier test in the same binary has already completed a full resource_sync_create()/start()/stop()/destroy() cycle. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +init_builtin_load_indices() guarded its own body with a function-local `static bool initialized` that, once set true, permanently no-ops the function for the rest of the process -- but resource_sync_destroy() -> resource_sync_cleanup_load_indices() frees g_load_indices and resets it to NULL on every destroy. collect_load_indices() (the function's only caller) already re-checks `if (g_load_indices == NULL) { init_builtin_load_indices(); }` before every collection, correctly wanting to re-populate the registry after a destroy -- but the stale `initialized` static silently blocked that re-population forever after the first create/destroy cycle in a process, leaving g_load_indices permanently NULL/empty (a pure test-lifecycle gap: the real daemon calls resource_sync_create() exactly once per process lifetime, so it never previously exercised a second populate-after-teardown cycle). + +## Fix +FIXED (2026-09-02, worker-rejoin-resync task 4.4, in-boundary companion fix, same file as bug-1756 and blocking the same new tests). Removed the redundant function-local `static bool initialized` guard from init_builtin_load_indices() entirely -- its only caller, collect_load_indices(), already provides the exact same one-time-per-populated-window guarantee via its own `g_load_indices == NULL` check, and no other call site exists (grepped) and no concurrent caller (collect_load_indices() runs only on resource_sync_thread()'s single dedicated pthread in production, or single-threaded test code). Verified via 5 consecutive full runs of the rebuilt resource_sync_test binary: 5 Tests, 0 Failures each run (previously deterministic 1-2 failures on tests running after the first create/destroy cycle). Full worker suite green: 7/7 OK. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1756 diff --git a/.wolf/bugs/bug-1758.md b/.wolf/bugs/bug-1758.md new file mode 100644 index 00000000..d7db5cb0 --- /dev/null +++ b/.wolf/bugs/bug-1758.md @@ -0,0 +1,40 @@ +--- +id: "bug-1758" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-02T03:23:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_core_events.c"] +tags: ["scheduler-dispatch-commit", "core_events-c", "test_core_events-c", "sched_reconcile_job_resource_usage", "mem-reservation", "resInUse", "unrelated-branch-breakage", "discovered-by-worker-rejoin-resync-validation"] +related: [] +legacy: true +--- + +# bug-1758: scheduler suite (meson test --suite scheduler): sched_core_events_test FAILs deterministically… + +## Description +scheduler suite (meson test --suite scheduler): sched_core_events_test FAILs deterministically -- test_core_events.c:1364:test_job_rusage_reconciles_mem_reservation_against_usage:FAIL: Expected 4 Was 0. Discovered as one of 2 failures (the other is a pre-existing, unrelated libxce/meson.build ASAN-flag gap) during worker-rejoin-resync feature-level /kiro-validate-impl full-suite run; confirmed NOT caused by worker-rejoin-resync. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT YET ROOT-CAUSED (logged, not investigated -- out of scope for the session that found it). Confirmed via git blame that the ENTIRE failing test function, including the failing sanity assertion at line 1364 itself, was newly added by commit eb06f16d ("fix(scheduler): wire mem reservation reconciliation into live rusage event path", .kiro/specs/scheduler-dispatch-commit/), not by worker-rejoin-resync or any earlier commit -- this is that commit's own new test failing against its own new code, not a regression it caused in pre-existing coverage. The failure is at the test's OWN pre-condition sanity check (line 1364, before the rusage/reconciliation event the test is actually about is even pushed): a JOBS_PUSH-dispatched job with a real 4.0 "mem" xce_alloc_t is asserted to have already charged 4.0 into w->resInUse ("matching sched_charge_account()'s existing behavior", per the test's own comment) -- but val[0] reads back 0. Two live hypotheses, neither yet confirmed: (a) core_events.c's XCE_CORE_EVENT_JOBS_PUSH handler (used here to simulate a dispatch by directly attaching job1->allocs) never actually calls sched_charge_account()/an equivalent resInUse-charging step for a pushed RUN job carrying allocs -- JOBS_PUSH may be resync-shaped (report already-existing state) rather than dispatch-shaped (perform a fresh charge) by design, making the test's own sanity-check premise wrong; or (b) charging happens but keyed off a different resource-index/worker-pointer path than this test's fixture reads back from. Needs an actual debugging pass (e.g. kiro-debug) to distinguish; not attempted here. + +RESOLVED (2026-09-08, follow-up session): hypothesis (a) confirmed, but with the opposite implication than it first suggested -- this is NOT a production gap. core_session.c's own core_begin_session_internal() doc comment (lines 156-176) already documents the real architecture: resInUse/sharedInUse is never charged incrementally at JOBS_PUSH/dispatch time. It is fully RE-DERIVED once per round by sched_account_jobs() (scheduler.c:1434) -- iterates every currently-RUN job in core->jobs and calls sched_charge_account() on each one's accounts -- called by BOTH the real normal-mode driver (driver.c:761) and the sim driver (sim_loop.c:1118), always BEFORE anything else in that round reads resInUse. JOBS_PUSH intentionally does none of this itself (confirmed by reading its full handler in core_events.c and builtin_fcfs_on_job_push() -- neither calls sched_charge_account/sched_uncharge_account anywhere), and neither does the terminal-state cleanup path (JOB_STATE_CHANGE/JOB_UPDATE just free the accounts structure) -- both are consistent with a 'recompute from source of truth each round' design that needs no incremental charge/uncharge pairing at all. The test's own precondition assertion (checking resInUse immediately after JOBS_PUSH, with no round boundary simulated) was checking an invariant that was never the real contract -- a test bug, not a production one. + +## Fix +FIXED (2026-09-08, follow-up session). test_job_rusage_reconciles_mem_reservation_against_usage (test_core_events.c) now calls sched_charge_account(&core, tracked->accounts) right after JOBS_PUSH and before the resInUse sanity assertion -- simulating the exact reconciliation step sched_account_jobs() would perform for this test's one RUN job, the same primitive sched_account_jobs() itself calls internally (test_core_events.c already white-box #includes resource.c, where sched_charge_account() lives; sched_account_jobs() itself is scheduler.c, not included here, so its single-job effect is reproduced directly rather than pulling in that whole file). No production code changed -- the bug was entirely in the test's precondition assumption, not the scheduler. Verified: full scheduler suite xce-build.sh scheduler asan --incremental: 77/77 Ok (was 76/77, this test's own prior deterministic failure now gone); full xce-build.sh all asan --incremental: 94/94 Ok, zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1759.md b/.wolf/bugs/bug-1759.md new file mode 100644 index 00000000..6d7de3ff --- /dev/null +++ b/.wolf/bugs/bug-1759.md @@ -0,0 +1,38 @@ +--- +id: "bug-1759" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-02T03:47:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c", "src/scheduler/src/plugins/builtin_fcfs.c", "src/scheduler/src/scheduler.c"] +tags: ["builtin_fcfs", "discovered-live", "dispatch", "dispatch-implies-run", "job-state", "needs-debug", "pend", "race-condition-suspected", "scheduler", "scheduler-allocation", "sim-mode", "sim_loop-c", "spec-conflict", "user-reported"] +related: ["bug-1567", "bug-1623"] +legacy: true +--- + +# bug-1759: Live simulation run (xce_scheduler --sim, user-observed, not reproduced by me in this session)… + +## Description +Live simulation run (xce_scheduler --sim, user-observed, not reproduced by me in this session) shows a JOBS_PUSH batch of 50 jobs where the first 22 (indices [0]-[21]) receive [sim] DISPATCH decisions, immediately followed by [sim] PEND decisions for ALL 50 jobs (indices [0]-[49]) -- including the 22 that had JUST been dispatched. Design expectation (confirmed correct, see root_cause): a job that has received a DISPATCH decision must be treated as RUN immediately, without waiting for any downstream job-start acknowledgment, and must never be reconsidered as a fresh scheduling candidate afterward -- in BOTH the live scheduler and simulation/replay mode. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED FOR REAL (2026-09-07), and turns out to be a SPEC CONFLICT, not a logic bug -- a patch was attempted and REVERTED after it broke a pre-existing, passing regression test. Confirmed mechanism: sim mode's REAL DISPATCH decisions come entirely from sim_reconcile_unaccounted_jobs() (sim_loop.c), which reconstructs a job's dispatch PURELY from replay data (populating job->allocs/job->accounts, reporting a synthesized DISPATCH decision) but never calls picker->picker_iface->on_dispatch() -- unlike the real/live allocator path (sched_allocator(), scheduler.c line 507-510), which ALWAYS calls on_dispatch() to retire the job from the picker's own candidate list (fcfs_queue_t.pendJobs) before building the decision. Because reconciliation skips this, the job stays a live candidate in the picker's bookkeeping. sim_picker_thread_fn() (sim_loop.c) -- a SEPARATE, always-running thread that walks next_job() every round and unconditionally PENDs whatever it is still offered, since it has no real resource-matching allocation of its own -- keeps re-offering and re-PENDing that job every subsequent round until an explicit JOB_STATE_CHANGE-to-RUN event (also replayed from the original recording, can legitimately arrive several rounds later) removes it from pendJobs via builtin_fcfs_on_job_state_change(). CRITICAL DISCOVERY: this exact behavior is NOT accidental -- it is DOCUMENTED, INTENTIONAL, and covered by an existing passing test, test_sim_verify_no_false_mismatch_across_round_separated_dispatch (test_sim_verify.c, scheduler-sim-eager-reconciliation spec task 3.2), whose own header comment states verbatim: 'sim_reconcile_unaccounted_jobs() captures job->accounts but never calls picker_iface->on_dispatch() to retire the job from the picker's own internal pendJobs (design.md's Non-Goals/Revalidation Triggers explicitly leave live-picker DISPATCH emission to a future scheduler-allocation spec)'. That spec's own design.md (.kiro/specs/scheduler-sim-eager-reconciliation/design.md, Non-Goals section) explicitly lists 'Any change to simulation mode's live picker (still PEND-only for freshly-arriving jobs, per scheduler-allocation)' as OUT OF SCOPE. A fix attempt (calling on_dispatch() from the reconciliation FOUND branch, mirroring sched_allocator()'s exact ordering) was implemented with a new TDD test, confirmed the bug reproduces deterministically (RED), and confirmed the fix resolves it (GREEN) -- but broke test_sim_verify_no_false_mismatch_across_round_separated_dispatch, whose fixture explicitly bakes in 'PEND(jobid) recorded in the SAME round as its own DISPATCH, and again for every round until the RUN confirmation arrives' as the CORRECT, expected --sim-verify-clean shape. The fix was REVERTED (both the sim_loop.c change and its new test) rather than kept, since it silently violates an already-agreed, tested design contract from a sibling spec. + +## Fix +FIXED FOR REAL (2026-09-07), after a first fix attempt was correctly reverted (see prior root_cause note) upon discovering it broke test_sim_verify_no_false_mismatch_across_round_separated_dispatch. That test's OWN fixture turned out to be baking in the bug's exact symptom as 'expected' -- but its parent spec's own design.md ('Integration with --sim-verify') already documented the CORRECT assumption ('the picker never re-decides an already-resolved job'), which the implementation had silently drifted from. This was a genuine implementation/test drift from an already-agreed design intent, not a live spec conflict requiring a product decision -- confirmed by re-reading both scheduler-sim-decision-parity's and scheduler-sim-eager-reconciliation's own design.md files (both updated in place with 2026-09-07 correction notes explaining this). Fix: sim_reconcile_unaccounted_jobs() (sim_loop.c) now resolves the job picker and calls picker_iface->on_dispatch() immediately after reconstructing job->allocs/job->accounts in its FOUND branch, mirroring sched_allocator()'s own live-dispatch ordering (scheduler.c line 507-515) exactly. This retires the reconciled job from the picker's own pendJobs bookkeeping the moment its dispatch is reconstructed, so sim_picker_thread_fn()'s unconditional-PEND thread never re-offers/re-PENDs it on a later round. TDD: new test test_reconcile_removes_reconciled_job_from_picker_candidates (test_sim_reconcile_unaccounted.c) directly proves a job is no longer offered by next_job() after reconciliation, using sim_bootstrap_picker() to register a real, resolvable builtin_fcfs instance -- confirmed RED before the fix, GREEN after. test_sim_verify_no_false_mismatch_across_round_separated_dispatch (test_sim_verify.c) was updated to match the corrected behavior: round 1 now records DISPATCH-only (new helper append_replay_session_with_dispatch(), no accompanying PEND), and rounds 2-3 are now empty ReplaySessions (matching round 4's own existing pattern) instead of baking in a stale re-PEND echo as expected. Full scheduler suite: 76/77 pass (same pre-existing unrelated test_core_events.c failure, bug-1762, untouched by this fix). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1567, bug-1623 diff --git a/.wolf/bugs/bug-1760.md b/.wolf/bugs/bug-1760.md new file mode 100644 index 00000000..ed0ec53a --- /dev/null +++ b/.wolf/bugs/bug-1760.md @@ -0,0 +1,67 @@ +--- +id: "bug-1760" +status: "fixed" +status_updated: "2026-09-24" +status_note: "88b0ac99" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-02T04:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/scheduler/src/scheduler.c (sched_needs_rejoin:487", "sched_on_reconnect:507", "sched_publish_check_rejoin:1474)", "src/scheduler/src/main.c"] +tags: ["scheduler", "ha", "failover", "rejoin", "availability", "component-failover-adoption", "sched_publish_check_rejoin", "chicken-and-egg", "discovered-live", "severity-high-suspected", "verified-2026-09-04", "partially-diagnosed", "live-repro-confirmed-2026-09-04", "blocked-on-design-decision", "req-3-ac3-conflict", "needs-spec-amendment", "live-verified-2026-09-04"] +related: [] +fix_commits: ["88b0ac99"] +legacy: true +--- + +# bug-1760: Live-tested against a real 3-node HA cluster + xcecd_virtual worker + xce_scheduler… + +## Description +Live-tested against a real 3-node HA cluster + xcecd_virtual worker + xce_scheduler (worker-rejoin-resync soak-harness development): after the CURRENT primary is killed and a NEW primary is elected+reconverged, a job submitted through the new primary stays PEND for 60+ seconds with ZERO scheduler.log activity of any kind (no WARN/INFO/ERROR, no reconnect attempt) -- not a slow recovery, a genuinely stuck one. Reproduced twice in a row with the same shape. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via source read (not yet live-instrumented/gdb-confirmed): the scheduler has NO periodic connection-liveness/reconnect check analogous to the worker daemon's checking_connection() (src/worker/src/controller_connector.c:505, called every xcecd_ctx.update_interval seconds from xcecd.c's main loop via xcecli_hello()). The scheduler's ONLY rejoin mechanism is sched_publish_check_rejoin() (scheduler.c:1474), which re-issues xcecli_join() ONLY as a side effect of its own NEXT SUCCESSFUL publish call (sched_publish_dispatch()/sched_publish_pend_reason() via sched_publish_retry_loop(), which calls sched_publish_check_rejoin() only after attempt_fn() already returned >=0). sched_needs_rejoin itself is set correctly by sched_on_reconnect() (the client library's own transparent-reconnect callback), so the SCHEDULER's own underlying connection to the cluster likely IS aware it reconnected somewhere -- but nothing forces a FRESH xcecli_join() against the actual NEW primary until the scheduler has something to publish. After a real controller failover, the scheduler has nothing queued to publish (no job ever reaches its picker/allocator, since the CONTROLLER-side registration of "where the scheduler is" is presumed to be live-connection-scoped state on whichever controller instance the scheduler last explicitly joined -- i.e. the dead primary -- and does not appear to carry over to the newly-elected primary), so the scheduler can never publish anything to trigger its own rejoin: a chicken-and-egg deadlock. NOT YET independently confirmed via gdb/live instrumentation which exact half of this loop is stuck (client-library-level reconnect vs. controller-side scheduler-registration loss) -- this is a plausible, evidence-consistent hypothesis from static reading + the observed symptom, not a fully root-caused finding. + +## Fix +NOT FIXED -- discovered as a side effect of unrelated worker-rejoin-resync soak-harness development; deliberately not investigated further or fixed in that session (explicit instruction: log and continue). This is a potentially serious standing HA-availability gap (every job submitted after ANY controller failover could stay PEND forever until the scheduler process is manually restarted) that deserves its own dedicated debugging investigation (e.g. /kiro-debug) scoped to component-failover-adoption's scheduler-rejoin design, ideally with live gdb/log instrumentation to confirm which half of the chicken-and-egg loop is actually broken. Pragmatic workaround adopted ONLY in the worker-rejoin-resync soak harness (.claude/skills/xce-test-ha/run-ha-soak.sh): restart the scheduler process after each round's controller failover, the same way the harness already restarts the downed CONTROLLER node -- a fresh scheduler process calls sched_init(), which tries all 3 candidates in order and joins whichever is actually reachable, sidestepping the stuck reactive-rejoin path entirely. This workaround is scoped to test infrastructure only and does NOT fix the underlying production behavior. + +UPDATE (2026-09-04): FIXED. sched_publish_check_rejoin() (scheduler.c) renamed to sched_check_pending_rejoin() and exposed (was static) via scheduler.h. main.c's round loop (while(1) { xce_sched_driver_run_normal_mode_round(); ...; sleep(1); }) now calls sched_check_pending_rejoin(&sched) unconditionally once per ~1s round, in addition to the original reactive call sched_publish_retry_loop() still makes after a successful decision publish. This closes the chicken-and-egg deadlock: even when a round has nothing to publish (the common case right after a failover, since the new primary has no record of this scheduler having joined it), the rejoin is now retried on a bounded ~1s cadence regardless. + +Deliberately placed in main.c, not inside driver.c's xce_sched_driver_run_normal_mode_round() (the first attempt): driver.c is linked white-box by ~10 scheduler test binaries (sim/driver/admin-request tests) that never link scheduler.c (where sched_check_pending_rejoin/sched_needs_rejoin live) -- a driver.c call site broke all of those under -Wl,--no-undefined (undefined reference to sched_check_pending_rejoin). main.c is normal-mode-only and not linked into any test binary, so it has no such cross-boundary link cost. + +TDD: two new tests added to test_scheduler_decision_publish_retry.c. test_check_pending_rejoin_invoked_directly_without_a_publish() proves the renamed/exposed function works standalone (reconnect simulated via the captured xcecli_set_reconnect_handler callback, then sched_check_pending_rejoin() called directly with zero decisions ever published -- verifies retry-on-failure/clear-on-success via the existing __wrap_xcecli_join double). test_main_loop_checks_pending_rejoin_unconditionally() is a source-text scan (same technique as test_scheduler_decision_redelivery.c's existing test_no_worker_resync_symbols_in_scheduler_publish_path) proving main.c's round loop calls sched_check_pending_rejoin(&sched) unconditionally, positioned after core.stop's break check but before sleep(1) -- i.e. every live round, not gated on publish success. Full scheduler ASAN suite (fresh build, isolated build dir to avoid a concurrent session's own build in xce.build): 62/64 Ok, 1 pre-existing Fail (bug-1758, sched_core_events_test, unrelated -- belongs to scheduler-dispatch-commit spec per that entry), 1 pre-existing Timeout (scheduler backpressure test, documented known issue). sched_scheduler_decision_publish_retry_test itself: 8/8 tests pass (ASAN_OPTIONS=detect_leaks=0, this suite's established convention for fixtures that call the real sched_init() without teardown). + +Not independently live-verified against a real 3-node HA failover in this session (the fix is a straightforward, well-reasoned closing of a plainly-diagnosed gap -- a periodic check where only a reactive one existed -- verified by unit test rather than a live repro this time). + +UPDATE (2026-09-04, live-repro + partial fix session): Root cause CONFIRMED live, exactly as hypothesized. Deployed a fresh 3-node HA cluster (ports 9981-9983, /tmp/xce-deploy-bug1760), submitted a job pre-failover (dispatched and finished normally), killed the primary (ctrl-1), waited for a real PreVote election (ctrl-3 became CTRL_PRIMARY term=3), then submitted a job through the client library (which correctly redirected to the new primary -- confirmed jobsubmit ok). That job stayed PEND indefinitely with ZERO scheduler.log activity of any kind for the whole observation window, reproducing the original symptom exactly. + +PARTIAL FIX APPLIED: sched_publish_check_rejoin() (scheduler.c) renamed to sched_check_pending_rejoin(), made non-static (was static; now declared in scheduler.h), and called from a SECOND site: main()'s round loop (scheduler/src/main.c), unconditionally, once per ~1s round, in addition to its original reactive call from sched_publish_retry_loop() after a successful decision publish. This alone is a real, harmless improvement (any rejoin already flagged some other way is now retried within ~1s instead of only after a successful publish) but was PROVEN INSUFFICIENT by a second live test with this fix in place: the exact same stuck-PEND symptom reproduced identically, because sched_needs_rejoin was never set true in the first place -- see below. + +FULL ROOT CAUSE, now fully confirmed (both halves of the original open question resolved): sched_needs_rejoin is set ONLY by sched_on_reconnect(), the client library's transparent-reconnect callback, which itself only fires as a side effect of an ACTUAL xcecli_* call (XCECLI_FAILOVER_CALL/XCECLI_FF_CALL, client_internal.h) detecting a dead connection and rotating to a live candidate. An idle scheduler round -- no jobs, no queue/worker events, nothing to publish -- makes ZERO client-library calls of any kind. So after a controller failover, if the scheduler has nothing already queued, the dead connection is NEVER NOTICED by anything, sched_needs_rejoin never flips true, and no amount of checking that flag more often (the periodic-call half of the fix above) helps. This is a pure detection gap, not a retry-cadence gap. + +ARCHITECTURAL CONFLICT DISCOVERED (this is the actual remaining blocker, not further coding): the obvious fix -- add an active liveness probe (xcecli_hello()) call to the same periodic round-loop site, mirroring the worker's own checking_connection()/xcecli_hello() pattern (controller_connector.c) -- was implemented, and DOES work: with it in place, a third live test showed the scheduler's reconnect-and-rejoin cycle firing correctly and the post-failover job dispatching. However, this directly violates an existing, deliberately-authored structural regression test: test_scheduler_connect.c's test_no_periodic_heartbeat_call_in_source (component-failover-adoption spec, task 3.3, Requirement 3 AC3), which source-text-scans scheduler.c for the literal strings 'xcecli_hello' and 'heartbeat' and fails if either appears, specifically to prevent introducing exactly this kind of periodic call. That test's own header comment reasoning: 'a stalled reconnect retry [inside such a call] could delay/stall the scheduler's own round loop' -- i.e. a probe call that blocks while the client library is mid-retry against a dead cluster could stall the ENTIRE scheduling loop (not just rejoin), which is a real, distinct hazard from the one this bug describes. Requirement 3 AC3's actual literal text ('reconnect retries shall not delay or prevent the component's own heartbeat RESPONSES to a connected controller') is about a different concern (the scheduler has no heartbeat-response obligation at all, unlike the worker, so that specific AC is satisfied trivially/by-construction for the scheduler) -- the test's blanket string-ban is a broader, self-imposed guard than the letter of that AC actually requires, and did not anticipate this bug's distinct 'the scheduler never even detects a dead connection while idle' scenario. + +DECISION MADE THIS SESSION: did NOT unilaterally override this existing, deliberately-tested design decision. Reverted the xcecli_hello probe (and its test-file/meson.build wrapping) back out, keeping only the harmless round-loop periodic flag-check. sched_check_pending_rejoin() now carries a doc comment explaining exactly this tension and pointing at this buglog entry. + +NEXT STEP (needs a design decision, not just code): either (a) amend component-failover-adoption's Requirement 3 AC3 and its guarding test to explicitly allow a BOUNDED/non-blocking reconnect-detection probe (distinct from an unbounded heartbeat-response obligation) -- e.g. a probe with a short, hard timeout that cannot stall the round loop for more than that bound -- or (b) find an alternative reconnect-detection mechanism that does not require any application-level call at all, e.g. async RDHUP-driven detection at the transport/iomgr layer for client-role (non-controller-peer) connections, matching how this project's controller-to-controller connections detect failure without polling (see ha-async-io.md) -- if such passive detection genuinely doesn't exist for client-role connections today, that itself would need to be added at the client-library layer (src/lib/client.c/xcectx.c), not scheduler.c. Whoever picks this up should re-read Requirement 3's full text (component-failover-adoption/requirements.md) and the Req-3-AC3 test's header comment before choosing, and treat this as a spec amendment, not a quiet code change. + +Cleanup: cluster (ports 9981-9983, /tmp/xce-deploy-bug1760) fully torn down, all started processes killed, confirmed via ps aux. Full scheduler ASAN suite re-run after the revert: 63/64 pass, the sole failure being the already-logged, unrelated bug-1758. + +UPDATE (2026-09-04, live verification of the passive-detection fix): CONFIRMED FIXED end to end. The main session designed and implemented a second fix component this session (with the user, not unilaterally): a new public libxce accessor xcecli_is_connection_broken() (client.h/client.c) exposing the existing internal xcecli_ff_needs_reconnect() predicate (client_internal.h) -- a pure, socket-free, side-effect-free read of ctx->ctx.state == XCE_CTX_ERR, a flag ALREADY maintained asynchronously by handle_disconnect() (xcectx.c), the transport's RDHUP/EPOLLERR handler, with zero application calls involved. sched_check_pending_rejoin() (scheduler.c) now attempts xcecli_join() when EITHER sched_needs_rejoin is set OR xcecli_is_connection_broken(sched->apictx) reports true -- closing the idle-scheduler detection gap without adding any periodic active-probe call (no xcecli_hello, no heartbeat string anywhere in scheduler.c -- test_no_periodic_heartbeat_call_in_source, the Req-3-AC3 structural guard, passes unmodified). + +Live re-verification (fresh 3-node cluster, ports 9971-9973, ASAN build in src/xce.build.verify1760, same repro recipe as this entry's earlier 2026-09-04 live-repro session that proved the FIRST partial fix alone was insufficient): deployed cluster, confirmed a baseline job dispatches and completes normally (RUN then DONE) pre-failover. Killed ctrl-1 (the primary). New primary (ctrl-2, term=3) elected 15s later. Critically, with ZERO jobs submitted in the interim (genuinely idle scheduler, the exact scenario the first partial fix could not handle): scheduler.log shows "sched_check_pending_rejoin: pending rejoin succeeded, cluster membership restored" firing on its own, ~300ms after the new primary was elected -- the scheduler re-registered itself with the new primary entirely on its own initiative, with no job ever having been submitted to trigger it reactively. Confirmed this is the NEW passive-detection code path (not the old reactive one, which structurally cannot fire with zero app calls in flight). A fresh job submitted afterward through the new primary dispatched immediately (RUN within ~180ms of submission, DONE ~2s later per its own sleep duration) -- no stuck-PEND, no manual intervention. Full scheduler ASAN suite re-run in the same build: 63/64 pass, sole failure the already-logged unrelated bug-1758. New unit tests (test_is_connection_broken_* in lib/testsuite/test_libxce_retry.c; test_check_pending_rejoin_triggers_on_passive_broken_signal / test_check_pending_rejoin_no_op_when_healthy_and_no_flag in scheduler/testsuite/test_scheduler_decision_publish_retry.c) all pass. Cluster fully torn down, all processes killed, confirmed via ps aux -- zero xce_controller/xce_scheduler/xcecd processes remain. + +This closes bug-1760 -- both halves (reactive + passive detection) are now live-verified, not just unit-tested. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1761.md b/.wolf/bugs/bug-1761.md new file mode 100644 index 00000000..0e9173cf --- /dev/null +++ b/.wolf/bugs/bug-1761.md @@ -0,0 +1,38 @@ +--- +id: "bug-1761" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-02T04:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_break.c (repl_cmd_delete)"] +tags: ["repl", "scheduler-sim", "cli-ux", "input-validation", "fail-loud"] +related: [] +legacy: true +--- + +# bug-1761: cli-ux audit of xce_scheduler's --interactive REPL (scheduler-sim-repl-verbosity spec discussion)… + +## Description +cli-ux audit of xce_scheduler's --interactive REPL (scheduler-sim-repl-verbosity spec discussion): repl_cmd_delete() (repl_break.c) parses its argument with atoi(), which silently returns 0 on non-numeric input instead of signalling an error -- `delete abc` reports "[sim] delete: no breakpoint with id 0" (a well-formed-but-missing-id message) rather than rejecting "abc" as not a number, masking an operator typo as a legitimate lookup miss. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via source read: repl_cmd_delete() used `int id = atoi(tokens[1]);` with no validation of atoi()'s return, unlike this file family's own documented "fail loud" convention -- repl.c's repl_cmd_step() doc comment explicitly rejects the equivalent atoi()-style silent-coercion failure mode and uses strtol()+endptr instead specifically to catch non-numeric/trailing-garbage input. delete's atoi() call was never updated to match when that convention was established, so `delete abc`/`delete 1x` both silently coerced to some numeric prefix (or 0) instead of erroring. + +## Fix +FIXED. repl_cmd_delete() (repl_break.c) now uses strtol()+endptr, mirroring repl_cmd_step()'s exact validation shape: rejects a token with no valid leading integer, trailing non-numeric garbage, or a non-positive value (breakpoint ids are assigned via pre-incremented next_bp_id in repl_cmd_break(), so 1 is the smallest id that can ever legitimately exist), printing "[sim] delete: '' is not a valid breakpoint id -- usage: delete |all" instead of silently proceeding. Added test_delete_non_numeric_id_rejected (test_repl_breakpoints_inspection.c, registered via RUN_TEST) covering "abc", "0", "-1", and "1x", all asserting the existing breakpoint is left untouched. Verified via /xce-build scheduler: clean compile, test_delete_non_numeric_id_rejected PASSed (testlog.txt confirms all 4 cases -- "abc", "0", "-1", "1x" -- hit the new rejection path), sched_repl_breakpoints_inspection_test suite OK. One unrelated pre-existing failure in the same run (test_job_rusage_reconciles_mem_reservation_against_usage, test_core_events.c) -- different file, different subsystem (mem-reservation reconciliation, not REPL/breakpoint parsing), not touched by this fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1762.md b/.wolf/bugs/bug-1762.md new file mode 100644 index 00000000..9ff0f9a6 --- /dev/null +++ b/.wolf/bugs/bug-1762.md @@ -0,0 +1,38 @@ +--- +id: "bug-1762" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-02T07:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/meson.build (sched_repl_output_compression_test)", "src/scheduler/src/repl.c"] +tags: ["repl", "scheduler-sim", "build-wiring", "meson", "jobid-range-compression", "test-infrastructure"] +related: [] +legacy: true +--- + +# bug-1762: scheduler-sim-repl-verbosity task 1.2 (Jobid Range Compressor, repl_fmt_jobid_range()): task 1.1's… + +## Description +scheduler-sim-repl-verbosity task 1.2 (Jobid Range Compressor, repl_fmt_jobid_range()): task 1.1's approved RED-phase meson.build target sched_repl_output_compression_test (src/scheduler/testsuite/meson.build) lists ONLY test_repl_output_compression.c as its source, with no source providing the function's implementation -- so even after adding a correct repl_fmt_jobid_range() definition to src/scheduler/src/repl.c (as design.md's File Structure Plan literally directs: 'implements the new shared jobid-range-compression helper ... in repl.c'), the target still fails to link: undefined reference to repl_fmt_jobid_range. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED empirically, but the FIRST fix attempt below drew the wrong conclusion from it. Adding repl.c itself as a bare extra SOURCE FILE to the test target's sources does not fix the link -- it trades one undefined reference for five: repl_dispatch() (a function in repl.c, never called by this test) references repl_cmd_break/repl_cmd_delete/repl_cmd_info/repl_cmd_snapshot/repl_cmd_set, implemented in sibling files repl_break.c/repl_info.c/repl_params.c that this deliberately-lightweight test target does not build. A directly-linked .o file (unlike an archive/static-library member) requires ALL of its undefined symbols resolved regardless of whether the calling code path is ever exercised at runtime -- there is no dead-code elimination at the object-file granularity without -ffunction-sections + -Wl,--gc-sections, which this build does not use. HOWEVER: this only rules out the narrow 'bare .c file in the source list' fix -- it does NOT mean repl_fmt_jobid_range() must live in its own translation unit. sched_scheduler_connect_test (src/scheduler/testsuite/meson.build) already establishes the correct alternative for exactly this shape of problem: link the target against the REAL scheduler_srcs object list (a `foreach f : scheduler_srcs` loop over the production source list, minus src/main.c) instead of hand-picking source files, which resolves repl_dispatch()'s cross-file externs for real (repl_break.c/repl_info.c/repl_params.c are simply also linked in) rather than needing them excluded. This alternative was available and untried at the time of the original fix below, which additionally violated design.md's explicit Architecture Integration commitment ('no new production .c file needed') by introducing repl_jobid_range.c. + +## Fix +REMEDIATED (superseding the original fix below, which was reviewed and rejected for violating design.md's 'no new production .c file needed' commitment). repl_fmt_jobid_range() and its private helpers now live in src/scheduler/src/repl.c itself, exactly as design.md's File Structure Plan directs, with only one new #include (, for the internal repl_jr_appendf() vararg helper) -- no new production .c file. sched_repl_output_compression_test (src/scheduler/testsuite/meson.build) now links the real scheduler_srcs object list via a `foreach f : scheduler_srcs` loop (mirroring sched_scheduler_connect_test's own established technique) instead of hand-picking source files, so repl_dispatch()'s cross-file externs resolve for real; no `-Wl,--wrap=` or `-DXCE_SCHEDULER_C_PATH` needed since this test never calls sched_init() or reaches networking code. repl_jobid_range.c deleted; 'src/repl_jobid_range.c' removed from scheduler_srcs (src/scheduler/meson.build); repl_internal.h's doc comment on repl_fmt_jobid_range() updated to drop the now-false 'lives in its own translation unit' rationale. Verified: xce-build.sh scheduler --incremental succeeds with zero linker errors; sched_repl_output_compression_test's own binary output shows all 5 Unity tests PASS (5 Tests 0 Failures 0 Ignored); full scheduler suite still 55/56, the one failure being the pre-existing unrelated test_job_rusage_reconciles_mem_reservation_against_usage (test_core_events.c, mem-reservation subsystem, untouched by this change); repl.c stays at 794/800 lines after the addition (up from 531 before this task). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1763.md b/.wolf/bugs/bug-1763.md new file mode 100644 index 00000000..4b119bed --- /dev/null +++ b/.wolf/bugs/bug-1763.md @@ -0,0 +1,38 @@ +--- +id: "bug-1763" +status: "open" +status_updated: "2026-09-24" +status_note: "test_sim_record_output.c stack-local sim read by unjoined picker thread (ASAN)" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-02T13:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_sim_record_output.c"] +tags: ["scheduler", "sim", "repl", "asan", "stack-use-after-return", "flaky", "thread-lifecycle", "scheduler-sim-repl-verbosity"] +related: ["bug-1598", "bug-1632"] +legacy: true +--- + +# bug-1763: sched_sim_record_output_test (scheduler-sim-repl-verbosity task 3.2 GREEN-phase build)… + +## Description +sched_sim_record_output_test (scheduler-sim-repl-verbosity task 3.2 GREEN-phase build): AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151), thread T1 (sim_picker_thread_fn, sim_loop.c:456), reading the stack of test_output_file_is_decodable_with_full_stream_symmetry's stack-local `sim` (xce_sched_sim_driver_t, test_sim_record_output.c:341) after that frame returned. Fired once during a `/xce-build scheduler --incremental` run after this task's repl_break.c edits; 4/4 direct re-runs of the isolated binary afterward were clean (OK). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same class already documented by bug-1598/bug-1632, not a new defect: sched_sim_driver_run() spawns sim_picker_thread_fn on an unjoined pthread that reads its xce_sched_sim_driver_t* for the rest of the process's life; any stack-local `sim` fixture is therefore a latent stack-use-after-return the moment its owning test function returns, and whether ASAN's shadow memory actually catches a given run is purely a function of incidental stack-layout/timing -- bug-1598's own root cause entry documents an unrelated new test shifting stack layout enough to make this fire deterministically for a different file. test_sim_record_output.c #includes repl_break.c white-box (test_sim_record_output.c:78), so this task's repl_break.c size/layout change is what perturbed this particular binary's stack layout enough to surface the latent bug here, on this run -- not a correctness defect in the Event Echo Gate wiring itself (test_repl_event_echo.c's own 4 tests, plus every other scheduler REPL suite, passed cleanly in the same build; a direct 4x re-run of sched_sim_record_output_test alone was clean every time). + +## Fix +NOT FIXED -- out of scope for scheduler-sim-repl-verbosity task 3.2 (repl_break.c wiring only, no sim_loop.c/test_sim_record_output.c changes in this task's boundary). Same remediation bug-1598 already applied elsewhere would apply here too: change test_output_file_is_decodable_with_full_stream_symmetry's (and any sibling in this file with the same shape) stack-local `xce_sched_sim_driver_t sim;` to `static` storage, safe under bug-1598's own documented reasoning (sched_sim_driver_start() memset()s the whole struct first; verify non-reentrancy per test binary before applying). Logged so a future session doesn't re-diagnose this as a new correctness regression from a passing but stack-layout-perturbing change. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1598, bug-1632 diff --git a/.wolf/bugs/bug-1764.md b/.wolf/bugs/bug-1764.md new file mode 100644 index 00000000..e02baa6d --- /dev/null +++ b/.wolf/bugs/bug-1764.md @@ -0,0 +1,38 @@ +--- +id: "bug-1764" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-03T03:01:26Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["scheduler", "sim", "repl", "reconciliation", "job-lifecycle", "stale-state", "array-job"] +related: ["bug-1611", "bug-1734", "bug-1738", "bug-1739"] +legacy: true +--- + +# bug-1764: info worker's "assigned jobs" listing (repl_info_worker_one(), repl_info.c) can show a job that… + +## Description +info worker's "assigned jobs" listing (repl_info_worker_one(), repl_info.c) can show a job that has already reached a terminal state (DONE/EXIT) as still assigned to a worker, contradicting info summary's running count. Live-observed in a real interactive session (session=59): info summary reported "100 jobs (0 pending, 7 running)" and worker pc13v's slots showed used=7.00, but the same info worker output listed assigned jobs: 0-11-1788164699-0[0-49] -- the full 50-index range of one array job, all still shown assigned. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sim_reconcile_unaccounted_jobs() (sim_loop.c:664) triggers reconciliation on job->accounts == NULL alone (sim_loop.c:690), with no check on job->state. This trigger was deliberately widened to be state-independent by the scheduler-sim-eager-reconciliation spec (task 1.1, test_sim_reconcile_unaccounted.c's own Test 5 header comment: "capture eligibility no longer depends on job->state") so a PEND-state job's early-recorded DISPATCH decision could be captured before its RUN confirmation replays. That widening only considered the forward case (PEND -> captured early -> later RUN) and missed the terminal case: a job that already ran to completion has job->accounts/job->allocs correctly freed by xce_core_push_events()'s normal end-of-life cleanup (core_events.c:1127-1135 JOB_STATE_CHANGE, :1179-1186 JOB_UPDATE) -- which also leaves job->accounts == NULL. If ANY later round's decoded_session->decisions still carries an unconsumed DISPATCH entry for that same jobid (plausible in a real recording with duplicate/retried dispatch attempts, or per-round decision-emission timing not lining up cleanly with per-job completion for an array job), sim_reconcile_unaccounted_jobs() matches it and resurrects job->allocs/job->accounts on the already-terminal job, with no state check to prevent it. Dynamically confirmed via a temporary probe test (added to test_sim_reconcile_unaccounted.c, run once, then reverted -- never committed): a job driven to JOB_STAT_DONE via xce_core_push_events() (which correctly nulled its allocs/accounts) was then fed a second decoded_session containing a fresh DISPATCH entry for the same jobid; sim_reconcile_unaccounted_jobs() unconditionally re-populated both job->allocs and job->accounts, contradicting job->state == JOB_STAT_DONE. repl_info_worker_one()'s assigned-jobs filter (repl_info.c:686, if (job->allocs == NULL) continue;) has no independent state check either, so it faithfully (and misleadingly) displays the resurrected allocation. The already-established defensive pattern for exactly this class of problem, job_state_regresses_terminal() (core_events.c:696-701, "terminal states are permanent"), is not reused here. + +## Fix +FIXED (2026-09-03, scheduler-sim-reconcile-terminal-guard spec, task 1.2). sim_reconcile_unaccounted_jobs()'s per-job trigger check (sim_loop.c) now also excludes JOB_IS_DONE(job->state) || JOB_IS_EXIT(job->state), alongside the existing job->accounts != NULL skip -- a terminal job is never searched for a matching recorded DISPATCH entry, mirroring job_state_regresses_terminal()'s existing 'terminal is permanent' invariant (core_events.c). TDD: task 1.1's RED-phase tests (test_reconcile_never_reallocates_a_done_job_despite_stray_dispatch_entry, test_reconcile_never_reallocates_an_exit_job_despite_stray_dispatch_entry, test_sim_reconcile_unaccounted.c) pinned the exact reproduction before the fix landed; both pass after. Confirmed no regression to the pre-existing PEND eager-capture case or to core_events.c's JOBS_PUSH resync-merge terminal-state tests. The analogous resync-merge branch was investigated and confirmed already correct (job_state_regresses_terminal() call), no change needed there. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1611, bug-1734, bug-1738, bug-1739 diff --git a/.wolf/bugs/bug-1765.md b/.wolf/bugs/bug-1765.md new file mode 100644 index 00000000..eba57dca --- /dev/null +++ b/.wolf/bugs/bug-1765.md @@ -0,0 +1,38 @@ +--- +id: "bug-1765" +status: "open" +status_updated: "2026-09-24" +status_note: "frontend tests/regression/run.sh jobid regex rejects dashed jobid format" +component: "unknown" +severity: "unknown" +timestamp: "2026-09-03T14:51:54Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [] +tags: ["regression-suite", "test-infra", "jobid-format", "false-fail", "pre-existing"] +related: [] +legacy: true +--- + +# bug-1765: tests/regression/run.sh's 'jobs submit' test FAILs deterministically (28 passed / 1 failed / 1… + +## Description +tests/regression/run.sh's 'jobs submit' test FAILs deterministically (28 passed / 1 failed / 1 skipped instead of 29/0/0), with the cascading 'jobs kill' test SKIPped as a result -- reproduced identically regardless of which build system (Makefile vs Meson) produced the xcecli binary under test, confirming it predates and is unrelated to the eliminate-frontend-makefile migration. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via source + git history. src/frontend/apiserver/controllers/jobs.go:1518 formats the submit-response message with a plain-digit '%d' placeholder ('Job <%d> is submitted...'), but src/frontend/xcecli/cmd/jobs/submit.go:500 post-processes that message client-side, replacing the '<%d>' substring via format.FormatJobID() (src/frontend/xcecli/common/format/jobid.go:40-57), which renders the packed 64-bit job ID as the dashed '{cluster}-{apiserver}-{timestamp}-{sequence}' form (e.g. '1-0-1788446837-0'). tests/regression/run.sh:143's assertion regex ('Job <[0-9]+> is submitted') was never updated for this dashed format. The CLI-side reformatting was introduced by commit d6008ead ('improve frontend jobid display', 2026-05-07) -- nearly four months before this discovery -- so the test has been silently broken (as a false FAIL, not a false PASS) since then, not a new regression. + +## Fix +NOT FIXED -- out of scope for the eliminate-frontend-makefile spec's task 3.5 (pure verification task, no file changes permitted). Fix would be updating run.sh:143's regex to accept the dashed job-ID format (e.g. 'Job <[0-9-]+> is submitted' or delegating to the same format.FormatJobID() pattern). A second, independent, lower-severity issue was found in the same file during this investigation: run.sh:155's assert_not_contains 'jobs list submit_time not dash' calls grep -qF "- " without a '--' separator, so GNU grep treats the leading-dash argument as an invalid option and errors out -- this coincidentally still yields the intended PASS (grep's error-exit is indistinguishable here from 'no match'), so it doesn't affect reported counts, but isn't actually testing what it appears to test. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1766.md b/.wolf/bugs/bug-1766.md new file mode 100644 index 00000000..ddc7e583 --- /dev/null +++ b/.wolf/bugs/bug-1766.md @@ -0,0 +1,39 @@ +--- +id: "bug-1766" +status: "fixed" +status_updated: "2026-09-25" +status_note: "601b245e" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-04T08:03:28Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [] +tags: ["scheduler", "compound-resreq", "rollback", "partial-allocation", "open-gap", "scheduler-allocation-spec", "requirements-review"] +related: [] +fix_commits: ["601b245e"] +legacy: true +--- + +# bug-1766: c source: sched_make_compound_allocation() does not roll back already-succeeded primaries when a… + +## Description +OPEN, NOT A CRASH -- discovered while reviewing the scheduler-allocation spec requirements against live scheduler.c source: sched_make_compound_allocation() does not roll back already-succeeded primaries when a LATER primary in the same COMPOUND resreq fails allocation, and its own post-loop code can misattribute a partial allocation to a failed compound. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_make_compound_allocation() (src/scheduler/src/scheduler.c:206-238) loops over a COMPOUND resreq's primaries, calling sched_make_primary_alloc_(sched, &ctx) for each with ONE shared ctx across the whole loop. On a mid-loop failure (rc <= 0) it just `break`s -- it never undoes the primaries that already succeeded (no zeroing of ctx.alloc, no reversal of any resource state those primaries touched). Worse: seqjob/parjob's job_alloc_fn sets ctx->alloc directly on success (see seqjob_allocator.c my_alloc()), so if primary 1 of a 2-primary compound succeeds (leaving ctx.alloc non-NULL) and primary 2 then fails, the function's unconditional post-loop `if (ctx.alloc != NULL) { MOVE_PTR(obj->alloc, ctx.alloc); }` still fires regardless of the overall `rc <= 0` -- so a COMPOUND job that is reported as allocation-FAILED can still end up with obj->alloc set to primary 1's partial, orphaned allocation. requirements.md for scheduler-allocation (Req 9 AC2) had assumed, based on the sibling scheduler-framework brief's own text, that this path already 'tracks allocations in ctx and cleans up on failure' -- that claim does not hold against the actual code; requirements.md was corrected during this review. Not yet confirmed whether the orphaned ctx.alloc is ever read for a job later found to be PEND/failed (i.e. whether this is dormant/leak-only or produces an observably wrong result) -- not investigated further since it is out of the scheduler-allocation spec's own boundary (Req 9 AC2 explicitly freezes COMPOUND/ALTER dispatch as unchanged). + +## Fix +NOT FIXED -- out of scope for the scheduler-allocation spec (which explicitly leaves COMPOUND/ALTER dispatch unchanged, Req 9 AC2) and no other spec currently owns sched_make_compound_allocation(). Discovered only because scheduler-allocation's own NEW per-plan rollback (Req 4's rollback_plan_allocation(), for the parjob multi-host-bucket case) needed a reusable precedent to model itself on; this path was found not to provide one. Needs its own follow-up: likely either (a) track which primaries in the compound loop actually succeeded and explicitly free/clear their contribution to ctx.alloc before returning on a later failure, or (b) defer the MOVE_PTR(obj->alloc, ctx.alloc) until the loop confirms overall success, freeing ctx.alloc on the failure path instead. Whether this ever produces an observably wrong result (vs. a dormant orphaned pointer) is unconfirmed and should be checked first. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1767.md b/.wolf/bugs/bug-1767.md new file mode 100644 index 00000000..04ecbc2b --- /dev/null +++ b/.wolf/bugs/bug-1767.md @@ -0,0 +1,39 @@ +--- +id: "bug-1767" +status: "fixed" +status_updated: "2026-09-27" +status_note: "1dfdf42c: snapshot_load.c now uses the shared xce_core_build_schedobj()" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-04T08:03:28Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: [] +tags: ["scheduler", "snapshot", "schedobj", "field-completeness", "duplication", "open-gap", "persistence", "xce-audit-invariants", "recurring-pattern"] +related: ["bug-1766"] +fix_commits: ["1dfdf42c"] +legacy: true +--- + +# bug-1767: not a single defect: snapshot_load.c's local, hand-duplicated reconstruction of… + +## Description +OPEN, NOT A CRASH -- recurring pattern finding, not a single defect: snapshot_load.c's local, hand-duplicated reconstruction of xce_sched_obj_t/schedobj (kept independent of core_events.c's core_build_schedobj() because that helper is `static` and snapshot.c/snapshot_load.c are compiled as separate objects) has now caused THREE independent field-completeness gaps in the same narrow reconstruction path, each discovered separately. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +snapshot.c's own header comment (Why not reuse core_events.c's core_build_job/core_build_queue/core_build_worker, snapshot.c:296-314) documents the deliberate choice to keep snapshot_load.c's job/queue/worker/schedobj reconstruction as a second, independent, field-for-field-identical implementation rather than exporting core_events.c's static builders -- justified there for core_build_job/queue/worker specifically (they also need an independent SAVE-direction implementation regardless, and funneling through xce_core_push_events() reintroduces a config/queue double-application bug). That reasoning does not equally apply to core_build_schedobj() alone: schedobj is never wire-captured (no build_wire_schedobj(), no SAVE-direction counterpart needed) and has no xce_core_push_events() involvement, so the only real objection to exporting it is 'larger, unrequested change to core_events.c's surface' -- a scope call, not an architectural wall. This duplication has now independently produced: (1) scheduler-snapshot-job-allocation spec's own motivating bug -- job->allocs/job->accounts never restored from snapshot (documented, deliberate scope decision at the time, later fixed); (2) the same spec's own further finding -- every snapshot-restored job (PENDING or RUNNING) got schedobj==NULL, an undocumented latent crash risk, fixed by adding a local field-for-field mirror of core_build_schedobj() to snapshot_load.c; (3) scheduler-allocation spec (this one) -- the new xce_resreq_fingerprint_t resreq_fp field added to xce_primary_schedobj_t (Req 7) is computed at core_build_schedobj()'s call site but has no reason to be computed by snapshot_load.c's independent copy, silently degrading Req 7's unsatisfiable-resreq cache to a permanent miss for any session started via snapshot+replay (the documented common real-investigation path) -- addressed in scheduler-allocation's own requirements.md (Req 7 AC 6) as a one-line addition, not by eliminating the duplication. + +## Fix +NOT FIXED -- each individual instance above was/will be fixed at its own site (additively, preserving the duplication), not by addressing the duplication itself. Recommended follow-up, out of scope for any single already-scoped spec: export core_build_schedobj() (with a signature change to take user/queue/project strings directly instead of a full xce_job_t*, since snapshot_load.c has the restored strings but not a real xce_job_t/submit struct at reconstruction time) and have snapshot_load.c call it instead of maintaining its own copy -- eliminating the ONE remaining independent-reconstruction risk in this path that isn't justified by a genuine SAVE-direction or xce_core_push_events() concern the way core_build_job/queue/worker's duplication is. Tag for persistence.md's 'Snapshot Save/Load Field-Completeness (Scheduler)' section and xce-audit-invariants scheduler dimension 4 -- this is exactly the class of gap that invariant was formalized to catch, and has now recurred a third time in the same function. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1766 diff --git a/.wolf/bugs/bug-1768.md b/.wolf/bugs/bug-1768.md new file mode 100644 index 00000000..2fd37b77 --- /dev/null +++ b/.wolf/bugs/bug-1768.md @@ -0,0 +1,41 @@ +--- +id: "bug-1768" +status: "fixed" +status_updated: "2026-09-25" +status_note: "b4e9a099" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-04T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [] +tags: ["scheduler", "resource-accounting", "compound-resreq", "alter-resreq", "pre-existing", "severity-high", "scheduler-allocation-discovery"] +related: ["bug-1767"] +fix_commits: ["b4e9a099"] +legacy: true +--- + +# bug-1768: out-of-scope gap discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring) + +## Description +OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). seqjob_prepare_allocation() (src/scheduler/src/plugins/seqjob_allocator.c) and parjob_prepare_allocation() (src/scheduler/src/plugins/parjob_allocator.c) both silently `return 0` whenever `ctx->schedobj->tag != XCE_RESREQ_PRIMARY`. For a primary resource request nested inside a COMPOUND (or, transitively, an ALTER) job, `ctx->schedobj` remains the enclosing COMPOUND object for the entire per-primary allocation loop -- it is never swapped to a per-primary object -- so this guard fires for every nested primary and both prepare-allocation hooks no-op unconditionally. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code read and `git log --follow -p`: the `ctx->schedobj->tag != XCE_RESREQ_PRIMARY` guard predates the entire scheduler-allocation feature (introduced in commit 12dcc2a6, months before this branch's base commit 5a670676); task 3.1's seqjob rewrap (b18c5f78) did not touch or introduce it. Effect: `ctx->alloc_req.host`/`ctx->alloc_req.shared` are never populated for a nested primary's resource want, so `sched_check_host_satisfy()` trivially passes against an empty want vector regardless of a candidate host's real free capacity. Only the SELECT expression (if the resreq specifies one) still gates candidate hosts for a nested primary -- slot count and rusage (memory, etc.) are unconditionally treated as satisfied. This means every COMPOUND job (and transitively every ALTER job, since ALTER delegates to COMPOUND dispatch) with a nested primary resource request has never had its real slots/rusage availability checked by the scheduler, since this guard was introduced. + +## Fix +NOT FIXED -- discovered and characterized during scheduler-allocation task 5.1's independent review (2026-09-04); deliberately left out of that task's scheduler.c-dispatch boundary. Fixing requires either (a) constructing/staging a transient per-primary schedobj view when dispatching a nested primary through prepare_allocation (mirroring task 5.1's own sched_primary_cache_obj_() stack-local-schedobj pattern used for cache-key synthesis), or (b) relaxing seqjob_prepare_allocation()/parjob_prepare_allocation()'s guard to operate directly on ctx->primary (already available on ctx regardless of the enclosing schedobj's tag) instead of gating on ctx->schedobj->tag at all. Needs its own dedicated bugfix task/spec given the severity and blast radius (all COMPOUND/ALTER jobs, not scoped to scheduler-allocation). + +UPDATE (2026-09-25): 2026-09-25 (bug-1769 fix 1ecd348f): with ALTER now allocating for real, this gap is what prevents a test where an ALTER alternative genuinely fails and the next one is used -- compound/alter-nested primaries never check slot/rusage availability, so every alternative 'fits'. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1767 diff --git a/.wolf/bugs/bug-1769.md b/.wolf/bugs/bug-1769.md new file mode 100644 index 00000000..71265df3 --- /dev/null +++ b/.wolf/bugs/bug-1769.md @@ -0,0 +1,39 @@ +--- +id: "bug-1769" +status: "fixed" +status_updated: "2026-09-25" +status_note: "1ecd348f" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-04T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [] +tags: ["scheduler", "alter-resreq", "type-confusion", "heap-oob-read", "pre-existing", "severity-high", "untested", "scheduler-allocation-discovery"] +related: ["bug-1768"] +fix_commits: ["1ecd348f"] +legacy: true +--- + +# bug-1769: out-of-scope type confusion discovered during scheduler-allocation task 5.1's review… + +## Description +OPEN, NOT A CRASH (but reachable) -- pre-existing, out-of-scope type confusion discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). core_build_alter_schedobj() (src/scheduler/src/core_events.c) stores raw `xce_compound_schedobj_t*` (sizeof == 24 bytes) into `obj->u.alters` (an xlist_t), but sched_make_alter_allocation() (src/scheduler/src/scheduler.c) reads each entry back cast as `xce_sched_obj_t*` (sizeof == 176 bytes, with `offsetof(xce_sched_obj_t, u) == 136`) and dereferences `obj->u.compound->resreq` through that mis-typed pointer. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code read at both the construction site (core_events.c) and the read site (scheduler.c), plus a compiled offsetof/sizeof probe against the real headers during task 5.1's review: this is a genuine struct-layout mismatch, not a false alarm -- the two types do not share a compatible prefix layout. Any real ALTER-tagged job (the resreq grammar's `{A} || {B}` alternate syntax legitimately produces XCE_RESREQ_ALTER per xce_resreq.peg) dereferencing through this cast reads ~112 bytes past the actual 24-byte allocation, an out-of-bounds heap read. Separately confirmed during the same investigation that test_snapshot_load_allocator_regression.c's header comment claiming to already exercise this ALTER path is itself mistaken: its `"type == any"` fixture has no `{`/`}` braces, so per xce_resreq.peg's grammar it parses as XCE_RESREQ_PRIMARY, not ALTER -- it never actually drives sched_make_alter_allocation()'s u.alters read path. No test anywhere in the tree currently constructs a real multi-entry u.alters list and drives it through sched_make_alter_allocation(), so this defect is both real and currently completely untested. + +## Fix +NOT FIXED -- discovered and characterized during scheduler-allocation task 5.1's independent review (2026-09-04); deliberately left out of that task's scheduler.c-dispatch boundary. Needs a dedicated bugfix task: either change core_build_alter_schedobj() to store properly-typed xce_sched_obj_t* entries into u.alters (matching what sched_make_alter_allocation() expects), or change sched_make_alter_allocation() to read xce_compound_schedobj_t* entries directly without the intermediate xce_sched_obj_t* cast. Should be paired with a new regression test that actually constructs an ALTER-tagged job (real `{A}||{B}` resreq syntax) and drives it end to end, since the existing test that was believed to cover this does not. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1768 diff --git a/.wolf/bugs/bug-1770.md b/.wolf/bugs/bug-1770.md new file mode 100644 index 00000000..302adab8 --- /dev/null +++ b/.wolf/bugs/bug-1770.md @@ -0,0 +1,39 @@ +--- +id: "bug-1770" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-04T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [] +tags: ["scheduler", "resreq-parsing", "job-submission", "parjob-allocator", "pre-existing", "severity-high", "scheduler-allocation-discovery", "task-5.3"] +related: ["bug-1766", "bug-1768", "bug-1769"] +fix_commits: ["799a71ec", "5c87f1cc"] +legacy: true +--- + +# bug-1770: FIXED (2026-09-06, resreq-maxtasks-wiring feature, commits 799a71ec..5c87f1cc) + +## Description +FIXED (2026-09-06, resreq-maxtasks-wiring feature, commits 799a71ec..5c87f1cc). Originally: pre-existing, out-of-scope gap discovered during scheduler-allocation task 5.3's end-to-end integration test. The REAL job-ingest path could NEVER produce a primary resource request with maxTasks > 1, regardless of what task count a real job submission specified -- parjob_allocator.c's my_alloc() (maxTasks > 1) was therefore UNREACHABLE from any real production job submission. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code read and a genuine RED-phase run of test_parjob_e2e_multislot.c (src/scheduler/testsuite/): parse_resreq_primary() (src/lib/resreq.c:431-432) unconditionally hardcodes `primary->minTasks = 1; primary->maxTasks = 1;` after parsing a resreq string's select/rusage/span/same/affinity clauses, regardless of their content -- there is no resreq-string syntax for task count at all (LSF-style task count is a job-submission-level parameter, `job_submit_data_t.mintasks/maxtasks`, sibling to the resreq string, not nested inside it). Neither of this codebase's two job-ingest constructors -- core_build_job()/core_build_primary_schedobj() (src/scheduler/src/core_events.c, the real production JOBS_PUSH path) nor make_sched_job() (src/scheduler/src/schedobj.c, the sibling non-event construction path) -- ever copies the job's own submitted job_submit_data_t.mintasks/maxtasks (correctly propagated onto the TOP-LEVEL xce_sched_job_t.mintasks/maxtasks by both functions, confirmed working) onto the NESTED xce_primary_resreq_t.minTasks/maxTasks that sched_make_primary_alloc_()'s maxTasks>1 dispatch check (scheduler.c:290) and parjob_allocator.c's my_alloc() actually read. Net effect: every job's primary resreq that parses successfully has maxTasks pinned to 1, so sched_make_primary_alloc_() always routes it to seqjob_allocator instead of parjob_allocator (scheduler.c:290-294) -- parjob_allocator's real combinatorial multi-host allocation (scheduler-allocation spec, tasks 4.1-4.3) is fully implemented and unit-tested but cannot be exercised by any real job submission as the system stands today. Live-confirmed: test_parjob_e2e_multislot.c's own RED-phase run (maxTasks patch temporarily disabled) showed the job routing to seqjob and dispatching onto exactly 1 host instead of the intended 2 (`Expected 2 Was 1`). + +## Fix +FIXED (2026-09-06, resreq-maxtasks-wiring feature, commits 799a71ec..5c87f1cc, 12 tasks). Added apply_job_task_count(xce_resreq_t*, uint32_t mintasks, uint32_t maxtasks) (src/lib/resreq.c/resreq.h) -- overwrites a standalone top-level primary's minTasks/maxTasks with the job's real requested count (no-op for COMPOUND/ALTER nested primaries, deliberately left out of scope per bug-1768; defensive min>max clamp). Wired at both live ingest call sites: core_build_job() (core_events.c, fresh submission) using job->submit->mintasks/maxtasks, and the snapshot-restore path (snapshot_load.c, --sim/replay driver) using the job's already-restored mintasks/maxtasks -- both called strictly before their respective schedobj/resreq_fp builders read the task count. make_sched_job() (schedobj.c) intentionally NOT wired -- confirmed dead code (unreachable via sched_update(), which has zero live callers). Verified end-to-end via a new xce-test-multihost skill: a real multi-task job (and a multi-task array-job element) submitted via plain jobsub is genuinely split across two real xcecd_virtual hosts by the scheduler's own decision and reaches DONE on both. Full regression clean (scheduler/controller/worker suites, only pre-existing unrelated bug-1758/bug-1723 remain) and HA soak clean (sequential + chaotic, zero job loss/divergence). test_parjob_e2e_multislot.c's PATCH_MAXTASKS_FOR_BUG_1770 workaround is removed; that test now drives its scenario entirely through unmodified real ingest. Sibling gaps deliberately NOT fixed by this feature, tracked separately: bug-1772 (snapshot-restore resreq_fp gap), bug-1773 (resource-request update never taking effect for scheduling), bug-1774 (make_sched_job()'s dead-code resreq_fp gap). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1766, bug-1768, bug-1769 diff --git a/.wolf/bugs/bug-1771.md b/.wolf/bugs/bug-1771.md new file mode 100644 index 00000000..35628a39 --- /dev/null +++ b/.wolf/bugs/bug-1771.md @@ -0,0 +1,39 @@ +--- +id: "bug-1771" +status: "fixed" +status_updated: "2026-09-25" +status_note: "1ecd348f" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-04T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [] +tags: ["scheduler", "fcfs", "legacy-plugin", "null-deref", "asan", "pre-existing", "severity-low", "scheduler-allocation-discovery", "task-5.3"] +related: [] +fix_commits: ["1ecd348f"] +legacy: true +--- + +# bug-1771: ASAN SEGV (READ of address 0x0) in fcfs_del_job() (src/scheduler/src/plugins/fcfs.c:318… + +## Description +ASAN SEGV (READ of address 0x0) in fcfs_del_job() (src/scheduler/src/plugins/fcfs.c:318, `job->queue->name`), reached via fcfs_commit() <- plugin_commit_decision() <- sched_commit_decision() <- commit_decision() <- sched_allocator(), the very first time a real DISPATCH decision was committed through the real, live sched_allocator() thread in this test directory. Discovered live during scheduler-allocation task 5.3's own end-to-end integration test construction (test_parjob_e2e_multislot.c) -- full ASAN backtrace captured. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code read and the live ASAN backtrace. plugin_commit_decision() (src/scheduler/src/plugin.c:442-475) unconditionally fans a committed decision out to EVERY registered plugin's own `.hooks.commit_hook`, regardless of which plugin is the currently-ACTIVE job picker -- this includes the legacy 'xce_sched_fcfs' plugin (src/scheduler/src/plugins/fcfs.c), still registered alongside the modern default 'builtin_fcfs' by sched_register_plugins() (scheduler.c) even when it is never the resolved picker. fcfs_commit() (fcfs.c:326-337) calls fcfs_del_job(plugin, decision->u.alloc->jobid) for every XCE_DECISION_DISPATCH, and fcfs_del_job() (fcfs.c:303-324) unconditionally dereferences `job->queue->name` with no NULL check on `job->queue` -- sched_get_queue() (core.c:257-271) is get-ONLY (never auto-creates), so any job whose xce_sched_job_t.queue is NULL (i.e. its submitted queue name was never registered via a real QUEUE_UPDATE/CONFIG_PUSH event before the job reached DISPATCH) crashes this unrelated legacy plugin's own housekeeping, even though that plugin never elected or tracked the job at all. Every prior real-scheduler_srcs test in this directory that reaches a real sched_commit_decision() (test_scheduler_cache_dispatch.c, test_scheduler_cache_dispatch_compound.c, test_snapshot_load_allocator_regression.c) only ever produces PEND decisions (fcfs_commit()'s `default: break;` branch, never touching job->queue), so this NULL-deref had never been triggered before. Not reachable in real production as the system stands today (a real job's queue is always registered via controller-pushed config before any job can target it), but IS a genuine, real, unguarded NULL dereference in a code path that runs unconditionally for every real DISPATCH decision regardless of test/production context. + +## Fix +NOT FIXED -- discovered during scheduler-allocation task 5.3's end-to-end test construction (2026-09-04). test_parjob_e2e_multislot.c's own fixture was corrected instead (added a real XCE_CORE_EVENT_QUEUE_UPDATE push registering the job's queue before the JOBS_PUSH event, matching real production sequencing where queues are always configured before any job can target them) -- this is the correct fix for that specific test, not a production defect this task's own boundary covers. The underlying hardening gap (fcfs_del_job() missing a `job->queue == NULL` guard before dereferencing `job->queue->name`) remains open; recommended minimal fix for a dedicated follow-up: `if ((job == NULL) || (job->queue == NULL)) { return 0; }` in fcfs_del_job(), mirroring the function's own existing `if (job == NULL) { return 0; }` early-return style. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1772.md b/.wolf/bugs/bug-1772.md new file mode 100644 index 00000000..958d30f1 --- /dev/null +++ b/.wolf/bugs/bug-1772.md @@ -0,0 +1,41 @@ +--- +id: "bug-1772" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-05T00:00:00Z" +last_seen: "2026-09-23T10:05:00Z" +occurrences: 1 +files: [] +tags: ["scheduler", "snapshot-restore", "resreq-fingerprint", "cache", "pre-existing", "schedobj-duplication", "severity-medium", "tdd", "audit-reopened"] +related: ["bug-1767", "bug-1770"] +legacy: true +gitea_issue: 52 +gitea_synced_status: "fixed" +--- + +# bug-1772: out-of-scope gap discovered during requirements scoping for the resreq-maxtasks-wiring spec… + +## Description +OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during requirements scoping for the resreq-maxtasks-wiring spec (bug-1770's fix). snapshot_build_primary_schedobj() (src/scheduler/src/snapshot_load.c:81-96) never computes primary->resreq_fp at all, unlike core_build_primary_schedobj() (src/scheduler/src/core_events.c:130-135), which sets resreq_fp.min_tasks/max_tasks/select_node_id/rusage_hash from the parsed xce_primary_resreq_t immediately after parsing. Every job reconstructed from a snapshot (post-restart or post-failover restore) therefore gets a zeroed resreq_fp -- the same class of gap as bug-1767's broader 'recurring schedobj-reconstruction-duplication' pattern (snapshot_load.c independently re-implements core_events.c's schedobj construction rather than sharing it, and each independent copy has repeatedly drifted field-by-field). TRIGGER CORRECTION (2026-09-05, resreq-maxtasks-wiring design review): this entry's original phrasing ('post-restart or post-failover restore') overstates the live trigger. xce_sched_core_load_snapshot()'s ONLY real (non-test) caller anywhere in src/ is sim.c's sim_locate_and_load_snapshot(), reachable only via main.c's --sim FILE offline replay flag. Normal-mode main.c has no snapshot-load call or flag at all; a live scheduler restart (including post-failover, e.g. run-ha-soak.sh's restart_scheduler_for_new_primary()) starts with empty in-memory state and rejoins via on_scheduler_join() -> the same core_build_job() path fresh ingest uses -- never touching snapshot_load.c. The accurate trigger is: a job in a .replay-recorded snapshot loaded by the --sim simulation/replay driver, not any live-cluster crash-recovery or failover scenario. This does not change the bug's validity or fix (the gap is real for --sim/replay fidelity, which matters per this project's driver-equivalence invariant since --sim drives the identical allocator dispatch) -- it only corrects when it's actually reachable, which matters for scoping test coverage (an HA soak does NOT exercise this path at all). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +snapshot_load.c's snapshot_build_primary_schedobj() is a hand-duplicated, field-for-field port of core_events.c's core_build_primary_schedobj() (kept independent because core_build_primary_schedobj() is static and the two files are compiled as separate objects -- see bug-1767's own root-cause text for the deliberate-duplication rationale). The port was written before task 1.2 (scheduler-allocation feature) added resreq_fp/hash_rusages() wiring to the core_events.c original, and the snapshot_load.c copy was never updated to match. Effect: a snapshot-restored job's unsatisfiable-resreq cache fingerprint (used by scheduler-allocation's alloc_cache.c) is always the zero value, which is a valid-looking but WRONG fingerprint -- it could produce a false cache hit against an unrelated job that also happens to have a zeroed fingerprint (extremely unlikely today since real jobs always populate resreq_fp correctly at first ingest, but a snapshot-restored job's own fingerprint stays wrong for its entire post-restore lifetime, and any code that later relies on comparing/keying by resreq_fp for a restored job silently gets 0/0/0/0). Discovered while scoping resreq-maxtasks-wiring (bug-1770's fix spec) because that fix will make resreq_fp carry real, meaningful, non-degenerate maxTasks values for the first time -- increasing the practical consequence of this gap versus its current all-zero-everywhere baseline. + +## Fix +FIXED (2026-09-23, TDD, /xce-audit-invariants scheduler dimension-4 audit -- independently re-found by both a dedicated subagent and direct investigation into bug-1845; this entry's own prior 'fixed'/'closed-by-scheduler-sim-live-allocation' tags were STALE/WRONG -- direct code inspection confirmed snapshot_build_primary_schedobj() still had no resreq_fp computation immediately before this fix). Added the same four-field resreq_fp computation core_build_primary_schedobj() (core_events.c) performs -- min_tasks/max_tasks/select_node_id/rusage_hash -- to snapshot_build_primary_schedobj() (snapshot_load.c), matching that function's own documented 'field-for-field port' contract, which had simply never included this field. Added test_snapshot_load_computes_resreq_fingerprint_for_restored_primary() (test_snapshot_save_load.c) -- a job restored from a snapshot with mintasks=4/maxtasks=6 must have schedobj->u.primary->resreq_fp.min_tasks/max_tasks reflect those real values, not {0,0,0,0}. Verified genuine red->green: temporarily reverted just the new resreq_fp assignment and reran -- scheduler snapshot save load test FAILED (exit 1); restored the fix and reran -- full 85/85 scheduler suite green. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1767, bug-1770 +- Gitea: #52 diff --git a/.wolf/bugs/bug-1773.md b/.wolf/bugs/bug-1773.md new file mode 100644 index 00000000..05d11d1c --- /dev/null +++ b/.wolf/bugs/bug-1773.md @@ -0,0 +1,39 @@ +--- +id: "bug-1773" +status: "fixed" +status_updated: "2026-09-27" +status_note: "41c3694d" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-05T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [] +tags: ["scheduler", "job-update", "resreq", "schedobj", "pre-existing", "severity-high", "resreq-maxtasks-wiring-discovery"] +related: ["bug-1770"] +fix_commits: ["41c3694d"] +legacy: true +--- + +# bug-1773: out-of-scope gap discovered during design review for the resreq-maxtasks-wiring spec (bug-1770's… + +## Description +OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during design review for the resreq-maxtasks-wiring spec (bug-1770's fix). A job's resource-request UPDATE (JOB_UPDATE core event / sched_update_existing_job()) never takes effect for scheduling purposes, for ANY field -- not just task count. sched_update_existing_job() (src/scheduler/src/schedobj.c:597-656) and core_events.c's JOB_UPDATE handler (src/scheduler/src/core_events.c:1200-1234) both do `free(job->resreq); job->resreq = newreq;` (a shallow free via FREEUP, utils.h:114) but never touch job->schedobj. ADDENDUM (2026-09-05, independent follow-up investigation): sched_update_existing_job() itself is dead code (unreachable -- its sole caller is scheduler.c's sched_update(), which has zero live callers, same root architecture as bug-1740). core_events.c's JOB_UPDATE handler IS live (traced main.c -> driver.c's xce_sched_driver_translate_pending_events() -> xce_core_push_events() -> the JOB_UPDATE case), reached routinely by ordinary job-lifecycle notifications (state/rusage/postdone), not only by a genuine resource-request change -- but grepping the whole controller finds no code path that ever mutates an existing job's submit->resreq after initial submission, so this live handler's parse_resreq() branch always re-parses the job's original, unchanged resreq string today. No real 'update a job's resource request' feature exists yet in this codebase; this bug describes what WOULD happen if one were added without also fixing job->schedobj reconstruction. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code read: the scheduler's actual dispatch reads job->schedobj (scheduler.c:483, sched_make_allocation(sched, job->schedobj)), not job->resreq. sched_make_primary_allocation() (scheduler.c:315) reads xce_primary_resreq_t* primary = obj->u.primary->resreq -- i.e. job->schedobj->u.primary->resreq, a pointer set exactly once, at ingest, by core_build_primary_schedobj()/make_primary_schedobj() (primary->resreq = resreq, aliasing job->resreq->u.primary at that moment). job->schedobj is built ONLY in core_build_job() (core_events.c:259-265) and make_sched_job() (schedobj.c:194-202) -- confirmed via grepping every `job->schedobj =` assignment in the scheduler; neither update path is among them. So after any resreq update, job->schedobj->u.primary->resreq keeps pointing at the ORIGINAL, ingest-time xce_primary_resreq_t forever, completely disconnected from the freshly-parsed newreq the update path installed into job->resreq. Any change an operator makes to a pending job's select/rusage/task-count via a resource-request update is silently invisible to the scheduler's own dispatch decision -- only a job's original, at-submission resreq is ever actually evaluated. + +## Fix +NOT FIXED -- discovered during resreq-maxtasks-wiring's design review (2026-09-05); deliberately kept out of that spec's scope (it is not about task-count wiring, and fixing it changes long-standing update semantics far beyond this feature's boundary). Needs its own dedicated bugfix task/spec: the update paths must rebuild (or re-point) job->schedobj's primary->resreq to the newly-parsed newreq, mirroring how core_build_job()/make_sched_job() construct it at ingest, probably by extracting a shared 'build/refresh primary schedobj from a resreq' helper both ingest and update can call. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1770 diff --git a/.wolf/bugs/bug-1774.md b/.wolf/bugs/bug-1774.md new file mode 100644 index 00000000..fdce70b3 --- /dev/null +++ b/.wolf/bugs/bug-1774.md @@ -0,0 +1,38 @@ +--- +id: "bug-1774" +status: "wontfix" +status_updated: "2026-09-24" +status_note: "accepted: dead code path (make_primary_schedobj resreq_fp), zero live impact" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-05T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [] +tags: ["scheduler", "resreq-fingerprint", "dead-code", "schedobj-duplication", "pre-existing", "severity-low", "resreq-maxtasks-wiring-discovery", "corrected"] +related: ["bug-1767", "bug-1770", "bug-1772"] +legacy: true +--- + +# bug-1774: gap discovered during design review for the resreq-maxtasks-wiring spec, sibling to bug-1772… + +## Description +OPEN, NOT A CRASH, DEAD-CODE-ONLY -- pre-existing gap discovered during design review for the resreq-maxtasks-wiring spec, sibling to bug-1772, CORRECTED 2026-09-05: this entry originally claimed make_sched_job()'s ingest path was "confirmed live, not legacy" -- that claim was WRONG and has been retracted (see root_cause). make_primary_schedobj() (src/scheduler/src/schedobj.c:45-63, backing make_sched_job()) never computes primary->resreq_fp, unlike core_build_primary_schedobj() (src/scheduler/src/core_events.c:130-135) which sets resreq_fp.min_tasks/max_tasks/select_node_id/rusage_hash from the parsed xce_primary_resreq_t immediately after parsing -- but make_sched_job() itself is unreachable in production today (see root_cause), so this gap currently has ZERO live impact. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code read that make_primary_schedobj() calloc's the primary schedobj and leaves resreq_fp all-zero, with no equivalent computation anywhere in the function -- identical failure shape to bug-1772 (snapshot_load.c's snapshot_build_primary_schedobj() has the same gap), a third instance of the recurring schedobj-reconstruction-duplication pattern already tracked by bug-1767. HOWEVER, independently re-verified 2026-09-05 (two separate investigations, exhaustive grep of every call site of sched_update(), sched_update_jobs(), and make_sched_job() across src/ outside testsuite/): sched_update(xce_scheduler_t*) (scheduler.c:1200) has ZERO live callers anywhere in this codebase -- only its own declaration/definition and doc-comments describing it as dead/superseded exist (matching bug-1740's already-established finding for the sibling sched_apply_job_rusage() case, same root architecture). sched_update_jobs() (scheduler.c:1109) is called only from inside sched_update()'s switch, and make_sched_job() (schedobj.c:153) is called only from sched_update_jobs() -- so the ENTIRE chain is unreachable in production. The real, live production job-ingest paths are exactly two: core_build_job() (core_events.c, fresh submission via the driver-translated XCE_CORE_EVENT_JOBS_PUSH path) and snapshot_load.c's restore path -- both confirmed live by the same grep rigor. This entry's original "confirmed live, not legacy" parenthetical was an unverified, incorrect aside and is hereby retracted; make_sched_job()'s resreq_fp gap is real but currently dead-code-only, with no production consequence unless/until something makes this path reachable again. + +## Fix +NOT FIXED, and NOT CURRENTLY WORTH FIXING given zero live impact -- discovered during resreq-maxtasks-wiring's design review (2026-09-05), corrected same day once the dead-code status was established. resreq-maxtasks-wiring's own Requirement 5.4 does NOT need an exception for this path (it can be excluded from consideration entirely, not merely excluded-and-documented, since it never ingests any real job) -- that spec's actual Req 5.4 exception is for bug-1772's snapshot-restore gap, which IS live. If make_sched_job()'s dead code path is ever revived or repurposed in a future change, revisit this fix then: add the same resreq_fp computation core_build_primary_schedobj() does to make_primary_schedobj(), or better, per bug-1767's broader recommended remediation, share one real schedobj-construction implementation across all three call sites instead of maintaining three independent, drift-prone copies (two live, one dead). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1767, bug-1770, bug-1772 diff --git a/.wolf/bugs/bug-1775.md b/.wolf/bugs/bug-1775.md new file mode 100644 index 00000000..0506fe09 --- /dev/null +++ b/.wolf/bugs/bug-1775.md @@ -0,0 +1,38 @@ +--- +id: "bug-1775" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-09-06T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [] +tags: ["test-infrastructure", "shell-script", "regex", "array-job", "xce-test-multihost", "severity-low"] +related: ["bug-1770"] +legacy: true +--- + +# bug-1775: xce-test-multihost's run-multihost-verify.sh (resreq-maxtasks-wiring task 4.3, array-job… + +## Description +xce-test-multihost's run-multihost-verify.sh (resreq-maxtasks-wiring task 4.3, array-job scenario): verify_task_started_on_host()'s grep -E pattern embedded a raw jobid string directly in an extended-regex alternation. A bracketed array-element jobid (e.g. "[0]") has its "[0]" misparsed by grep -E as a one-character bracket-expression class (matching a literal '0') instead of the literal 3-character substring "[0]" -- causing a false negative: `grep -qE 'Job base[0] ntasks is:'` does not match the literal log line `"Job base[0] ntasks is: 5"`. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Extended regular expressions treat unescaped `[...]` as a bracket-expression (character class), not a literal substring. Array-element jobids introduced by task 4.3 are the first jobid format in this harness containing ERE metacharacters (plain jobids are digits/hyphens only); the pre-existing plain-job code path never hit this because grep -E on a digits-and-hyphens-only string has no metacharacters to misparse. + +## Fix +FIXED (2026-09-06, resreq-maxtasks-wiring task 4.3): added ere_escape() to run-multihost-verify.sh, applied to any jobid string before use in a grep -E pattern (escapes '[' and ']', the only metacharacters a jobid can contain per fmt_jobid_ex/REGEX_JOBARRAY_ID's digit/hyphen/bracket-only format). Confirmed a byte-identical no-op for plain (non-array) jobids. Reproduced the false negative standalone before the fix and confirmed the escaped pattern matches correctly after, independently re-verified by review. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1770 diff --git a/.wolf/bugs/bug-1776.md b/.wolf/bugs/bug-1776.md new file mode 100644 index 00000000..42bf03d8 --- /dev/null +++ b/.wolf/bugs/bug-1776.md @@ -0,0 +1,38 @@ +--- +id: "bug-1776" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-06T21:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_info.c"] +tags: ["scheduler", "repl", "sim", "assigned-jobs", "ux-improvement"] +related: ["bug-1601", "bug-1591"] +legacy: true +--- + +# bug-1776: User improvement request (simulator REPL feedback): 'info worker's assigned jobs' listing gives no… + +## Description +User improvement request (simulator REPL feedback): 'info worker's assigned jobs' listing gives no indication of how many 'slots' each assigned job actually consumes on that worker -- every assigned job looks identical regardless of whether it occupies 1 slot or many (e.g. a fully-packed 22-slot worker running one 2-slot job shows the exact same line shape as a 1-slot job). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT A BUG -- feature gap. repl_info_worker_one()'s assigned-jobs loop already walks each job's xce_host_alloc_t (per-host allocation) to match it against the current worker's name, but only extracted job->jobid and discarded the xce_host_alloc_t itself (`break`) without ever inspecting its `allocs[]` array, which already carries a per-resource allocation list (same struct/pattern the controller's own get_nslots_in_alloc(), job.c, already scans for the identical 'slots' resname) -- so the slot count was always available with no new wire/plumbing needed, just never read at this call site. + +## Fix +FIXED (2026-09-06). Added repl_worker_alloc_nslots() (repl_info.c) mirroring get_nslots_in_alloc()'s resname=="slots" scan (duplicated rather than shared, since that helper is file-static to job.c and this is a read-only REPL display concern). The assigned-jobs loop now also collects a parallel assigned_slots[] array (repl_append_int(), new int-array counterpart to the existing repl_append_u64()). When every assigned job on a worker consumed the SAME slot count and that count is >1, the existing compressed jobid-range line gets a trailing '*N' (e.g. 'assigned jobs: 0-14-1788751435-0*2'); a single-slot-per-job worker (the common case) renders identically to before. When assigned jobs on the SAME worker have DIFFERING slot counts, a single '*N' suffix on a compressed range would be ambiguous, so the listing falls back to the pre-existing uncompressed per-job lines (same fallback tier a too-small compression buffer already used), each annotated with its own count. New tests in test_repl_resource_table.c: test_info_worker_assigned_job_listing_appends_slot_count_when_uniform, test_info_worker_assigned_job_listing_falls_back_per_job_when_slots_differ. Full scheduler suite: 76/77 pass (the one failure, test_job_rusage_reconciles_mem_reservation_against_usage in test_core_events.c, is a pre-existing unrelated mem-reservation-subsystem failure already documented in bug-1762, untouched by this change). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1601, bug-1591 diff --git a/.wolf/bugs/bug-1777.md b/.wolf/bugs/bug-1777.md new file mode 100644 index 00000000..dc83353a --- /dev/null +++ b/.wolf/bugs/bug-1777.md @@ -0,0 +1,38 @@ +--- +id: "bug-1777" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test workerlist MaxSwap ignores hosts.yaml override" +component: "controller" +severity: "unknown" +timestamp: "2026-09-06T21:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c"] +tags: ["controller", "worker", "resource", "maxswap", "workerlist", "config-override", "deferred"] +related: ["bug-1554"] +legacy: true +--- + +# bug-1777: xce_test workerlist's MaxSwap column can permanently disagree with the scheduler's own… + +## Description +xce_test workerlist's MaxSwap column can permanently disagree with the scheduler's own resmgr/replay view for the same worker: workerlist always shows the join-time hardware_monitor stub/detected value, never a hosts.yaml per-host maxswap override, while the scheduler always uses the override. User noticed this while comparing a scheduler simulator replay (maxswap=10000) against xce_test workerlist (MaxSwap=1.00 GB) for the same host and asked whether the two should be consistent. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via code trace (not yet reproduced against a live pc13v/replay artifact -- none existed on disk at investigation time). xce_worker_t carries two separate copies of a host-attrs-style resource: attrs->maxswap (hostattrs_t, written ONLY by xcecd.c's init_host_attrs() at join time -- rejoin_worker(), worker.c:440, just re-copies the same join-time value) and resvals[idx("maxswap")] (resmgr-indexed zarray, seeded from attrs->maxswap by add_hostattrs_resvals() at apply_join_state() start, worker.c, then OVERWRITTEN by mark_host_res()/update_val_res() if a hosts.yaml resource override exists for that host, worker.c:79 `array = &(w->host.resvals)`). cmd_workerlist() (test.c:1478) reads attrs->maxswap directly -- structurally incapable of ever reflecting a config override, since nothing writes an override back into attrs. The scheduler-facing path (collect_resvals_workers() -> resvals_to_sched -> xcesched_push_workers(), dataex.c/sched.c) reads resvals, which DOES pick up the override. So whenever a maxswap override is configured, the scheduler correctly uses it (and a replay/snapshot reflects it) while xce_test workerlist permanently shows the stale join-time value regardless of timing/staleness -- not a caching/race artifact, a structural gap. The Go apiserver/xcecli path (convert.go) also just casts attrs.maxswap with no override-awareness, and xcecli's hosts.yaml model doesn't even expose a MaxSwap column today. + +## Fix +NOT FIXED -- user explicitly deferred (2026-09-06) to prioritize bug-1554 (real swap/tmp hardware detection, now fixed) and the assigned-jobs slot-count display improvement first. Two candidate fix directions discussed: (a) make cmd_workerlist()/the Go apiserver path read the effective resmgr/resvals value instead of attrs directly, or (b) make mark_host_res() write the override back into attrs too, so every attrs consumer (maxcpus/maxmem/maxswap/maxtmp alike) stays in sync at the source -- (b) was the recommended direction since it fixes the whole class rather than patching each display path individually, but neither is implemented yet. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1554 diff --git a/.wolf/bugs/bug-1778.md b/.wolf/bugs/bug-1778.md new file mode 100644 index 00000000..1463007c --- /dev/null +++ b/.wolf/bugs/bug-1778.md @@ -0,0 +1,38 @@ +--- +id: "bug-1778" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-07T06:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_info.c"] +tags: ["job-lifecycle", "stale-state", "repl", "snapshot", "scheduler", "related-to-bug-1776", "assigned-jobs", "persistence", "needs-live-repro", "related-to-bug-1764", "interim-fix", "sim"] +related: ["bug-1764", "bug-1776"] +legacy: true +--- + +# bug-1778: Live-observed (interactive sim session=29, replaying a real captured trace): three plain… + +## Description +Live-observed (interactive sim session=29, replaying a real captured trace): three plain (non-array) jobs already at JOB_STAT_DONE (0-14-1788751409-0, 0-14-1788751435-0, 0-14-1788751504-0) still appear in `info worker pc13v`'s 'assigned jobs' listing (each annotated '*2' via bug-1776's slot-count fix), alongside the genuinely-running array job elements. User: 'The finished job(exit/done) should be de-assign from worker.' + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +PARTIALLY CONFIRMED, NOT FULLY ROOT-CAUSED this session (user asked to record, not fix). Confirmed facts: (1) repl_info_worker_one()'s assigned-jobs loop (repl_info.c, ~line 686) gates solely on `job->allocs != NULL`, with no independent job->state check -- if allocs is non-NULL for a terminal job, it will always display, by design (mirrors the exact structural gap bug-1764 already documented for this same function). (2) The worker's ACTUAL resource accounting is NOT confused by this: the same session's 'resource slots: value=22.00 used=20.00' correctly reflects only the 5 genuinely-running 4-slot array elements (5*4=20), excluding the 3 DONE jobs entirely -- proving core.resmgr/resInUse (the real admission-control state) is fine; only the assigned-jobs DISPLAY loop (which walks job->allocs directly, a separate field) is stale for these 3 jobs. (3) This is NOT a regression of bug-1764's fix: verified sim_loop.c:713-714 still correctly excludes JOB_IS_DONE(job->state)||JOB_IS_EXIT(job->state) from sim_reconcile_unaccounted_jobs()'s resurrection path, so that specific documented mechanism is not what's reintroducing these allocs. (4) core_events.c's XCE_CORE_EVENT_JOB_STATE_CHANGE (line ~1155-1163) and XCE_CORE_EVENT_JOB_UPDATE (line ~1207-1215) handlers both correctly free_JobAlloc_ptr()+null job->allocs atomically alongside the state write whenever `!JOB_IS_RUN(new_state)` -- so if either of THESE handlers was what drove these 3 jobs to DONE, allocs should already be NULL; something else must be true. Leading unconfirmed hypothesis: snapshot_load.c:490-493 deep-copies wj->allocs from the .replay snapshot's wire payload UNCONDITIONALLY whenever wj->allocs != NULL, with no check against the snapshot's own wj->state -- if the live production recording's snapshot-capture moment (snapshot_wire.c's save side) captured these 3 jobs' allocs and state in a mutually inconsistent way (e.g. a save-time field-read ordering gap, or the snapshot genuinely predates their JOB_STATE_CHANGE/JOB_UPDATE terminal cleanup while a separately-recorded later event updated state to DONE without a matching allocs-clearing event reaching this replay stream), the replay would faithfully reproduce that inconsistency verbatim. NOT verified with a live trace/gdb this session -- flagged per this project's own persistence.md field-completeness concern (a state that 'recovers correctly' while an unrelated field goes stale is the exact class of bug that steering calls out), not asserted as confirmed. + +## Fix +INTERIM DISPLAY-LAYER MITIGATION APPLIED (2026-09-07, per user request). repl_info_worker_one()'s assigned-jobs loop (repl_info.c) now also excludes JOB_IS_DONE(job->state) || JOB_IS_EXIT(job->state), in addition to the existing job->allocs != NULL check -- mirrors the same 'terminal is permanent' guard job_state_regresses_terminal()/sim_reconcile_unaccounted_jobs() already apply elsewhere. TDD: new test test_info_worker_assigned_job_listing_excludes_terminal_job_despite_stale_allocs (test_repl_resource_table.c) directly assigns the SAME allocs struct to a DONE job and a RUN job on the same worker, forcing the DONE job's state after assigning allocs (reproducing the exact state/allocs inconsistency without needing to root-cause how it arises live) -- confirms the DONE job is excluded while the RUN job still shows. Full scheduler suite: 76/77 pass (same pre-existing unrelated test_core_events.c failure as before, bug-1762). NOTE: this masks the symptom at the display layer only -- the underlying question of WHY job->allocs stays non-NULL for a terminal job (suspected snapshot save/load inconsistency) remains unresolved; a real fix should also prevent/clear the stale allocs at its source. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1764, bug-1776 diff --git a/.wolf/bugs/bug-1779.md b/.wolf/bugs/bug-1779.md new file mode 100644 index 00000000..ed93e61d --- /dev/null +++ b/.wolf/bugs/bug-1779.md @@ -0,0 +1,38 @@ +--- +id: "bug-1779" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-07T06:58:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["array-job", "builtin_fcfs", "decision", "dispatch", "job-lifecycle", "needs-live-repro", "pend", "picker", "scheduler", "scheduler-allocation", "sim", "spec-conflict"] +related: ["bug-1778"] +legacy: true +--- + +# bug-1779: Live-observed (interactive sim session=19, replaying a real captured trace, 'step 9'): within the… + +## Description +Live-observed (interactive sim session=19, replaying a real captured trace, 'step 9'): within the SAME batch of scheduler decisions, jobs 0-12-1788762745-0[0..4] receive a DISPATCH decision, then immediately ALSO receive a PEND decision in the same step's output -- and `info jobs` afterward shows all 10 array elements (0-12-1788762745-0[0..9]) as PEND, including the 5 that were just dispatched. User: 'when a job has been dispatched (in scheduler), we should treat it as RUN, so it should be excluded from PEND ... we need to remove 0-12-1788762745-0[0-4] from PEND but show in RUN.' + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED FOR REAL (2026-09-07), and turns out to be a SPEC CONFLICT, not a logic bug -- a patch was attempted and REVERTED after it broke a pre-existing, passing regression test. Confirmed mechanism: sim mode's REAL DISPATCH decisions come entirely from sim_reconcile_unaccounted_jobs() (sim_loop.c), which reconstructs a job's dispatch PURELY from replay data (populating job->allocs/job->accounts, reporting a synthesized DISPATCH decision) but never calls picker->picker_iface->on_dispatch() -- unlike the real/live allocator path (sched_allocator(), scheduler.c line 507-510), which ALWAYS calls on_dispatch() to retire the job from the picker's own candidate list (fcfs_queue_t.pendJobs) before building the decision. Because reconciliation skips this, the job stays a live candidate in the picker's bookkeeping. sim_picker_thread_fn() (sim_loop.c) -- a SEPARATE, always-running thread that walks next_job() every round and unconditionally PENDs whatever it is still offered, since it has no real resource-matching allocation of its own -- keeps re-offering and re-PENDing that job every subsequent round until an explicit JOB_STATE_CHANGE-to-RUN event (also replayed from the original recording, can legitimately arrive several rounds later) removes it from pendJobs via builtin_fcfs_on_job_state_change(). CRITICAL DISCOVERY: this exact behavior is NOT accidental -- it is DOCUMENTED, INTENTIONAL, and covered by an existing passing test, test_sim_verify_no_false_mismatch_across_round_separated_dispatch (test_sim_verify.c, scheduler-sim-eager-reconciliation spec task 3.2), whose own header comment states verbatim: 'sim_reconcile_unaccounted_jobs() captures job->accounts but never calls picker_iface->on_dispatch() to retire the job from the picker's own internal pendJobs (design.md's Non-Goals/Revalidation Triggers explicitly leave live-picker DISPATCH emission to a future scheduler-allocation spec)'. That spec's own design.md (.kiro/specs/scheduler-sim-eager-reconciliation/design.md, Non-Goals section) explicitly lists 'Any change to simulation mode's live picker (still PEND-only for freshly-arriving jobs, per scheduler-allocation)' as OUT OF SCOPE. A fix attempt (calling on_dispatch() from the reconciliation FOUND branch, mirroring sched_allocator()'s exact ordering) was implemented with a new TDD test, confirmed the bug reproduces deterministically (RED), and confirmed the fix resolves it (GREEN) -- but broke test_sim_verify_no_false_mismatch_across_round_separated_dispatch, whose fixture explicitly bakes in 'PEND(jobid) recorded in the SAME round as its own DISPATCH, and again for every round until the RUN confirmation arrives' as the CORRECT, expected --sim-verify-clean shape. The fix was REVERTED (both the sim_loop.c change and its new test) rather than kept, since it silently violates an already-agreed, tested design contract from a sibling spec. + +## Fix +FIXED FOR REAL (2026-09-07), after a first fix attempt was correctly reverted (see prior root_cause note) upon discovering it broke test_sim_verify_no_false_mismatch_across_round_separated_dispatch. That test's OWN fixture turned out to be baking in the bug's exact symptom as 'expected' -- but its parent spec's own design.md ('Integration with --sim-verify') already documented the CORRECT assumption ('the picker never re-decides an already-resolved job'), which the implementation had silently drifted from. This was a genuine implementation/test drift from an already-agreed design intent, not a live spec conflict requiring a product decision -- confirmed by re-reading both scheduler-sim-decision-parity's and scheduler-sim-eager-reconciliation's own design.md files (both updated in place with 2026-09-07 correction notes explaining this). Fix: sim_reconcile_unaccounted_jobs() (sim_loop.c) now resolves the job picker and calls picker_iface->on_dispatch() immediately after reconstructing job->allocs/job->accounts in its FOUND branch, mirroring sched_allocator()'s own live-dispatch ordering (scheduler.c line 507-515) exactly. This retires the reconciled job from the picker's own pendJobs bookkeeping the moment its dispatch is reconstructed, so sim_picker_thread_fn()'s unconditional-PEND thread never re-offers/re-PENDs it on a later round. TDD: new test test_reconcile_removes_reconciled_job_from_picker_candidates (test_sim_reconcile_unaccounted.c) directly proves a job is no longer offered by next_job() after reconciliation, using sim_bootstrap_picker() to register a real, resolvable builtin_fcfs instance -- confirmed RED before the fix, GREEN after. test_sim_verify_no_false_mismatch_across_round_separated_dispatch (test_sim_verify.c) was updated to match the corrected behavior: round 1 now records DISPATCH-only (new helper append_replay_session_with_dispatch(), no accompanying PEND), and rounds 2-3 are now empty ReplaySessions (matching round 4's own existing pattern) instead of baking in a stale re-PEND echo as expected. Full scheduler suite: 76/77 pass (same pre-existing unrelated test_core_events.c failure, bug-1762, untouched by this fix). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1778 diff --git a/.wolf/bugs/bug-1780.md b/.wolf/bugs/bug-1780.md new file mode 100644 index 00000000..00679ffb --- /dev/null +++ b/.wolf/bugs/bug-1780.md @@ -0,0 +1,38 @@ +--- +id: "bug-1780" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-09-07T07:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/convert.go"] +tags: ["apiserver", "convert.go", "execuser", "job-submit", "jobstartinfo", "nobody", "permission-denied", "worker", "xcecli"] +related: [] +legacy: true +--- + +# bug-1780: User-reported (xcecli): 'xcecli jobs submit -n 2 --env /run/current-system/sw/bin/sleep 100'… + +## Description +User-reported (xcecli): 'xcecli jobs submit -n 2 --env /run/current-system/sw/bin/sleep 100' submits successfully but the job immediately fails with Status . xcecd.log shows: '[WARN][controller_connector.c:822] No execuser provided in startjob request, defaulting to nobody', then 'Starting worker as user nobody to process command...', then 'Worker 301734 exited with code 126'. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +modelJobToSubmit() (src/frontend/apiserver/backend/convert.go, converts a Go models.JobInfo into the C job_startinfo_t sent to the controller) populated si.cmdline/launch_type/job_pre_exec/job_post_exec but never set si.execuser, even though job_startinfo_t (src/include/xce/types.h) has an execuser field and models.JobInfo already has an ExecUser field (used only on the read/display path, in convert_pure.go's rawJobToModel). CreateJob() (controllers/jobs.go) also never populated JobInfo.ExecUser when building the job to submit -- there is no --exec-user flag on xcecli either, so the field was simply always empty at submission. The controller (src/controller/job.c) forwards job.submit->start_info verbatim to the worker without ever touching execuser. xcecd's controller_connector.c (lines 817-822) defaults an empty/NULL execuser to the literal Linux user 'nobody', then (lines 1037-1040) copies that into job_ctx->limits->user_name, which xcecd_dbus_manager.c uses as systemd-run's User= for the job's transient unit. 'nobody' has no home directory/login shell and typically cannot execute or access the real submitting user's binaries/cwd, so the transient unit fails with 'Permission denied' -- surfacing to xcecd_worker_utils.c as exit_code=126, which xcecli then reports as job Status . Every job submitted via xcecli/apiserver was silently running as 'nobody' regardless of who submitted it. + +## Fix +FIXED (2026-09-07), TDD RED->GREEN. Added resolveExecUser(execUser, submitUser string) string to convert_pure.go -- a pure-Go helper (no CGO types) that returns the caller-specified exec user if present, else defaults to the submitting user; this mirrors the existing pure/CGO split in this file (rawJobToModel etc. already keep pure field-mapping logic separate from the CGO struct-building in convert.go specifically so it's testable without CGO -- and because a _test.go file can never itself import "C" (golang/go#13938), this split is the only way to unit-test any part of modelJobToSubmit's logic). modelJobToSubmit() now sets si.execuser = C.CString(resolveExecUser(job.ExecUser, job.SubmitUser)); freeJobSubmit() now also frees si.execuser (previously would have leaked once set). New tests in convert_test.go: TestResolveExecUser_DefaultsToSubmitUserWhenEmpty, TestResolveExecUser_PrefersExplicitExecUser, TestResolveExecUser_EmptyWhenBothEmpty -- pass under CGO_ENABLED=0 go test ./backend/... (RED confirmed by the function not existing before this change). Verified with a full /xce-build apiserver asan --incremental: exit 0, all frontend Go suites pass including apiserver/backend (7.595s, real CGO build), fresh xce_apiserver/xcecli binaries produced. Live re-run of the exact reported repro (xcecli jobs submit ... /run/current-system/sw/bin/sleep 100) not yet performed in this session -- recommend a live smoke test to confirm the job now runs as the submitting user and reaches DONE/RUN instead of EXIT before closing. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1781.md b/.wolf/bugs/bug-1781.md new file mode 100644 index 00000000..40c7b52e --- /dev/null +++ b/.wolf/bugs/bug-1781.md @@ -0,0 +1,41 @@ +--- +id: "bug-1781" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-09-07T08:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/clib.go (ControlJob)", "src/lib/request.c (xce_jobctrl_req)", "src/lib/controller.capnp.h (JobControl_u_which)", "src/include/xce/types.h (xce_jobctrl_t)", "src/controller/job.c (job_ctrl)"] +tags: ["apiserver", "capnp", "clib.go", "job-control", "kill", "stop", "resume", "checkpoint", "union-discriminant", "undefined-behavior", "gitea-45", "live-verified"] +related: ["bug-1742"] +legacy: true +gitea_issue: 45 +gitea_synced_status: "fixed" +--- + +# bug-1781: Gitea issue #45 REOPENED after bug-1742's fix + +## Description +Gitea issue #45 REOPENED after bug-1742's fix. User-reproduced: after bug-1742 fixed the earlier panic/empty-error-body symptom, `xcecli jobs kill ` (and stop/resume/checkpoint) now fails with a real, non-empty error instead: `request to /xce/v1/jobs//control failed: [failed to write data: xcecli_jobctrl failed: -22]` (-22 = -EINVAL). Reproduced live for kill/stop/resume/checkpoint, all failing. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ControlJob() (clib.go) set xce_jobctrl_t.action to the job_ctrl_type_t action code (models.JOB_STOP=1, JOB_RESUME=2, JOB_KILL=3, JOB_CHKPNT=8, src/include/xce/types.h job_ctrl_type_t) -- but that field is actually the capnp JobControl_u_which UNION DISCRIMINANT (JobControl_u_siginfo=0, JobControl_u_ckptoption=1, src/lib/controller.capnp.h), a completely different enum that says which branch of xce_jobctrl_t.u (siginfo vs ckptoption) is populated. xce_jobctrl_req() (src/lib/request.c:2200) and the controller's job_ctrl() (src/controller/job.c:1972, for a job with no allocs yet) both switch on ctrinfo->action expecting the discriminant values 0/1, not the job_ctrl_type_t codes. Consequences per action: kill(3) and resume(2) never match 0 or 1, so xce_jobctrl_req's `default: return -EINVAL` fires client-side before the request is even sent to the controller -- this is the exact -22 in the bug report. checkpoint(8) hits the same default-EINVAL path, so checkpoint was ALSO silently broken (never reached the controller's real checkpoint-eligibility check). stop(1) is the worst case: it numerically collides with JobControl_u_ckptoption(1), so xce_jobctrl_req took the ckptoption branch and read `ctrinfo->u.ckptoption->method` through a pointer that was actually populated as `xce_signal_t*` (via jobctrl_set_siginfo, clib.go) -- a wrong-typed union-member read, undefined behavior in the apiserver's own address space, not merely a clean rejection. The `signo`-based dispatch already correctly threaded through job_ctrl()'s inner switch (SIGCONT->resume, SIGSTOP/SIGTSTP->stop, SIGKILL/SIGTERM/default->kill on the not-yet-started-job branch) proves the intended design: `action` is the union tag, and `u.siginfo->signo` is what actually distinguishes stop/resume/kill -- ControlJob() conflated the two enums when translating the caller's action into the wire struct. + +## Fix +FIXED and LIVE-VERIFIED (2026-09-07). Added jobCtrlUnionDiscriminant(action int) int to convert_pure.go (pure Go, no CGO types, alongside resolveExecUser from bug-1780) -- returns JobControl_u_ckptoption(1) for JOB_CHKPNT, JobControl_u_siginfo(0) for JOB_STOP/JOB_RESUME/JOB_KILL. ControlJob() (clib.go) now sets `ctrl.action = C.int(jobCtrlUnionDiscriminant(action))` instead of the raw action code; the existing jobctrl_set_siginfo/jobctrl_set_ckptoption C helpers (already correct, only ever touched ctrl.u) are unchanged. TDD: TestJobCtrlUnionDiscriminant_StopResumeKillUseSiginfo and TestJobCtrlUnionDiscriminant_ChkptUsesCkptOption (convert_test.go), passing under CGO_ENABLED=0 go test (RED before the function existed). Verified with a full /xce-build apiserver asan --incremental (exit 0, apiserver/backend real-CGO suite green). LIVE-VERIFIED beyond static analysis (learning from bug-1742's own gap, which was static-analysis-only and left this exact defect in place): deployed a real single-node cluster (controller+scheduler+worker+apiserver) via deploy-ha.sh, submitted a job that stays PEND (unsatisfiable select[] so it never dispatches -- exercises job_ctrl()'s not-yet-started branch, the exact code path this bug hits, without needing job execution), and confirmed via the real built xcecli binary: `jobs kill` succeeds (job removed from listing, previously -22), `jobs stop`/`jobs resume` succeed with no apiserver crash (previously read through the wrong union member -- confirmed apiserver process survived and stayed responsive), and `jobs checkpoint` on an unstarted job now correctly gets a clean business-rule rejection ("cannot be ") instead of a spurious client-side -EINVAL. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1742 +- Gitea: #45 diff --git a/.wolf/bugs/bug-1782.md b/.wolf/bugs/bug-1782.md new file mode 100644 index 00000000..7dcc900e --- /dev/null +++ b/.wolf/bugs/bug-1782.md @@ -0,0 +1,41 @@ +--- +id: "bug-1782" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "apiserver" +severity: "unknown" +timestamp: "2026-09-07T08:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/frontend/apiserver/backend/convert_pure.go (ParseHumanJobID", "new)", "src/frontend/apiserver/controllers/jobs.go (ControlJobByID)"] +tags: ["apiserver", "api-design", "job-control", "human-readable-id", "gitea-45", "enhancement", "live-verified"] +related: ["bug-1781"] +legacy: true +gitea_issue: 45 +gitea_synced_status: "fixed" +--- + +# bug-1782: Not a defect -- API ergonomics follow-up discussed while fixing bug-1781/gitea#45. The job control… + +## Description +Not a defect -- API ergonomics follow-up discussed while fixing bug-1781/gitea#45. The job control REST endpoint (`/xce/v1/jobs/{jobids}/control`) only accepted the packed numeric job ID; xcecli already translates the human-readable form (e.g. `1-0-1788768547-0`, as printed by `jobs list`) to that numeric form client-side (format.ToAPIJobID) before calling the API, so a caller hitting the endpoint directly (not through xcecli) had to reimplement that bit-packing itself. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +N/A -- enhancement, not a bug. Existing numeric-only behavior was correct and intentional. + +## Fix +ADDED (2026-09-07), additive/backward-compatible -- the numeric form is untouched and still works exactly as before. Added ParseHumanJobID(s string) (uint64, bool) to convert_pure.go: parses a plain human-readable job ID ("{cluster}-{apiserver}-{timestamp}-{sequence}", no array brackets -- controllers/jobs.go's parseArrayName already strips any "[idx]" suffix before this runs) back into the packed numeric ID, reusing the bit-layout constants this file already duplicates from xcecli/common/format/jobid.go for the read path (FormatJobID/ExtractArrayBaseID). Works correctly for both a plain job's own ID and an array child's dash-quad prefix (verified equal to ExtractArrayBaseID(original) in the array case -- the two enums are NOT ambiguous here, unlike bug-1781). ControlJobByID (jobs.go) now falls back to ParseHumanJobID when jobid.KeyID isn't numeric, before erroring; numeric IDs skip this path entirely. TDD: TestParseHumanJobID_NonArrayRoundTripsWithFormatJobID, TestParseHumanJobID_ArrayChildPrefixMatchesBaseID, TestParseHumanJobID_RejectsNumeric, TestParseHumanJobID_RejectsGarbage (convert_test.go), passing under CGO_ENABLED=0 go test. Verified with a full /xce-build apiserver asan --incremental (exit 0). LIVE-VERIFIED: against the same deployed cluster used for bug-1781, called POST /xce/v1/jobs/{jobids}/control directly via curl with the URL-encoded human-readable ID (bypassing xcecli's own translation entirely) and confirmed the job was correctly found and killed (HTTP 202, job removed from listing). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1781 +- Gitea: #45 diff --git a/.wolf/bugs/bug-1784.md b/.wolf/bugs/bug-1784.md new file mode 100644 index 00000000..25ea297c --- /dev/null +++ b/.wolf/bugs/bug-1784.md @@ -0,0 +1,41 @@ +--- +id: "bug-1784" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-09-07T09:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/group.c (groupmgr_load_configs", "new)", "src/include/group.h", "src/scheduler/src/scheduler.c (sched_init_config)"] +tags: ["scheduler", "limit-plugin", "usergroup", "hostgroup", "group-manager", "gitea-46", "live-verified"] +related: ["bug-1743", "bug-1785", "bug-1786", "bug-1787", "bug-1788", "bug-1789"] +legacy: true +gitea_issue: 46 +gitea_synced_status: "fixed" +--- + +# bug-1784: Discovered while live-verifying gitea#46's Part 1 (dispatch-stops-at-8, previously hypothesized in… + +## Description +Discovered while live-verifying gitea#46's Part 1 (dispatch-stops-at-8, previously hypothesized in bug-1743 to be limits.yaml's ug1_normal_slots cap=8 working as designed, never live-confirmed). Live test: submitted 10 single-slot jobs as user1 (a genuine ug1 member per users.yaml) on queue normal against a 22-slot worker -- all 10 ran, the supposed cap=8 usergroup limit never blocked anything. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Named usergroups/hostgroups (users.yaml's top-level "usergroups:", hosts.yaml's "hostgroups:") are carried down to the scheduler via xce_conf4sched_t.ugroups/hgroups (src/controller/sched.c populates these from c->myconf->cluster_config->ugroups/hgroups), but nothing on the scheduler side ever registered them into a group manager by name: groupmgr_add() (src/lib/group.c) -- the obvious candidate -- had ZERO callers anywhere in production code (confirmed via full-tree grep). When a limit's own per-limit consumer group (plugins/limit.c's build_consumer()) references a named group like "ug1" as a member, make_g_member() (group.c) calls get_group(mgr, "ug1"), finds nothing (since it was never registered), and falls back to treating "ug1" as a literal, ungrouped member name that will never strcmp-match a real username via is_in_group(). Confirmed via source inspection (build_group(), a plausible alternative entry point declared in group.h, has NO implementation anywhere either -- a second dead prototype) and via live behavior (user1, a real ug1 member, ran unthrottled against a configured cap). + +## Fix +FIXED (2026-09-07). Added groupmgr_load_configs(var mgr, int count, xce_group_config_t** configs) to group.c/group.h: iterates the config list and registers each named entry into mgr via the existing, already-correct makeGroup(mgr, config, config->name). Wired into scheduler.c's sched_init_config(): groupmgr_load_configs() for both core.ugmgr/config->ugroups and core.hgmgr/config->hgroups, called BEFORE plugin_config_update() so any limit's own build_consumer() call (which runs inside plugin_config_update -> limit_update_config -> build_limit) can resolve a named-group reference at build time. TDD: 3 new tests in src/lib/testsuite/test_group.c -- test_load_configs_registers_each_group_by_name, test_load_configs_enables_group_name_as_member_reference (the exact regression: a limit-style unnamed consumer group referencing a named group by member, mirroring build_consumer()'s own call shape), test_load_configs_zero_count_is_a_noop. All pass (313 Tests 0 Failures under CGO... err, under ASAN_OPTIONS=detect_leaks=0 xcelib_test). Full scheduler suite: 76/77 (pre-existing unrelated bug-1762 test_core_events.c failure). LIVE-VERIFIED: fresh single-node deploy, user1 (ug1 member) submitting jobs on queue normal now correctly has match_limit() report a match (previously always rc=0/no-match) for ug1_slots and ug1_normal_slots -- confirmed via scheduler trace logs. (Full cap enforcement required two further fixes in the same session -- see bug-1786/1788 -- and one remaining gap not yet fixed, see bug-1789.) + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1743, bug-1785, bug-1786, bug-1787, bug-1788, bug-1789 +- Gitea: #46 diff --git a/.wolf/bugs/bug-1785.md b/.wolf/bugs/bug-1785.md new file mode 100644 index 00000000..229fe2c1 --- /dev/null +++ b/.wolf/bugs/bug-1785.md @@ -0,0 +1,41 @@ +--- +id: "bug-1785" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-09-07T09:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/xcelimit.c (build_consumer)"] +tags: ["scheduler", "limit-plugin", "capnp", "group-config", "default-semantics", "gitea-46"] +related: ["bug-1784", "bug-1786", "bug-1787", "bug-1788", "bug-1789"] +legacy: true +gitea_issue: 46 +gitea_synced_status: "fixed" +--- + +# bug-1785: Found immediately after fixing bug-1784: with named-usergroup resolution working, a limit scoped… + +## Description +Found immediately after fixing bug-1784: with named-usergroup resolution working, a limit scoped to only SOME of the four dimensions (users.yaml's demo limits.yaml sets users+queues on ug1_normal_slots but never projects/hosts) still matched no one at all. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +build_consumer() (xcelimit.c) is documented to default an unconfigured scope to the builtin "all" group (matching everyone) when g_config is NULL. But capnp encodes LimitConfig's users/queues/projects/hosts as embedded GROUPS, not nullable pointers (xceadmin's mapping.py: _build_membership_group() no-ops on an absent YAML key, but the embedded capnp struct is still present on the wire as an empty-but-non-NULL group) -- so an unconfigured scope arrives here as a non-NULL xce_group_config_t with zero members/ex_members, never as NULL. build_consumer()'s g_config!=NULL branch then builds a real, empty, UNNAMED group via makeGroup() -- a group that (per is_in_group()'s own logic) matches no one, the exact OPPOSITE of the intended "unrestricted" default. Every limit in this repo's demo limits.yaml configures only a subset of the four scopes, so every one of them was silently rejecting every job on its unconfigured dimensions. + +## Fix +FIXED (2026-09-07). build_consumer() now normalizes an empty-but-present g_config (members.count==0 AND ex_members.count==0) to NULL before the existing g_config!=NULL/NULL branch runs, restoring the documented "fall back to all" behavior for a genuinely-unconfigured scope. An operator who wants "no one" for a scope still has the builtin "empty" group to reference by name explicitly, so this doesn't remove any real capability. TDD: new test_build_consumer_empty_config_defaults_to_all (src/lib/testsuite/test_limit.c) -- builds a zero-valued xce_group_config_t{0}, asserts build_consumer() falls back to a group where is_in_group() matches an arbitrary name. Passes (313 lib tests, 0 failures). Full scheduler suite: 76/77 (pre-existing bug-1762). LIVE-VERIFIED as part of the same trace as bug-1784/1786/1787/1788 -- combined, these let match_limit() and instantiate_limit() run to completion for all 7 limits in limits.yaml on every job submission (previously it silently rejected on the FIRST unconfigured scope it hit). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1784, bug-1786, bug-1787, bug-1788, bug-1789 +- Gitea: #46 diff --git a/.wolf/bugs/bug-1786.md b/.wolf/bugs/bug-1786.md new file mode 100644 index 00000000..ecdb9762 --- /dev/null +++ b/.wolf/bugs/bug-1786.md @@ -0,0 +1,41 @@ +--- +id: "bug-1786" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-07T09:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugins/limit.c (match_job_global_limit_)"] +tags: ["scheduler", "limit-plugin", "kvmgr", "error-code-convention", "not-found-vs-error", "gitea-46", "live-verified"] +related: ["bug-1495", "bug-1784", "bug-1785", "bug-1787", "bug-1788", "bug-1789"] +legacy: true +gitea_issue: 46 +gitea_synced_status: "fixed" +--- + +# bug-1786: Found immediately after bug-1784/1785: with group resolution and default-scope semantics both… + +## Description +Found immediately after bug-1784/1785: with group resolution and default-scope semantics both fixed, match_limit() finally reported a genuine match (rc=1) for the FIRST time in this codebase's live history -- and the very next step (usage/threshold accounting) immediately broke: limit_job_election()'s loop aborted after the first matching limit, so no limit past it in iteration order was ever evaluated for that job. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +match_job_global_limit_(), right after a successful match_limit(), does `rc = key_find(privdata->instances, vresname, &vres); if (rc <= 0) { return rc; }` before the `if (vres == NULL) { instantiate_limit(...) }` block below it. On the FIRST-EVER lookup of a given vresname (always true immediately after bug-1784/1785 made matching work for the first time), key_find() signals "not found" as a NEGATIVE rc (confirmed live: rc=-2/-ENOENT, not 0) -- this codebase's kvmgr convention elsewhere (get_group(), is_host_based_limit()) treats rc<=0 uniformly as "not found", so this specific early-return, written assuming rc==0 meant not-found and only negative meant a real error, unconditionally caught EVERY not-found case and returned before instantiate_limit() could ever run. limit_job_election()'s own loop breaks on any rc<0 from match_job_global_limit_, so this also silently aborted evaluation of every remaining limit for that job. Same defect class already fixed once before in this codebase, in a different pair of functions: bug-1495 (sched_get_user()/sched_get_project(), core.c) hit and fixed the identical rc<0-vs-not-found conflation. + +## Fix +FIXED (2026-09-07). Removed the erroneous `if (rc <= 0) { return rc; }` guard entirely; `vres` (calloc'd NULL, only ever reassigned by a successful key_find) is already the correct, sufficient signal for "needs instantiation" via the existing `if (vres == NULL)` block immediately below -- mirrors this file's own build_vres_name/key_add error-handling style, just without conflating a legitimate not-found miss with failure. No new unit test added for this specific control-flow fix (see bug-1788's fix note for why: exercising limit_job_election() end-to-end requires a fully-wired xce_sched_core_t fixture -- resmgr, sharedInst, a real xce_sched_obj_t with initialized xlist_t/zarray_t fields -- judged not worth the fragility for a single control-flow correction already covered by full live E2E verification). Full scheduler suite: 76/77 (pre-existing bug-1762). LIVE-VERIFIED: after this fix (combined with bug-1787/1788, needed to get all the way through), all 7 limits in limits.yaml now appear in match_job_global_limit_'s own trace log for every job submission, where previously only the first one in iteration order ever did. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1495, bug-1784, bug-1785, bug-1787, bug-1788, bug-1789 +- Gitea: #46 diff --git a/.wolf/bugs/bug-1787.md b/.wolf/bugs/bug-1787.md new file mode 100644 index 00000000..14b93b64 --- /dev/null +++ b/.wolf/bugs/bug-1787.md @@ -0,0 +1,41 @@ +--- +id: "bug-1787" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-07T09:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugins/limit.c (match_job_global_limit_", "match_job_host_limit_)"] +tags: ["scheduler", "limit-plugin", "rtti", "pointer-indirection", "type-confusion", "gitea-46", "live-verified"] +related: ["bug-1784", "bug-1785", "bug-1786", "bug-1788", "bug-1789"] +legacy: true +gitea_issue: 46 +gitea_synced_status: "fixed" +--- + +# bug-1787: Found immediately after bug-1786: match_job_global_limit_'s own key_find rc=-2 masking fixed, but… + +## Description +Found immediately after bug-1786: match_job_global_limit_'s own key_find rc=-2 masking fixed, but the FOLLOWING step -- instantiate_limit() -- now returned rc=-22/-EINVAL outright, still aborting the loop after the first limit. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +is_host_based_limit() and instantiate_limit() (src/lib/xcelimit.c) both take a plain `var resmgr` parameter -- core.h declares core.resmgr itself as a plain `var` (a handle, not var*), matching how every other manager consumer in this codebase (get_group(), is_in_group(), key_find() itself) takes its manager by value. All four call sites in limit.c (2 in match_job_global_limit_, 2 in the host-scoped sibling match_job_host_limit_) instead passed &(sched->core.resmgr) -- the ADDRESS of the field, one level of indirection too many. RTTI's key_find() dispatch received what amounts to a var* misinterpreted as a var (a meta_obj_t* pointing at raw, non-RTTI memory) and rejected it: silently inside is_host_based_limit() (whose own rc<=0 guard turns ANY key_find failure, including this type-confusion, into a uniform -ENOENT -- see bug-1786's note on why that specific defensive pattern isn't itself wrong), but fatally inside instantiate_limit(), whose own key_find() call on the SAME mistyped resmgr returned -EINVAL and (before this session's other fixes) propagated straight out. + +## Fix +FIXED (2026-09-07). Removed the erroneous & at all 4 call sites (2 in match_job_global_limit_, 2 in match_job_host_limit_): `is_host_based_limit(sched->core.resmgr, limit)` / `instantiate_limit(&vres, sched->core.resmgr, limit, ...)`. Confirmed via live diagnostic instrumentation (temporary, removed before landing) that instantiate_limit()'s own top-of-function NULL-argument dump showed every argument non-NULL yet still failing -- proving the -EINVAL came from RTTI's own internal type/handle validation inside key_find(), not the function's explicit guard clause, which directly confirmed the type-confusion theory before touching production code. Full scheduler suite: 76/77 (pre-existing bug-1762). LIVE-VERIFIED as part of the combined bug-1784/1785/1786/1788 trace. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1784, bug-1785, bug-1786, bug-1788, bug-1789 +- Gitea: #46 diff --git a/.wolf/bugs/bug-1788.md b/.wolf/bugs/bug-1788.md new file mode 100644 index 00000000..037d6941 --- /dev/null +++ b/.wolf/bugs/bug-1788.md @@ -0,0 +1,41 @@ +--- +id: "bug-1788" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-09-07T10:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/xcelimit.c (instantiate_limit)"] +tags: ["scheduler", "limit-plugin", "kvmgr", "error-code-convention", "not-found-vs-error", "truthiness-bug", "gitea-46", "live-verified"] +related: ["bug-1495", "bug-1784", "bug-1785", "bug-1786", "bug-1787", "bug-1789"] +legacy: true +gitea_issue: 46 +gitea_synced_status: "fixed" +--- + +# bug-1788: Found immediately after bug-1787: with the resmgr type-confusion fixed, instantiate_limit() still… + +## Description +Found immediately after bug-1787: with the resmgr type-confusion fixed, instantiate_limit() still returned rc=-2/-ENOENT, still aborting limit_job_election()'s loop after the first limit -- reproduced identically on a scheduler's FIRST-EVER join to a freshly-started controller, ruling out any rejoin/resync-specific explanation. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The EXACT same defect class as bug-1786, recurring twice more, plus a related truthiness bug, all inside instantiate_limit() itself: (1) `rc = key_find(resmgr, vresname, &(vres->vres)); if (rc < 0) { return rc; } if (rc == 0) { add_vres(...); }` -- live-confirmed via targeted diagnostic instrumentation that this key_find's own "not found" signal is rc=-2/-ENOENT (not 0) for THIS resmgr-backed lookup too, so the `if (rc<0) return rc;` guard caught every first-ever vresname lookup before the add_vres()-creates-it branch could run. (2) The immediately following lookup of the limit's underlying resource (limit->val.resname, e.g. "slots") had the identical `if (rc==0) return -ENOENT;` pattern, vulnerable to the same rc convention mismatch (not hit in this specific trace only because "slots" was already registered by the time it was reached, but latent). (3) `if (is_host_based_limit(resmgr, limit)) { shared = 0; }` checked bare truthiness of a function that can return a negative errno -- a non-zero, therefore truthy, value -- so a transient is_host_based_limit() failure would have incorrectly flagged a limit as host-based. + +## Fix +FIXED (2026-09-07). All three corrected in instantiate_limit(): (1) `if (vres->vres == NULL) { add_vres(...); }` replaces the rc<0/rc==0 checks -- vres->vres is calloc'd NULL and only ever reassigned by a successful key_find, the same reliable-signal pattern as bug-1786's fix. (2) `if (vres->res == NULL) { return -ENOENT; }` replaces the equivalent rc<0/rc==0 pair for the resource-existence check. (3) `is_host_based_limit(resmgr, limit) > 0` replaces the bare truthiness check. No new unit test added (same reasoning as bug-1786: this function's correctness is now covered by full live E2E verification, and hand-building a resmgr+sched_core fixture for a unit test was judged not worth the fragility versus the live proof already obtained -- see bug-1786/1789 discussion). Full scheduler suite: 76/77 (pre-existing bug-1762). LIVE-VERIFIED: combined with bug-1784/1785/1786/1787, all 7 limits in limits.yaml (ug1_slots, ug2_slots_per_user, night_mem_pct, hg1_slots_per_host, ug1_normal_slots, limit1, limits2) now correctly appear in the trace and complete matching+instantiation for every job submission on a completely fresh, never-before-touched cluster deploy -- confirmed reproducible across 3 independent from-scratch redeploys, ruling out any deploy-specific or rejoin-specific artifact. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1495, bug-1784, bug-1785, bug-1786, bug-1787, bug-1789 +- Gitea: #46 diff --git a/.wolf/bugs/bug-1789.md b/.wolf/bugs/bug-1789.md new file mode 100644 index 00000000..f738ec05 --- /dev/null +++ b/.wolf/bugs/bug-1789.md @@ -0,0 +1,41 @@ +--- +id: "bug-1789" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-07T10:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/resource.c (sched_fill_limit_requests)"] +tags: ["scheduler", "limit-plugin", "resource-allocation", "gitea-46", "live-verified"] +related: ["bug-1784", "bug-1785", "bug-1786", "bug-1787", "bug-1788", "bug-1790"] +legacy: true +gitea_issue: 46 +gitea_synced_status: "fixed" +--- + +# bug-1789: After bug-1784 through bug-1788 all fixed and live-verified (all 7 limits in limits.yaml now… + +## Description +After bug-1784 through bug-1788 all fixed and live-verified (all 7 limits in limits.yaml now correctly match and instantiate usage tracking for every job), the actual numeric cap still does not reject anyone: 10 single-slot jobs submitted as user1 on queue normal against ug1_normal_slots's configured value:8 cap all still reach RUN, none PEND. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT YET ROOT-CAUSED TO A FIX -- traced to a specific, likely-defective computation, but the correct intended semantics were not established this session (deliberately stopped here rather than guess at production logic). Chain traced and CONFIRMED CORRECT up to one point: match_job_global_limit_ populates obj->glimits (fixed, bug-1784-1788) -> plugin.c's plugin_prepare_allocation calls sched_fill_limit_requests(sched, &(ctx->alloc_req.shared), &(obj->glimits)) when ctx->worker==NULL (the global/non-host-scoped case, correct for ug1_normal_slots) -> seqjob_allocator.c/parjob_allocator.c both correctly call sched_check_shared_satisfy(sched, uncomit, &(ctx->alloc_req.shared), &(ctx->unmeet)) and correctly reject (PREASON_NO_RESOURCE, rc=0) when a shared resource's wanted amount would exceed its tracked total (itself correctly set to limit->val.val=8 by update_limit_res_value(), which DOES run correctly now). The break is inside sched_fill_limit_requests() itself (resource.c:912-931): for each matched limit, it computes the job's "wanted" quantity of the limit's virtual resource via `val = get_resval_by_id(vec, vres->res->idx)` where `vec` is `ctx->alloc_req.shared` ITSELF -- i.e. it looks up how many units of the limit's UNDERLYING resource (e.g. "slots") the job already requests, inside the SAME shared-resource-request vector it is about to write the limit's own synthetic entry into. But "slots" is inherently a per-host resource, requested via ctx->alloc_req.HOST (task/slot count per host), not ctx->alloc_req.shared -- so this lookup always misses, get_resval_by_id's own not-found fallback returns 0.0, and every job is recorded as requesting ZERO units of ug1_normal_slots regardless of how many slots it actually asks for. sched_check_shared_satisfy then trivially "satisfies" a wanted=0 request against any total, no matter how many jobs are already running -- the cap can never bind. NOT CONFIRMED whether this is a straightforward bug (should read from ctx->alloc_req.host or from resreq directly) or reflects a genuine, unaddressed design gap in how a GLOBAL (non-host-scoped) limit is meant to interact with a per-host-scoped underlying resource like slots -- the fix requires understanding what "consumption count" SHOULD mean for this combination, not just correcting an obvious return-code mistake like bug-1784-1788 all were. + +## Fix +FIXED (2026-09-07, follow-up session). Root-caused via codegraph trace of plugin_prepare_allocation()'s full hook loop (not just the two call sites the prior session had traced): seqjob_prepare_allocation()/parjob_prepare_allocation() (the allocator plugins' own prepare_allocation_hook) ALWAYS stage the job's wanted slot count into ctx->alloc_req.host -- even during the ctx->worker==NULL/global pass -- because "slots" is deliberately excluded from calc_wanted_global_resvals()/calc_wanted_host_resvals() (it's per-host, not XCE_RESF_GLOBAL) and is staged by a separate, unbranched follow-up store. This confirmed the prior session's NEXT-STEPS item (1): the job's real wanted slot count (ctx->nslots, set earlier in that SAME prepare_allocation call) is already available and correct for both phases -- 1/seqjob or primary->maxTasks/parjob-global or the current bucket's count/parjob-per-host -- it just was never read from the right place. FIRST ATTEMPT (reverted): tried mirroring the ctx->worker branch into the seqjob/parjob "add slots" store itself (writing into ctx->alloc_req.shared during the global phase) -- this broke 6 previously-passing tests (seqjob/parjob dispatch tests going from DISPATCH to PEND) because ctx->alloc_req.shared is also read by sched_check_shared_satisfy() against sched->core.sharedInst, the GLOBAL capacity pool, which never tracks "slots" (a per-host-only resource) -- injecting a spurious "slots" want there made EVERY job's general satisfy check fail, not just limit-scoped ones. REAL FIX: read-side only, confined to limit evaluation. sched_fill_limit_requests() (resource.c) now takes an added xce_sched_ctx_t* ctx parameter; when a matched limit's underlying resource is "slots" (by name, matching this codebase's existing convention of hardcoding "slots" by name in seqjob/parjob's own key_find() calls), it uses (float)ctx->nslots directly as the wanted quantity instead of the always-missing get_resval_by_id(vec, ...) lookup; every other resource keeps the original vec-lookup behavior unchanged. plugin.c's two call sites updated to pass ctx; scheduler.h's extern declaration updated; all 23 testsuite files providing a link-satisfying no-op stub for the old 3-arg signature mechanically updated to the new 4-arg one (added (void)ctx; each, zero behavior change). TDD: 2 new focused unit tests in test_resource_accounting.c (white-box #include of resource.c) -- test_fill_limit_requests_global_slots_limit_uses_ctx_nslots (builds a bare xce_vres_t{res="slots", vres="limit=X"}, ctx.nslots=6, empty vec, asserts the limit's own vres entry gets stored with val=6.0) and test_fill_limit_requests_non_slots_resource_reads_from_vec (same shape for a "mem"-resname'd limit, vec pre-staged with 3.0, ctx.nslots=99 deliberately mismatched, asserts the mem path is unaffected and still reads 3.0 from vec, not ctx->nslots). Full scheduler suite: 76/77 (only pre-existing, unrelated bug-1762 failure remains -- test_job_rusage_reconciles_mem_reservation_against_usage, mem-reservation subsystem, untouched by this change). LIVE-VERIFIED: fresh single-node deploy (localhost:19997), 10 single-slot jobs submitted as user1 (ug1 member) on queue normal. hosts.yaml's default host template has max_slot: -1 (unlimited), ruling out raw host capacity as a factor. Scheduler trace confirms match_job_global_limit_ matching ug1_slots/ug1_normal_slots/limit1 (all three global, slots-resname'd) for every job, and resource.c's unsatisfy_() now logging a REAL non-zero wanted value for the limit's virtual resource on every charge (previously always 0.0, per this bug's original root_cause) -- e.g. 'unsatisfy_ resource 35 total 3.000000 inuse 3.000000 uncomit 0.000000 wanted 1.000000' for limit1 (cap 3.0, matches every non-rshen user including user1) once its 3rd job committed, correctly PENDing the 4th and all subsequent jobs with a genuine shortfall. limit1 (cap 3, broader scope: all users except rshen) bound before ug1_normal_slots (cap 8, narrower: ug1+normal only) could, since both match user1 identically and limit1's cap is lower -- expected overlapping-limit interaction (flagged in an earlier session's own observation), not a gap in this fix; both run through the identical, now-fixed code path. REVISED (2026-09-07, same session, user feedback): the name-based `strcmp(vres->res->name, "slots")` special-case above was flagged as unreliable (correctly -- it silently fails to generalize to any future per-host resource that isn't literally named "slots"). Replaced with a flag-based dispatch: sched_fill_limit_requests() now takes xce_sched_alloc_req_t* alloc_req instead of xce_sched_ctx_t* ctx (nothing else from ctx was ever used), and for each matched limit reads its underlying resource's want from whichever of alloc_req->shared/alloc_req->host that resource's own XCE_RESF_GLOBAL flag indicates -- the SAME flag calc_wanted_global_resvals()/calc_wanted_host_resvals() (resreq.c) themselves already dispatch on to decide where THEY write a resource's want, so this is not a new rule, just the existing one applied consistently. plugin.c's two call sites, scheduler.h, and all 23 testsuite stubs updated accordingly (mechanical param rename). Replaced the 2 name-based unit tests with 3 flag-based ones in test_resource_accounting.c: test_fill_limit_requests_global_limit_on_host_resource_reads_host (bug-1789's actual case: a GLOBAL limit's own write target is alloc_req->shared, but its per-host underlying resource's want must be read from alloc_req->host), test_fill_limit_requests_host_limit_on_host_resource_reads_host (already-working sibling case, unaffected), and test_fill_limit_requests_host_limit_on_global_resource_reads_shared (the new symmetric case this refactor adds real coverage for: a HOST-scoped limit's write target is alloc_req->host, but a genuinely-XCE_RESF_GLOBAL-flagged underlying resource's want must be read from alloc_req->shared instead -- proves the dispatch is a real bidirectional rule, not a one-off special case). Full scheduler suite re-run: 76/77 (same pre-existing bug-1762 failure, no new failures). Live re-verified end to end with a fully fresh redeploy (not just a scheduler restart against old job state, to keep the signal clean): identical result to the first live verification, byte-identical scheduler trace line ('unsatisfy_ resource 35 total 3.000000 inuse 3.000000 uncomit 0.000000 wanted 1.000000'), confirming the refactor is behavior-preserving for the current config. KNOWN RESIDUAL GAP (explicitly out of scope, not silently solved): a hypothetical NEW global limit on some per-host resource OTHER than "slots" (e.g. an absolute-value global mem cap, as opposed to the existing percent-based host-scoped mem limits) would still hit an analogous bug -- calc_wanted_global_resvals() itself skips any non-XCE_RESF_GLOBAL-flagged resource entirely, so such a resource's want is never computed anywhere during the global phase; "slots" only works because seqjob/parjob_prepare_allocation() stage it via a separate, always-run step outside the normal rusages loop, which no other resource has. Fixing that would mean teaching calc_wanted_global_resvals() (or an equivalent) to also compute a non-global resource's want when a currently-matched limit needs it -- a materially larger change, not needed by any limit in the current limits.yaml (the two mem-based limits, night_mem_pct/limits2, are both host-scoped, unaffected either way) and deliberately not attempted here. FURTHER SIMPLIFIED (2026-09-07, same session, user request): sched_fill_limit_requests()'s 1st parameter (xce_scheduler_t* sched) was entirely unused in the function body (only alloc_req and limits are referenced) -- a leftover from an earlier signature shape, not flagged by the prior same-day refactor that removed the old vec parameter. Dropped it: signature is now sched_fill_limit_requests(xce_sched_alloc_req_t* alloc_req, xlist_t* limits). Updated scheduler.h's extern declaration, both call sites in plugin.c's plugin_prepare_allocation() (the sched local var stays -- still used elsewhere in that function for plugin iteration), all 23 link-satisfying testsuite stubs (mechanical Python replace, byte-identical body across all 23), and the 3 real call sites in test_resource_accounting.c (the local `sched` var in each test stays too -- still needed for xce_sched_core_init()/makeResource() resource setup, just no longer passed to this call). Full scheduler suite re-verified after this change: 76/77 (same pre-existing, unrelated bug-1762 failure, no new failures). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1784, bug-1785, bug-1786, bug-1787, bug-1788, bug-1790 +- Gitea: #46 diff --git a/.wolf/bugs/bug-1790.md b/.wolf/bugs/bug-1790.md new file mode 100644 index 00000000..1f3955cc --- /dev/null +++ b/.wolf/bugs/bug-1790.md @@ -0,0 +1,46 @@ +--- +id: "bug-1790" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-09-07T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/resreq.c"] +tags: ["scheduler", "resource-accounting", "limits", "global-limit", "pre-existing", "severity-low", "live-verified"] +related: ["bug-1789"] +legacy: true +--- + +# bug-1790: filed for its own tracking per user request + +## Description +OPEN, NOT A CRASH -- known residual gap explicitly carved out of bug-1789's fix, filed for its own tracking per user request. A hypothetical NEW global limit defined on a per-host resource OTHER than "slots" (e.g. a future absolute-value global mem cap, as opposed to the existing percent-based host-scoped mem limits in limits.yaml) would silently under-enforce the same way bug-1789's slots limits did, for a different reason than bug-1789's root cause. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +bug-1789 fixed sched_fill_limit_requests()'s READ side: it now picks whichever of alloc_req->shared/alloc_req->host to read a limit's underlying resource's want from by checking that resource's own XCE_RESF_GLOBAL flag on vres->res (and separately picks the WRITE destination for the limit's own vres entry by checking vres->vres's flag) -- see resource.c:928-955. That fix only works because alloc_req->host already has SOME value for "slots" to read, staged by seqjob_prepare_allocation()/parjob_prepare_allocation() via a separate, unconditional step outside the normal rusages loop, in EVERY phase including the global one. No other per-host resource gets this special treatment. The general population path -- calc_wanted_global_resvals() (src/lib/resreq.c:734) and calc_wanted_host_resvals() (src/lib/resreq.c:793), both called from seqjob_allocator.c/parjob_allocator.c's prepare_allocation hooks -- each unconditionally skip any resource whose XCE_RESF_GLOBAL flag doesn't match their own phase (calc_wanted_global_resvals() line 763: `if ((r->flags & XCE_RESF_GLOBAL) == 0) continue;`; calc_wanted_host_resvals() line 822: `if ((r->flags & XCE_RESF_GLOBAL) != 0) continue;`), and only calc_wanted_global_resvals() runs during the ctx->worker==NULL global phase. So for any per-host resource other than "slots", alloc_req->host is simply never populated with a want during the global phase -- sched_fill_limit_requests()'s now-correct read-side dispatch would read a genuine zero from an empty vector, not a population bug on its own side, but the net effect (a global limit on a per-host resource never seeing a nonzero want) is the same silent under-enforcement bug-1789 fixed for slots. + +## Fix +FIXED (2026-09-07, follow-up session, user-driven design). Corrected an earlier, wrong candidate fix (calling calc_wanted_host_resvals() during the global phase) identified during design discussion as invalid: that function's PER_HOST formula returns one host's own share (v->value, no multiplier) and, for parjob specifically, its nslots argument is reset PER BUCKET (parjob_allocator.c's parjob_try_plan_(), ctx->nslots = plan->bucket_slots[b], set right before each candidate host is tried) -- neither matches what a GLOBAL limit needs, which is the job-WIDE total, not any one host's/bucket's share. + +REAL FIX: added calc_wanted_host_resval_job_totals() (src/lib/resreq.c), a new function for resources NOT flagged XCE_RESF_GLOBAL that computes the JOB-WIDE TOTAL want using the SAME formulas calc_wanted_global_resvals() already uses for GLOBAL resources (PER_TASK: value*nslots, PER_JOB: value, PER_HOST: value*nhosts -- nslots/nhosts being the whole job's totals, set during the global phase, not any bucket's share). Both functions now share a common static helper (calc_wanted_job_total_resvals_()) parameterized only by which flag direction to match, avoiding duplicating the ~30-line loop body. seqjob_prepare_allocation()/parjob_prepare_allocation()'s global-phase branch (ctx->worker == NULL) now calls this new function into ctx->alloc_req.host immediately alongside the existing calc_wanted_global_resvals() call into .shared. + +SAFETY, traced precisely rather than assumed: plugin_prepare_allocation() (plugin.c:607-613) resets ONLY .shared at the top of a global-phase call and ONLY .host at the top of a per-host call -- never the other -- so .host is NOT reset when this new write happens. sched_fill_limit_requests() for obj->glimits runs later in that SAME plugin_prepare_allocation() invocation (plugin.c:644-647), reading and transferring the value into the limit's own permanent .shared record BEFORE any later per-host call ever resets .host. The value is therefore fully consumed and irrelevant by the time .host gets correctly wiped and repopulated with real per-host/per-bucket data for the actual capacity check -- two different consumers at two different times, not a conflict, confirmed by reading parjob_allocator.c's own per-bucket reset code directly (not inferred). + +TDD: two new regression tests calling seqjob_prepare_allocation()/parjob_prepare_allocation() directly with ctx.worker == NULL and a "mem"-named (non-"slots", non-global) rusage entry -- test_seqjob_prepare_allocation_global_phase_stores_host_resource_job_total (test_seqjob_allocator.c) and test_parjob_prepare_allocation_global_phase_stores_host_resource_job_total (test_parjob_combinatorial.c, maxTasks=3 specifically to prove the formula uses the job's WHOLE nslots=3, giving 2.0*3=6.0, not any bucket's share). Both assert .shared stays empty (mem correctly excluded, not globally flagged) and .host carries both the new mem entry and the pre-existing unconditional "slots" entry. Both PASS standalone with full Unity PASS lines confirmed (not just suite-level OK). + +Full `xce-build.sh all asan --incremental`: 92/94 Ok. The 2 failures are both pre-existing and unrelated, confirmed by cross-referencing this exact buglog: test_job_rusage_reconciles_mem_reservation_against_usage (bug-1762) and libxce test's 456-byte/29-allocation leak in test_xcealloc.c's make_alloc()/make_host_alloc() calls (bug-1733's fix note already confirms this exact leak signature -- 456 bytes, 29 allocations -- present byte-for-byte identically via an independent git-stash A/B rebuild in an earlier, unrelated session). LIVE-VERIFIED (2026-09-08, follow-up session). Added ug1_global_mem to limits.yaml -- a permanent, real GLOBAL limit on "mem" (a per-host resource, not "slots"), capping ug1's total cluster-wide mem usage at 5,000,000,000 -- exactly the previously-hypothetical scenario this bug was about, now real example config instead of untested. Fresh single-node deploy, 3 jobs submitted as user1 (ug1 member) each requesting rusage[mem=2000000000] (2GB) on queue normal: scheduler trace confirms ug1_global_mem matching (rc=1) and, for the first time, correctly computing a nonzero job-wide-total wanted value via calc_wanted_host_resval_job_totals() -- 'unsatisfy_ resource 36 total 5000000000.000000 inuse 4000000000.000000 uncomit 0.000000 wanted 2000000000.000000' / 'unsatisfy_ unmet resource 36 (limit=ug1_global_mem) shortfall 1000000000.000000' -- correctly detecting the 3rd job (4GB inuse + 2GB wanted = 6GB > 5GB cap) and PENDing it until an earlier job's mem freed up. Deploy config validated cleanly with zero new warnings. This closes the residual live-verification gap entirely -- bug-1790 is now proven correct for a resource other than "slots", not just unit-tested. REFACTORED (2026-09-08, same follow-up session, user-driven design review): the fix above worked and was live-verified, but put limit-plugin-specific logic (computing a job-total for a per-host resource, needed only when some GLOBAL limit references it) inside the ALLOCATOR plugins (seqjob_prepare_allocation()/parjob_prepare_allocation()) -- the wrong owner: those plugins should only ever compute what the job itself wants for capacity-satisfy purposes, and the duplication across two allocator plugins (a third would need it too) plus unconditionally computing totals for every per-host resource regardless of whether any limit needs one were both symptoms of that. Moved ownership to sched_fill_limit_requests() (resource.c) itself, called once centrally from plugin_prepare_allocation() regardless of which allocator ran: removed calc_wanted_host_resval_job_totals() and its calls from both allocator plugins entirely; added calc_wanted_job_total_resval() (resreq.c) -- a single-resource on-demand lookup (by name, no resmgr/key_find needed since the caller already has the resolved xce_resource_t), computing the SAME job-wide-total formula only for the one (scope, underlying-resource) combination that needs it (a GLOBAL limit on a per-host resource) -- the other three combinations are unchanged, still reading straight out of alloc_req. sched_fill_limit_requests() gained back rusages/nhosts/nslots parameters (undoing part of this session's earlier signature-trimming, but for a genuine need this time, not a re-added unused one); plugin.c's two call sites pass obj->u.primary->resreq->rusages (guarded on obj->tag == XCE_RESREQ_PRIMARY -- NULL for COMPOUND/ALTER, matching this codebase's existing scope convention for those tags) and ctx->nhosts/ctx->nslots (already correctly set by whichever allocator just ran, same invocation). All 23 testsuite link-satisfying stubs mechanically updated to the new 5-arg signature; the two allocator-level regression tests added by the first pass (test_seqjob_allocator.c/test_parjob_combinatorial.c) removed -- the behavior they proved no longer exists by design -- and test_resource_accounting.c's test_fill_limit_requests_global_limit_on_host_resource_reads_host rewritten to exercise the new on-demand path directly (job rusages instead of pre-staged .host, nslots=3 to prove the job-WIDE total is used, matching what the removed parjob-specific test proved). Full xce-build.sh all asan --incremental: 94/94 Ok, zero regressions. Live re-verified against the SAME ug1_global_mem deploy/scenario: byte-identical trace ('unsatisfy_ resource 36 total 5000000000.000000 inuse 4000000000.000000 uncomit 0.000000 wanted 2000000000.000000' / 'shortfall 1000000000.000000') to the pre-refactor result, confirming behavior-preserving. REGRESSION CAUGHT AND FIXED (2026-09-08, same follow-up session, user prompted "should we regression the test of gitea#46" after the ownership refactor above). calc_wanted_job_total_resval() scans `rusages` by name -- but "slots" is NEVER a declared rusage entry in this codebase: seqjob_prepare_allocation()/parjob_prepare_allocation() derive its want directly from the job's own task count (1.0 for seqjob, always maxTasks==1; (float)ctx->nslots for parjob), via their own unconditional key_find("slots", ...) + store, entirely outside the normal rusages loop -- confirmed by the user ("all resources except slots should be included in the rusage[] for request"). The refactor's on-demand lookup therefore always missed for "slots" and silently left `val` at its 0.0 default. Live-confirmed against a fresh deploy of the ORIGINAL gitea#46 scenario (9 plain sleep jobs, user1, queue normal, no -R flag -- exactly the issue's own repro): all 3 slots-based limits (limit1/ug1_slots/ug1_normal_slots) matched correctly (rc=1), but ZERO unsatisfy_ lines appeared anywhere and all 9 jobs dispatched with no cap enforcement at all -- gitea#46's exact original symptom, reintroduced by the refactor. The refactor's own new unit test had masked this: it artificially added a "slots" entry to its test rusages, which doesn't reflect production reality. FIX: sched_fill_limit_requests() (resource.c) now falls back to `nslots` directly (the SAME value both allocators already store as "slots"'s own per-host want) whenever the rusages lookup misses AND the resource is named "slots" -- the one documented exception to the rusages-lookup rule, matching this codebase's own existing "slots" special-casing convention (both allocators already key_find it by literal name, outside their normal rusages-driven computation). New test test_fill_limit_requests_global_limit_on_slots_uses_nslots (test_resource_accounting.c) proves the fallback with NO "slots" entry in rusages at all, matching the real shape; the original test renamed/reworked to use "mem" instead, so it now genuinely exercises the general on-demand path rather than accidentally exercising (and masking a gap in) the special case. Full xce-build.sh all asan --incremental: 94/94 Ok. Live re-verified against BOTH the original gitea#46 scenario (byte-identical to the pre-refactor trace: 'unsatisfy_ resource 35 total 3.000000 inuse 3.000000 uncomit 0.000000 wanted 1.000000' / 'shortfall 1.000000' for limit1) and the ug1_global_mem scenario from the refactor's own prior verification -- both correct. SIMPLIFIED (2026-09-08, same follow-up session, user insight: "alloc_req should have enough information? slots should be added into it?"). The nslots-fallback fix above worked but was an unnecessary name-based special case: alloc_req->host ALREADY holds exactly "slots"'s job-wide total during the global phase -- both allocators' own unconditional "add slots" step stages it there regardless of phase, and plugin_prepare_allocation() (plugin.c) never resets .host during the global phase (confirmed: sched_ctx_init(), schedobj.c, gives every job a fresh, empty alloc_req, so by the time sched_fill_limit_requests() runs -- right after the allocator's own hook, same invocation -- .host contains exactly "slots"'s total and nothing else). Replaced the strcmp(vres->res->name, "slots") check with reading alloc_req->host FIRST (get_resval_by_id()) and falling back to the rusages-based on-demand computation only when nothing is staged there (val stays the function's 0.0 default, get_resval_by_id()'s own "not found" signal -- safe here since "slots" is never legitimately 0: seqjob only runs for maxTasks==1, parjob only for maxTasks>1, so nslots is always >=1). This is data-driven, not name-based -- matching the same design philosophy bug-1789's own earlier flag-based-dispatch revision established -- and generalizes for free to any future resource an allocator plugin similarly pre-stages outside its normal rusages loop, with no further sched_fill_limit_requests() changes needed. Test renamed test_fill_limit_requests_global_limit_on_slots_reads_host, now pre-staging alloc_req->host exactly as the allocators do (rather than relying on the removed nslots fallback) with rusages left empty, matching production exactly. Full xce-build.sh all asan --incremental: 94/94 Ok. Live re-verified against BOTH the original gitea#46 scenario and the ug1_global_mem scenario: byte-identical scheduler traces to every prior verification round, confirming this simplification is behavior-preserving. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1789 diff --git a/.wolf/bugs/bug-1791.md b/.wolf/bugs/bug-1791.md new file mode 100644 index 00000000..78a15889 --- /dev/null +++ b/.wolf/bugs/bug-1791.md @@ -0,0 +1,38 @@ +--- +id: "bug-1791" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-09-09T08:50:58.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/interruptcall.c"] +tags: ["pthread", "mutex", "EPERM", "cross-thread-unlock", "ownership-handoff", "xce-test-cli-ux", "task-1.5", "interruptcall.c", "design-review-caught-by-test", "posix-ub", "sem_t", "semaphore-migration"] +related: [] +legacy: true +--- + +# bug-1791: controller_unit_test: interruptcall.c:142: xcetest_worker_main: Assertion \`unlock_rc == 0' failed… + +## Description +controller_unit_test: interruptcall.c:142: xcetest_worker_main: Assertion \`unlock_rc == 0' failed (pthread_mutex_unlock returned EPERM) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce-test-cli-ux task 1.5's per-connection lock registry initialized each connection's pthread_mutex_t with PTHREAD_MUTEX_ERRORCHECK, intended to catch double-unlock/never-unlocked regressions. But this design's own ownership-handoff protocol (design.md/research.md's 'per-connection lock' decision) REQUIRES a legitimate cross-thread release on the abandonment path: the calling (REPL) thread acquires the lock via pthread_mutex_trylock(), but on abandonment the still-running detached WORKER thread is the one that must release it later, once fn() finishes -- ownership of 'release the lock' passes to the worker at the same moment ownership of the heap call block does. PTHREAD_MUTEX_ERRORCHECK (and _RECURSIVE) mutexes track a single owning thread and reject pthread_mutex_unlock() from any other thread with EPERM, which fired immediately on the very first same-key-serialization test. + +## Fix +Switched the per-connection lock registry (xcetest_conn_lock_for() in interruptcall.c) to initialize each mutex with the DEFAULT type (pthread_mutex_init(&node->mutex, NULL)) instead of PTHREAD_MUTEX_ERRORCHECK. glibc default (PTHREAD_MUTEX_NORMAL-equivalent) mutexes do not track a per-thread owner, so a worker thread may validly unlock a mutex the REPL thread locked. Kept the assert(pthread_mutex_unlock(...) == 0) safety-net checks at every release site (computed outside the assert() expression itself, so the unlock call is never skipped under NDEBUG) as a lower-cost guard against genuine corruption, without relying on error-checking-mutex semantics that conflict with the intentional cross-thread handoff. SUPERSEDED 2026-09-09 (task 1.5 remediation round 1, independent review): an external reviewer correctly pointed out that POSIX classifies non-owning-thread pthread_mutex_unlock() as undefined behavior for PTHREAD_MUTEX_NORMAL/DEFAULT too, not just ERRORCHECK/_RECURSIVE -- NORMAL is merely *permitted* not to detect the violation, not declared well-defined for it. The DEFAULT-mutex fix above was therefore empirically safe on XCE's glibc/NPTL-exclusive Nix build but relied on non-portable, POSIX-UB behavior, under-documented as such. Replaced the pthread_mutex_t per-connection lock with a POSIX semaphore (sem_t, binary, sem_init(&sem, 0, 1)) throughout: sem_trywait() in place of pthread_mutex_trylock() (EAGAIN in place of EBUSY) in the interruptible poll loop, and sem_post() in place of pthread_mutex_unlock() at every release site (both the caller-normal-completion path and the worker-abandoned-cleanup cross-thread path). sem_t has no ownership concept, so this cross-thread release is now fully POSIX-defined with no UB reliance at all, at the same code complexity. All 1285 controller_unit_test tests (including the 3 task-1.5-specific tests) continued to pass unchanged after the migration. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1792.md b/.wolf/bugs/bug-1792.md new file mode 100644 index 00000000..22b374f8 --- /dev/null +++ b/.wolf/bugs/bug-1792.md @@ -0,0 +1,38 @@ +--- +id: "bug-1792" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-09-09T12:40:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_test_cli.c"] +tags: ["libedit", "readline", "tab-completion", "pty", "ECHO", "test-harness", "xce-test-cli-ux", "task-2.3", "test_test_cli.c", "silent-input-mode"] +related: [] +legacy: true +--- + +# bug-1792: test_test_cli.c tab-completion test (xce-test-cli-ux task 2.3): Tab keystroke produced zero effect… + +## Description +test_test_cli.c tab-completion test (xce-test-cli-ux task 2.3): Tab keystroke produced zero effect (no bell, no redraw, no completion) when driven through the existing clitest_spawn() pty harness -- 'wo' + Tab + Enter submitted literally "wo" ("wo: No such command for xce test."), never "worker". + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +clitest_spawn() (task 1.7) deliberately clears kernel ECHO on the pty slave BEFORE forking, documented at the time as affecting ONLY the line discipline's own re-echo of injected bytes, 'not readline()/libedit's own prompt/redraw output, which stays intact'. An A/B probe (standalone C program linked against this project's exact pinned libedit, driven both ways via a raw openpty()+fork() harness, controlling for controlling-terminal presence too) showed that assumption does not hold for THIS libedit build: when the slave's ECHO bit already reads off at the moment readline() is invoked, libedit treats the session as a silent/non-interactive (password-style) input mode and suppresses ALL of its own live editing feedback, not just kernel echo -- Tab is accepted as a no-op, the line is only ever consumed whole on Enter, and no bell/redraw/candidate-list ever appears. With kernel ECHO left at its default-on state (libedit reconfigures the tty itself once its own readline() call takes over, taking on echo/redraw duties without any raw kernel double-echo reaching the captured stream), the identical Tab sequence produced the expected bell + in-place completion + ambiguous-match candidate listing. Controlling-terminal setup (setsid()/TIOCSCTTY) was also probed and made no observable difference -- ECHO state was the sole deciding factor. + +## Fix +Refactored clitest_spawn() into clitest_spawn_ex(sess, path, argv, disable_echo) with the ECHO-clearing block made conditional on the new parameter; clitest_spawn() itself is preserved as a one-line wrapper calling clitest_spawn_ex(..., true), so every existing (output-only) caller is unaffected. The new xce-test-cli-ux task 2.3 tab-completion test calls clitest_spawn_ex(..., false) directly to keep kernel ECHO on, which restored full interactive completion behavior end-to-end (single-match auto-complete, ambiguous-match double-Tab listing, all verified via the real xce_test binary). Documented the finding in both clitest_spawn_ex()'s own comment and the file's top-of-file API summary so future tasks in this spec (3.5, 4.1-4.4) that add more interactive-feedback-dependent tests to this same file know to use clitest_spawn_ex(..., false) rather than rediscovering this the hard way. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1793.md b/.wolf/bugs/bug-1793.md new file mode 100644 index 00000000..3b597c8e --- /dev/null +++ b/.wolf/bugs/bug-1793.md @@ -0,0 +1,38 @@ +--- +id: "bug-1793" +status: "open" +status_updated: "2026-09-24" +status_note: "xce_test join maxswap/maxtmp defaults are stale byte values (join_worker_args.c)" +component: "controller" +severity: "unknown" +timestamp: "2026-09-09T16:24:19Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/join_worker_args.c (join_worker_parse_attrs", "hostattrs_t default initialization)"] +tags: ["controller", "testsuite", "xce_test", "cmd_join_worker", "stale-default", "units", "deferred"] +related: ["bug-1557"] +legacy: true +--- + +# bug-1793: xce_test's worker->join command's maxswap (1024) and maxtmp (2048) defaults have the same… + +## Description +xce_test's worker->join command's maxswap (1024) and maxtmp (2048) defaults have the same stale-MB-scale bug as bug-1557's maxmem -- discovered while fixing bug-1557 (xce-test-cli-ux task 2.6) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same root cause as bug-1557: maxswap/maxtmp were reasonable defaults (1KB/2KB) back when these fields were a smaller unit, but an earlier session converted maxswap/maxtmp to bytes-canonical (confirmed via resources.yaml 'max swap/tmp disk space in bytes' and hardware_monitor.c's raw-byte sysinfo()/statvfs() computations) without updating this test-tool default. 1024 bytes of swap / 2048 bytes of tmp are just as nonsensical as the old maxmem=64 bytes. + +## Fix +NOT FIXED -- deliberately deferred (xce-test-cli-ux task 2.6). USAGE_JOIN_WORKER's help text currently documents these exact stale values accurately, so fixing only the code without also fixing the docs would create a new code/docs mismatch outside task 2.6's Requirement-12.1-scoped mandate (which covers maxmem only). Recommended: a coordinated fix changing both defaults (e.g. maxswap to a byte-scale value matching typical swap sizing, maxtmp similarly) AND USAGE_JOIN_WORKER's documented defaults, together. A regression test (test_join_worker_sibling_defaults_unchanged in test_join_worker_args.c) currently locks in today's stale values specifically so a future fix here doesn't collide with bug-1557's maxmem-specific test assertions -- update/remove that test's maxswap/maxtmp assertions when this bug is actually fixed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1557 diff --git a/.wolf/bugs/bug-1794.md b/.wolf/bugs/bug-1794.md new file mode 100644 index 00000000..71726b9e --- /dev/null +++ b/.wolf/bugs/bug-1794.md @@ -0,0 +1,38 @@ +--- +id: "bug-1794" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-09-09T19:03:51Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c (cmd_jobrun", "fill_job_alloc", "jobrun_call_t/jobrun_thunk)"] +tags: ["controller", "testsuite", "xce_test", "cmd_jobrun", "fill_job_alloc", "uninitialized-memory", "heap-allocation", "asan", "interruptcall", "xce-test-cli-ux", "task-3.2"] +related: ["bug-1791"] +legacy: true +--- + +# bug-1794: xce_test cmd_jobrun crashes (ASAN allocation-size-too-big abort in alloc_packet, requested size… + +## Description +xce_test cmd_jobrun crashes (ASAN allocation-size-too-big abort in alloc_packet, requested size ~0xffffffffc3c3c4b0) when run live against a real controller, after xce-test-cli-ux task 3.2 wrapped its xcecli_jobdispatch() call through the interruptible-call adapter (heap-allocated ctx block instead of a stack-local xce_job_alloc_t). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +PRE-EXISTING, latent: fill_job_alloc() (test.c) only ever sets xce_job_alloc_t.nhosts/.allocs -- it never initializes the struct's other two fields, nres_shared and allocs_shared (xce/types.h). Before task 3.2, xce_job_alloc_t jalloc was a plain stack-local in cmd_jobrun, so those two never-set fields held whatever incidental value that stack frame happened to have -- harmless in practice, apparently usually small/zero. Task 3.2 moved this struct into a heap-allocated jobrun_call_t block (required by the interruptible-call adapter's contract: fn must only touch heap-owned memory, never the caller's stack, since an abandoned call's worker thread can outlive the caller's own stack frame) allocated via plain malloc(), which leaves ALL bytes as genuine heap garbage -- confirmed live (real single-node deploy, jobrun -j -m worker-1:1): the garbage nres_shared/allocs_shared corrupted xce_jobdispatch_req's wire-size computation in lib/request.c, producing a multi-exabyte allocation request that ASAN correctly aborted on. + +## Fix +FIXED (2026-09-09, xce-test-cli-ux task 3.2). Every per-call-site heap block (all ~26 _call_t typedefs test.c's interrupt-wrapping introduces) now allocates via calloc(1, sizeof(*call)) instead of malloc(sizeof(*call)), zero-initializing every field uniformly rather than auditing each wrapped request struct for a similarly-never-fully-set field one at a time -- the same defensive default an accidentally-zero stack frame was already silently relied on for. Verified by re-running the exact live repro (jobrun against a real single-node deploy) after the fix: completes normally with no ASAN error. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1791 diff --git a/.wolf/bugs/bug-1795.md b/.wolf/bugs/bug-1795.md new file mode 100644 index 00000000..48244f1f --- /dev/null +++ b/.wolf/bugs/bug-1795.md @@ -0,0 +1,38 @@ +--- +id: "bug-1795" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-09-09T20:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c (leave_ctx)", "src/controller/testsuite/interruptcall.c/.h (xcetest_wait_connection_idle", "new)", "src/lib/client.c (xcecli_wakeup", "xcecli_stop", "xcecli_ctx_fini", "read-only for this fix)"] +tags: ["controller", "testsuite", "xce_test", "leave_ctx", "cmd_up", "cmd_quit", "interruptcall", "use-after-free", "asan", "concurrency", "semaphore", "xce-test-cli-ux", "task-3.2", "kiro-review-finding"] +related: ["bug-1794"] +legacy: true +--- + +# bug-1795: leave_ctx() (test.c, called by cmd_up/cmd_quit) could free a connection's api_ctx_t… + +## Description +leave_ctx() (test.c, called by cmd_up/cmd_quit) could free a connection's api_ctx_t (xcecli_stop()+xcecli_ctx_fini(), src/lib/client.c) while an abandoned worker thread from a prior Ctrl-C-interrupted RPC on that SAME connection was still running and still reading/writing it -- a genuine use-after-free, found by kiro-review during xce-test-cli-ux task 3.2's own round-1 review (not reproduced live/under ASAN on that run; identified structurally by tracing the code paths). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Task 3.2's own interruptible-call adapter (tasks 1.4/1.5, interruptcall.c) runs every blocking xce_* RPC on a detached worker thread; when the operator interrupts (Ctrl-C) a blocked call, xcetest_run_interruptible() returns -EINTR to the REPL immediately, but the worker thread it spawned keeps running -- possibly for up to its own ~10s cbuf_timedget deadline -- reading/writing whatever connection state its ctx closes over. Before this task, the REPL was fully synchronous, so leave_ctx() (test.c, shared by cmd_up's '..' and cmd_quit) could never run while an RPC was outstanding on that connection; task 3.2's adapter broke that invariant by letting control return to the prompt WHILE a worker is still abandoned-but-running. leave_ctx() already called xcecli_wakeup(active_ctx->ctx, -EINTR) before tearing the connection down, but that is only a best-effort nudge (it wakes the underlying condvar wait sooner than the worker's own deadline) -- nothing forced leave_ctx() to actually WAIT for that woken worker to finish before proceeding to xcecli_stop() (tears down the live socket + failover lock) and xcecli_ctx_fini() (unconditionally free()s the whole api_ctx_t). If the operator's very next action after abandoning a hung command is 'up' or 'quit' -- an entirely ordinary thing to do -- the still-running worker can be mid-read/write on memory that just got freed out from under it. This directly contradicts the Interruptible Call Adapter's own documented invariant ('at most one worker thread -- running or abandoned-but-still-running -- is ever active per connection at a time', interruptcall.h/design.md) -- that invariant was enforced only BETWEEN xcetest_run_interruptible() calls (via the per-connection semaphore registry, tasks 1.4/1.5), never against connection TEARDOWN, since leave_ctx() never itself goes through xcetest_run_interruptible(). + +## Fix +FIXED (2026-09-09, xce-test-cli-ux task 3.2 remediation round 1). Added xcetest_wait_connection_idle(void* conn_key) to interruptcall.c/.h, reusing the EXACT SAME per-connection semaphore registry lookup (xcetest_conn_lock_for()) that xcetest_run_interruptible() already builds and maintains -- not a new/parallel mechanism. It does a plain BLOCKING sem_wait() (deliberately not the interruptible sem_trywait()-poll variant used elsewhere -- this is a narrow, safety-critical wait where correctness outranks Ctrl-C responsiveness, and it resolves quickly in practice since xcecli_wakeup() already nudges the worker) then immediately sem_post()s the semaphore back rather than holding it -- the semantic is 'prove no worker is active on this connection right now', not 'hold the connection locked'. leave_ctx() now calls xcetest_wait_connection_idle(ctx->ctx) right after the existing xcecli_wakeup() call and right before xcecli_stop()/xcecli_ctx_fini() -- since both cmd_up and cmd_quit call this SAME shared leave_ctx(), both paths are covered by a single call site. New black-box regression test test_interrupt_then_immediate_teardown_no_uaf (test_test_cli.c, reuses the existing clitest_start_blackhole_listener() harness helper) interrupts a blocked hello RPC against a black-hole TCP peer and sends 'up' IMMEDIATELY afterward -- with no settling wait for the interrupted prompt to redisplay first, unlike the sibling interrupt test -- to race leave_ctx()'s teardown against the abandoned worker on purpose; asserts the session exits only via a normal (non-signal) status and that stderr never contains an 'AddressSanitizer' diagnostic anywhere across the whole session. Verified under the asan build profile. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1794 diff --git a/.wolf/bugs/bug-1796.md b/.wolf/bugs/bug-1796.md new file mode 100644 index 00000000..d1994b57 --- /dev/null +++ b/.wolf/bugs/bug-1796.md @@ -0,0 +1,38 @@ +--- +id: "bug-1796" +status: "open" +status_updated: "2026-09-24" +status_note: "qctrl/hostctrl perror() prints stale errno instead of RPC rc (src/harness/test.c)" +component: "controller" +severity: "unknown" +timestamp: "2026-09-09T22:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test.c (cmd_qctrl", "cmd_hostctrl", "cmd_jobkill RPC-error return paths)"] +tags: ["controller", "testsuite", "xce_test", "cmd_qctrl", "cmd_hostctrl", "cmd_jobkill", "perror", "errno", "misleading-diagnostic", "xce-test-cli-ux", "task-4.2", "pre-existing", "deferred"] +related: [] +legacy: true +--- + +# bug-1796: cmd_qctrl/cmd_hostctrl/cmd_jobkill's RPC-failure branches perror() a fixed label (e.g + +## Description +cmd_qctrl/cmd_hostctrl/cmd_jobkill's RPC-failure branches perror() a fixed label (e.g. "xcecli_queuectrl") without ever setting errno to reflect the RPC's actual returned status, so the printed reason (e.g. "Interrupted system call", "Resource temporarily unavailable", "Inappropriate ioctl for device") is whatever errno happened to be left over from an unrelated earlier libc call in the same process, not the real failure reason. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Each of these commands does `if (rc < 0) { perror("xcecli_..."); return rc; }` where `rc` is the RPC's own negative-errno-style return value (e.g. a clean -ENOENT for "no such queue/host"), but perror() always reads the CURRENT process errno, not `rc` -- and nothing in this branch ever does `errno = -rc` first. Found during xce-test-cli-ux task 4.2's live confirmation-gate verification: a bypassed qctrl/hostctrl RPC against a deliberately bogus (nonexistent) queue/host name printed different, unrelated-looking error text across different runs of the SAME scenario, root-caused (and independently reproduced 3 times by the task 4.2 reviewer) to stale errno left over from prior, unrelated syscalls/signal-decline paths in the same process -- e.g. a destructive-command Ctrl-C decline explicitly sets errno = EINTR at its own call site and never resets it, so a LATER unrelated RPC failure in the same session then misreports itself as "Interrupted system call". Confirmed pre-existing (commit 41f56fa6e, 2025-11-05), predating the whole xce-test-cli-ux spec; NOT a defect in the interruptible-call adapter (tasks 1.4/3.2) or the confirmation gate (task 2.1/3.3) -- both were independently confirmed to behave correctly in every case tested, this is purely a misleading-diagnostic-text bug. + +## Fix +NOT YET FIXED (found and deliberately deferred -- out of boundary for task 4.2, a pure live-verification task with no source changes permitted). Recommended: set `errno = -rc;` immediately before each `perror(...)` call on these RPC-error paths (or replace with `fprintf(stderr, "...: %s\n", strerror(-rc));` to avoid the errno indirection entirely), and audit other cmd_* handlers for the same perror()-without-errno-set pattern. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1797.md b/.wolf/bugs/bug-1797.md new file mode 100644 index 00000000..6ab1edc6 --- /dev/null +++ b/.wolf/bugs/bug-1797.md @@ -0,0 +1,38 @@ +--- +id: "bug-1797" +status: "open" +status_updated: "2026-09-24" +status_note: "cmd_qctrl leaks strdup'd queue name on RPC error (src/harness/test.c)" +component: "controller" +severity: "leak" +timestamp: "2026-09-09T22:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/testsuite/test.c (cmd_qctrl)"] +tags: ["controller", "testsuite", "xce_test", "cmd_qctrl", "leak", "asan", "xce-test-cli-ux", "task-4.2", "pre-existing", "deferred"] +related: ["bug-1796"] +legacy: true +--- + +# bug-1797: cmd_qctrl leaks its strdup()'d queue name on the xcecli_queuectrl() RPC-error return path (ASAN… + +## Description +cmd_qctrl leaks its strdup()'d queue name on the xcecli_queuectrl() RPC-error return path (ASAN LeakSanitizer-confirmed, 14 bytes). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +cmd_qctrl does `queue = strdup(optarg)` while parsing -q, then on `if (rc < 0) { perror("xcecli_queuectrl"); return rc; }` (the RPC-error branch) returns without `free(queue)` -- only the success path frees it. Distinct from the adjacent, intentional non-free on the EINTR/abandoned-worker path (ownership legitimately transferred there per the interruptible-call adapter's contract). Found during xce-test-cli-ux task 4.2's live confirmation-gate verification (a bypassed qctrl against a bogus queue name triggered this exact RPC-error branch under the asan build). Confirmed pre-existing (commit 41f56fa6e, 2025-11-05), predating the whole xce-test-cli-ux spec and untouched by task 3.1's flagspec migration; not caused by any change in this spec. + +## Fix +NOT YET FIXED (found and deliberately deferred -- out of boundary for task 4.2, a pure live-verification task with no source changes permitted). Recommended: add `free(queue);` on the RPC-error return path (or restructure with a single cleanup label), and check `cmd_hostctrl`/`cmd_jobkill`'s sibling strdup'd-argument fields for the same pattern. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1796 diff --git a/.wolf/bugs/bug-1798.md b/.wolf/bugs/bug-1798.md new file mode 100644 index 00000000..b815025d --- /dev/null +++ b/.wolf/bugs/bug-1798.md @@ -0,0 +1,41 @@ +--- +id: "bug-1798" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-09-10T06:57:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/meson.build"] +tags: ["meson", "build-system", "capnp", "controller.capnp.h", "ninja-race", "missing-dependency-edge", "libxce", "src/lib/meson.build", "root-cause-fix"] +related: ["bug-1439"] +legacy: true +gitea_issue: 267 +gitea_synced_status: "fixed" +--- + +# bug-1798: Fresh (from-scratch) build fails intermittently with `fatal error: controller.capnp.h: No such… + +## Description +Fresh (from-scratch) build fails intermittently with `fatal error: controller.capnp.h: No such file or directory`, reported live by the user running `./tools/xce-build.sh all debug` -- first for a libxce-internal consumer (utils.c.o), then AGAIN after the first fix landed, this time for scheduler/src/sim_record.c.o and scheduler/src/core_events.c.o (files entirely outside libxce). This is the root-cause fix for a class of failures already recorded across many prior recurrences in this buglog (see related_bugs) as an unexplained/unfixed 'ninja-race' -- this entry identifies and fixes the actual missing-dependency-edge cause at the meson.build level, in three parts (see fix), and additionally audited the whole project for other instances of this same bug class. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +THREE separate but related missing-dependency-edge gaps, all in src/lib/meson.build, all following the identical shape -- a multi-output custom_target() (a .c + a paired .h) whose .h output is reached by some OTHER file only via #include, with libxce_src or libxce_dep only ever referencing the .c output's index [0]: (1) the capnp code-generation custom_target (`gen_srcs`, output controller.capnp.c/.h) -- `libxce_src` only listed `gen_srcs[0]`, so libxce's OWN other .c files (utils.c, hook.c, threshold.c, xlist.c, window.c, xceapi.c, wrap.c, zarray.c, htab.c, xstring.c, etc.) that #include controller.capnp.h had no build edge forcing codegen to finish first. (2) The SAME gen_srcs custom_target was also unreachable, via any dependency edge, from EXTERNAL consumers: `libxce_dep = declare_dependency(link_with: libxce)` had no `sources:` kwarg, so `link_with` only ordered a consumer's LINK step after libxce was built, never a consumer's own COMPILE steps -- and controller/worker/scheduler all reach controller.capnp.h directly via a raw `-Ilib`/`-I../lib` include path, formally depending on nothing. Fix (1) alone only closed the race for files inside libxce itself, which is why the user's SECOND from-scratch `all debug` run still failed, now in scheduler instead. (3) Found via a follow-up audit (user asked 'check source code generated by peg'): the SAME shape recurs for all four packcc PEG-parser custom_targets (section_parser, rusage_parser, xce_resreq_parser, xce_select_parser -- each producing a .c + a paired .h: section.h, rusage.h, xce_resreq.h, xce_select_peg.h). `libxce_src` only listed each parser's `[0]` (.c) index; `lib/resreq.c` (confirmed via a project-wide grep to be the ONLY consumer of any of these four headers, entirely inside src/lib/, so no libxce_dep-level fix was needed for this part) #includes all four headers but has no declared build edge to any of the four codegen steps. On an INCREMENTAL build all three gaps are invisible (the generated files from a prior build already exist on disk), which is why this session's own extensive controller-only `--incremental` rebuilds throughout the xce-test-cli-ux spec never hit any of them -- they only manifest on a genuinely from-scratch build (`rm -rf` + fresh `meson setup`, which `xce-build.sh ` does by default without `--incremental`), and even then only when ninja's scheduler happens to pick an unlucky order under parallelism -- explaining the long history of the capnp variant being reported as an intermittent, unfixed 'ninja-race' across at least 5 prior recorded recurrences (2026-07-16 through 2026-07-26) with no meson.build-level fix ever applied, only the workaround 'retry the build once'. A project-wide search confirmed these are the ONLY multi-output C/H code-generation custom_target()s anywhere in the codebase (every other custom_target() -- xceadmin, frontend/apiserver, frontend/xcecli, autoscaler/providers -- produces a single Go-binary/stamp output with no equivalent header-only-reached-via-#include hazard). + +## Fix +FIXED (2026-09-10), in three parts, all in src/lib/meson.build. Part 1: changed `libxce_src`'s `gen_srcs[0]` element to the unindexed `gen_srcs`, closing gap (1) for capnp. Part 2 (the fix that actually closes the general, project-wide capnp race): added `sources: [gen_srcs]` to `libxce_dep`'s `declare_dependency()` call, closing gap (2) -- `declare_dependency()`'s `sources:` kwarg exists specifically for this case (generated files that must be built before ANY consumer of the dependency compiles), and since controller/worker/scheduler all already list `libxce_dep` in their own `dependencies:`, this single change propagates the missing build-order edge to every current and future consumer with no per-component changes needed. Part 3: changed `libxce_src`'s `section_parser[0]`/`rusage_parser[0]`/`xce_resreq_parser[0]`/`xce_select_parser[0]` elements to their unindexed forms, closing gap (3) for all four PEG parsers (no libxce_dep change needed here since nothing outside lib/ consumes these headers). Every part uses the same pattern: passing a multi-output custom_target unindexed into a `sources:`-accepting list still compiles only its recognized-source (.c) output, while ordering the WHOLE enclosing target's (or dependency's every consumer's) build after ALL of that custom_target's outputs are generated. (Tried first, and reverted: a `depends: [gen_srcs]` kwarg directly on the `static_library()` call -- rejected immediately by this project's pinned Meson 1.9.1 with 'Got unknown keyword arguments "depends"', not supported on `static_library()`/`build_target()` in this version.) Verified incrementally as each gap was found: two consecutive from-scratch `lib debug` builds passed with Part 1 alone (313/313 lib tests) -- but this was INSUFFICIENT, confirmed live by the user's own second `./tools/xce-build.sh all debug` run still failing in scheduler; after Part 2, two consecutive full from-scratch `./tools/xce-build.sh all debug` builds (1173/1173 targets each, the user's own exact repro command) both succeeded with zero failures and no retry; after adding Part 3 (found via the follow-up PEG audit), a third consecutive full from-scratch `all debug` build also succeeded clean (1173/1173), plus a fresh `lib debug` build + full `xcelib_test` run (313/313) and a fresh incremental `controller asan --incremental` rebuild. That incremental rebuild's FIRST run showed `1331 Tests 1 Failures` in `test_smoke_launch_send_echo_capture_stdout_and_exit_code` ('xce_test did not exit within the harness's safety timeout after quit') -- investigated and confirmed to be an unrelated, system-load-induced timing flake (the controller build ran concurrently with/immediately after the just-completed full `all debug` build): re-running the SAME binary standalone on an otherwise-idle system passed this exact test and the full suite cleanly (1331/1331) twice in a row; this is a pre-existing PTY-harness timing sensitivity, not a defect introduced by any of the three meson.build parts, all of which only affect build-graph ordering, never runtime process-exit behavior. Three consecutive full-project clean-build successes do not, by the race's own probabilistic nature, provide absolute certainty against some other unlucky ninja ordering recurring -- but Parts 1-3 address the actual general-case structural root cause for every multi-output codegen step in the project (every consumer of libxce_dep, plus every internal libxce_src consumer, now has a real build-graph edge on the relevant codegen step), so this is expected to eliminate this entire race class project-wide rather than merely reduce its likelihood. Does not address the separate, distinct apiserver<-libxce.a CGO-link race noted in one prior recurrence (2026-07-26) -- that is a different meson target (frontend/apiserver's go_build custom_target referencing @BUILD_ROOT@/lib/libxce.a via an env-var string, not a `dependencies:`/`libxce_dep` relationship) and remains unfixed; also worth a follow-up ticket for the test_test_cli.c PTY-harness safety-timeout flakiness under system load, observed here but out of this bug's own scope. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1439 +- Gitea: #267 diff --git a/.wolf/bugs/bug-1799.md b/.wolf/bugs/bug-1799.md new file mode 100644 index 00000000..57826902 --- /dev/null +++ b/.wolf/bugs/bug-1799.md @@ -0,0 +1,42 @@ +--- +id: "bug-1799" +status: "fixed" +status_updated: "2026-09-24" +status_note: "25fb5c11" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-11T07:32:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/scheduler.c", "src/scheduler/src/core_session.c"] +tags: ["scheduler", "sim", "replay", "resource-accounting", "resInUse", "sched_account_jobs", "worker-rejoin-resync", "IS_WORKER_RECONCILING", "over-commit", "slots", "max-capacity-exceeded", "core_reset_workers", "bug-1751-interaction", "user-reported", "found-live"] +related: ["bug-1751", "bug-1739", "bug-1734", "bug-1624", "bug-1727", "bug-1613"] +fix_commits: ["25fb5c11"] +legacy: true +gitea_issue: 268 +gitea_synced_status: "fixed" +--- + +# bug-1799: User-reported (xce_scheduler -i -s , live cluster-captured replay… + +## Description +User-reported (xce_scheduler -i -s , live cluster-captured replay /tmp/scheduler-replay-20260911-064117.replay, worker pc13v, 22 real slots): `info worker pc13v` at session=98 shows `resource slots: value=22.00 used=22.00` while the printed `assigned job:` list for that same worker sums to 26 slots (jobs [5]-[9] each *4 = 20, plus six single-slot jobs from a second batch = 6); the very next session (99, no new DISPATCH/PEND events, same assigned-jobs list) shows `used=26.00` -- exceeding the worker's own `value=22.00` capacity, an admission-invariant violation (max slots exceeded). Session 98's own event stream shows two back-to-back `WORKERS_PUSH`+`JOB_PUSH` (njobs=20) events for the same worker/jobids, consistent with a live scheduler-connection resync/reconnect being faithfully replayed (matching bug-1734/bug-1739's documented resync scenario), plus `sched_add_job failed ... rc=-17, merging state into existing tracked job instead of discarding` warnings for nearly every job in the resync batch, and one `resync claims RUN for jobid ...[7]/[8] with no allocation data -- refusing to merge a phantom RUN` (bug-1739's guard firing as designed). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Interaction between two independently-reasonable pieces of code that were never reconciled with each other. (1) bug-1751 (2026-09-01, worker-rejoin-resync task 5) deliberately added an `IS_WORKER_RECONCILING(w->status)` skip to `core_reset_workers()` (core_session.c:136-142, the sim/normal-mode-shared per-round resInUse-reset helper called from `xce_core_begin_session()`): a worker mid-resync has its `w->resInUse` zeroing SKIPPED that round, on purpose, so its in-flight usage isn't wiped while reconciliation is still in progress. (2) `sched_account_jobs()` (scheduler.c:1434-1461) is the ONLY thing that re-derives `resInUse`/`sharedInUse` each round -- it walks every job in `core->jobs` and, for each one with `JOB_IS_RUN(job->state) && job->accounts != NULL`, calls `sched_charge_account()` (an unconditional ADD via `sched_update_res_accounts(..., charge=1)`, resource.c:608-633) -- with NO equivalent RECONCILING check and NO reset of its own. Its own documented contract (core.h:311-322) is explicit that callers 'must call this strictly after the session's resval reset' -- a precondition bug-1751's reset-skip silently violates for exactly the RECONCILING window it targets. Consequence: for every round a worker stays RECONCILING (entered via a WORKERS_PUSH re-announce during a scheduler reconnect/resync, exactly what session 98's two WORKERS_PUSH events reflect), every currently-RUN job's slot charge on that worker gets RE-ADDED on top of the NOT-reset resInUse from the prior round, instead of the intended zero-then-resum. This produces exactly the observed shape: `used=` (accumulating additively) trails one round behind the true `assigned jobs` sum at first (22 vs 26 at session 98, since only some of the newly-RUN jobs' charges had been double-counted-in by that round's pass), then overtakes real capacity outright (26 at session 99). This is NOT the same mechanism as bug-1739 (JOBS_PUSH resync-merge phantom-RUN guard, which governs whether `job->allocs`/`state` get attached at all) nor bug-1624 (already-fixed dispatch-to-RUN synchronization gap) nor bug-1727 (already-fixed `info worker` display double-count of `value=`) -- all four live in the same resource-accounting neighborhood but are structurally distinct bugs. Root cause identified via static trace only (codegraph_explore + direct read of core_session.c/scheduler.c/core.h); not yet reproduced with an isolated minimal repro or instrumented trace confirming the exact round-by-round resInUse progression. + +## Fix +NOT YET FIXED -- investigation only, per user request (this session's live replay-driven Q&A, not yet escalated to an implementation task). Likely fix direction: give `sched_account_jobs()` the SAME `IS_WORKER_RECONCILING` awareness `core_reset_workers()`/`sched_reset_worker_()` already have (bug-1751) -- either (a) skip charging jobs whose host is currently RECONCILING (mirroring the reset-skip so the pair stays consistent: neither reset nor resum touches a RECONCILING worker's resInUse until it clears), or (b) have `core_reset_workers()` NOT skip the reset for a RECONCILING worker but instead have `sched_account_jobs()` re-derive from job state alone regardless of RECONCILING (i.e. resInUse is always a clean per-round sum, and RECONCILING-safety is enforced elsewhere, e.g. admission control refusing new dispatches to a RECONCILING host, which per bug-1751/bug-1745 already exists as a separate candidate-filter check). Needs the worker-rejoin-resync spec's own design intent re-read (.kiro/specs/worker-rejoin-resync/) before picking (a) vs (b), plus a live repro (re-run this exact replay file with an instrumented sched_account_jobs() logging resInUse before/after each round) to confirm the round-by-round mechanism before implementing, and full TDD (scheduler-driver-equivalence steering requires normal-mode/sim-mode parity for any fix here). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1751, bug-1739, bug-1734, bug-1624, bug-1727, bug-1613 +- Gitea: #268 diff --git a/.wolf/bugs/bug-1800.md b/.wolf/bugs/bug-1800.md new file mode 100644 index 00000000..80f5ed24 --- /dev/null +++ b/.wolf/bugs/bug-1800.md @@ -0,0 +1,41 @@ +--- +id: "bug-1800" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-11T07:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_info.c"] +tags: ["scheduler", "repl", "info-worker", "info-resource", "resource-accounting", "units-mode", "bytes-human-format", "mem", "swp", "tmp", "consumable", "display-feature", "user-requested", "tdd"] +related: ["bug-1727", "bug-1601", "bug-1613"] +legacy: true +gitea_issue: 269 +gitea_synced_status: "fixed" +--- + +# bug-1800: User request (live replay session, worker pc13v): `info worker`'s CONSUMABLE value=/used= line for… + +## Description +User request (live replay session, worker pc13v): `info worker`'s CONSUMABLE value=/used= line for a bytes-valued resource (mem/swp/tmp) printed raw byte counts with no unit scaling -- e.g. `resource mem: value=20684476416.00 used=87960930222080.00` -- unlike every OTHER bytes-valued field in the same listing (maxmem/maxswap/maxtmp already render `value=33182072832.000000 (30.90 GB)`, and mem/tmp/swp themselves render human-scaled via the Load Metrics path when NOT flagged consumable). bug-1727's own header comment had explicitly scoped this out: 'no bytes-valued CONSUMABLE resource is exercised anywhere in this codebase's tests today.' + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl_info_worker_one()'s and repl_info_resource_one()'s CONSUMABLE Value_u_fvalue branches both hardcoded `%.2f`/`%.2f` for value=/used=, deliberately bypassing repl_format_numeric_resource_value()'s existing units_mode-aware/repl_is_bytes_valued_resource()-gated formatting that every other numeric field in this listing already goes through -- correct for a non-bytes-valued consumable (slots, licenses) but wrong for mem/tmp/swp, which are bytes-valued CONSUMABLE resources in live practice even though no test exercised that combination before this session. + +## Fix +FIXED (2026-09-11). Added a new shared static helper repl_format_consumable_value_and_used() (repl_info.c, right after repl_format_numeric_resource_value()) that routes value=/used= through the existing repl_format_numeric_resource_value() formatter (units_mode-aware, plus the swp MB->bytes special case) when repl_is_bytes_valued_resource(rname) or rname=="swp" is true, and keeps the exact pre-existing "%.2f"/"%.2f" formatting otherwise (slots/licenses unaffected). Wired into both repl_info_worker_one()'s per-host CONSUMABLE case and repl_info_resource_one()'s GLOBAL/SHARED CONSUMABLE case, which share the fix via this one helper so they can't disagree. Updated the stale doc comments at repl_info.c's Resource Value Formatter section header and both call sites (previously asserted this pair was 'NOT units_mode-aware' and that no bytes-valued CONSUMABLE resource existed in tests -- no longer true). TDD: added 4 new tests to test_repl_resource_table.c (test_info_worker_units_mode_both_renders_raw_and_human_for_consumable_bytes_resource, test_info_worker_units_mode_raw_renders_bare_values_for_consumable_bytes_resource, test_info_worker_units_mode_converts_swp_megabytes_for_consumable_resource, test_info_resource_units_mode_both_renders_raw_and_human_for_consumable_bytes_resource) covering both/raw modes, the swp MB special case, and the info-resource twin. RED confirmed first (4 new failures, 76 pre-existing tests still green, `xce-build.sh scheduler asan xce.build --incremental`); GREEN confirmed after the fix (77/77 Ok, zero regressions). All existing pinned CONSUMABLE value=/used= tests (bug-1601/1613/1727, using "slots"/"licenses"/"gpu_licenses") were verified to use non-bytes-valued resource names before the fix, so they were unaffected by construction, not just by accident. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1727, bug-1601, bug-1613 +- Gitea: #269 diff --git a/.wolf/bugs/bug-1801.md b/.wolf/bugs/bug-1801.md new file mode 100644 index 00000000..418083b4 --- /dev/null +++ b/.wolf/bugs/bug-1801.md @@ -0,0 +1,41 @@ +--- +id: "bug-1801" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-11T08:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/resource.c"] +tags: ["scheduler", "resource-accounting", "resInUse", "sched_account_jobs", "sched_update_account", "worker-rejoin-resync", "IS_WORKER_RECONCILING", "IS_WORKER_UNAVAIL", "over-commit", "slots", "bug-1751-interaction", "tdd", "user-requested"] +related: ["bug-1799", "bug-1751", "bug-1739", "bug-1734", "bug-1624", "bug-1727"] +legacy: true +gitea_issue: 270 +gitea_synced_status: "fixed" +--- + +# bug-1801: Follow-up fix for bug-1799 (user asked to 'fix bug-1799', this session): the resInUse over-commit… + +## Description +Follow-up fix for bug-1799 (user asked to 'fix bug-1799', this session): the resInUse over-commit caused by core_reset_workers()'s RECONCILING/UNAVAIL reset-skip (bug-1751) having no matching skip in sched_account_jobs()'s per-round re-derive pass. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +See bug-1799's own entry for the full mechanism. Confirmed exactly as diagnosed: sched_update_account() (resource.c, the function both sched_charge_account() and sched_account_jobs()'s per-round resum ultimately call) had no IS_WORKER_UNAVAIL/IS_WORKER_RECONCILING check in its per-host charge loop, so a job on a worker mid-resync got its charge re-added every round on top of the not-reset total. + +## Fix +FIXED (2026-09-11). Added `if (IS_WORKER_UNAVAIL(w->status) || IS_WORKER_RECONCILING(w->status)) { continue; }` to sched_update_account()'s per-host loop (resource.c), right after resolving the worker `w`, before the sched_update_res_accounts() charge call -- the exact same combined status check core_reset_workers()/sched_reset_worker_() already use for the reset half (bug-1751), restoring the 'resum only what reset also touched' symmetry. Scoped to the per-host branch only: core->sharedInUse's own reset (core_reset_shared_resval()) is unconditional with no per-worker gating, so the shared-resource charge is untouched. This is a single-point fix (sched_update_account() backs both sched_charge_account()/sched_uncharge_account()), so it also applies (harmlessly/defensively -- a RECONCILING/UNAVAIL worker should never be a fresh DISPATCH target per the existing bug-1745/1751 candidate-filter exclusion) to the dispatch-time charge path via sched_commit_job_alloc(). TDD: added test_charge_account_skips_reconciling_or_unavail_worker_host_charge (test_resource_accounting.c) -- charges a job account against a worker flagged RECONCILING then UNAVAIL (per-host resInUse must stay 0 both times, shared resInUse still charges normally), then clears the status bit and re-charges (must resume normally on the very next call, proving this is a per-call skip, not a permanent block). RED confirmed by temporarily reverting the skip and rebuilding (`xce-build.sh scheduler asan xce.build --incremental`): exactly this one new test failed (`Expected 0 Was 2.5`), 76 other tests green. GREEN confirmed after restoring the fix: 77/77 Ok, zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1799, bug-1751, bug-1739, bug-1734, bug-1624, bug-1727 +- Gitea: #270 diff --git a/.wolf/bugs/bug-1802.md b/.wolf/bugs/bug-1802.md new file mode 100644 index 00000000..d70a52c6 --- /dev/null +++ b/.wolf/bugs/bug-1802.md @@ -0,0 +1,40 @@ +--- +id: "bug-1802" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-11T08:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_break.c", "src/scheduler/src/repl_internal.h"] +tags: ["scheduler", "repl", "sim", "replay", "breakpoints", "break", "ANY", "dummy-session", "cli-ux", "feature", "user-requested", "tdd"] +related: [] +legacy: true +gitea_issue: 271 +gitea_synced_status: "fixed" +--- + +# bug-1802: User request (same live replay session as bug-1799/1800): 'how can we step to the next session… + +## Description +User request (same live replay session as bug-1799/1800): 'how can we step to the next session which is not dummy(something occurs)?' -- the REPL's `break ` vocabulary required setting all ten per-event breakpoint kinds individually to catch 'the next session with ANY event at all,' with no single convenience kind for that common workflow. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not a bug -- a missing convenience feature. BP_KIND_TABLE (repl_break.c) only offered the ten specific per-event kinds (CONFIG_PUSH/JOB_PUSH/.../JOB_RUSAGE) plus BEGIN_ROUND/DECISION; repl_breakpoint_matches_event() compared bp->kind for exact equality against repl_bp_kind_for_core_event(ev->type), with no wildcard option. + +## Fix +FIXED (2026-09-11). Added a new XCE_REPL_BP_ANY enum member (repl_internal.h) and a `{"ANY", XCE_REPL_BP_ANY}` BP_KIND_TABLE entry (listed first for discoverability). repl_breakpoint_matches_event() (repl_break.c) now special-cases bp->kind == XCE_REPL_BP_ANY to match any event whose type maps to a real per-event kind (repl_bp_kind_for_core_event() returning nonzero) instead of requiring exact kind equality -- still subject to the breakpoint's own filters (though a bare `break ANY`, the intended usage, has none). Deliberately does NOT match BEGIN_ROUND (fires on every session including dummy ones -- matching it would defeat the whole point) or DECISION (a separate drained-decision hook). `break ANY` + `continue` now stops the REPL at the very next session that decodes at least one real event, transparently skipping any number of dummy (virtual-clock-only) sessions in between, in one command instead of ten. TDD: added test_any_breakpoint_skips_dummy_sessions_to_first_real_event (test_repl_breakpoints_inspection.c, full replay-file + REPL-script Group D fixture) -- two dummy sessions (rounds 1000/2000, zero events each) followed by one JOB_STATE_CHANGE event, `break ANY` + `continue` must land exactly at event #1 having processed both dummy rounds as 'idle' (round-rollup evidence), and must never print BEGIN_ROUND. Also extended the two existing BP_KIND_TABLE token-enumeration tests (test_repl_help.c's help-text test, test_repl_breakpoints_inspection.c's unrecognized-event-type-error test) to include "ANY" so both stay genuinely exhaustive against the real table. RED confirmed (the new test failed on the FIRST attempt for an unrelated reason -- the hit message names the breakpoint's own user-typed token, "ANY", not the underlying event's type name "JOB_STATE_CHANGE", which the test's first draft wrongly assumed; corrected the expected string, not the implementation, which was already correct). GREEN confirmed: 77/77 Ok, zero regressions, alongside bug-1801's fix in the same build. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #271 diff --git a/.wolf/bugs/bug-1803.md b/.wolf/bugs/bug-1803.md new file mode 100644 index 00000000..dfe65550 --- /dev/null +++ b/.wolf/bugs/bug-1803.md @@ -0,0 +1,41 @@ +--- +id: "bug-1803" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-11T09:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_break.c", "src/scheduler/src/repl_internal.h", "src/scheduler/src/sim.c", "src/scheduler/src/repl_info.c"] +tags: ["scheduler", "repl", "sim", "replay", "break-ANY", "reverted", "info-summary", "next-non-dummy-session", "sim_prescan_next_nonempty_session", "cli-ux", "user-requested", "tdd", "design-mismatch"] +related: ["bug-1802"] +legacy: true +gitea_issue: 272 +gitea_synced_status: "fixed" +--- + +# bug-1803: User feedback (live replay session, immediately after bug-1802's `break ANY` shipped): 'It seems… + +## Description +User feedback (live replay session, immediately after bug-1802's `break ANY` shipped): 'It seems that break any command will introduce too many break in single session' -- live transcript showed `break ANY` + repeated `step` firing 'hit breakpoint 1 @ event #N: any' on EVERY event (JOB_UPDATE, JOB_STATE_CHANGE, ...) inside an already-busy session (session 40), forcing many single-event `step`s just to get back OUT of that session -- the opposite of the intended 'skip past dummy sessions in one hop' workflow. User explicitly requested removing `break ANY` and instead having `info summary` display the next non-dummy session id so `step N` can jump there directly. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Design mismatch, not a code defect: a breakpoint is inherently a persistent, repeatedly-firing mechanism (correct behavior for its stated purpose -- it fires on every matching event until deleted), but the user's actual need ('skip past a run of dummy sessions to the next one worth looking at') is a ONE-TIME navigation query, not a recurring pause condition. Modeling it as a breakpoint kind (bug-1802) meant it kept firing on every subsequent matching event for the rest of the session, well past the point it had already served its purpose, with no natural way to say 'just this once.' + +## Fix +REVERTED `break ANY` entirely (bug-1802's XCE_REPL_BP_ANY enum member, BP_KIND_TABLE entry, repl_breakpoint_matches_event() special case, and its 3 tests) back to the exact pre-bug-1802 code, confirmed via `grep -rn XCE_REPL_BP_ANY` returning nothing but the historical doc-comment note left in repl_internal.h. REPLACED with a one-time informational display: `info summary` now also prints 'next non-dummy session: N -- run `step M` to reach it' (or 'none remaining'), computed by a new sim_prescan_next_nonempty_session() (sim.c, exported via sim.h) that peeks ahead in sim->replay_file from the current read position (same ftell()/fseek()-around-a-shape-only-scan technique sim_prescan_session_count() already establishes for the startup session-count banner) for the next ReplaySession boundary preceded by at least one real event, then rewinds -- a pure read-only query, never a recurring pause. Returns a RELATIVE 'sessions ahead' count (not an absolute session number) specifically to avoid needing xce_sched_repl_state_t's full definition inside sim.c (an incomplete type there by the project's own Simulation Isolation boundary -- caught by a real compile error on the first attempt, `invalid use of incomplete typedef`, when the function tried to read sim->repl_state->session_count directly); repl_info.c (which has the full repl_internal.h definition) adds the relative count to rs->session_count for the absolute display number. TDD: added test_info_summary_reports_next_nondummy_session_when_present and test_info_summary_reports_none_remaining_when_all_dummy (test_repl_breakpoints_inspection.c). RED confirmed by temporarily hardcoding the summary's prescan result to a sentinel non-0/non-1 value and rebuilding: exactly these 2 new tests failed, 75 others green. GREEN confirmed after restoring: 77/77 Ok, zero regressions. The existing test_info_summary_reports_counts (a bare in-memory fixture with no `.replay` file, sim->replay_file==NULL) is correctly unaffected -- the new line is silently omitted whenever the prescan can't run, mirroring sched_sim_driver_start()'s own 'purely informational, silently skipped on failure' precedent for its sibling prescan function. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1802 +- Gitea: #272 diff --git a/.wolf/bugs/bug-1804.md b/.wolf/bugs/bug-1804.md new file mode 100644 index 00000000..1ff0a136 --- /dev/null +++ b/.wolf/bugs/bug-1804.md @@ -0,0 +1,41 @@ +--- +id: "bug-1804" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-11T09:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: [".kiro/specs/scheduler-sim-eager-reconciliation/{requirements", "design", "research", "tasks", "spec.json}", "src/scheduler/src/sim_loop.c"] +tags: ["scheduler", "sim", "replay", "sim_reconcile_unaccounted_jobs", "sched_commit_decision", "job-state", "job-lifecycle", "resource-accounting", "resInUse", "design-fork", "scheduler-sim-eager-reconciliation", "spec-revision", "user-directed", "bug-1624", "driver-equivalence", "tdd"] +related: ["bug-1613", "bug-1611", "bug-1624", "bug-1799", "bug-1801"] +legacy: true +gitea_issue: 273 +gitea_synced_status: "fixed" +--- + +# bug-1804: User design assertion (same live replay session as bug-1799/1800/1803): 'according to the design… + +## Description +User design assertion (same live replay session as bug-1799/1800/1803): 'according to the design, when a job has been scheduled by scheduler, we can say job is running in scheduler angle' -- directly challenging this session's own earlier explanation that a job showing DISPATCH-then-still-PEND for multiple sessions was 'by design, not a bug' (deferred-until-confirmation semantics from the scheduler-sim-eager-reconciliation spec). User asked for confirmation and, once confirmed as a genuine design fork (not a bug in the then-current implementation), explicitly directed adopting the OTHER interpretation: revise the spec to match normal-mode's own synchronous dispatch-commit semantics (bug-1624). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not a bug in the sense of an implementation defect -- a genuine, previously-researched design fork between two defensible readings of 'job->state during replay': (a) reconstruct what the ORIGINAL live scheduler process's own job->state field actually held (RUN, synchronously, the instant sched_commit_decision() decided to dispatch -- bug-1624), vs (b) track the SEPARATELY-recorded JOB_STATE_CHANGE/worker-confirmation event's own timing as the operative signal (the scheduler-sim-eager-reconciliation spec's originally-shipped, explicitly-researched choice -- its research.md's own 'Does bug-1624's normal-mode fix already close this gap?' entry correctly noted bug-1624 is a live in-memory write, not a recordable event, so a recording's own JOB_STATE_CHANGE event genuinely CAN land rounds after its DISPATCH decision even post-bug-1624 -- a true fact, but the spec then chose interpretation (b) on top of that fact without it being forced). Live operator use surfaced interpretation (b)'s practical cost directly: a job showing PEND in `info jobs`/`info worker` for multiple sessions after its own `[sim] DISPATCH` line had already printed, reading as a scheduler bug even though it was working exactly as the shipped spec specified. + +## Fix +FIXED (2026-09-11, user-directed). Adopted interpretation (a): `sim_reconcile_unaccounted_jobs()` (sim_loop.c)'s FOUND branch now sets `job->state = JOB_STAT_RUN;` synchronously, in the same operation as the allocation capture (`job->allocs`/`job->accounts`), mirroring `scheduler.c`'s `sched_commit_decision()` (bug-1624) exactly. `sched_account_jobs()`'s own unchanged `JOB_IS_RUN` gate therefore charges the job's resInUse starting the SAME round as capture, not a later confirmation round. A later, separately-recorded `JOB_STATE_CHANGE`/`JOB_UPDATE`-to-RUN event, when it replays, is now a safe idempotent re-affirmation (RUN->RUN, `core_events.c`'s handler reassigns `job->state` unconditionally with no charging side effect) rather than the sole state-transition trigger. Revised the full `scheduler-sim-eager-reconciliation` spec in place (requirements.md Requirement 1's ACs 1.2-1.5 rewritten, Boundary Context's out-of-scope bullet corrected; design.md's Overview/Non-Goals/Boundary Commitments/Architecture flowchart/System Flows sequence diagram/Requirements Traceability/Service Interface doc/Testing Strategy all updated with dated 'Correction (2026-09-11)' notes, not silently rewritten; research.md's own 'Does bug-1624 already close this gap' entry got a superseding revision note explaining the (a) vs (b) trade-off explicitly, since the original factual finding was correct and remains true -- only the DECISION built on top of it changed; tasks.md got a new task 4/4.1/4.2 documenting the revision, both marked done; spec.json's updated_at bumped). TDD: renamed/inverted test_captured_pend_job_excluded_from_resinuse_until_run to test_captured_job_included_in_resinuse_same_round_as_capture (test_sim_reconcile_unaccounted.c) -- asserts job->state==RUN and resInUse charged immediately after capture (no separate job->state write needed), plus a new Requirement-1.4 companion check that a redundant RUN->RUN confirmation event (driven through the REAL XCE_CORE_EVENT_JOB_STATE_CHANGE handler) doesn't touch the allocation and doesn't change what gets charged after a proper core_reset_workers()-then-sched_account_jobs() round boundary (the test's own first draft skipped that reset before its second sched_account_jobs() call, an artificial double-charge that can never happen in real per-round production flow -- caught by RED/GREEN, fixed in the test, not the production code). Also rewrote test_sim_contention_parity.c's entire second test (test_contention_capacity_check_correct_across_multi_round_dispatch_confirm_gap) -- a 300+ line fixture/assertion block built around the OLD 'deferred until confirmation, partial-then-full usage across rounds 1/4/5' narrative -- to the NEW 'both dispatched jobs saturate capacity immediately at round 1; the later confirmation events are safe no-ops' narrative, since the old assertions (capacity free at round 1, only-A-charged at round 4) are now factually backwards. Audited every other file calling/depending on sim_reconcile_unaccounted_jobs() (test_snapshot_save_load.c, test_sim_snapshot_reconcile_gap.c, test_repl_resource_table.c, test_sim_round_boundary_reconciliation.c, test_sim_e2e_original_motivation.c, test_sim_verify.c) and confirmed each is unaffected: their fixtures either put the DISPATCH and JOB_STATE_CHANGE in the SAME round (a no-op change under the new synchronous-capture semantics, since the job was already RUN via the explicit event before reconciliation ever ran) or never rely on job-state/resInUse timing at all (pure --sim-verify decision-content comparisons, or jobs that are never captured by this mechanism in the first place). RED confirmed (temporarily reverted the `job->state = JOB_STAT_RUN;` line, rebuilt: exactly the 2 updated tests failed for the expected reasons, 75 others green). GREEN confirmed after restoring: full scheduler suite 77/77 Ok, zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1613, bug-1611, bug-1624, bug-1799, bug-1801 +- Gitea: #273 diff --git a/.wolf/bugs/bug-1805.md b/.wolf/bugs/bug-1805.md new file mode 100644 index 00000000..3589fdd1 --- /dev/null +++ b/.wolf/bugs/bug-1805.md @@ -0,0 +1,38 @@ +--- +id: "bug-1805" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-09-14T05:07:17.752283Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/meson.build"] +tags: ["meson", "symlink", "custom_target", "relative-path", "build-system", "xce-harness-component"] +related: [] +legacy: true +--- + +# bug-1805: xce-harness-component task 1.1: controller_unit_test had 29 test failures (all in… + +## Description +xce-harness-component task 1.1: controller_unit_test had 29 test failures (all in test_test_cli.c/test_xcetest_process_entry.c) after xce_test moved from src/controller/testsuite/ to src/harness/ -- Expected 0 Was 127 and missing xce> prompt + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +A bridging custom_target (xce_test_controller_link) created a symlink via command: ['ln', '-sf', '@INPUT0@', '@OUTPUT0@']. Meson expands @INPUT0@ to a build-root-relative path (harness/xce_test), but a relative symlink TARGET is resolved relative to the SYMLINK OWN DIRECTORY at dereference time, not the ln invocation cwd or the string reference frame it was written in. The resulting symlink at /controller/xce_test therefore looked for /controller/harness/xce_test, which does not exist, causing every subprocess-spawning test (resolve_xce_test_path()) to exec a missing binary (exit 127). + +## Fix +FIXED (2026-09-14, xce-harness-component task 1.1). Changed the custom_target command to use the literal relative path ['ln', '-sf', '../harness/xce_test', '@OUTPUT0@'] instead of @INPUT0@, so the symlink text correctly resolves relative to controller/ up to harness/xce_test. Verified: controller_unit_test dropped from 29 failures to 0 (1228 Tests 0 Failures 0 Ignored), confirmed via the Unity binary's own printed summary line, not just meson's aggregate Ok/Fail count. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1806.md b/.wolf/bugs/bug-1806.md new file mode 100644 index 00000000..d170344e --- /dev/null +++ b/.wolf/bugs/bug-1806.md @@ -0,0 +1,38 @@ +--- +id: "bug-1806" +status: "wontfix" +status_updated: "2026-09-24" +status_note: "accepted Meson limitation (user decision 2026-09-14)" +component: "harness" +severity: "unknown" +timestamp: "2026-09-14T09:20:27.103747+00:00" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/harness/meson.build", "src/tools/xce-build.sh"] +tags: ["meson", "test-runner", "build-dependency", "xce-harness-component", "accepted-limitation"] +related: ["bug-1805"] +legacy: true +--- + +# bug-1806: xce-harness-component task 4.3: /xce-build controller alone still rebuilds harness/xce_test on… + +## Description +xce-harness-component task 4.3: /xce-build controller alone still rebuilds harness/xce_test on disk under the default asan profile, apparently violating requirement 2.2 (controller-only build shall not produce xce_test as a side effect) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Deterministic, not a flake -- confirmed via build-log line analysis. meson test (run by /xce-build controller's default RUN_TESTS=1 asan profile via meson test --suite controller) rebuilds every registered test()'s build-order dependencies PROJECT-WIDE before filtering which tests actually RUN by --suite. harness_unit_test's test() registration (src/harness/meson.build) has depends: [xce_test] (added task 2.1, verified load-bearing in task 4.2), so any meson test invocation -- even one scoped to --suite controller -- transitively relinks xce_test. The explicit step is clean (never references xce_test); only the SUBSEQUENT meson test step touches it. Task 3.1's own validation missed this because it checked for the OLD path (controller/xce_test, correctly absent) rather than the NEW path (harness/xce_test, which does get rebuilt). + +## Fix +NOT FIXED -- accepted as a documented Meson platform limitation (user decision 2026-09-14). Requirement 2.2 narrowed to scope the build system's guarantee to what it directly controls: the explicit compile-target list never names xce_test (still true and verified). Rejected alternatives: (a) add --no-rebuild to xce-build.sh's meson test calls -- non-trivial, since controller's OWN test suite already has 4 small standalone test binaries (sched_push_unit_test, sched_worker_change_unit_test, apiserver_push_unit_test, host_runusage_unit_test) not in its explicit compile target list, relying on this same auto-rebuild; fixing this properly means auditing and completing every component's explicit target list, with real regression risk; (b) restructure harness_unit_test's pty-spawning tests to drop the depends: edge and locate xce_test at runtime instead -- directly reverses the build-order guarantee task 4.2 just verified is real and load-bearing, reintroducing the stale/missing-binary risk it was added to prevent. requirements.md/design.md/tasks.md updated to reflect the narrowed, accurate guarantee. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1805 diff --git a/.wolf/bugs/bug-1807.md b/.wolf/bugs/bug-1807.md new file mode 100644 index 00000000..b1052149 --- /dev/null +++ b/.wolf/bugs/bug-1807.md @@ -0,0 +1,38 @@ +--- +id: "bug-1807" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "harness" +severity: "unknown" +timestamp: "2026-09-14T15:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/harness/test.c (all ~20 cmd_* handlers)", "src/harness/join_worker_args.c"] +tags: ["confirmed", "getopt", "getopt_long", "cli-parsing", "xce_test", "harness_unit_test", "test_test_cli.c", "flagspec"] +related: [] +legacy: true +--- + +# bug-1807: harness_unit_test's test_test_cli.c: 4 of 132 tests deterministically FAIL every run… + +## Description +harness_unit_test's test_test_cli.c: 4 of 132 tests deterministically FAIL every run -- test_migrated_commands_sample_reject_unrecognized_flag (jobkill unrecognized-flag rejection), test_qctrl_hostctrl_confirm_gate_destructiveness_determination (qctrl -c or hostctrl -c prompt missing -- WHICH one shifts between builds), test_destructive_commands_prompt_by_default_and_yes_bypasses (upgradeconfig prompt missing), test_confirm_gate_session_toggle_skips_and_restores ('set +y' restore prompt missing). Each failing assertion burns its full 5000ms clitest_wait_for_text() timeout (total suite runtime ~28-29s vs ~9s once fixed) -- not a race, a genuine miss every time. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via isolated pty repro (bypassing Unity/gdb entirely): every cmd_*() REPL command handler resets getopt_long() state with `optind = 1; opterr = 0;` before its own parse loop. glibc's getopt_long only re-initializes its internal static parser state (nextchar, first_nonopt/last_nonopt) when optind==0 OR on the process's very first-ever getopt_long() call (guarded by the internal __getopt_initialized flag) -- optind=1 alone is NOT sufficient once any earlier call in the same process has already run once. Since xce_test's REPL dispatches many independent cmd_*() calls (each with its OWN, differently-sized argv array) across the lifetime of one process, a command whose parse ends by consuming a short no-argument flag as its LAST token (e.g. `qctrl -q normal -a`, `hostctrl -h node01 -o`) leaves stale internal state that corrupts the NEXT unrelated command's getopt_long() call -- observed corrupting the NEXT command so badly that argv[0] (the command's own name, e.g. 'upgradeconfig' or 'jobkill') is misread as an unrecognized flag via flagspec_reject(). 100% reproducible in isolation: `qctrl -q normal -a` immediately followed by ANY other command (upgradeconfig, jobkill) corrupts that next command every time; the same session's commands run individually/first are unaffected. Explains why the specific victim test shifted between builds (bug-1807 was first seen failing on hostctrl -c, then qctrl -c after an unrelated same-file edit) -- runtime sequence-dependent, not random. + +## Fix +FIXED. Changed all 20 occurrences of `optind = 1;` in src/harness/test.c and the 1 occurrence in src/harness/join_worker_args.c to `optind = 0;` (the documented glibc idiom that forces getopt_long to fully reinitialize, including nextchar/first_nonopt/last_nonopt, on every independent parse). Verified: (1) the 3 isolated minimal pty repros that previously reproduced the corruption 100% of the time now pass cleanly; (2) harness_unit_test run 10x consecutively under the debug profile: 132 Tests 0 Failures every run (was 4 Failures every run pre-fix), runtime dropped from ~28-29s to ~8-9s per run, confirming the failures were full timeout burns, not near-miss races; (3) full `/xce-build harness asan` (meson test, ASAN-instrumented) also reports 5/5 OK including harness_unit_test, no sanitizer findings. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1808.md b/.wolf/bugs/bug-1808.md new file mode 100644 index 00000000..df94a77c --- /dev/null +++ b/.wolf/bugs/bug-1808.md @@ -0,0 +1,38 @@ +--- +id: "bug-1808" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "harness" +severity: "leak" +timestamp: "2026-09-14T15:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/harness/testsuite/test_test_cli.c", "src/harness/testsuite/harness_test.c"] +tags: ["process-leak", "fd-leak", "unity", "longjmp", "test-infrastructure", "test_test_cli.c", "clitest"] +related: ["bug-1807"] +legacy: true +--- + +# bug-1808: src/harness/testsuite/test_test_cli.c: a spawned xce_test child process… + +## Description +src/harness/testsuite/test_test_cli.c: a spawned xce_test child process (clitest_spawn/clitest_spawn_ex) plus its pty master fd and stderr pipe fd leak whenever a TEST_ASSERT_* assertion fails partway through the owning test function, because every test relies on its OWN clitest_kill_and_reap()+clitest_close() calls placed inline at the bottom of the function body -- code that Unity's longjmp-based TEST_PROTECT()/TEST_ABORT() abort mechanism skips entirely on assertion failure. Directly observed: 4 live, idle `xce_test --norc` child processes still running (blocked forever on their own prompt) under one in-progress harness_unit_test invocation, discovered while investigating bug-1807's failures. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Design gap, not a logic error in any specific test: clitest_kill_and_reap()/clitest_close() were documented as a mandatory safety net ('every caller is required to always end a session with clitest_kill_and_reap() regardless of which path was taken') but placed only at each test function's own normal-exit code path. Unity's RUN_TEST() macro DOES guarantee tearDown() runs after every test (pass, fail, or aborted-via-longjmp), but harness_test.c's shared tearDown() (used by all 13 Unity test groups linked into this binary, most unrelated to clitest) was a no-op beyond a cosmetic separator print -- it never reaped a still-open clitest session. + +## Fix +FIXED. Added a static clitest_session_t mirror (g_last_session) in test_test_cli.c updated by clitest_spawn_ex() (set on successful spawn), clitest_kill_and_reap() (reaped flag synced), and clitest_close() (cleared) -- a STATIC-storage mirror was required rather than tracking a pointer into the aborted test's own local `sess`, since that stack frame is unwound by the longjmp and would leave a dangling pointer. Exposed a new clitest_reap_leaked_session() function, called from harness_test.c's tearDown() (now declared extern there), which force-kills+reaps+closes any session left open when a test aborted before reaching its own inline cleanup. Only one clitest session is ever open at a time in this file (verified), so a single mirror slot is sufficient. Verified via ASAN build (xce.build): no LeakSanitizer or other sanitizer findings from the new code across a clean 5/5 harness suite run. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1807 diff --git a/.wolf/bugs/bug-1809.md b/.wolf/bugs/bug-1809.md new file mode 100644 index 00000000..20a1ef25 --- /dev/null +++ b/.wolf/bugs/bug-1809.md @@ -0,0 +1,38 @@ +--- +id: "bug-1809" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-14T16:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge branch", "~line 845)"] +tags: ["scheduler", "logging", "observability", "core_events-c", "jobs_push", "sched_add_job", "eexist", "log-level", "noise-reduction", "not-a-defect", "session-sibling-of-bug-1734", "session-sibling-of-bug-1714", "DBG-macro", "utils-h"] +related: ["bug-1734", "bug-1714", "bug-1739", "bug-1738"] +legacy: true +--- + +# bug-1809: User-reported: live scheduler logs are flooded with WARN lines for every ordinary… + +## Description +User-reported: live scheduler logs are flooded with WARN lines for every ordinary XCE_CORE_EVENT_JOBS_PUSH resync redelivery: "xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid (rc=-17), merging state into existing tracked job instead of discarding" (src/scheduler/src/core_events.c). rc=-17 is -EEXIST -- the expected, routine outcome of on_scheduler_join()'s full resync (bug-1734) or the array-push dedup race (bug-1714), already correctly handled by the bug-1734/bug-1738/bug-1739 merge logic in the same branch, not a functional defect. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not a defect -- a UX/observability request. sched_add_job() (core.c) wraps key_add()->kvtab_add() (lib/kv.c), whose only failure modes for the core->jobs table (KVMGR_KEY_INT, no keyops->assign) are -EEXIST (duplicate jobid) or -ENOMEM (allocation failure). The EEXIST case is therefore the near-universal, routine shape of this branch under a reconnect/resync storm (confirmed live in bug-1714's own investigation: 700+ of these WARNs fired from a single 8-reconnect burst) -- WARN was the wrong level for an outcome the code already handles correctly by design; it was originally added at WARN by bug-1734's own fix pass without considering steady-state log volume under the reconnect-storm conditions that same session's investigation had just captured. + +## Fix +FIXED (2026-09-14). Split the single WARN call in the JOBS_PUSH EEXIST branch (core_events.c) on add_rc: when add_rc == -EEXIST, log via a new DBG(...) macro (utils.h, wraps tlog_debug() -- DEBUG level, silent by default, visible with -vvv) instead of WARN -- preserves the message (still valuable forensic evidence per bug-1734/bug-1714) without flooding default-verbosity operational logs. Any OTHER negative add_rc (e.g. -ENOMEM) still logs at WARN, since that would be a genuine anomaly, not a routine resync artifact. The new macro is named DBG, not DEBUG, matching the existing TRACE/INFO/WARN/ERROR one-line wrapper convention in utils.h: a first attempt to name it DEBUG(...) collided with the build's own bare -DDEBUG command-line define (src/meson.build, set for BOTH the debug and asan buildtypes -- confirmed via build warning "DEBUG redefined ... this is the location of the previous definition " across every file that includes utils.h under either profile), so DBG was used instead per user decision after the tradeoff was surfaced. Verified: xce-build.sh scheduler debug --incremental and xce-build.sh scheduler asan --incremental both compile clean (zero DBG/DEBUG-redefinition warnings in either profile); full scheduler suite 77/77 Ok, zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1734, bug-1714, bug-1739, bug-1738 diff --git a/.wolf/bugs/bug-1810.md b/.wolf/bugs/bug-1810.md new file mode 100644 index 00000000..f12ca712 --- /dev/null +++ b/.wolf/bugs/bug-1810.md @@ -0,0 +1,38 @@ +--- +id: "bug-1810" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "hang" +timestamp: "2026-09-15T03:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/mylock.c (cbuf_put/cbuf_get/cbuf_timedget)", "src/lib/vtaskmgr.c (vcpu_idle", "vcpu_entry)"] +tags: ["libxce", "mylock", "vtaskmgr", "cbuf", "pthread_cancel", "pthread_cond_wait", "cleanup-handler", "orphaned-mutex", "deadlock", "hang", "xce_test", "harness", "ctrl-d", "eof", "quit", "teardown", "race-condition", "tdd-red-green", "live-gdb-repro"] +related: [] +legacy: true +--- + +# bug-1810: User-reported live hang: xce_test (harness REPL) never exits on Ctrl-D (EOF) after entering a live… + +## Description +User-reported live hang: xce_test (harness REPL) never exits on Ctrl-D (EOF) after entering a live connection context (client/worker/scheduler). Live gdb attach on the user's machine showed the main thread parked forever in leave_ctx() -> xcecli_ctx_fini() -> xce_ctx_fini() -> vtaskmgr_fini() -> vcpu_fini()'s pthread_join(vcpu->vcpu, ...); a sibling vcpu thread was simultaneously deadlocked in cbuf_put() at mylock.c:101's pthread_mutex_lock(&(b->mutex)) on vtaskmgr->wbuf.mutex, whose __owner TID had already exited. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Two compounding defects in src/lib's shared task-pool primitives (used by every daemon + xce_test's own client/worker/scheduler REPL contexts, each of which owns a vtaskmgr_t for async I/O dispatch). (1) mylock.c's cbuf_put()/cbuf_get()/cbuf_timedget() lock b->mutex, then block in pthread_cond_wait()/pthread_cond_timedwait() with NO pthread_cleanup_push() handler installed. POSIX mandates pthread_cond_wait() reacquire its mutex before honoring a pending cancellation and running cleanup handlers -- with no handler registered, a thread cancelled while parked there terminates HOLDING the mutex, orphaning it locked forever; every other thread that later calls pthread_mutex_lock() on that same cbuf deadlocks uninterruptibly (plain mutex lock is not a cancellation point in glibc's NPTL). (2) vtaskmgr.c's teardown protocol races directly into that hazard from up to three independent angles, all torn down via pthread_cancel()+pthread_join(): (a) vtaskmgr_fini() cancels the dispatcher thread FIRST -- if it lands while the dispatcher is blocked in pick_idle_vcpu()'s cbuf_get() on the shared vtaskmgr->wbuf (empty, e.g. because real I/O work had every vcpu momentarily busy), that orphans wbuf's mutex immediately, before any vcpu teardown even starts; (b) vcpu_fini() then wakes each idle vcpu via bind_vtask(NULL, vcpu) (a 'poison pill' through the vcpu's own 1-capacity wbuf) and immediately pthread_cancel()s it, but vcpu_entry()'s loop could not distinguish that pill from 'no task yet' -- it looped straight back into vcpu_idle(), which unconditionally re-announced the vcpu into the shared (by then permanently undrained, since the dispatcher is already dead) vtaskmgr->wbuf, racing the pending cancellation against that second cbuf_put(); (c) the same poison pill arriving while a vcpu was still mid-execution of a real task queued it silently for the NEXT vcpu_idle() call, which still re-announced into the shared queue first before ever consuming it. Any of the three could orphan vtaskmgr->wbuf's mutex; every vcpu processed afterward that still touched the shared queue then deadlocked forever on pthread_mutex_lock(), and vtaskmgr_fini()'s pthread_join() for it never returned -- hanging the whole process (xce_test on Ctrl-D quit; identically reachable from any daemon that tears down a vtaskmgr_t under real concurrent I/O, e.g. controller/scheduler/worker shutdown). + +## Fix +FIXED (2026-09-15), TDD RED->GREEN. Two-part fix, both required (confirmed empirically -- the vtaskmgr.c-only fix alone left the dispatcher-orphan path (2a above) unaddressed and still reproduced the hang under stress). (1) mylock.c: wrapped the pthread_cond_wait()/pthread_cond_timedwait() loops in cbuf_put()/cbuf_get()/cbuf_timedget() with pthread_cleanup_push(pthread_mutex_unlock, &b->mutex) / pthread_cleanup_pop(0) -- a cancelled waiter now always releases the mutex on unwind, so no cbuf mutex can ever be left orphaned-locked by a cancellation, regardless of which caller (dispatcher or vcpu) hits it. cbuf_timedget()'s early ETIMEDOUT return was restructured to break+flag instead of returning from inside the pushed region, since push/pop must be a matched pair at the same lexical nesting level. (2) vtaskmgr.c, defense-in-depth / closes the underlying logic bug independently of (1): vcpu_idle() now does a non-blocking cbuf_tryget() on its own wbuf BEFORE announcing itself into the shared vtaskmgr->wbuf, so an already-queued poison pill (queued while the vcpu was mid-task) is consumed directly instead of triggering a needless re-announce; vcpu_entry()'s loop now checks vcpu->task after vcpu_idle() returns and breaks (exits the thread) on NULL instead of looping back into idle -- a NULL task is unambiguously vcpu_fini()'s shutdown signal, since the dispatcher (vdispatcher_entry) never calls bind_vtask() with a NULL task. New regression test src/lib/testsuite/test_vtaskmgr_teardown.c (wired into src/lib/meson.build + testsuite/test.c): floods a burst of real VTASK_LIVENESS tasks across a 16-vcpu vtaskmgr then calls vtaskmgr_fini() on a helper thread with a pthread_timedjoin_np() 2s bounded-timeout join, self-pinning the test to a single CPU (sched_setaffinity, inherited by every spawned vtaskmgr thread at clone()) since the race needs preemptive time-slicing to reproduce reliably -- confirmed 0/hundreds of hangs on stock multi-core scheduling (true parallelism lets a vcpu almost always win the race to its own pthread_testcancel() before cancellation is even issued) vs. reliable reproduction (hung on iteration 0) once pinned to one core. Verified via: (a) this new test, RED before the fix (Expected 0 Was 110/ETIMEDOUT) and GREEN after; (b) a standalone single-core-pinned stress harness outside the Unity suite, 2000/2000 iterations clean post-fix (vs. hanging on iteration 0 with no fix and again on iteration 3 with only the vtaskmgr.c half of the fix, confirming both parts were needed); (c) full libxce test suite (314/314) and full harness suite (132/132, including the pty-based clitest tests that exercise real client-context connect+quit) both green; (d) a live pty-driven repro of the ORIGINAL user report -- launching the real xce_test binary, entering the client context, sending a real Ctrl-D (EOT) byte -- now exits in <0.1s (previously would hang indefinitely); (e) full project meson test matrix (98/98 suites) green on a clean run -- two earlier full-matrix runs showed unrelated, non-reproducing timeouts (one in harness's own quit-wait, one in controller_unit_test) that did not reproduce on isolated reruns, consistent with this sandbox's pre-existing parallel-test-suite CPU-contention flakiness (see bug-1429/bug-1580), not a regression from this fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1811.md b/.wolf/bugs/bug-1811.md new file mode 100644 index 00000000..f28328a0 --- /dev/null +++ b/.wolf/bugs/bug-1811.md @@ -0,0 +1,38 @@ +--- +id: "bug-1811" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-15T07:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c", "src/scheduler/src/sim_loop.c", ".kiro/specs/scheduler-sim-decision-parity/{requirements", "design}.md"] +tags: ["not-a-bug", "design-decision", "scheduler", "sim-replay", "recorded-decision-reconciliation", "reverted-fix"] +related: ["bug-1739", "bug-1804", "bug-1778"] +legacy: true +--- + +# bug-1811: User-reported (interactive sim session=35, `info jobs`/`info worker pc13v`): job… + +## Description +User-reported (interactive sim session=35, `info jobs`/`info worker pc13v`): job 0-14-1789369200-0[21] shows RUN in `info jobs` but is absent from `info worker pc13v`'s assigned-jobs listing (which lists [0-20,22], skipping 21) -- user's working theory was slot overcommitment (23 running jobs vs. 22-slot worker capacity). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT A BUG -- initial diagnosis was WRONG and the attempted fix was reverted. Root-caused (correctly, mechanism-wise) to XCE_CORE_EVENT_JOB_STATE_CHANGE/XCE_CORE_EVENT_JOB_UPDATE (core_events.c) setting job->state=RUN unconditionally even when job->allocs/job->accounts are still NULL -- repl_info_worker_one() (repl_info.c) walks job->allocs to build the assigned-jobs listing, so an unaccounted RUN job is correctly displayed as RUN in `info jobs` but silently absent from `info worker`'s listing. Initially misdiagnosed this AS a defect (a 'phantom RUN' analogous to bug-1739's JOBS_PUSH resync gap) and implemented a guard refusing the RUN transition without an existing allocation. That guard was WRONG: this exact 'RUN with allocs==NULL' transient window is the INTENTIONAL, designed, tested trigger condition for Recorded-Decision Reconciliation (scheduler-sim-decision-parity spec, sim_loop.c's sim_reconcile_unaccounted_jobs(), extended by bug-1804's 2026-09-11 revision to set job->state=RUN synchronously via JOB_STATE_CHANGE/JOB_UPDATE ahead of allocation capture, matching the original live scheduler's own sched_commit_decision() semantics). Confirmed via test_sim_e2e_original_motivation.c's own header comment and ANCHOR_ID fixture: the SAME core_events.c code path is used, by design, to construct a job that is 'already running, with an unaccounted allocation, when a scheduling round's own live decision-making reaches it' -- 'Recorded-Decision Reconciliation's entire reason for existing'. My guard broke this fixture (test_e2e_original_motivation_long_replay) plus test_push_events_fires_each_hook_once_per_event_with_picker_idle and test_drain_translates_and_pushes_all_ten_op_types, both of which legitimately exercise a bare JOB_STATE_CHANGE/JOB_UPDATE(RUN) with no allocation yet. Further confirmed against scheduler-sim-decision-parity/requirements.md Requirement 4 ('Graceful degradation when recorded decision data is unavailable'): AC 4.1/4.2 EXPLICITLY anticipate and accept that a job's originating DISPATCH decision may not be found in the recorded replay data, in which case the scheduler reports that job's usage as unavailable rather than fabricating a value, and continues processing -- this is precisely job 21's observed symptom, not an accident. The user's overcommitment theory is also independently wrong on the accounting side: sched_account_jobs() only charges resInUse for JOB_IS_RUN(state) && accounts != NULL, so an unaccounted RUN job is correctly EXCLUDED from the real slot-usage total (used=22.00 already correctly reflects only the 22 genuinely-accounted jobs) -- there is no real overcommitment, only a display gap for a job whose allocation reconciliation hasn't (yet, or ever, depending on whether its DISPATCH decision exists later in this replay stream) caught up. + +## Fix +REVERTED. Initial fix (guard in core_events.c's XCE_CORE_EVENT_JOB_STATE_CHANGE/JOB_UPDATE refusing a RUN claim without an existing allocation, plus 6 new tests in test_core_events.c and a fixture change in test_driver_translate.c) was implemented, RED/GREEN-tested, then found to break test_sim_e2e_original_motivation.c's test_e2e_original_motivation_long_replay (the live-verify for the entire Recorded-Decision Reconciliation mechanism) plus two pre-existing test_core_events.c/test_driver_translate.c tests. All changes reverted via `git checkout --` before any commit. No production code changed. Left as an open question whether job 21 (and the user's follow-up observation that this 'occurs more frequently') ever actually reconciles later in the same replay session -- if a job's RUN-but-unaccounted display state NEVER resolves even after the replay completes, that would indicate its DISPATCH decision is genuinely missing from the recorded stream (the same category of gap scheduler-snapshot-job-allocation's PREEXIST_ID addresses for pre-recording dispatches, via snapshot-based restoration instead of replay-based reconciliation) -- worth a follow-up investigation with concrete evidence (does the job ever show up in `info worker`'s listing on a later round?) before attempting any further fix. A safe, smaller follow-up actually worth considering: improve `info worker`'s assigned-jobs display (repl_info.c) to explicitly distinguish a RUN-but-unaccounted job (e.g. an annotation) from silent omission, so operators aren't misled into an overcommitment theory the way this session's user was -- proposed to the user, not yet implemented. SUPERSEDED (2026-09-15, same session): the 'not a bug' conclusion above was itself only half right. The transient RUN-before-allocation window IS legitimate (Recorded-Decision Reconciliation), but the specific PERMANENT loss the user kept observing turned out to be a real, separate defect in the JOBS_PUSH duplicate-jobid resync merge -- see bug-1812 for the actual root cause and fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1739, bug-1804, bug-1778 diff --git a/.wolf/bugs/bug-1812.md b/.wolf/bugs/bug-1812.md new file mode 100644 index 00000000..b278e9b4 --- /dev/null +++ b/.wolf/bugs/bug-1812.md @@ -0,0 +1,38 @@ +--- +id: "bug-1812" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-15T08:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge", "bug-1734/bug-1739's own code)"] +tags: ["scheduler", "resync", "jobs-push", "allocation-loss", "race-condition"] +related: ["bug-1734", "bug-1738", "bug-1739", "bug-1811"] +legacy: true +--- + +# bug-1812: User-reported (interactive sim session=34->35 stepped live, AND independently confirmed via a real… + +## Description +User-reported (interactive sim session=34->35 stepped live, AND independently confirmed via a real production scheduler.log from rshen@pc13v /t/xce-ha-deploy): job 0-14-1789369200-0[21] holds allocs=1 (state=RUN) at the end of session 34; stepping into session 35 replays a JOBS_PUSH resync (njobs=60, including all of array job 0-14-1789369200-0[0-39]) followed by a JOB_UPDATE and a redundant JOB_STATE_CHANGE(RUN) for [21]; immediately after, `info job [21]` shows state=RUN but allocs=0. The real scheduler.log shows the identical mechanism live: `sched_job_state_change ... state 2` (schedStartJob confirmation) at 15:00:05.196, followed 16ms later by `JOBS_PUSH: sched_add_job failed for jobid ...[21] (rc=-17), merging state into existing tracked job instead of discarding` -- repeating in bursts on later reconnects (six WARNs in ~2s at 15:01:51-53). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED, via direct code reading plus independent live evidence from both a sim replay and a real production scheduler.log. xce_core_push_events()'s JOBS_PUSH handler, on a duplicate jobid (sched_add_job() -EEXIST, a resync redelivery of an already-tracked job), branches three ways once job_state_regresses_terminal() is ruled out: (1) resync claims RUN with real allocation data -> transfer it in (bug-1739's fix); (2) resync claims RUN with NO allocation data -> refuse, keep existing (bug-1739's own companion guard); (3) resync claims anything else (the bare `else` branch, bug-1734's original fix) -> UNCONDITIONALLY frees existing->accounts/existing->allocs and overwrites existing->state with the resync's own claim, with NO check for whether `existing` already holds a REAL, already-captured allocation the resync claim doesn't support. on_scheduler_join()'s full resync (xcesched_push_jobs(), src/controller/sched.c:214, fired on every scheduler (re)connect via join.c) is built from collect_jobs_committed() -- a committed-data snapshot gated only on 'job submission is committed', NOT on whether this specific job's DISPATCH transition has been committed yet. When a scheduler reconnects (or the connection blips) around the same moment it makes/receives confirmation of its own fresh dispatch decision for a job, the resync's committed-data snapshot can legitimately still show that job as PEND (its own dispatch commit hasn't propagated into the snapshot query yet) even though the scheduler's own live state (already set via sched_commit_decision() or the schedStartJob/JOB_STATE_CHANGE confirmation path) correctly has it RUN with a real allocation. Branch (3) blindly trusts the resync's stale PEND claim over the scheduler's own more-current, already-allocated state, freeing the real allocation and (in the observed cases) leaving state briefly PEND until a later, in-flight JOB_UPDATE/JOB_STATE_CHANGE for the SAME jobid (already queued in the same event batch, confirming the SAME dispatch) re-sets state=RUN -- but with no mechanism to restore the allocation branch (3) just destroyed. This exactly matches both the sim replay symptom (info jobs shows RUN, info worker's allocs-driven assigned-jobs listing omits it) and the production log's precise 16ms collision. Not the same mechanism as bug-1811's original (correct, but incomplete) 'Recorded-Decision Reconciliation transient window' explanation -- that window is real and self-healing, but this is a genuinely separate, permanent-until-this-fix data-loss bug triggered by resync/dispatch timing, not the reconciliation mechanism itself. + +## Fix +FIXED (2026-09-15), TDD RED->GREEN. Added a guard in JOBS_PUSH's duplicate-jobid merge (core_events.c), inserted between bug-1739's `else if (JOB_IS_RUN(job->state))` branch and the original bare `else`: `else if ((existing->allocs != NULL) && !JOB_IS_DONE(job->state) && !JOB_IS_EXIT(job->state))` -- refuses the resync claim (WARNs, keeps existing->state/allocs/accounts untouched) whenever `existing` already holds a real allocation AND the resync's claimed state is not genuinely terminal. A terminal claim (DONE/EXIT) still falls through to the original branch, correctly releasing the allocation -- this is bug-1734's own legitimate 'job finished while disconnected' case and must not be blocked. Symmetric in spirit to bug-1739's own phantom-RUN guard (refuse to fabricate an unsupported RUN; now also refuse to destroy a supported RUN). TDD: added test_jobs_push_duplicate_jobid_run_with_allocation_survives_stale_nonrun_resync (test_core_events.c) -- job tracked RUN with a real allocation, redelivered via JOBS_PUSH as PEND with no allocation, must keep its RUN state and real allocation intact; and its companion test_jobs_push_duplicate_jobid_terminal_resync_releases_real_allocation -- the SAME RUN-with-real-allocation job, redelivered as DONE, must still release the allocation and transition (proves the new guard doesn't block legitimate terminal resyncs). RED confirmed (`xce-build.sh scheduler asan xce.build --incremental`): exactly the new stale-resync test failed (Expected 2 [RUN] Was 1 [PEND]), the terminal-resync companion already passed (unaffected by the unfixed code), 75 other tests green. GREEN confirmed after the fix: full scheduler suite 77/77 Ok, zero regressions -- including test_sim_e2e_original_motivation.c (the Recorded-Decision Reconciliation live-verify a FIRST, incorrect fix attempt this same session had broken and had to be reverted) and test_driver_translate.c, both untouched by this fix and green throughout. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1734, bug-1738, bug-1739, bug-1811 diff --git a/.wolf/bugs/bug-1813.md b/.wolf/bugs/bug-1813.md new file mode 100644 index 00000000..c51c1903 --- /dev/null +++ b/.wolf/bugs/bug-1813.md @@ -0,0 +1,38 @@ +--- +id: "bug-1813" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-15T08:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/include/scheduler.h", "src/scheduler/src/handler.c", "src/scheduler/src/driver.c", "src/scheduler/testsuite/test_job_array_expand.c", "src/scheduler/testsuite/test_job_array_wire_e2e.c", "src/scheduler/testsuite/test_driver_translate.c"] +tags: ["scheduler", "vtaskmgr", "async-io", "connection-stability", "job-array", "architecture", "user-directed"] +related: ["bug-1812", "bug-1734", "bug-1739", "bug-1714"] +legacy: true +--- + +# bug-1813: Follow-up to bug-1812's investigation (user-directed architectural fix, not a live-reproduced… + +## Description +Follow-up to bug-1812's investigation (user-directed architectural fix, not a live-reproduced crash): while investigating why a scheduler reconnect/resync storm collides with in-flight array-job dispatches (bug-1812's own root cause), user asked to investigate the connection-loss trigger itself. Traced the scheduler's array-job receive path (handler.c's sched_on_job_array(), reached via scheduler_req_handler() on the shared vtaskmgr thread pool, default 8 threads) and found it does the FULL per-element expansion (dedup check via sched_get_job() + N independent deep clones via clone_job_submit_data()/clone_job_startinfo()) inline, synchronously, on whatever vtask thread handles the incoming push -- the exact 'long/unbounded work on a pool that must stay free for liveness-critical messages' anti-pattern .kiro/steering/ha-async-io.md's Thread-role contract warns about (the same pool is shared with the scheduler's own controller connection traffic). A prior, independent finding (2026-09-04 discovery, this project) already showed a burst of 6x50-element array pushes (300 job events, landing within ~2ms of each other) caused the scheduler to lose its connection to the primary controller within 245ms and enter a failover/reconnect loop -- directly consistent with this mechanism. User proposed the fix directly: the request-handling thread should just requeue the raw request, with the actual (expensive) work deferred to run alongside 'begin session' -- i.e. the round-processing thread's own xce_sched_driver_translate_pending_events()/xce_core_begin_session() sequence (main.c's xce_sched_driver_run_normal_mode_round(), driver.c), which carries no connection-liveness responsibility and already establishes this exact deferred-dispatch pattern for XCE_SCHED_OP_ADMIN_RECORD. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via direct code reading (not yet live-reproduced end-to-end with gdb/ASAN under a real burst -- this fix is the same corrective direction bug-1714's own still-open investigation was heading, applied proactively here rather than waiting for a full live repro). handler.c's sched_on_job_array() (the XceData_data_jobArray case's handler, invoked by scheduler_req_handler() on a vtaskmgr thread) did the ENTIRE per-element expansion synchronously and inline: for each of up to array_size elements, a sched_get_job() dedup lookup plus a full deep clone (clone_job_submit_data()/clone_job_startinfo(), several calloc()+strdup() calls per element) before ever enqueueing anything. For a large concurrent burst (6 arrays x 50 elements landing within ~2ms, the 2026-09-04 finding), this ties up several of the vtaskmgr pool's 8 threads for the duration of 300 elements' worth of cloning -- the same pool the scheduler's own controller connection traffic depends on, a direct instance of ha-async-io.md's documented anti-pattern (a project-wide DESIGN INVARIANT, 'applies to ALL components: controller, worker, scheduler'). + +## Fix +FIXED (2026-09-15), TDD RED->GREEN, user-directed design. Added XCE_SCHED_OP_UPDATE_JOB_ARRAY (scheduler.h) mirroring XCE_SCHED_OP_ADMIN_RECORD's own established deferred-dispatch pattern, with a new xce_sched_update_request_t.u.jobarrayupdate union member carrying the raw, still-unexpanded xce_job_submit_t* descriptor plus (post-expansion) expanded_njobs/expanded_jobs for release_translated_payload()'s later cleanup. handler.c's sched_on_job_array() now does O(1) work only: validate the descriptor shape, wrap it in a request, enq_pend_update() -- no loop, no clone, no dedup check, ownership of the descriptor transferred to the queue on success (sched_handle_dataset()'s XceData_data_jobArray case updated to null dataset->rows on success instead of unconditionally freeing it). The full per-element expansion logic (clone_job_startinfo/clone_job_submit_data/make_array_element, moved verbatim) plus its dedup check now live in driver.c's translate_one_pending_update(), a new XCE_SCHED_OP_UPDATE_JOB_ARRAY case producing the same XCE_CORE_EVENT_JOBS_PUSH shape the old handler.c code did -- this runs on the round-processing thread (main.c's xce_sched_driver_run_normal_mode_round(), invoked before xce_core_begin_session() every round), which carries no connection-liveness responsibility. Side benefit: the dedup check's own pre-existing TOCTOU race (documented in the original code as bug-1714 follow-up 'Bug B', a genuine cross-thread race between the vtask thread's check and the driver thread's own core->jobs mutations) is now eliminated for the single-request case, since only the round-processing thread ever runs this check -- the remaining same-batch-double-delivery edge case is unchanged and was already documented as safe-by-design (sched_add_job()'s key_add() plus the JOBS_PUSH duplicate-merge logic, core_events.c, is the actual correctness boundary, untouched by this fix). release_translated_payload() (driver.c) gained a matching cleanup case: frees the original job_array descriptor unconditionally (never itself pushed into core) plus the N expanded clones (same unconditional free XCE_SCHED_OP_UPDATE_JOB's own jobupdate.jobs already gets). TDD: 4 new unit tests in test_driver_translate.c directly on translate_one_pending_update() (expands into N events with independent clones; skips already-existing elements against a REAL core->jobs kvmgr; all-duplicate redelivery returns 0/no-op; malformed payload returns 0) plus 1 new full drain-push-release end-to-end test (test_drain_translates_and_pushes_job_array_op) proving ownership correctness under real ASAN instrumentation. test_job_array_expand.c's and test_job_array_wire_e2e.c's own tests (which previously asserted the OLD inline-expansion behavior, including test_job_array_wire_e2e.c's tlog early-print-callback log-count capture, since the confirmation log line moved to driver.c) were revised to assert the NEW deferred-enqueue behavior instead -- redelivery dedup and log-firing-count coverage moved to test_driver_translate.c's own tests (log-firing-count itself was not re-added at the new location, a deliberate small scope trim, noted here rather than silently dropped). RED confirmed at each stage (xce-build.sh scheduler asan xce.build --incremental): first pass caught the 4 new driver-level unit test failures as expected plus 2 pre-existing test_job_array_wire_e2e.c failures this refactor had NOT yet been applied to (a test file this investigation initially missed, found only via the full suite run -- both asserted 'Expected 2 Was 11', i.e. the old handler-level XCE_SCHED_OP_UPDATE_JOB=2 check now correctly seeing the new XCE_SCHED_OP_UPDATE_JOB_ARRAY=11). GREEN confirmed after revising that file too: full scheduler suite 77/77 Ok, zero regressions, including test_sim_e2e_original_motivation.c (verified explicitly, given this same session's earlier unrelated mistake on bug-1811/1812's own investigation broke it once already) and all 10 new/revised job-array tests individually confirmed PASS via the raw test log. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1812, bug-1734, bug-1739, bug-1714 diff --git a/.wolf/bugs/bug-1814.md b/.wolf/bugs/bug-1814.md new file mode 100644 index 00000000..1314509d --- /dev/null +++ b/.wolf/bugs/bug-1814.md @@ -0,0 +1,40 @@ +--- +id: "bug-1814" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-15T09:31:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/xcecd_cgroups.c (collect_job_rusages() -- the fix)", "src/scheduler/src/resource.c (sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage() -- the consumer that sums the corrupted per-job value into w->resInUse", "unmodified", "already correct on its own terms)", "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_RUSAGE handler -- confirmed the single", "shared call site both normal-mode (driver.c) and sim/replay-mode (sim_loop.c) events funnel through", "so this bug is not replay-specific)"] +tags: ["worker", "xcecd", "xcecd_cgroups", "collect_job_rusages", "systemd", "dbus", "sd-bus", "cgroup", "UINT64_MAX", "sentinel-value", "resource-accounting", "mem", "memory_rss", "resInUse", "scheduler", "sim", "replay", "normal-mode-also-affected", "unit-mismatch-flagged-not-fixed", "found-live", "user-reported", "tdd", "environment-limitation", "needs-real-deployment-verification", "confirmed-persistent-not-transient", "cgroup-config-suspect"] +related: ["bug-1735", "bug-1736", "bug-1740", "bug-1799", "bug-1801"] +legacy: true +--- + +# bug-1814: User-reported, live replay session (/tmp/scheduler-replay-20260915-091847.replay, worker pc13v)… + +## Description +User-reported, live replay session (/tmp/scheduler-replay-20260915-091847.replay, worker pc13v): `info worker pc13v` at session=2 (right after a burst of 19 JOB_UPDATE/JOB_RUSAGE event pairs for jobid[0]-[18]) showed `resource mem: value=22346629120.00 (20.81 GB) used=334251534843904.000000 (304.00 TB)` -- a used figure ~15000x larger than the worker's own total capacity, for a 22-slot/20.81GB host. Session 1 (before any JOB_RUSAGE events) showed a sane used=225280.000000 (220.00 KB) baseline. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +collect_job_rusages() queries 4 systemd cgroup resource counters (CPUUsageNSec, MemoryCurrent, MemorySwapCurrent, TasksCurrent) via get_unit_property()/sd_bus_get_property_trivial(). get_unit_property()'s success return (r>=0) means only that the D-Bus round trip itself worked -- systemd's own convention (undocumented in this codebase before now) is to report UINT64_MAX for a counter that isn't actually being tracked yet for a unit (accounting not yet active, or queried before the unit's cgroup is fully set up), NOT an error and NOT 0. The pre-fix code never checked for this sentinel. For MemoryCurrent specifically: (float)UINT64_MAX / (1024*1024) [MB unit conversion] rounds, under float32, to exactly 17592186044416.0 (2^44) -- a single job's fabricated 'memory usage'. sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage() (resource.c, unmodified and correct on its own terms -- resource-accounting.md's max(reserved,actual) policy) then charges this per-job figure into the worker's w->resInUse pool once per RUN job whose rusage tick reports it; the replay session had 19 such jobs report in the same burst, all apparently hitting the same not-yet-tracked-by-systemd condition (freshly-started array-job siblings), so the pool summed to exactly 19 * 2^44 = 334251534843904.0 -- matching the reported figure to the bit. Verified this arithmetic numerically (Python) before making any code change. sched_reconcile_host_resource_usage()'s own reconciliation math (resource.c:802-834) is not itself buggy -- garbage in, garbage charged, exactly as designed for a real (non-sentinel) report. + +## Fix +FIXED (2026-09-15). Added a new static predicate sd_resource_counter_is_available(uint64_t val) (xcecd_cgroups.c, right after get_unit_property()) that returns false only for the exact UINT64_MAX sentinel, and gated all four collect_job_rusages() property reads (CPUUsageNSec/MemoryCurrent/MemorySwapCurrent/TasksCurrent) on `get_unit_property(...) >= 0 && sd_resource_counter_is_available(u64_val)` before trusting the value -- an unavailable counter is now silently skipped (the xce_rusage_t entry keeps its calloc-zeroed 0.0 default) rather than treated as a real reading. Applied uniformly to all four properties (not just MemoryCurrent) since systemd uses the identical UINT64_MAX-for-unavailable convention for each of them. Did NOT change memory_rss's existing MB unit convention (bug-1735's established wire contract, also relied on by the controller's own passthrough mem_usage/max_mem/avg_mem/used_mem display pipeline, misc.c/job.c) -- a candidate compensating fix in resource.c's sched_reconcile_job_resource_usage() (multiplying ru->value by 1024*1024 before charging, to match w->resInUse's apparent byte-native convention per resource-accounting.md) was drafted, then reverted: it would have broken test_resource_accounting.c's existing, currently-passing fixtures (test_reconcile_job_resource_usage_maps_memory_rss_to_mem_account and siblings), which assert memory_rss values flow through to w->resInUse with NO unit conversion -- contradicting the codebase's own current tested contract for this function. Left as an OPEN, FLAGGED, NOT-FIXED question (not resolved by this session): whether the scheduler's 'mem' resource job-level reserved/used accounting is actually MB-native end to end (matching the test fixtures and resource-accounting.md's own 'charge 4MB' worked example) or byte-native (matching w->resvals' real mem_free-in-bytes capacity reading) -- these two readings of the same document appear to conflict, and resolving it requires tracing what unit resreq's rusage[mem=...] parsing actually produces in production, which this session did not do. TDD: new src/worker/testsuite/test_xcecd_cgroups_rusage.c (white-box #include of xcecd_cgroups.c, same convention as hardware_monitor_test/resource_sync_test; a trivial make_job_sd_unit_name() stub satisfies the one linker dependency this file pulls in from xcecd_worker_utils.c that this test never actually calls) -- 5 tests covering the sentinel predicate (UINT64_MAX rejected, 0 and ordinary readings accepted, UINT64_MAX-1 accepted i.e. only the exact sentinel is special-cased) plus one test reproducing the exact pre-fix arithmetic (UINT64_MAX/1MB == 17592186044416.0, times 19 == 334251534843904.0) as a documented magnitude proof. The real D-Bus/systemd round trip inside collect_job_rusages() itself could not be live-verified in this sandbox -- no D-Bus/systemd daemon is reachable here (same environment limitation bug-1735 already established for this file); this is a static-analysis-plus-arithmetic-proof fix, needs live confirmation against a real cgroup-backed xcecd deployment. Wired into src/worker/testsuite/meson.build (new 'worker xcecd_cgroups rusage test' target). Full worker suite build+test in progress at time of this entry. + +ADDENDUM (2026-09-15, same session, user's own multi-session replay trace): user reported that across multiple replay sessions, a job's corrupted 'used' figure NEVER self-corrects -- it stays elevated until the job finishes, not just at its first rusage tick. This resolves the open 'is this a one-time startup race or a persistent condition' question this entry originally left unanswered: sched_reconcile_host_resource_usage()'s delta-based math (resource.c) means a SUBSEQUENT poll returning a real, small value would compute a large NEGATIVE delta against the still-huge acct_entry->used, self-correcting the pool almost immediately -- that never happening across many sessions proves MemoryCurrent keeps returning the UINT64_MAX sentinel (or something producing an equally-unavailable reading) on EVERY 10-second poll (job_rusage_collect_interval, xcecd_worker.c:50) for that job's entire lifetime, not just once at unit creation. Confirmed MemoryAccounting=yes IS explicitly requested at transient-unit creation (xcecd_cgroups.c:270-291, start_systemd_job()) -- so this is not a 'forgot to enable accounting' bug; something about how MemoryCurrent is actually served back for these units on the live pc13v host (cgroup v1 vs v2, controller delegation, a systemd/D-Bus quirk) appears to be structurally broken, not transiently racy. Separately noted: CPUAccounting is NEVER explicitly requested (only TasksAccounting/MemoryAccounting are) -- CPUUsageNSec's availability depends entirely on that host's system-wide DefaultCPUAccounting= default, a related but distinct potential gap. NOT verified live (no D-Bus/systemd in this sandbox, same limitation as bug-1735) -- needs a check on the real pc13v host (`systemctl show -p MemoryCurrent -p MemoryAccounting` and/or `cat /sys/fs/cgroup//memory.current`) to distinguish a systemd-layer reporting bug from a kernel/cgroup-delegation-level gap. This fix's guard is still correct and necessary regardless of which it turns out to be -- it stops the garbage charge -- but the practical effect for affected jobs is that 'mem used' will sit at the flat reservation baseline for their whole runtime rather than tracking real usage, since real data genuinely never arrives for them; this is a host/cgroup-configuration question, out of this codebase's scope to resolve further. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1735, bug-1736, bug-1740, bug-1799, bug-1801 diff --git a/.wolf/bugs/bug-1815.md b/.wolf/bugs/bug-1815.md new file mode 100644 index 00000000..498c9043 --- /dev/null +++ b/.wolf/bugs/bug-1815.md @@ -0,0 +1,38 @@ +--- +id: "bug-1815" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-15T09:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/resource.c (sched_reconcile_host_resource_usage() -- the fix)", "src/scheduler/src/core_events.c (JOBS_PUSH resync-merge RUN branch", "lines ~913-924 -- confirmed the account-object-replacement mechanism that interacts with this gap", "not itself modified)", "src/scheduler/testsuite/test_resource_accounting.c (new regression test)"] +tags: ["scheduler", "resource-accounting", "resInUse", "sched_reconcile_host_resource_usage", "IS_WORKER_RECONCILING", "IS_WORKER_UNAVAIL", "worker-rejoin-resync", "bug-1801-gap", "bug-1814-followup", "JOBS_PUSH", "resync", "mem", "never-self-corrects", "user-reported", "live-verified", "found-live", "tdd"] +related: ["bug-1814", "bug-1799", "bug-1801", "bug-1751", "bug-1812", "bug-1739"] +legacy: true +--- + +# bug-1815: Follow-up to bug-1814: user reported that tracing a job's 'mem used' figure across multiple live… + +## Description +Follow-up to bug-1814: user reported that tracing a job's 'mem used' figure across multiple live replay sessions showed it never self-corrects, staying elevated until the job finishes -- not a one-time blip. User also live-verified on the real pc13v host (`systemctl show -p MemoryCurrent -p MemoryAccounting`) that systemd genuinely reports correct, sane MemoryCurrent values for a healthy running job (MemoryCurrent=5525504, MemoryAccounting=yes) -- disproving the hypothesis that bug-1814's UINT64_MAX sentinel is a persistent/structural condition on this host. This left the 'why does it never self-correct' question open. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_reconcile_host_resource_usage() (the JOB_RUSAGE-driven per-job mem-usage reconciliation path, resource.c) was missing the exact IS_WORKER_UNAVAIL/IS_WORKER_RECONCILING guard that bug-1801 (2026-09-11, same file) already added to sched_update_account() (the regular per-round charge/uncharge path). core_reset_workers() (bug-1751) deliberately FREEZES a RECONCILING/UNAVAIL worker's w->resInUse each round (skips the reset), on the understanding that NOTHING touches it until a clean per-round reset+resum can run once the worker's real state is confirmed; bug-1801 extended that freeze to sched_update_account() but this THIRD, independent resInUse-mutating path was never covered. Consequence: every JOB_RUSAGE tick that lands while a worker is RECONCILING keeps applying its computed delta to resInUse regardless, breaking the freeze. Compounding this: core_events.c's JOBS_PUSH resync-merge RUN branch (lines 913-924) can replace an already-tracked job's entire xce_job_account_t (existing->accounts = job->accounts, after free_job_account()-ing the old one) with a freshly-built object whose 'mem' xce_res_account_t has used=0.0 (make_res_account(), xcealloc.c) -- safe ONLY because the very next clean per-round rebuild (sched_account_jobs(), which fully re-derives resInUse from scratch each round) is supposed to correct any resulting discontinuity. That safety net is exactly what stays skipped while the worker remains RECONCILING. So a rusage tick landing after such a resync-merge, during the same still-RECONCILING window, computes its delta against the FRESH account's 0.0 baseline and adds the FULL usage on top of whatever was already frozen in the pool (potentially including bug-1814's own garbage charge, or simply a prior legitimate charge that now gets double-counted) -- with nothing left to ever subtract it back out until the worker escapes RECONCILING for good (letting a clean rebuild run) or the job reaches a terminal state (whose own uncharge path removes whatever the CURRENT account object holds). This produces exactly the observed 'used' figure that grows and never self-corrects for the job's entire remaining lifetime -- matching the user's live multi-session trace precisely, and consistent with this session's own bug-1812 finding that large array-job bursts cause repeated scheduler-to-controller connection loss (i.e. repeated/extended RECONCILING windows), a plausible trigger for this specific replay's affected jobs. + +## Fix +FIXED (2026-09-15). Added `if (IS_WORKER_UNAVAIL(w->status) || IS_WORKER_RECONCILING(w->status)) { return 0; }` to sched_reconcile_host_resource_usage() (resource.c), immediately after the existing NULL-arg guard, before any resInUse/acct_entry mutation -- mirroring bug-1801's exact guard on sched_update_account() verbatim (same two status bits, same early-return-as-no-op semantics). Deliberately a full skip (neither the pool NOR acct_entry->used is touched) so the account's own tracked baseline stays consistent with whatever the still-frozen pool holds; once the worker's status clears, the very next rusage tick (or the next clean sched_account_jobs() rebuild) reconciles cleanly with no special-casing needed. TDD: added test_reconcile_host_resource_usage_skips_reconciling_or_unavail_worker (test_resource_accounting.c), mirroring test_charge_account_skips_reconciling_or_unavail_worker_host_charge's (bug-1801) own structure -- asserts a RECONCILING worker's resInUse AND acct_entry->used both stay untouched across a rusage tick, same for UNAVAIL, then confirms reconciliation resumes correctly (overage math verified) on the very next call once status clears. RED confirmed by temporarily disabling the new guard (`if (0 && (...))`) and rebuilding (`xce-build.sh scheduler asan xce.build --incremental`): exactly the 1 new test failed ('Expected 0 Was 5'), 76 other scheduler tests stayed green. GREEN confirmed after restoring the guard: full scheduler suite 77/77 Ok, zero regressions. NOT separately fixed here (out of scope, flagged only): the JOBS_PUSH resync-merge's own free-then-replace-without-uncharge pattern (core_events.c:913-924) remains as-is -- it is safe BY DESIGN as long as the following per-round rebuild is guaranteed to run, which is now consistently true again for all three resInUse-mutating paths (charge/uncharge/reconcile) once this fix closes the last gap; revisiting that pattern itself was judged unnecessary once this symmetry is restored. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1814, bug-1799, bug-1801, bug-1751, bug-1812, bug-1739 diff --git a/.wolf/bugs/bug-1816.md b/.wolf/bugs/bug-1816.md new file mode 100644 index 00000000..6954772b --- /dev/null +++ b/.wolf/bugs/bug-1816.md @@ -0,0 +1,38 @@ +--- +id: "bug-1816" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-09-15T13:05:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/deploy-ha.sh (port pre-flight check", "cleanup.sh generation fix", "new stop-all.sh generation", "start-all.sh wait_for_pidfile race fix)", ".claude/skills/xce-deploy-ha/SKILL.md (Step 2", "Clean restart", "Troubleshooting documentation updates)"] +tags: ["deploy-ha", "xce-deploy-ha-skill", "deploy-ha.sh", "ha-failover", "stale-process", "zombie-process", "port-conflict", "cleanup.sh", "stop-all.sh", "start-all.sh", "race-condition", "pidfile", "tooling", "user-requested", "live-verified", "found-live", "not-a-code-regression"] +related: ["bug-1814", "bug-1815"] +legacy: true +--- + +# bug-1816: User request, following the bug-1814/1815 mem-accounting investigation: a live 3-node HA cluster… + +## Description +User request, following the bug-1814/1815 mem-accounting investigation: a live 3-node HA cluster on pc13v went through a repeated PreVote-failover storm (term climbing 2->6, standbys losing primary_ctrl_fd RDHUP repeatedly, xceadmin scheduler record status failing with 'scheduler not connected'/'too many redirects') while load was confirmed low (loadavg 0.5-1.6 on 22 cores). Root cause found live by the user: `ps axwww | grep control` on pc13v showed SIX xce_controller processes -- two full sets of ctrl-1/ctrl-2/ctrl-3 (PIDs 294848/294869/294905 and 610530/610546/610561) all bound to the identical ports (9901-9903) and identical --workdir/--certdir/--logfile paths under /tmp/xce-deploy-ha. The user's own hand-typed cleanup command (`ps axwww | grep xce-ha-deploy | ... | kill -9`) had a typo -- the actual workdir is /tmp/xce-deploy-ha/... ('xce-deploy-ha'), not 'xce-ha-deploy' (words transposed) -- so it matched nothing and silently failed to kill the first deploy's processes before a second deploy started on the same ports. User asked to enhance the xce-deploy-ha skill's shutdown-all mechanism so this class of mistake becomes structurally hard to repeat. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Two independent, compounding structural gaps let a hand-typed-command typo turn into a genuine HA failover storm instead of a clean, obvious failure: (1) deploy-ha.sh had NO pre-flight check that the target ports were actually free before starting a new cluster -- it would happily bind a second, fully independent 3-node Raft group to the exact same ports an existing, still-alive cluster already held, producing real cross-talk (mixed vote traffic, corrupted term/leader tracking between two unrelated groups) rather than a clean bind() failure with a clear error. (2) start.sh/start-all.sh already HAD a correct, working 'already running' guard (reads $TOP/run/*.pid, checks kill -0, refuses to start a duplicate) -- but the pre-existing cleanup.sh unconditionally did `rm -f $TOP/run/*.pid` as pure state cleanup, with NO check that the PID each file pointed to was actually dead first. Running cleanup.sh (or a fresh deploy-ha.sh invocation that regenerates it) without ALSO successfully killing the live processes first silently erased the ONLY signal that guard depends on, letting a stale process survive completely undetected by every existing safety net in the codebase. The user's specific typo was the proximate trigger, but the underlying design let a single hand-typed-command mistake cascade into two independent Raft clusters, sharing ports, cross-talking -- a much worse failure mode than the typo alone should have caused. + +## Fix +FIXED (2026-09-15). Three changes to src/tools/deploy-ha.sh, live-verified end-to-end against real xce_controller/xceadmin/xcecd_virtual/xce_scheduler/xce_apiserver binaries (not just syntax-checked): (1) A port pre-flight check, added immediately after PEER_ARRAY is parsed and before ANY directory/cert/cluster.bin state is created -- for each peer's port, checks via `ss -tln` whether something is already listening; if so, prints the offending PID/process (via `ss -tlnp`, best-effort) and refuses to proceed with an actionable error (the same kill-by-name commands Step 2 documents), exiting before touching disk. Soft-skips (warns, doesn't fail) if `ss` isn't on PATH. Live-verified: a redeploy attempt against a still-running cluster's ports was correctly refused, identifying the exact blocking PIDs, and created no directory at all under the attempted $TOP. (2) cleanup.sh's generated body now stops every live PID in $TOP/run/*.pid (kill -9, with a warning to `sudo kill -9` if the real root-run worker doesn't die from a plain kill) BEFORE removing any pidfile or other state, closing the exact gap in root_cause (2) -- restoring the invariant that start.sh/start-all.sh's own guard depends on: no pidfile is ever removed while pointing at a still-alive process. stop-all.sh (new, generated alongside start.sh/cleanup.sh) is the same pidfile-based stop logic as a standalone script that leaves state untouched -- the canonical, typo-proof way to stop everything a specific deploy started, since it reads that deploy's own tracked PIDs instead of matching process names/paths by hand. (3) Found and fixed a related race while live-testing (1) and (2) together: start-all.sh launched worker/scheduler/apiserver via backgrounded `setsid nohup ... &` with no wait for each to actually write its own pidfile before declaring itself done -- running cleanup.sh/stop-all.sh immediately after start-all.sh returned could catch worker/scheduler BEFORE their pidfile existed, defeating the very fix in (2)/(3) for those two components specifically (reproduced live: cleanup.sh correctly stopped ctrl-1/2/3 but left worker-1/scheduler running with their pidfiles already wiped -- the identical zombie-process class this whole fix targets, via a narrower timing window instead of an operator typo). Added a small wait_for_pidfile() poll (up to 5s, 0.2s interval) after each of the three backgrounded launches; start-all.sh now only reports 'done' once every component it started has a live pidfile on disk (or warns by name if one didn't appear in time). Re-verified live after this fix: cleanup.sh run IMMEDIATELY after start-all.sh returned correctly stopped all 6 processes (3 controllers + worker + scheduler + apiserver), confirmed via `ps`/`ss` showing nothing left running and no listeners on the deployed ports. SKILL.md updated: Step 2 now prefers a known deploy's own stop-all.sh over the hand-typed blanket kill (documents exactly why, citing this incident), Clean restart section documents cleanup.sh's new stop-then-wipe behavior and stop-all.sh as the no-wipe alternative, and two new Troubleshooting rows cover the port-pre-flight error message and the 'inexplicable HA failover despite low load, multiple PIDs per role in logs' symptom pattern directly, matching this incident. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1814, bug-1815 diff --git a/.wolf/bugs/bug-1817.md b/.wolf/bugs/bug-1817.md new file mode 100644 index 00000000..0b8a0723 --- /dev/null +++ b/.wolf/bugs/bug-1817.md @@ -0,0 +1,38 @@ +--- +id: "bug-1817" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-09-15T19:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/zarray.c (zarray_fini() -- primary fix)", "src/lib/xlist.c (xlist_fini() -- identical fix", "same bug class)", "src/scheduler/src/schedobj.c (reset_sched_obj() -- the reachable production caller that exposed the bug", "NOT itself modified: correct by construction once the library primitives are fixed)", "src/scheduler/src/plugins/limit.c (limit_job_election()/match_job_host_limit_() -- the call chain that reaches the torn array", "not modified)", "src/lib/testsuite/test_zarray.c", "src/lib/testsuite/test_xlist.c (new regression tests)"] +tags: ["scheduler", "zarray", "xlist", "libxce", "zarray_fini", "xlist_fini", "reset_sched_obj", "schedobj", "limit-plugin", "match_job_host_limit_", "sigsegv", "null-pointer-dereference", "torn-state", "use-after-free-adjacent", "config-version", "job-election", "allocator-thread", "user-reported", "found-live", "core-dump", "gdb", "tdd", "library-primitive-fix", "recurring-bug-class", "previously-unroot-caused"] +related: ["bug-1629", "bug-1578", "bug-1490"] +legacy: true +--- + +# bug-1817: User-reported live core dump: xce_scheduler (build.dbg, HA-connected normal mode, --peers… + +## Description +User-reported live core dump: xce_scheduler (build.dbg, HA-connected normal mode, --peers ctrl-1/2/3) crashed with SIGSEGV in its dedicated allocator thread (sched_allocator, scheduler.c:484). gdb backtrace: zarray_get_ex (lib/zarray.c:332, `data[i] = array->data[idx]`) <- zarray_get <- match_job_host_limit_ (plugins/limit.c:370) <- limit_prepare_allocation <- plugin_prepare_allocation <- sched_try_alloc_on_host_ (resource.c:528) <- parjob_try_plan_ <- my_alloc <- sched_make_parjob_allocation <- sched_make_primary_alloc_ <- sched_make_primary_allocation <- sched_make_allocation <- sched_allocator <- start_thread. gdb locals at the crash frame: i=0, array->data=0x0 (NULL), while the bounds check `idx (0) >= array->caps` had evidently passed (i.e. array->caps was nonzero) -- a zarray_t with data==NULL but caps>0, an internally inconsistent/torn state that should be structurally impossible for a correctly-initialized array. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +zarray_fini(array) (lib/zarray.c) freed array->data and set it to NULL, but left array->caps/count/pos at whatever nonzero values they held before the call -- NOT reset to the safe (0,0,0) state zarray_init() establishes. xlist_fini() (lib/xlist.c) had the byte-for-byte identical gap (FREEUP(l->data) alone, l->caps/l->count left stale). Both functions' own bounds-checked readers (zarray_get_ex's `idx >= array->caps`, xlist_get's `idx >= l->caps`) treat a nonzero caps as a promise that array->data/l->data is a valid, sized allocation -- a promise zarray_fini()/xlist_fini() broke for any caller that reused the SAME struct after fini without an intervening re-init. schedobj.c's reset_sched_obj(xce_sched_obj_t* obj) does exactly that: `xlist_fini(&(obj->glimits)); zarray_fini(&(obj->hlimits));` with no matching re-init, called from plugins/limit.c's limit_job_election() every time a job's schedobj.configver falls behind the cluster's current core.configver (i.e. whenever the cluster config has been reloaded/pushed since this job's limits were last computed) -- `obj` here is the job's own PERSISTENT xce_sched_obj_t (job->schedobj, allocated once at job ingest, reused for the job's entire lifetime), not a disposable scratch object. Once at least one host-limit match has grown obj->hlimits' caps beyond 0 (zarray_put() inside match_job_host_limit_(), the very next line after the crash site) and the config version later advances again while that job is still PEND, limit_job_election()'s reset_sched_obj() call tears obj->hlimits down to (data=NULL, caps=stale-nonzero) -- and the next allocation attempt for that same job (in the same or a later scheduling round, from a completely different call path: the allocator thread's plugin_prepare_allocation -> limit_prepare_allocation -> match_job_host_limit_ -> zarray_get(&(obj->hlimits), worker->idx)) dereferences the freed array, since worker->idx (0 for a single-worker deployment) is below the stale caps. This exact crash class had already caused at least one earlier, never-successfully-root-caused crash near the same call site (memory observation from 2026-08-17, 'crash occurs in driver/decision processing after limit_job_election returns successfully' -- that investigation correctly localized the area but concluded the crash was downstream in the decision pipeline, not inside reset_sched_obj()/zarray_fini() itself, for lack of as clean a gdb backtrace as this session's core dump provided). reset_sched_obj() itself was ALSO nearly lost entirely in an earlier refactor (bug-1490, 're-added reset_sched_obj() verbatim (unchanged body)' after a wholesale schedobj.c rewrite dropped it) -- its buggy body has survived unchanged through at least that refactor. + +## Fix +FIXED (2026-09-15). Fixed at the library-primitive level, not at reset_sched_obj()'s call site: zarray_fini() now resets array->count/array->caps/array->pos to 0 (matching zarray_init()'s own defaults) immediately after freeing/NULLing array->data; xlist_fini() now resets l->count/l->caps to 0 (matching xlist_init()) after FREEUP(l->data). This restores the invariant both functions' own doc comments now state explicitly: _fini() leaves the struct in the SAME safe, empty, immediately-reusable state _init() does, not just 'data freed, everything else however it happened to be.' schedobj.c's reset_sched_obj() and every other _fini()-without-reinit caller across the codebase are now correct by construction, with zero code changes needed there. Scoped the fix to the library primitives (not schedobj.c-side re-init calls) deliberately: audited every zarray_fini()/xlist_fini() call site outside testsuite/ (5 zarray_fini, ~25 xlist_fini, all in controller/lib) and confirmed every one either discards the struct immediately after (a stack-local scratch array/list, function returns right after) or is tearing down a struct being destroyed -- none relies on caps/count surviving fini, so this is a pure safety improvement for the whole 68+-caller zarray_init/all xlist_init blast radius, not a behavior change for any correct caller. TDD: added test_zarray_fini_resets_bounds_for_reuse (test_zarray.c) and test_xlist_fini_resets_bounds_for_reuse (test_xlist.c), both driving the raw zarray_init/zarray_put/zarray_fini/zarray_get and xlist_init/xlist_store/xlist_fini/xlist_get API directly (not the xzarray RTTI wrapper test_zarray.c's other tests use) since that's the exact API schedobj.c calls -- put an item (grows caps>0), fini WITHOUT re-init (exactly reset_sched_obj()'s pattern), assert caps/count reset to 0 and data is NULL, assert a get() for the previously-valid index now safely returns NULL instead of what used to be a NULL-pointer dereference, then assert the struct is genuinely reusable (a subsequent put/get round-trips correctly). RED confirmed by temporarily short-circuiting both fixes with `if (0) { ... }`: both new tests failed cleanly ('Expected 0 Was 4' / 'Expected 0 Was 2', the stale pre-fini caps value) -- Unity's assertion-failure longjmp aborts that one test function before reaching the actual NULL-deref line, so RED was captured as a clean, deterministic assertion failure rather than a process crash, letting the rest of the 300+-test suite still run and report cleanly around it. GREEN confirmed after restoring both fixes: full xcelib_test suite 314/314 individual test cases across all aggregated test files, zero regressions. Full scheduler suite also re-verified: 77/77, zero regressions (this library change is upstream of essentially all scheduler resource/allocation code). NOT investigated/fixed here, flagged only: a related, ALREADY-DOCUMENTED-BUT-UNAPPLIED bug (bug-1578, controller/sched.c's sched_worker_change()) has a DIFFERENT mechanism (a dangling ALIAS to a since-freed xlist_t backing array surviving on a separate struct field, not the same struct's own stale bookkeeping) that this fix does not touch or fix -- genuinely unrelated, noted only because it surfaced during this investigation's own zarray/xlist call-site audit. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1629, bug-1578, bug-1490 diff --git a/.wolf/bugs/bug-1818.md b/.wolf/bugs/bug-1818.md new file mode 100644 index 00000000..94226f2d --- /dev/null +++ b/.wolf/bugs/bug-1818.md @@ -0,0 +1,38 @@ +--- +id: "bug-1818" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "harness" +severity: "unknown" +timestamp: "2026-09-16T08:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/harness/test.c (cmd_ctrlstatus -- fix)", ".kiro/specs/controller-status/design.md (Design Delta -- Requirement 12", "Locked Decision 3 -- corrected)"] +tags: ["controller-status", "ctrlstatus", "cmd_ctrlstatus", "ha", "election", "live-e2e-found", "display-layer", "dead-code-gate", "primaryName"] +related: [] +legacy: true +--- + +# bug-1818: Live 3-node HA E2E validation (controller-status spec, Requirement 12 delta, task 7.6)… + +## Description +Live 3-node HA E2E validation (controller-status spec, Requirement 12 delta, task 7.6): cmd_ctrlstatus (xce_test client) printed electedTerm: unknown / electedAt: unknown for the new primary-election-term/time fields on EVERY standby query, both before and after a real primary kill/failover (confirmed after ~4 heartbeat intervals, ruling out a convergence-timing gap) -- even though the same query correctly showed non-stale, converged term/committedLsn/appliedLsn values for the same standby at the same moment. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +cmd_ctrlstatus gated the new electedTerm/electedAt print lines on NULLSTR(status->primaryName), reusing the same condition the pre-existing Primary: line already uses -- per the delta design's own Locked Decision 3, which assumed primaryName would be "known" on a standby that has learned the primary. But primaryName is UNCONDITIONALLY empty on every non-primary responder in this build: ctrlstatus_fill (src/controller/ctrlstatus.c) only self-reports the primary's own name when the responder IS the primary (ha->primary_host is vestigial/never written, a pre-existing behavior from Requirement 2 / commit ade0081, predating this delta). So Requirement 2.3's "standby knows the primary" branch is dead code via the plain ctrlstatus command, and gating the new fields on the same condition made them permanently unreachable on any standby -- even though status->elected_at_ms (populated unconditionally by ctrlstatus_fill, task 7.4, and correctly heartbeat-learned by task 7.3) was present and correct the whole time. Caught only by a real live cluster run; all unit tests (which test ctrlstatus_fill directly, not cmd_ctrlstatus's print gate) passed throughout, since the bug was entirely in the untested CLI display layer. + +## Fix +FIXED (2026-09-16, controller-status Requirement 12 delta). Changed cmd_ctrlstatus's gate from NULLSTR(status->primaryName) to status->elected_at_ms == 0. This is both necessary and sufficient: elected_at_ms stays exactly 0 until a real promotion stamps it (either promotion road, ha_do_promote / raft_cb_become_leader) or a heartbeat carries a learned value from an actual primary (ha_consume_heartbeat_committed) -- unlike term, which can legitimately bump from mere vote/candidacy activity before any primary is actually elected, so term == 0 would have been the wrong gate (verified via a dedicated candidacy-in-progress test case: elected_at_ms == 0 but term != 0 must still print unknown). ctrlstatus_fill itself was NOT touched -- it correctly remained an unconditional, no-role-branch read (Locked Decision 2), unaffected by this bug. Verified via a standalone scratch program (against the real ctrlstatus_fmt.h header, 4 cases: standby/bug case, never-elected, candidacy-in-progress, primary/regression) plus a repeat live 3-node E2E run confirming the fix converges end-to-end after a real primary kill. design.md's Locked Decision 3 and its dependent traceability/service-contract text were corrected in the same commit to document this live-E2E finding, mirroring the spec's existing Req 10/11 delta convention for recording live findings. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1819.md b/.wolf/bugs/bug-1819.md new file mode 100644 index 00000000..bdc66759 --- /dev/null +++ b/.wolf/bugs/bug-1819.md @@ -0,0 +1,38 @@ +--- +id: "bug-1819" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-09-16T17:15:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ctrlstatus.c (ctrlstatus_fill", "primary-side peer-status matching loop)", "src/controller/testsuite/test_ctrlstatus.c"] +tags: ["controller-status", "ctrlstatus", "ctrlstatus_fill", "repl", "standby_conn_t", "slot-reuse", "first-match-wins", "stale-slot", "live-e2e-found", "task-8.6", "tdd"] +related: ["bug-1474"] +legacy: true +--- + +# bug-1819: controller-status spec Requirement 13 delta, task 8.6 (bug found live during task 8.5's optional… + +## Description +controller-status spec Requirement 13 delta, task 8.6 (bug found live during task 8.5's optional E2E attempt): ctrlstatus_fill's primary-side peer-status loop (src/controller/ctrlstatus.c) scanned replicator_t.standbys[] for a host/port match and stopped at the FIRST match by slot index. When a standby was killed and later reconnected into a DIFFERENT replicator slot than the one it originally occupied (its old slot keeps host/port populated with ep==NULL after the kill), the lower-indexed stale slot won the scan and the peer was reported dead PERMANENTLY -- even minutes after it had genuinely reconnected and fully caught up. Task 8.5's live run observed this directly: primary log showed acked_lsn converging to the committed tip at a specific timestamp while ctrlstatus kept reporting dead from restart through +5m45s. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The primary-side branch of ctrlstatus_fill's per-peer loop scanned replicator_t.standbys[] with a `for (j = 0; j < MAX_STANDBYS; j++) { if (host/port match) { ...; break; } }` pattern -- first host/port match wins, regardless of whether that slot is live (ep != NULL) or stale (ep == NULL). repl_add_standby's slot allocation is keyed by ep identity, not by configured-peer name, so a killed-then-reconnected standby's NEW connection can land in a different slot than its old, now-stale one (confirmed live this session via a related but distinct mechanism -- see bug-1474 cross-reference below); once that happens, the stale, lower-indexed slot permanently wins ctrlstatus_fill's first-match scan even though the peer is genuinely live and caught up in its new slot. + +## Fix +FIXED (2026-09-16, TDD RED->GREEN). Changed the matching loop to scan ALL MAX_STANDBYS slots for a given configured peer (no `break` on the first match) and prefer any slot with ep != NULL over a slot with ep == NULL when multiple slots match the same host:port -- tracked via a `live_match` flag; the first slot with ep != NULL found during the scan determines the reported status (ok/catching-up per its own in_catch_up), and a stale (ep == NULL) match only sets the dead fallback if no live match has been seen yet. If more than one live slot somehow matches the same peer, any one of them is an acceptable source (design.md task 8.6 note) since they would all observe the same live peer. Scope: read-only, same MAX_STANDBYS==3 bounded scan and same repl->lock scope task 8.2 already established -- no lock-scope change (explicitly out of this task's scope per its own text). RED: 2 new unit tests (test_ctrlstatus_peer_status_prefers_live_slot_over_stale_ok, test_ctrlstatus_peer_status_prefers_live_slot_catching_up) added to test_ctrlstatus.c, each constructing a fixture with a lower-index stale slot (ep==NULL) and a higher-index live slot (ep!=NULL) matching the SAME configured peer's host:port -- failed pre-fix with 'Expected 1 Was 3' and 'Expected 2 Was 3' (the permanently-dead symptom, reproduced exactly). GREEN: same two tests pass post-fix; full controller_unit_test suite = 1216 Tests, 0 Failures (both the meson aggregate and the standalone controller_unit_test binary's own summary line cross-checked, per testing.md convention) -- no regressions. LIVE VERIFIED: fresh 3-node cluster (deploy-ha.sh, /tmp/xce-deploy-ctrlstatus86), killed standby ctrl-2, submitted an 80-job backlog while it was down (committedLsn advanced 1->161, confirming genuine WAL activity), restarted ctrl-2, polled ctrlstatus on the primary 8 times over ~24s -- ctrl-2 showed ok immediately on every poll after reconnect (never stuck at dead), while the primary's own log confirmed a genuine WAL catch-up stream (standby[1] acked 1->161) was still converging underneath, proving the fix reads the live registry state correctly rather than merely getting lucky on timing. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1474 diff --git a/.wolf/bugs/bug-1820.md b/.wolf/bugs/bug-1820.md new file mode 100644 index 00000000..c42acaaa --- /dev/null +++ b/.wolf/bugs/bug-1820.md @@ -0,0 +1,38 @@ +--- +id: "bug-1820" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-09-17T02:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/deploy-ha.sh (generated scripts/stop-all.sh and scripts/cleanup.sh kill loops; generated scripts/start-worker.sh's sudo branch)"] +tags: ["deploy-ha", "stop-all.sh", "cleanup.sh", "start-worker.sh", "xcecd", "sudo", "orphaned-process", "real-worker", "sigkill", "live-verified"] +related: ["bug-1720"] +legacy: true +--- + +# bug-1820: User report: after deploying a real (root, sudo-launched) xcecd worker via /xce-deploy-ha and… + +## Description +User report: after deploying a real (root, sudo-launched) xcecd worker via /xce-deploy-ha and stopping the cluster with scripts/stop-all.sh, xcecd keeps running -- ps shows a leftover xcecd process (a different PID than the one stop-all.sh reported stopping) still bound to the deploy dir's logs after stop-all.sh reported "Stopping worker-1 (PID )" with no warning. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +start-worker.sh's real-worker branch writes the pidfile via `echo $$ > "$PIDFILE"` and then does `exec sudo "$WORKER_BIN" ...` -- `exec` replaces the pidfile-writing shell's own process image with sudo IN PLACE (same PID), so the PID recorded in the pidfile is sudo's own PID, not the actual worker daemon's. sudo then forks a SEPARATE, root-owned child process that execs and runs the real xcecd daemon; sudo itself stays alive as a thin parent/monitor. stop-all.sh's (and cleanup.sh's, identical logic) kill loop did `kill -9 "$PID"` against the recorded (sudo) PID -- SIGKILL terminates sudo instantly but is never forwarded to its child (SIGKILL cannot be caught/relayed by the dying parent), so the real xcecd process is silently orphaned (reparented to init) and keeps running. The scripts' own pre-existing "still alive after kill -9" warning branch also never fired for this exact case, because it re-checks the SAME (sudo) PID, which genuinely is dead by then -- so the orphaned child produced zero warning output, making the bug fully silent. + +## Fix +FIXED (2026-09-17). In both generated scripts/stop-all.sh and scripts/cleanup.sh (src/tools/deploy-ha.sh), before killing a pidfile'd PID, check whether that PID's own /proc/$PID/cmdline starts with "sudo " (the telltale sign left by `exec sudo ...`); if so, find its child process(es) via `pgrep -P "$PID"` and kill them first via `sudo kill -9` (they run as root), before killing the sudo PID itself. The post-kill "still alive" check was also extended to additionally test for the child PID's continued existence (via `/proc/$CHILD` presence, which needs no kill permission, since `kill -0` on a root-owned PID from a non-root caller returns EPERM regardless of whether the process is alive). Live-verified end-to-end: deployed a fresh 3-node cluster with XCE_REAL_WORKER=1 (forcing the sudo xcecd path), confirmed the exact sudo-parent/root-child PID pair via `ps axwww`, ran the fixed stop-all.sh and confirmed both processes died with the new "is a sudo wrapper -- also stopping its child" line printed and zero processes remaining afterward; repeated the same reproduction against cleanup.sh with the same clean result. No unit test exists for this bash tooling (deploy-ha.sh has no meson-registered test suite, consistent with other shell tooling in this project e.g. xce-build.sh); verification is this live end-to-end reproduction. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1720 diff --git a/.wolf/bugs/bug-1821.md b/.wolf/bugs/bug-1821.md new file mode 100644 index 00000000..dbc0a23b --- /dev/null +++ b/.wolf/bugs/bug-1821.md @@ -0,0 +1,38 @@ +--- +id: "bug-1821" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-09-17T04:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/src/xceadmin/discovery.py (new resolve_controllers/follow_redirects_any)", "src/xceadmin/src/xceadmin/push.py", "src/xceadmin/src/xceadmin/scheduler.py (both migrated off their own duplicated _split_controller)", "src/xceadmin/src/xceadmin/cli.py (--controller help text)"] +tags: ["xceadmin", "discovery.py", "push.py", "scheduler.py", "cli.py", "--controller", "failover", "master-rotation", "user-reported", "tdd", "live-verified"] +related: ["bug-1566", "bug-1822"] +legacy: true +--- + +# bug-1821: User-reported: xceadmin's --controller option (config push and all 3 scheduler record… + +## Description +User-reported: xceadmin's --controller option (config push and all 3 scheduler record start/stop/status commands) accepted only a SINGLE host:port -- passing a comma-separated list of all configured controllers (the natural way to query an HA cluster without knowing in advance which node is alive/primary, e.g. `--controller "localhost:9901,localhost:9902,localhost:9903"` with 9901 dead) failed outright: `Error: cannot connect to localhost:9901,localhost:9902,localhost:9903: [Errno -2] Name or service not known` -- the whole comma-joined string was parsed as one bogus hostname. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +push.py and scheduler.py each had their own private _split_controller(controller: str) -> (host, port) that did a single controller.rsplit(':', 1) -- designed only ever to see one host:port. A comma-separated value was never split on ',' at all; it fell through as one literal (unparseable) hostname straight into socket.create_connection via xcecommon.wire.connect, which propagates the resulting socket.gaierror (a plain OSError subclass) unwrapped by design. + +## Fix +FIXED (2026-09-17, TDD RED->GREEN). Added xceadmin.discovery.resolve_controllers(controller) -> list[(host,port)] (comma-split, whitespace-stripped, per-entry bare-host default port) and xceadmin.discovery.follow_redirects_any(candidates, send_fn) -> (host,port,reply_bytes), which tries each candidate in order via the existing follow_redirects(), catching per-candidate OSError (connection refused, DNS failure) and NoPrimaryError (discovery-level timeout) and moving to the next candidate -- but NOT catching ReplyDecodeError/TooManyRedirectsError, since a genuinely-reachable candidate's protocol-level failure isn't fixed by trying a different node. push.py/scheduler.py's own _split_controller removed; both now call resolve_controllers + follow_redirects_any. cli.py's --controller help text updated on all 4 commands to document the comma-list form. 12 new unit tests in test_discovery.py (single/list/whitespace parsing; first-reachable-wins; skip-refused/skip-DNS-failure/skip-timeout-then-succeed; all-fail raises NoPrimaryError naming every candidate; redirect-following composes correctly with failover; a reachable candidate's decode error does NOT fall through to the next). 2 existing tests per file (test_config_push_mock.py, test_scheduler.py) updated to call the new shared function instead of the removed private one. Full suite 304/304. Live-verified: deployed a 3-node cluster, killed the first-listed controller, confirmed `--controller "dead,live1,live2"` correctly skips the dead one and reaches a live one (both healthy-cluster and killed-primary scenarios). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1566, bug-1822 diff --git a/.wolf/bugs/bug-1822.md b/.wolf/bugs/bug-1822.md new file mode 100644 index 00000000..35635ea8 --- /dev/null +++ b/.wolf/bugs/bug-1822.md @@ -0,0 +1,38 @@ +--- +id: "bug-1822" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "xceadmin" +severity: "unknown" +timestamp: "2026-09-17T04:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request)"] +tags: ["xceadmin", "scheduler.py", "_send_admin_request", "NoPrimaryError", "retry", "heartbeat_timeout_ms", "election", "live-verified", "tdd", "bug-1713", "bug-1715"] +related: ["bug-1713", "bug-1715", "bug-1821"] +legacy: true +--- + +# bug-1822: Found live while verifying bug-1821's --controller failover-list fix: querying `xceadmin scheduler… + +## Description +Found live while verifying bug-1821's --controller failover-list fix: querying `xceadmin scheduler record status --controller "host1,host2,host3"` immediately after killing the primary hard-failed with `Error: no primary available: none of the configured controllers responded (...)` instead of riding out the cluster's own re-election, even though _send_admin_request() already has a bounded retry loop (bug-1713/1715) built for exactly this 'cluster still settling' condition. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Right after a primary is killed, a standby does not even START a new election until its own heartbeat_timeout_ms elapses (ha.h: HA_DEFAULT_HEARTBEAT_TIMEOUT_MS = 10000, i.e. 10s) -- until then it keeps redirecting every schedAdmin request straight back to the now-dead old primary. follow_redirects_any() (bug-1821) correctly detects that dead redirect target is unreachable and tries the next configured candidate, but if EVERY candidate is in this same stale-redirect state, the whole candidate list is exhausted and follow_redirects_any raises NoPrimaryError -- an exception _send_admin_request()'s existing retry loop never caught. That loop only retried on a REPLY that arrived and said not-ready (RequestReply.status in _TRANSIENT_STATUSES, or SchedAdminReply.ok=false with a known-transient reason) -- a total connection failure (no reply at all) propagated immediately with zero retries, even though it is the identical 'cluster still settling' condition the existing retry logic was built to ride out, just signaled differently. + +## Fix +FIXED (2026-09-17, TDD RED->GREEN). _send_admin_request()'s retry loop restructured from a single bounded for-loop sharing one attempt counter into a `while True` loop with TWO independent counters/budgets: the pre-existing status_attempt (_MAX_TRANSIENT_ATTEMPTS=4, _TRANSIENT_RETRY_DELAY_S=1.5s, unchanged, still shared by the reply.status and SchedAdminReply.ok=false branches exactly as before) and a new no_primary_attempt (_MAX_NO_PRIMARY_ATTEMPTS=8, _NO_PRIMARY_RETRY_DELAY_S=2.0s -- deliberately larger, since this failure mode's worst case is the ~10s heartbeat-timeout detection window plus real election time, categorically longer than the RPC-level 'already electing, replies not-ready' scenario the original budget was tuned for) that catches NoPrimaryError from follow_redirects_any and retries the whole candidate list fresh. 2 new unit tests (monkeypatching follow_redirects_any directly: retries-then-succeeds, and exhaustion-still-propagates-NoPrimaryError after exactly _MAX_NO_PRIMARY_ATTEMPTS attempts). Full suite 304/304. Live-verified: killed a settled cluster's primary, immediately queried the failover-list, confirmed (in one repro) the retry loop visibly cycled the candidate list 5+ times over ~15s while standbys still redirected to the dead primary, then successfully reached the genuinely newly-elected primary once the standbys' own election completed -- the query result correctly transitioned from NoPrimaryError to the new primary's real (if scheduler-daemon-absent-in-this-test-env) reply, proving the retry rode out the real HA re-election, not just a mocked delay. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1713, bug-1715, bug-1821 diff --git a/.wolf/bugs/bug-1823.md b/.wolf/bugs/bug-1823.md new file mode 100644 index 00000000..b9dbf835 --- /dev/null +++ b/.wolf/bugs/bug-1823.md @@ -0,0 +1,38 @@ +--- +id: "bug-1823" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-09-17T04:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/tools/deploy-ha.sh (generated scripts/stop-all.sh and scripts/cleanup.sh)"] +tags: ["deploy-ha", "stop-all.sh", "cleanup.sh", "xcecd", "sudo", "orphaned-process", "real-worker", "sudo-monitor", "process-tree-depth", "live-verified", "synthetic-repro"] +related: ["bug-1820"] +legacy: true +--- + +# bug-1823: User-reported (regression on top of bug-1820's own fix, live on the user's real host, 'pc13v')… + +## Description +User-reported (regression on top of bug-1820's own fix, live on the user's real host, 'pc13v'): after ./scripts/stop-all.sh reported 'Stopping worker-1 (PID N)' with no warning, the real xcecd worker was STILL running afterward as a different PID -- and a second `sudo ./scripts/stop-all.sh` run immediately reported 'Nothing was running for this deploy' (the pidfile had already been deleted by the first run despite the kill not actually succeeding). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +bug-1820's own fix assumed sudo always has exactly ONE level of indirection (sudo forks exactly one direct child that IS the real target), and looked for that child via `pgrep -P $PID` (direct children of the pidfile'd PID only). On the user's real host, sudo interposes an ADDITIONAL 'monitor' process between the invoked sudo and the actual target (sudo -> monitor -> target, THREE processes, not two -- standard sudo 1.8.7+ architecture for proper signal/exit-status relay, not present in this sandbox's own simpler sudo build, which is why bug-1820's fix appeared to work here but not on the user's real box). `pgrep -P $PID` found and killed the MONITOR (a direct child of the pidfile'd PID) while the actual worker -- the monitor's OWN child, not a direct child of the pidfile'd PID -- survived completely untouched, silently orphaned exactly as in the original bug-1820 symptom. Confirmed and reproduced deterministically via a synthetic 3-level process tree (fake-outer -> fake-monitor -> fake-worker, no real sudo needed) built specifically to isolate the tree-walking logic from any particular sudo version's actual behavior. + +## Fix +FIXED (2026-09-17). Replaced the parent/child tree-walking approach entirely with a depth-independent identity-based re-scan: after attempting to kill the pidfile'd PID, both generated scripts/stop-all.sh and scripts/cleanup.sh now search system-wide (`pgrep -f`) for ANY still-running process whose own cmdline matches the worker's distinguishing identity -- its `-n $NAME` flag plus this specific deploy's `-l $TOP/logs` flag, the same two args start-worker.sh's own 'already running' guard already checks -- and kills every match directly (both a plain `kill -9` and a `sudo kill -9`, since ownership can't be known in advance), regardless of how many hops deep in the process tree it landed. No longer needs to know or guess sudo's internal process-tree depth. A no-op for ctrl-N/scheduler/apiserver pidfiles, which use neither flag. Live-verified: (1) regression-checked against this sandbox's own real (2-level) sudo worker deploy -- still cleans up correctly; (2) built and killed a synthetic 3-level tree (fake-outer/fake-monitor/fake-worker, exact process-tree shape modeled on the user's live report) via the exact identity-pgrep mechanism the fix uses, confirming the deepest (target) process is found and killed directly, and the intermediate 'monitor' hop then exits on its own once its child is gone (it was just wait()-ing/relaying) -- proving the fix generalizes to arbitrary sudo process-tree depth rather than being tuned to one specific sudo version's behavior. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1820 diff --git a/.wolf/bugs/bug-1824.md b/.wolf/bugs/bug-1824.md new file mode 100644 index 00000000..0b98649b --- /dev/null +++ b/.wolf/bugs/bug-1824.md @@ -0,0 +1,38 @@ +--- +id: "bug-1824" +status: "open" +status_updated: "2026-09-24" +status_note: "repl_info.c bounds check uses zarray count instead of caps" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-17T02:45:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/repl_info.c"] +tags: ["scheduler", "repl", "zarray", "bounds-check", "out-of-bounds-read", "latent", "discovered-during-scheduler-sim-diagnostics"] +related: [] +legacy: true +--- + +# bug-1824: out-of-scope latent bug discovered while implementing scheduler-sim-diagnostics task 3.1 (new… + +## Description +NOT A CRASH -- pre-existing, out-of-scope latent bug discovered while implementing scheduler-sim-diagnostics task 3.1 (new repl_info_limit() live-usage display). repl_info_resource_one() (src/scheduler/src/repl_info.c ~line 996) bounds-checks a resource's zarray index via `(r->idx < 0) || (r->idx >= sim->core.sharedInst.count)` before calling zarray_get() -- but zarray_t.count tracks the number of POPULATED slots (incremented only when a previously-NULL slot becomes non-NULL via zarray_put_ex/zarray_store), not the array's allocated capacity (.caps, grown by zarray_ensure_items). A sparsely-populated zarray (e.g. only a high index ever zarray_put() -- realistic if resources are registered out of index order, or an index is skipped) can have count=1 while a LOWER, unpopulated index is itself < count and would incorrectly pass this check and be read as though populated, or a HIGHER populated index can exceed .count and be incorrectly rejected as absent even though zarray_get()'s own .caps-based check would have safely returned it (or correctly returned NULL for a genuinely-unpopulated-but-in-capacity slot, since zarray_ensure_items() NULL-initializes newly-grown slots). Net effect: repl_info_resource_one() can report a resource as having no value ('value=(none)') when a real, populated value exists at that index but the array is sparse, OR (less likely) skip a real bounds-safety net zarray_get_ex() would otherwise provide. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl_info_resource_one()'s pre-check compares r->idx against sim->core.sharedInst.count (populated-slot count) instead of relying on zarray_get_ex()'s own bounds check, which is against .caps (allocated capacity) -- these two counters diverge for any sparsely-populated zarray. Confirmed by direct reading of src/lib/zarray.c: zarray_get_ex() (~line 341-362) checks only `idx >= array->caps` before dereferencing array->data[idx]; zarray_put_ex()/zarray_store() are the only writers of .count, incrementing it exclusively when filling a previously-NULL slot -- so .count is a 'high-water mark of contiguous-from-zero fill,' not a true capacity bound. Separately, and more seriously: neither the .count-based check NOR zarray_get_ex()'s own .caps check safely rejects a NEGATIVE idx -- `idx >= caps` is false for any negative idx when caps >= 0, so a negative idx falls through to `array->data[idx]`, a genuine out-of-bounds read. A negative idx is reachable in this codebase: add_vres() (src/lib/xcelimit.c ~407-435) unconditionally sets a newly-allocated xce_resource_t's ->idx to -1 before key_add() runs, only overwriting it with the real slot index on success -- a caller that reaches a resource-value read before/without that overwrite succeeding would pass a negative idx through. repl_info_resource_one()'s existing `r->idx < 0` half of its check does happen to guard this specific negative case correctly (by luck of ordering, not by understanding zarray's real contract) -- the count-vs-caps confusion is the part that is a genuine, distinct latent bug. + +## Fix +NOT FIXED -- discovered and characterized during scheduler-sim-diagnostics task 3.1 (limit-usage display), out of that task's scope (task 3.1 only touches its OWN new code path, repl_print_limit_usage(), which was written from scratch to avoid this exact mistake: it omits the imprecise .count pre-check entirely, relies on zarray_get_ex()'s own .caps bound for the upper end, and adds an explicit idx<0 guard of its own -- verified correct and reviewed independently, see this session's git history for scheduler-sim-diagnostics task 3.1). repl_info_resource_one() itself was deliberately left untouched to keep task 3.1 in its declared boundary. Recommended fix when picked up: replace the `r->idx >= sim->core.sharedInst.count` half of the pre-check with reliance on zarray_get()'s own .caps-based bound (mirroring task 3.1's new repl_print_limit_usage() pattern), keeping the `r->idx < 0` guard. Likely benign in practice today only because resource registration order in this codebase's current production paths tends to be dense/contiguous from index 0, so .count and .caps rarely diverge for sim->core.sharedInst specifically -- but this is an incidental property of current callers, not a guarantee, and worth a regression test with a deliberately sparse fixture (zarray_put() at a high index with no lower indices ever populated) before relying on it. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1825.md b/.wolf/bugs/bug-1825.md new file mode 100644 index 00000000..13c4bdd2 --- /dev/null +++ b/.wolf/bugs/bug-1825.md @@ -0,0 +1,48 @@ +--- +id: "bug-1825" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-17T07:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_picker.c"] +tags: ["scheduler", "sim-mode", "repl", "limit-plugin", "plugin-bootstrap", "architectural-gap", "discovered-during-scheduler-sim-diagnostics", "driver-parity"] +related: [] +legacy: true +--- + +# bug-1825: verified architectural gap discovered while closing out scheduler-sim-diagnostics task 5 (final… + +## Description +NOT A CRASH -- significant, verified architectural gap discovered while closing out scheduler-sim-diagnostics task 5 (final validation task for the new `info limit` REPL command). The limit plugin is never registered in ANY simulation-mode driver path, so `info limit ` will ALWAYS report "no limits configured"/"no such limit" for a real operator running `xce_scheduler -i`/`-s ` against a genuine recording -- even one whose original live run had limits configured and actively enforced. The entire feature built across scheduler-sim-diagnostics tasks 1-5 is correctly implemented, thoroughly tested, and read-only/deterministic as designed, but is currently INERT in real production usage: every positive-display test path (tasks 2-3, showing a limit's configured setting and live usage) only exercises the code via a test-only helper that hand-registers the plugin into core->plugins directly, which has no production equivalent in sim mode. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +register_limit() (plugins/limit.c:609) has exactly one call site in the entire codebase: scheduler.c:605, inside sched_register_plugins(), which is itself called only from scheduler.c:827 -- the normal-mode (controller-connected, live) driver's sched_init() path. Sim mode's own plugin-bootstrap mechanism, sim_bootstrap_picker() (sim_picker.c), exists specifically to register a picker plugin (builtin_fcfs, via plugin_register_core()) for simulation/replay mode, since sim mode deliberately does not run the full normal-mode init sequence -- but it only ever registers the PICKER plugin category. It has no equivalent step for the limit plugin (or, by the same reasoning, potentially other plugin categories registered by sched_register_plugins() that were never audited for this gap). Confirmed via full-codebase grep: register_limit()/sched_register_plugins() are never referenced anywhere under sim.c/sim_loop.c/sim_picker.c/driver.c/repl*.c, and no write into core->plugins occurs anywhere in the sim-mode driver files. This is the same class of gap scheduler-driver-equivalence.md already documents for OTHER driver-parity properties (e.g. bug-1613's sched_account_jobs() gap) -- a sim-mode driver feature silently missing something the normal-mode driver does, discovered only when a REPL feature tried to read state that assumes it. + +## Fix +FIXED (2026-09-17, scheduler-sim-diagnostics task 6). Added limit_new_plugin() factory (extracted from register_limit(), mirroring builtin_fcfs_new_plugin()'s established pattern) and a new sim_bootstrap_limit_plugin() (new file src/scheduler/src/sim_limit.c, mirroring sim_picker.c's structure exactly): registers the limit plugin via plugin_register_core() and seeds it with sim->core.config (the snapshot's own borrowed-reference config, which already carries the original session's limit configuration -- no recording/replay format change needed) via a new limit_seed_config(plugin, xce_sched_core_t* core, config) entry point. Wired into sim_loop.c alongside the existing sim_bootstrap_picker(sim) call. + +First attempt (round 1) set plugin->sched = (void*)sim (a disguised xce_sched_sim_driver_t*) to reuse the existing config_hook path -- REJECTED on independent review: this repeats the exact pointer-to-first-member cross-cast pattern that already corrupted memory once in this codebase (see this bug's own root_cause note and core.h's xce_core_cache_clear_cb_t doc comment for that prior incident). Round 2 eliminated the cast entirely: limit_seed_config() takes xce_sched_core_t* core as an honest parameter (a real field both xce_scheduler_t and xce_sched_sim_driver_t genuinely have), delegating to a shared limit_seed_config_impl() also used by normal-mode limit_update_config(). No cast, no aliasing risk -- confirmed structurally sound by two independent reviewers, each reading the actual struct definitions and full call chain rather than trusting the report. + +Also closed: sim_limit.c added to sched_sim_isolation_lib's source list (meson.build) so Req 5 AC 17's mechanical controller-I/O-isolation check covers it (round 1 omitted this; round 2 fixed it, sched_sim_isolation_test independently re-verified passing with the new file included). + +Ripple: two waves of an identical, minimal link-satisfying stub pattern across the ~24 white-box test files that #include "../src/sim_loop.c" (one for limit_new_plugin(), one for limit_seed_config()) -- same established 'duplicate, don't share' idiom this file family already used for the earlier scheduler-sim-diagnostics tasks' repl_info.c ripple. + +TDD: RED->GREEN verified for both the config-seed test and the remediation itself (a temporary limit_seed_config() no-op stub reproduced the exact expected single-test failure, reverted cleanly). Full scheduler suite: 78/78 Ok, 0 Fail; sched_repl_limit_table_test 26/26; sched_limit_plugin_test 5/5 (zero regression to normal-mode registration); sched_sim_isolation_test 5/5 (genuinely covering the new file). + +Deliberately NOT fixed (separate, larger, tracked as a distinct follow-up item in .kiro/specs/scheduler-sim-diagnostics/tasks.md, not this bug): limit *matching* (limit_job_election()) is still never invoked during replay -- it's only reachable via plugin_job_election(), which is normal-mode-only (one caller, scheduler.c) and would itself need a core-safe variant before it could be wired into sim_loop.c's decision path. So 'info limit' now shows real CONFIGURED settings against a real replay, but live USAGE will still correctly show 'no active usage instance yet' until that separate, larger fix lands. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1826.md b/.wolf/bugs/bug-1826.md new file mode 100644 index 00000000..8be71892 --- /dev/null +++ b/.wolf/bugs/bug-1826.md @@ -0,0 +1,38 @@ +--- +id: "bug-1826" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-17T21:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugin.c"] +tags: ["scheduler", "sim-mode", "repl", "limit-plugin", "limit-matching", "plugin-job-election", "architectural-gap", "discovered-during-scheduler-sim-diagnostics", "driver-parity", "follow-up-to-bug-1825"] +related: [] +legacy: true +--- + +# bug-1826: deliberately out-of-scope follow-up discovered while fixing bug-1825 (sim-mode limit plugin… + +## Description +NOT A CRASH -- known, deliberately out-of-scope follow-up discovered while fixing bug-1825 (sim-mode limit plugin bootstrap, scheduler-sim-diagnostics task 6). Bug-1825's fix makes the limit plugin's CONFIGURED SETTING visible in sim mode (info limit now shows a real replay's actual limit configuration). But limit *matching* -- the mechanism that actually populates live usage instances (xce_vres_t) as jobs get evaluated -- is still never invoked during replay. So info limit's live-usage display will correctly show 'no active usage instance yet' for every limit against any real replay, even one whose original live session actively enforced and charged against that limit. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +limit_job_election() (plugins/limit.c, the function that calls match_job_global_limit_()/match_job_host_limit_() and populates xce_vres_t usage instances) is only ever invoked via plugin_job_election() (plugin.c:554), which -- like plugin_config_update()/plugin_prepare_allocation() -- takes a `void* arg` cast unconditionally to `xce_scheduler_t*` and reads `sched->core.plugins`. plugin_job_election() has exactly ONE call site in the entire codebase: scheduler.c (normal-mode only). Nothing in sim_loop.c's replay decision path calls plugin_job_election() or any core-safe equivalent, so no limit ever gets matched against any job during simulation/replay, regardless of whether the limit plugin is registered (bug-1825 fixes registration+config, not this). + +## Fix +FULLY FIXED (2026-09-19, scheduler-sim-live-allocation tasks 1.1-5.2). The core-safe plugin_job_election() variant this entry called for was built in task 1.2 (retyped to take xce_sched_core_t* directly, iterating core->plugins); it is now genuinely invoked in sim mode via two paths: (a) the unified sched_job_picker()/sched_allocator() thread pair (task 3.1, wired into sim mode in task 4.1) calls the full core-safe election/allocation chain for every job the live picker reaches, which includes limit matching via limit_job_election() (task 2.3, also fixed bug-1828, a pre-existing host-scoped-limit enforcement defect blocking this same path); (b) sim_reconcile_unaccounted_jobs() (task 4.2) additionally calls plugin_job_election() directly for any job only recovered via recorded-decision reconciliation, so a job never misses limit registration regardless of which path resolves it. Host-scoped limit fan-out during live seqjob/parjob dispatch specifically required a further fix (task 4.3, closing a throwaway-wrapper-registry seam gap) before it worked end-to-end. Proven via the full TDD + contention-fixture discipline this entry called for: bare-core contention fixtures (task 5.1, test_sim_contention_limit_driver_parity.c) and this entry concrete acceptance test -- a genuine real capnp-encoded .replay driven through the real, unmodified sched_sim_driver_run(), with both a global and a host-scoped limit configured and matched, both showing real non-zero usage via the same limit_get_usage_instances() accessor info limit reads (task 5.2, test_sim_e2e_limit_visibility.c) -- independently reviewed with a targeted revert-to-pre-task-4.3 RED-phase reproduction confirming the host-scoped half genuinely regresses without the fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1827.md b/.wolf/bugs/bug-1827.md new file mode 100644 index 00000000..edce629c --- /dev/null +++ b/.wolf/bugs/bug-1827.md @@ -0,0 +1,38 @@ +--- +id: "bug-1827" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-18T07:15:00Z" +last_seen: "2026-09-19T14:30:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test_core_isolation.c"] +tags: ["scheduler", "core-isolation", "test-coverage-gap", "test_core_isolation", "core_clock", "audit-finding", "discovered-during-xce-audit-invariants"] +related: [] +legacy: true +--- + +# bug-1827: ), while investigating driver state-equivalence for bug-1826. test_core_isolation.c's own header… + +## Description +NOT A CRASH -- coverage gap found during an /xce-audit-invariants scheduler run (dimension 2: core module I/O isolation), while investigating driver state-equivalence for bug-1826. test_core_isolation.c's own header comment claims core_clock.c joins ONLY the nm(1) undefined-symbol enforcement layer, not the wall-clock text-scan check -- implying it is still covered by the OTHER two text-scan checks (forbidden #include list, forbidden symbol substrings). In reality, core_clock.c is absent from all three text-scan functions (test_core_never_includes_forbidden_headers(), test_core_never_calls_forbidden_symbols(), and the wall-clock scan), not just the wall-clock one -- no XCE_CORE_CLOCK_C_PATH macro is defined anywhere in test_core_isolation.c or testsuite/meson.build's c_args for that test. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +core_clock.c was added to the mechanically-enforced sched_core_isolation_lib archive (meson.build) and given a documented single exception (the wall-clock text scan, for its one sanctioned clock_gettime() call), but the two OTHER text-scan functions (forbidden-headers, forbidden-symbols) were never extended to also read core_clock.c's source text -- only core.h/core.c/core_events.c/core_session.c are read by those two checks (via XCE_CORE_H_PATH/_C_PATH/_EVENTS_C_PATH/_SESSION_C_PATH macros; no clock-file variant exists). The nm(1) undefined-symbol layer (which DOES include core_clock.c in the archive) only catches an actual undefined-symbol reference at link time -- it cannot catch a bare forbidden #include (e.g. client.h/netio.h/xceapi.h/delay.h/idgen.h/xce/cluster.h) that resolves to no undefined symbol (a macro-only or inline-only header), so the doc comment's implied coverage is narrower in practice than stated. No evidence found that core_clock.c's actual current content violates isolation today -- this is a gap in what the mechanism would CATCH if it did, not a confirmed live violation. + +## Fix +FIXED (2026-09-19). Added XCE_CORE_CLOCK_C_PATH (testsuite/meson.build c_args for sched_core_isolation_test, mirroring the existing XCE_CORE_H_PATH/_C_PATH/_EVENTS_C_PATH/_SESSION_C_PATH pattern) and extended both test_core_never_includes_forbidden_headers() and test_core_never_calls_forbidden_symbols() (test_core_isolation.c) to also scan core_clock.c -- it remains excluded ONLY from the wall-clock scan (test_core_never_calls_wallclock_functions), for the same sanctioned-single-clock_gettime()-call reason as before. TDD: confirmed RED by temporarily inserting #include "netio.h" into core_clock.c and rebuilding -- the extended test correctly failed with the message "core_clock.c directly includes forbidden header '#include "netio.h"'" (test_core_isolation.c:226) while every other test stayed green (84/85); reverted (git diff confirms core_clock.c byte-identical to HEAD) and reconfirmed GREEN (85/85). No live violation existed in core_clock.c -- this closes the coverage gap the mechanism itself had, matching what its own header/doc comments already claimed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1828.md b/.wolf/bugs/bug-1828.md new file mode 100644 index 00000000..5a2f53bb --- /dev/null +++ b/.wolf/bugs/bug-1828.md @@ -0,0 +1,37 @@ +--- +id: "bug-1828" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugins/limit.c (match_job_host_limit_)", "src/scheduler/testsuite/test_repl_limit_table.c (new test)"] +tags: ["scheduler", "limit-plugin", "kvmgr", "error-code-convention", "not-found-vs-error", "match_job_host_limit_", "sibling-of-bug-1786", "scheduler-sim-live-allocation", "task-2.3"] +related: [] +legacy: true +--- + +# bug-1828: IMPACT CORRECTED 2026-09-18 (independent review of scheduler-sim-live-allocation task 2.3): this… + +## Description +IMPACT CORRECTED 2026-09-18 (independent review of scheduler-sim-live-allocation task 2.3): this is NOT just a display/accounting gap -- it meant host-scoped resource limits were NEVER ACTUALLY ENFORCED, in ANY mode (normal or simulation), for this function's entire history. Originally discovered while writing scheduler-sim-live-allocation spec task 2.3's own required TDD test (a host-scoped limit match, via the simulation-mode-registered limit plugin, must create a real usage instance verifiable through limit_get_usage_instances()). limit_prepare_allocation()/match_job_host_limit_() consistently returned rc=-2 (-ENOENT) instead of >= 0, and no usage instance was ever created, even though match_limit() itself genuinely matched (rc=1) and is_host_based_limit() correctly identified the limit as host-based. Tracing the call chain (plugin.c's plugin_prepare_allocation() hook loop -> sched_fill_limit_requests()) shows the missing usage instance meant a host-scoped limit's vres was never added to obj->hlimits, so sched_fill_limit_requests() always iterated an empty list -- the limit's numeric cap was silently never applied to any job's resource request, in production, since this code existed. This does not corrupt the dispatch/pend decision machinery itself (plugin_prepare_allocation()'s hook-calling loop discards the hook's return value, so the bug never fed into a control-flow branch) -- it is a dormant-limit-never-binds defect, but a real enforcement gap, not merely a cosmetic one. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +match_job_host_limit_() (src/scheduler/src/plugins/limit.c) is the host-scoped sibling of match_job_global_limit_() and has the same defect CLASS bug-1786 fixed in that function alone (misinterpreting key_find()'s negative "not found" return as fatal, rather than the correct not-yet-instantiated signal) -- not byte-for-byte identical wording (bug-1786's own guard was `if (rc <= 0) { return rc; }`; this sibling's was `if (rc < 0) { return rc; }`), same underlying mistake. Right after a successful match_limit(), it does `rc = key_find(privdata->instances, vresname, &vres); if (rc < 0) { return rc; }` before the `if (vres == NULL) { instantiate_limit(...) }` block below it. On the FIRST-EVER lookup of a given vresname (always true the first time a host-scoped limit ever matches), key_find() signals "not found" as a NEGATIVE rc (-ENOENT), not 0 -- this codebase's established kvmgr convention (get_group(), is_host_based_limit(), and bug-1786/bug-1495's own root causes) -- so this early-return unconditionally caught the expected miss and returned before instantiate_limit() could ever run. bug-1786 (2026-09-07) fixed this exact defect class in match_job_global_limit_() but never touched its host-scoped sibling, which retained the bug because no test exercised match_job_host_limit_()/limit_prepare_allocation() with real matching until this task added one (verified via grep: zero prior test call sites for either function anywhere in the test suite). + +## Fix +FIXED (2026-09-18, scheduler-sim-live-allocation task 2.3). Removed the erroneous `if (rc < 0) { return rc; }` guard in match_job_host_limit_() entirely, mirroring match_job_global_limit_()'s own bug-1786 fix exactly: `vres` (calloc'd NULL, only ever reassigned by a successful key_find) is already the correct, sufficient signal for "needs instantiation" via the existing `if (vres == NULL)` block immediately below. TDD: new test test_limit_prepare_allocation_matches_host_limit_via_sim_registered_plugin (test_repl_limit_table.c) drives the real limit_prepare_allocation()/match_job_host_limit_() path against a plugin registered via plugin_register_core() and a real percent-based host-scoped limit; RED (rc=-2, confirmed via temporary debug instrumentation tracing is_host_based_limit()=1, match_limit()=1, but limit_prepare_allocation()=-2) before this fix, GREEN (rc>=0, usage instance created and visible via limit_get_usage_instances()) after. Full scheduler suite: 79/79, zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-1829.md b/.wolf/bugs/bug-1829.md new file mode 100644 index 00000000..26367f82 --- /dev/null +++ b/.wolf/bugs/bug-1829.md @@ -0,0 +1,38 @@ +--- +id: "bug-1829" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "crash" +timestamp: "2026-09-19T04:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/include/plugin.h (seqjob_allocator_new_plugin()/parjob_allocator_new_plugin() factory declarations, task 4.3)", "src/scheduler/src/plugins/seqjob_allocator.c (task 4.3: init_fn + prepare_allocation_hook + job_alloc_fn retyped to read plugin->core; new seqjob_allocator_new_plugin() factory -- genuinely modified, no longer just the root-cause file)", "src/scheduler/src/plugins/parjob_allocator.c (task 4.3: same retyping + new parjob_allocator_new_plugin() factory)", "src/scheduler/src/sim_alloc.c (task 4.1: new file, initial bootstrap via throwaway wrapper; task 4.3: rewritten to register directly against sim->core via plugin_register_core(), no wrapper in the registration/fan-out path)", "src/scheduler/src/sim_loop.c (task 4.1: sched_sim_driver_run() call sites -- unmodified by task 4.3)", "src/scheduler/meson.build (task 4.1: scheduler_srcs, sched_sim_isolation_lib -- unmodified by task 4.3)", "src/scheduler/testsuite/test_sim_live_allocation.c (task 4.1: new file, live-dispatch RED/GREEN proof)", "src/scheduler/testsuite/test_sim_alloc_limit_fanout.c (task 4.3: new file, host-scoped-limit-fan-out RED/GREEN proof)", "src/scheduler/testsuite/test_seqjob_allocator.c, test_seqjob_rewrap_equivalence.c, test_parjob_combinatorial.c (task 4.3: white-box fixture ripple -- plugin instances built directly in these tests needed their own .core field set alongside .sched)"] +tags: ["scheduler", "sim-mode", "asan", "segv", "null-deref", "seqjob_allocator", "parjob_allocator", "plugin-sched-vs-plugin-core", "scheduler-sim-live-allocation", "task-4.1", "follow-up-tracked", "task-4.3", "fully-fixed"] +related: ["bug-1828"] +legacy: true +--- + +# bug-1829: AddressSanitizer: SEGV (NULL pointer read) in sched_make_seqjob_allocation… + +## Description +AddressSanitizer: SEGV (NULL pointer read) in sched_make_seqjob_allocation (src/scheduler/src/scheduler.c:59, `plugin->job_alloc_fn` where `plugin = core->seqjob_allocator` is NULL), called from sched_make_primary_alloc_ -> sched_make_primary_allocation -> sched_make_allocation -> sched_allocator (sched_election.c) -> the real pthread spawned by sched_sim_driver_run() (sim_loop.c). Discovered while writing the mandated TDD test for scheduler-sim-live-allocation task 4.1 (its own Observable: 'a simulation-mode session ... produces a genuine dispatch or pend decision for a job based on its actual resource availability, instead of unconditionally pending it') -- the crash is 100% reproducible for ANY job reaching primary resource allocation in simulation mode, regardless of resreq shape or dispatch/pend outcome (sched_make_parjob_allocation has the identical NULL-core->parjob_allocator crash for maxTasks>1 jobs). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +core->seqjob_allocator/core->parjob_allocator (xce_sched_core_t fields, core.h) are never populated for the simulation driver, anywhere. Normal mode populates them via sched_init()'s sched_register_plugins() -> register_seqjob_allocator()/register_parjob_allocator() (plugins/seqjob_allocator.c, plugins/parjob_allocator.c), whose init_fn does `((xce_scheduler_t*)plugin->sched)->core.seqjob_allocator = plugin` -- REQUIRING a genuine, live xce_scheduler_t*. sched_sim_driver_start() (sim.c) registers no plugin at all (sim.h's own documented 'Known scope boundary'), and xce_sched_sim_driver_t deliberately has no xce_scheduler_t*/xce_scheduler_t field of its own (Req 5 AC 17, simulation isolation). This is NOT limited to the init_fn: every one of these two plugins' RUNTIME hooks (seqjob_prepare_allocation()/my_alloc(), parjob_prepare_allocation()/my_alloc(), and several internal static helpers taking xce_scheduler_t* sched as a parameter) ALSO read `plugin->sched->core.*` (resmgr for the 'slots'/rusage lookups, workers for host_iter, and sched_make_allocs()/sched_try_alloc_on_host_() in scheduler.c/resource.c which ALSO take xce_scheduler_t*/need `sched->core.resmgr`) on every single allocation attempt, not just at registration. This gap was already discovered and explicitly deferred by scheduler-sim-live-allocation task 3.1 (see test_sched_election_sim_fixture.c's own header comment: 'a pre-existing, tracked, OUT-OF-SCOPE gap ... NOT one of task 3.1's five permitted changes -- fixing it would be scope creep into a different, not-yet-assigned task') -- task 3.1's own fixture never reached this crash only because IT registers seqjob_allocator directly against a real, throwaway xce_scheduler_t wrapper whose OWN .core IS the core under test (single-core fixture), a shape simulation mode itself can never have (its core lives inside xce_sched_sim_driver_t, never xce_scheduler_t). Task 4.1 is the first task whose OWN production change (spawning the real sched_job_picker()/sched_allocator() thread pair against simulation mode's own core, replacing the old always-PEND sim_picker_thread_fn() placeholder that never called sched_make_allocation() at all) makes this code path reachable in simulation mode for the first time ever. + +## Fix +FULLY FIXED (2026-09-19, scheduler-sim-live-allocation task 4.3, closing this bug's own tracked follow-up). Root cause: seqjob_allocator_init()/parjob_allocator_init() (plugins/seqjob_allocator.c, plugins/parjob_allocator.c) and every runtime hook these plugins register (seqjob_prepare_allocation()/my_alloc(), parjob_prepare_allocation()/my_alloc()/parjob_try_plan_()) now read the plugin's own honestly-typed `core` field (task 1.2's xce_sched_plugin_t.core) instead of deriving an xce_scheduler_t* from plugin->sched -- mirroring task 2.3's plugins/limit.c migration exactly. Two new factory functions, seqjob_allocator_new_plugin()/parjob_allocator_new_plugin() (plugin.h, mirroring limit_new_plugin()/builtin_fcfs_new_plugin()), let sim_alloc.c's sim_bootstrap_seqjob_allocator()/sim_bootstrap_parjob_allocator() register the allocator plugin DIRECTLY against &(sim->core) via the existing plugin_register_core() -- the same call sim_bootstrap_picker()/sim_bootstrap_limit_plugin() already use -- so plugin->core honestly points at sim->core and every later plugin_prepare_allocation() fan-out (my_alloc() -> sched_try_alloc_on_host_() -> plugin_prepare_allocation(), resource.c/plugin.c) reaches the SAME core->plugins registry the limit plugin (and picker) are registered in. Task 4.1's throwaway xce_scheduler_t wrapper is gone from the plugin-registry/fan-out path entirely -- no separate registry stands in for it anymore. One narrow, necessary exception remains: my_alloc() still keeps a small, long-lived, heap-allocated xce_scheduler_t wrapper (never freed, same accepted-leak convention) whose ONLY purpose is satisfying sched_make_allocs()'s (scheduler.c, out of this task's file boundary, unmodified) own real xce_scheduler_t* requirement for the FINAL allocation record of an already-decided winning host -- that function's body only ever reads sched->core.resmgr/lock/nextid, so the wrapper's .core.resmgr is ALIASED (a `var`, not copied) to sim->core's own; it is set on plugin->sched ONLY AFTER plugin_register_core() has already run (init_fn no longer reads plugin->sched at all) and is never used for the plugin registry or any prepare_allocation_hook fan-out, so it cannot reintroduce the separate-registry gap this fix closes. Verified live: new test_sim_alloc_limit_fanout.c (real scheduler_srcs link, bare xce_sched_sim_driver_t fixture, real sched_job_picker()/sched_allocator() thread pair) registers a host-scoped limit directly against sim->core (sim_bootstrap_limit_plugin()) alongside the seqjob allocator plugin (sim_bootstrap_seqjob_allocator()), drives a genuine live dispatch to a real DISPATCH decision (JOB_STAT_RUN), and confirms the limit's usage instance is created (limit_get_usage_instances()) -- RED before this fix (job dispatches correctly but zero usage instances; the fan-out never reached the limit plugin), GREEN after. Full scheduler suite: 82/82, zero regressions -- three white-box test files that construct an xce_sched_plugin_t directly without going through plugin_register()/plugin_register_core() (test_seqjob_allocator.c, test_seqjob_rewrap_equivalence.c, test_parjob_combinatorial.c) needed their fixture plugin instances' own `.core` field set alongside the pre-existing `.sched` field, plus parjob_try_plan_()'s one direct white-box call site (test_parjob_combinatorial.c) updated from &sched to &(sched.core) for its now-honest xce_sched_core_t* signature -- the same established ripple-fix pattern this feature's own tasks.md Implementation Notes already document for prior signature migrations. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1828 diff --git a/.wolf/bugs/bug-1830.md b/.wolf/bugs/bug-1830.md new file mode 100644 index 00000000..3e527301 --- /dev/null +++ b/.wolf/bugs/bug-1830.md @@ -0,0 +1,42 @@ +--- +id: "bug-1830" +status: "cannot-reproduce" +status_updated: "2026-09-24" +status_note: "downgraded 07b5c989; user's own retest showed mem limit usage correctly" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-20T09:00:00Z" +last_seen: "2026-09-20T09:30:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugins/limit.c"] +tags: ["scheduler", "sim-mode", "repl", "limit-plugin", "limit-matching", "mem-resource", "live-verification-finding", "related-to-bug-1825", "related-to-bug-1826", "discovered-during-scheduler-sim-diagnostics-requirement-6-live-test", "corrected", "not-reproduced-by-user", "needs-better-repro"] +related: ["bug-1825", "bug-1826"] +repro_attempts: "user retest 2026-09-20, not reproduced" +legacy: true +gitea_issue: 53 +gitea_synced_status: "cannot-reproduce" +--- + +# bug-1830: CORRECTED 2026-09-20 (user's own independent retest) -- original framing was too strong + +## Description +CORRECTED 2026-09-20 (user's own independent retest) -- original framing was too strong. A mem-based limit (limits2, from the real src/configs/limits.yaml) DID correctly populate a live usage instance in a user-run retest: `info limit limits2` against a real replay showed `usage limit=limits2/u=rshen/q=xce/p= host=pc13v: accounted=18022400.000000 (17.19 MB) capacity=4635344384.000000 (4.32 GB)` -- fully populated and correctly human-formatted on both accounted and capacity. This directly contradicts this entry's original claim that mem-based limits 'never' populate a usage instance via replay. ORIGINAL FINDING (retained for context, now understood to be narrower than stated): an earlier live-verification session (agent-run, single short-lived deploy+capture cycle) saw every mem-based limit show 'no active usage instance yet' while a slots-based limit in the same replay populated correctly. Given the user's retest succeeded, the real condition is likely session-shape/timing-dependent (e.g. elapsed replay rounds, WORKERS_PUSH arrival timing relative to job dispatch, or how long the recording ran) rather than a categorical mem-vs-slots failure. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +- Attempts: user retest 2026-09-20, not reproduced + +## Root cause +Suspected, not confirmed: match_job_global_limit_()/match_job_host_limit_() (plugins/limit.c) key_find() the limit's constrained resource by name against core->resmgr at match time. For `slots`, this resource is always already present in core->resmgr from session start. For `mem`, the resource may only become known to core->resmgr via a WORKERS_PUSH replay event reporting the worker's live mem value -- if that push event is decoded/applied AFTER the job's own dispatch decision is replayed (or never lands in the resmgr in a way key_find() can see before the match attempt), the mem-based limit's match silently misses, leaving no xce_vres_t/usage instance ever created for it. test_sim_e2e_limit_visibility.c's own acceptance test (scheduler-sim-live-allocation task 5.2) only proves this works when the mem resource is hand-baked directly into the snapshot's resmgr BEFORE any replay event is processed -- it does not cover the ordinary real-deployment shape (resource known only via a live WORKERS_PUSH event that arrives as part of the replay stream itself, not pre-seeded in the snapshot). This needs direct tracing of match timing vs. WORKERS_PUSH application order in sim_loop.c's real round sequence to confirm. + +## Fix +STATUS DOWNGRADED to investigation-needed, not a confirmed defect. The original agent-run live test's single short capture window may not be representative -- the user's own longer/different retest shows mem-based limit usage population DOES work under real replay conditions. If this is ever seen to fail again, the reproduction needs to capture the SPECIFIC session shape that fails (replay round count, elapsed time between record-start and the job's dispatch, WORKERS_PUSH cadence) since a bare 'mem-based limits don't populate' framing has now been shown to be incorrect as a categorical claim. Not fixed because there is no longer a confirmed, reproducible defect to fix -- downgrading rather than closing, since the ORIGINAL single agent-run observation was real and unexplained, just not reproduced by the user's own retest. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1825, bug-1826 +- Gitea: #53 diff --git a/.wolf/bugs/bug-1831.md b/.wolf/bugs/bug-1831.md new file mode 100644 index 00000000..dc05b83f --- /dev/null +++ b/.wolf/bugs/bug-1831.md @@ -0,0 +1,41 @@ +--- +id: "bug-1831" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-09-20T10:00:00Z" +last_seen: "2026-09-20T10:00:00Z" +occurrences: 1 +files: ["src/lib/xcelimit.c"] +tags: ["scheduler", "lib", "limit-plugin", "display-formatting", "percent-limit", "info-limit", "requirement-6", "discovered-during-live-retest", "related-to-bug-1832"] +related: ["bug-1832"] +legacy: true +gitea_issue: 56 +gitea_synced_status: "fixed" +--- + +# bug-1831: Live-verification finding (user's own retest of scheduler-sim-diagnostics Requirement 6): a… + +## Description +Live-verification finding (user's own retest of scheduler-sim-diagnostics Requirement 6): a correctly-configured 20% memory limit (limits2, resname: mem, value: 0.2, percent: true) displayed as 'mem = 0.200000%' in `info limit` -- reading as 0.2 percent to a human, not the 20 percent it actually computes and enforces. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce_limit_val_t.val is genuinely a 0-1 fraction when percent != 0 (confirmed end-to-end: no /100 or *100 scaling exists anywhere in the config->capnp->runtime pipeline -- src/xceadmin/src/xcecommon/mapping.py, controller.capnp's LimitValue, its C encode/decode, build_limit(), and update_limit_res_value() (plugins/limit.c) all pass the value through unchanged, multiplying it DIRECTLY against the resource's current value for capacity computation -- confirmed authoritative by the original hand-written C literal this codebase's oldest percent-based limit was migrated from, `.val = 0.2` for a documented '20% mem cap', sched.c's limit2_config, commit ecf1916d). fmt_limit()/fmt_limit_units() (src/lib/xcelimit.c) formatted the percent branch as a bare `snprintf(..., "%f%%", limit->val.val)` -- echoing the raw 0-1 fraction with a literal '%' appended, instead of converting to conventional 0-100 percent notation first. + +## Fix +FIXED (2026-09-20). fmt_limit_units()'s percent branch now formats `limit->val.val * 100.0` instead of the raw `limit->val.val`, so a correctly-configured 0.2 (20%) limit displays as '20.000000%'. This is display-only -- update_limit_res_value()'s actual capacity computation was already correct and is untouched; only what a human reads changed. fmt_limit() (a thin XCE_UNITS_RAW wrapper around fmt_limit_units()) inherits this fix for its own percent-branch output too -- a deliberate, intentional behavior change from its pre-fix output for percent-based limits specifically (its 5 pre-existing direct unit tests only assert `rc > 0`, never the exact percent string, so this was safe). Updated test_limit.c's test_fmt_limit_units_percent_limit_ignores_units_mode fixture from an unrealistic `.val = 80.0` to a realistic `.val = 0.8` (80%) -- assertion text needed zero change since 0.8*100=80 either way. Updated test_repl_limit_table.c's test_info_limit_named_percent_limit_own_percentage_value_never_scaled_by_units_mode to expect the shared add_configured_percent_limit() fixture's (deliberately unchanged, .val=50.0) raw display as '5000.000000%' -- mathematically correct given that fixture's arbitrary test constant was never meant to represent a realistic percent value. RED confirmed by grepping the raw Unity testlog for :FAIL: (the aggregate meson summary masks this, a known Unity-harness quirk in this project) before the fix; GREEN confirmed after, both lib (0 Unity FAILs) and scheduler (85/85, 0 Unity FAILs) suites. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1832 +- Gitea: #56 diff --git a/.wolf/bugs/bug-1832.md b/.wolf/bugs/bug-1832.md new file mode 100644 index 00000000..53d04516 --- /dev/null +++ b/.wolf/bugs/bug-1832.md @@ -0,0 +1,41 @@ +--- +id: "bug-1832" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "config" +severity: "unknown" +timestamp: "2026-09-20T10:00:00Z" +last_seen: "2026-09-20T10:00:00Z" +occurrences: 1 +files: ["src/configs/limits.yaml"] +tags: ["scheduler", "config", "limit-plugin", "percent-limit", "night_mem_pct", "config-authoring-bug", "discovered-during-live-retest", "related-to-bug-1831", "behavior-change"] +related: ["bug-1831"] +legacy: true +gitea_issue: 58 +gitea_synced_status: "fixed" +--- + +# bug-1832: src/configs/limits.yaml's night_mem_pct limit (resname: mem, value: 50, percent: true) is… + +## Description +src/configs/limits.yaml's night_mem_pct limit (resname: mem, value: 50, percent: true) is functionally dead -- it can never bind or deny any job, since its actual computed capacity is 50x (5000%) the resource's current value, not 50% of it. Surfaced while investigating bug-1831's percent-display discrepancy. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce_limit_val_t.val is a 0-1 fraction convention when percent is set (see bug-1831's root_cause for full pipeline trace -- no /100 scaling exists anywhere). night_mem_pct was authored directly in YAML for a docs/example config (commit a1478510, 'docs(admin-config-tool): commit spec docs and example configs') with no C-literal ancestry to cross-check against, unlike limits2 (migrated verbatim from a hand-written C literal `.val = 0.2` for '20% mem cap', sched.c's limit2_config) -- so it used the human-intuitive but incorrect 0-100 scale convention ('50' meaning 50%) instead of the pipeline's real 0-1 fraction convention. + +## Fix +FIXED (2026-09-20, user-directed). Changed limits.yaml's night_mem_pct value from 50 to 0.5, matching the 0-1 fraction convention -- it now correctly computes and enforces a genuine 50% cap on the night queue's mem usage instead of a non-binding 5000% one. This is a production config behavior change, not merely a display fix: night_mem_pct will now actually constrain job admission for the first time in any real deployment using this config, where previously it silently did nothing. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1831 +- Gitea: #58 diff --git a/.wolf/bugs/bug-1833.md b/.wolf/bugs/bug-1833.md new file mode 100644 index 00000000..e92740eb --- /dev/null +++ b/.wolf/bugs/bug-1833.md @@ -0,0 +1,41 @@ +--- +id: "bug-1833" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "config" +severity: "crash" +timestamp: "2026-09-20T11:00:00Z" +last_seen: "2026-09-20T11:00:00Z" +occurrences: 1 +files: ["src/configs/schema/limits.schema.json"] +tags: ["xceadmin", "config-validation", "json-schema", "limit-plugin", "percent-limit", "proactive-hardening", "related-to-bug-1831", "related-to-bug-1832", "build-gotcha"] +related: ["bug-1831", "bug-1832"] +legacy: true +gitea_issue: 59 +gitea_synced_status: "fixed" +--- + +# bug-1833: user-requested, following bug-1831/bug-1832. Nothing in xceadmin's config validation caught a… + +## Description +NOT A CRASH -- proactive hardening, user-requested, following bug-1831/bug-1832. Nothing in xceadmin's config validation caught a percent-based limit's value being outside the valid 0.0-1.0 fraction range (e.g. 'value: 50, percent: true', meant to express 50% but computing as 5000% of the resource) -- both night_mem_pct in the real src/configs/limits.yaml (bug-1832) and, discovered while adding this validation, an identical stale value in the xceadmin test suite's own checked-in golden fixture (src/xceadmin/tests/fixtures/full/configs/limits.yaml) had this exact defect. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +limits.schema.json's Limit definition had `"value": {"type": "number"}` with no range constraint at all, and no conditional (if percent then range) constraint existed anywhere in this codebase's 7 bundled JSON schemas -- confirmed no if/then usage exists in any of them prior to this fix. + +## Fix +FIXED (2026-09-20). Added an if/then JSON Schema (draft 2020-12) conditional constraint to the Limit definition: `if percent == true, then value must be a number in [0.0, 1.0]`. Empirically verified against the real jsonschema==4.26.0 library this repo uses -- a violation surfaces with the exact same file/json_pointer/constraint contract validate_yaml()'s existing ValidationError already provides (constraint='maximum'/'minimum'), so zero changes were needed to validation.py's own error-reporting code. This is now enforced at real `xceadmin config build` time (deploy-ha.sh's actual cluster.bin generation path), not merely a unit-test-only check. IMPORTANT BUILD GOTCHA discovered while implementing this: this repo has TWO copies of every bundled JSON schema -- src/configs/schema/*.schema.json (the REAL canonical source) and src/xceadmin/src/xcecommon/schema_files/json/ (a checked-in mirror). meson-build.sh:104 ALWAYS overwrites the xceadmin copy from src/configs/schema/ during any real build (`cp "$SRC_DIR/configs/schema/"*.schema.json "$SCHEMA_DST/json/"`) -- editing ONLY the xceadmin-tree copy is silently discarded on the next build. Both copies were edited to stay in sync. Also fixed the same night_mem_pct value: 50 -> 0.5 defect the new constraint caught in the xceadmin test suite's own checked-in 'full' golden fixture, and regenerated its checked-in expected.bin golden artifact to match (via build.build_cluster_config()). Added 3 new tests to test_validation.py covering: reject out-of-range percent value, accept in-range percent value, and confirm a non-percent value is never range-checked regardless of magnitude. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1831, bug-1832 +- Gitea: #59 diff --git a/.wolf/bugs/bug-1834.md b/.wolf/bugs/bug-1834.md new file mode 100644 index 00000000..4203d10f --- /dev/null +++ b/.wolf/bugs/bug-1834.md @@ -0,0 +1,41 @@ +--- +id: "bug-1834" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-20T09:35:00Z" +last_seen: "2026-09-20T10:15:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugins/limit.c"] +tags: ["scheduler", "limit-plugin", "resource-accounting", "mem-vs-maxmem", "swap-vs-maxswap", "percent-limit-capacity", "update_limit_res_value", "instantiate_limit", "design-gap", "user-identified", "needs-owner", "related-to-bug-1831", "related-to-bug-1832", "night_mem_pct"] +related: ["bug-1831", "bug-1832"] +legacy: true +gitea_issue: 61 +gitea_synced_status: "fixed" +--- + +# bug-1834: User-identified design gap (not yet reproduced as a live symptom, code-traced and confirmed): for… + +## Description +User-identified design gap (not yet reproduced as a live symptom, code-traced and confirmed): for a percent-based limit whose `resname` is a dynamic/consumable resource (e.g. `mem`, `swap`), the limit's CAPACITY is computed as percent * the resource's current LIVE value, not percent * the resource's static total. This means the same limit's enforced cap drifts up and down as unrelated memory pressure on the host changes, rather than staying pinned to a stable fraction of total capacity (e.g. 'this queue may use up to 20% of the host's total memory' should not shrink because some other job is currently using more memory). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +update_limit_res_value() (limit.c:142-165) computes `val = get_resvals(array, 1, &(vres->res->idx), &val); val = val * vres->limit->val.val;` for any percent!=0 limit -- `vres->res` is resolved in instantiate_limit() (xcelimit.c:574) via a single key_find(resmgr, limit->val.resname, ...), i.e. exactly the resource named by the limit's OWN `resname` field. `resname` today serves two roles that this code conflates: (1) selecting the accounting/consumable resource the limit's virtual resource (`vres->vres`) is keyed against for usage tracking (correctly `mem`, since jobs consume `mem`), and (2) supplying the live base value multiplied by percent to derive capacity (currently ALSO `mem`, the dynamic worker-reported load index per .kiro/steering/resource-accounting.md -- 'mem is not maxmem... mem is a dynamic load index... refreshed on the worker's regular load-report cadence'). Role (2) should read a static/non-consumable resource instead (`maxmem` for `mem`, `maxswap` for `swap`) -- both already exist as registered per-host resources (src/configs/resources.yaml:108 `maxmem`, :112 `maxswap`, 'self-reported once by the worker at join time and never refreshed' per resource-accounting.md) so no new resource type is needed, only a change to which resource the percent multiplication reads. xce_limit_val_t/xce_limit_config_t (types.h:800-818) have no field today to name this second resource explicitly -- resname is single-purpose in the struct. + +## Fix +FIXED (2026-09-20). Added xce_vres_t.percent_res (src/include/xce/types.h) -- a resolved xce_resource_t* pointer for the percent-branch capacity read, kept separate from `res` (which stays the limit's own accounting resource, unchanged) so the two roles are no longer conflated. instantiate_limit() (src/lib/xcelimit.c) resolves it ONCE at instantiation time via a small table-driven static lookup (percent_capacity_resource_map[]: mem->maxmem, swap->maxswap) -- table-driven per user request, so a future dynamic/static pair (e.g. tmp/maxtmp) is a one-line addition. A missing/unregistered counterpart is NOT a hard failure: percent_res stays NULL and update_limit_res_value() (src/scheduler/src/plugins/limit.c) falls back to `res`, preserving today's behavior for any resource with no known static pair -- this is what keeps every pre-existing percent-limit test passing unchanged (they never register 'maxmem' in their resmgr fixtures, and none of them exercise update_limit_res_value()'s real computation anyway -- its existing white-box harness, test_repl_limit_table.c, stubs get_resvals()/put_resvals() as no-ops). New coverage: 4 unit tests in src/lib/testsuite/test_limit.c proving percent_res resolves to maxmem/maxswap for mem/swap, stays NULL for percent=0, and stays NULL (graceful fallback, not -ENOENT) for a percent-based limit on a resource with no known static counterpart (e.g. ncpus). Extended the shared src/lib/testsuite/test_resmgr_fixture.c/.h with maxmem/swap/maxswap test resources for this. Full lib (314 tests) and scheduler (85 test binaries) suites pass with zero regressions -- confirmed via raw testlog grep for ':FAIL:', not just the meson aggregate summary. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1831, bug-1832 +- Gitea: #61 diff --git a/.wolf/bugs/bug-1835.md b/.wolf/bugs/bug-1835.md new file mode 100644 index 00000000..9051b249 --- /dev/null +++ b/.wolf/bugs/bug-1835.md @@ -0,0 +1,41 @@ +--- +id: "bug-1835" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-09-21T03:47:10Z" +last_seen: "2026-09-21T03:47:10Z" +occurrences: 1 +files: ["src/schema/controller.capnp"] +tags: ["controller", "ha", "capnp-schema", "codegen-limitation", "sigsegv", "uninitialized-memory", "controller.capnp", "ctrl-hello", "raft", "leader-election", "systemic", "user-reported", "live-crash", "related-to-bug-1504", "related-to-bug-1505", "related-to-bug-1461"] +related: ["bug-1504", "bug-1505", "bug-1461"] +legacy: true +gitea_issue: 64 +gitea_synced_status: "fixed" +--- + +# bug-1835: Live 3-node HA cluster crash: xce_controller SIGSEGV in __memcpy_avx_unaligned_erms, called from… + +## Description +Live 3-node HA cluster crash: xce_controller SIGSEGV in __memcpy_avx_unaligned_erms, called from capnp's copy_ptr/capn_setp, called from write_CtrlHelloRequest (lib/controller.capnp.c:22821) via encode_CtrlHelloRequest_ptr (lib/controller.capnp.c:22935), via xce_encode/xce_request, via ha_send_ctrlhello_to_all_standbys (controller/cluster.c:2432), via raft_cb_become_leader (controller/ha.c:2113), via raft_on_vote_reply -> vote_reply_dispatch -> peer_ctrl_msg_handler_inline -> peer_ctrl_vtask_entry, on the vcpu thread handling the winning vote reply. Core confirmed built from commit 1e43345b (xce v0.0.1-1e43345b, 2026-09-21T03:10:39Z); ctrl-1 had just won its Raft election (term 2, votes_granted=2) and crashed while broadcasting CtrlHello to its 2 standbys. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CtrlHelloRequest.key was declared `Data` in controller.capnp -- the exact same latent bug class as bug-1504 (SnapshotData.internalState) and bug-1505 (WalCatchupEntry.payload), and explicitly named as a known-but-unfixed instance of it in bug-1505's own writeup ('Known/suspected other Data fields with the same latent gap: ... CtrlHelloRequest.key'). This project's capnp-c backend silently no-ops the encoding of any Data-typed field inside the generated encode_XXX() composite helper (see encode_CtrlHelloRequest's ' /* key key */' stub, lib/controller.capnp.c) without ever zero-initializing the caller's local `struct CtrlHelloRequest d;` first (encode_CtrlHelloRequest_ptr, lib/controller.capnp.c:22926-22938) -- so d.key.p is left as genuinely uninitialized stack garbage. write_CtrlHelloRequest (lib/controller.capnp.c:22821) then unconditionally calls capn_setp(p.p, 1, s->key.p) with that garbage capn_ptr, and capn_setp's copy_ptr dereferences the garbage segment/data pointer, SIGSEGV'ing in memcpy. ha_send_ctrlhello_to_all_standbys (controller/cluster.c:2419-2424) and ha.c:9495 both always set ctrl_hello_req_t.key = NULL (the field has never had a real producer anywhere in the codebase -- confirmed via project-wide grep, no call site ever sets it to non-NULL nor reads it back), so this is genuine undefined behavior whose manifestation depends entirely on whatever garbage bytes happen to occupy that vtask thread's stack slot at each call -- exactly the same 'lucky vs unlucky stack layout' nondeterminism documented for bug-1504/1505, which is why this had not crashed before despite the gap being known since 2026-08-07. Also confirmed the C struct itself compounded the risk: ctrl_hello_req_s.key (typemap.h) was typed `xce_data_t*` -- the local-Data-struct-shadowing wrapper type from bug-1461's own root cause -- rather than a real byte-blob type, even though it was never actually used as one. + +## Fix +FIXED (2026-09-21), same pattern as bug-1504/bug-1505: changed CtrlHelloRequest.key from `Data` to `List(UInt8)` in src/schema/controller.capnp (which this capnp-c backend DOES correctly auto-generate real, non-no-op encode/decode/free for -- proven precedent: FetchFileReply.content/bytesRead, SnapshotData.internalState, WalCatchupEntry.payload), with a new keyLen (@4, UInt32) companion field as its maplistcount. Updated typemap.h's ctrl_hello_req_s.key from `xce_data_t*` to `uint8_t*` and added a new `key_len` (uint32_t) field. No consumer-code changes were needed: project-wide grep confirmed .key is never set to non-NULL nor read anywhere in production (ha_send_ctrlhello_to_all_standbys and ha.c's hello builder both always pass NULL), so the fix is purely structural -- it eliminates the uninitialized-capn_ptr class of UB for this field rather than papering over one occurrence. Added regression test test_ctrl_hello_request_key_round_trip (controller/testsuite/test_e2e_capnp.c), covering both the actual production shape (key==NULL/key_len==0, the exact shape that used to SIGSEGV -- now proven to encode/decode cleanly via the real generic codec) and a real non-NULL key payload (proves content genuinely round-trips now, which the old no-op'd Data codec never did). A deterministic RED test reproducing the SIGSEGV itself was not attempted -- by its own nature as reads of uninitialized stack memory, this bug class is not ASAN-catchable and its manifestation is stack-layout-dependent (bug-1504's own writeup makes the identical observation); the fix instead structurally eliminates the uninitialized-read entirely rather than chasing a flaky repro. Verified: full controller_unit_test suite (5/5 meson test binaries, 0 failures, including the new round-trip test and the pre-existing test_ctrl_hello_req_t_layout/test_ha_ctrlhello.c suite) via a clean ASAN rebuild (`xce-build.sh controller asan`) that also regenerated controller.capnp.c/.h from the updated schema with no codegen errors. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1504, bug-1505, bug-1461 +- Gitea: #64 diff --git a/.wolf/bugs/bug-1836.md b/.wolf/bugs/bug-1836.md new file mode 100644 index 00000000..dfc79fb6 --- /dev/null +++ b/.wolf/bugs/bug-1836.md @@ -0,0 +1,41 @@ +--- +id: "bug-1836" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-20T22:00:00Z" +last_seen: "2026-09-21T05:46:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["scheduler", "sim", "replay", "resource-accounting", "sim_reconcile_unaccounted_jobs", "scheduler-driver-equivalence", "over-commit", "slots", "capacity-check", "xce-audit-invariants", "user-reported", "found-live"] +related: ["bug-1611", "bug-1613", "bug-1799", "bug-1764", "bug-1759", "bug-1779"] +legacy: true +gitea_issue: 65 +gitea_synced_status: "fixed" +--- + +# bug-1836: User-reported live over-dispatch in xce_scheduler -i -s : replaying a captured session… + +## Description +User-reported live over-dispatch in xce_scheduler -i -s : replaying a captured session (JOB_PUSH of a 50-element array job) against a real single worker with 22 slots dispatched 44 concurrent single-slot jobs (`assigned jobs: 0-3-1789963203-0[0-43]`), exactly double the host's real capacity, while `info worker` simultaneously showed `resource slots: value=22.00 used=22.00` -- the accounting display looked self-consistent (used == value) even though double the job count the display's own 'used' figure implied was actually running. User confirmed normal mode (the live controller-connected driver) does not exhibit this under equivalent conditions, isolating the defect to simulation/replay mode specifically. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sim_reconcile_unaccounted_jobs() (the 'Recorded-Decision Reconciliation' mechanism, scheduler-sim-decision-parity/scheduler-sim-eager-reconciliation specs) unconditionally force-applied any recorded DISPATCH decision it found in decoded_session->decisions for a job with accounts==NULL -- setting job->state=JOB_STAT_RUN and job->allocs/job->accounts straight from the recording -- with ZERO call to sched_check_host_satisfy()/sched_check_shared_satisfy() or any other admission/capacity check. This directly contradicted the scheduler-sim-decision-parity spec's own design.md assumption underpinning its Requirement 2 ('capacity awareness'): 'sched_check_host_satisfy_()/sched_check_shared_satisfy_() already read resInUse/sharedInUse unconditionally; once those fields are correctly populated in simulation mode, the existing (unmodified) capacity-check code path produces the same outcome normal mode would' -- an assumption that only holds for jobs decided via the LIVE picker/allocator path (sched_election.c's sched_allocator(), which DOES call these checks via sched_try_alloc_on_host_()), not for jobs captured via this SEPARATE reconciliation backstop, which bypasses that code path entirely. Confirmed via the round sequence in sched_sim_driver_run(): xce_core_push_events() creates all 50 freshly-pushed jobs (PEND, accounts==NULL) BEFORE sim_reconcile_unaccounted_jobs() runs for that same round, so ALL 50 are eligible reconciliation candidates the moment the function starts -- for however many of them (44, in the live report) the session's OWN originally-recorded decisions said DISPATCH, reconciliation force-applied all of them with no regard for the CURRENT sim snapshot's actual worker capacity (22 slots), before the live picker/allocator (which runs afterward, per the fixed round order) ever got a chance to independently admission-check any of them -- explaining both the 44 dispatches and the 6 correctly-PENDed remainder (jobs whose OWN recorded decision was PEND, never matched by reconciliation's ReplayDecision_body_dispatch-only filter). This is a distinct mechanism from bug-1611/bug-1613 (both already fixed/superseded by scheduler-sim-live-allocation and scheduler-sim-eager-reconciliation) -- those closed a TIMING gap in when reconciliation's job->state transition happens; this bug is a CAPACITY-VALIDATION gap in whether it happens at all, independent of timing. Uncovered via /xce-audit-invariants scheduler, dimension 1 (driver state-equivalence). + +## Fix +FIXED (2026-09-21). Added sim_reconciled_alloc_fits_capacity() (sim_loop.c, static) -- re-validates a candidate recorded allocation's full resource footprint (every per-host allocation plus any shared/global allocation) against LIVE resInUse/sharedInUse using the SAME sched_check_host_satisfy()/sched_check_shared_satisfy() admission-check functions (resource.c) the live picker/allocator path already calls, resolving each allocation's resnames against core->resmgr fresh (the decoded allocation carries only resnames/values, not resolved indices). sim_reconcile_unaccounted_jobs()'s FOUND branch now calls this BEFORE taking ownership of a matched entry; a capacity miss is handled exactly like 'no matching recorded decision' (job left unaccounted this round, entry NOT consumed/freed, naturally retried later or correctly PENDed by the live picker once it runs). A newly-admitted candidate's usage is charged into the LIVE resInUse/sharedInUse immediately via sched_charge_account() (mirroring sched_commit_decision()'s own immediate charge for a live dispatch) so a LATER candidate in the SAME reconciliation pass correctly sees an EARLIER accepted candidate's usage -- essential for the exact multi-job-contention scenario this bug exercises. To make the capacity check see genuinely prior-round (already-running) jobs' usage too, added sim_reconcile_toggle_prior_charges() (sim_loop.c, static) -- seeds resInUse/sharedInUse from every already-RUN+accounted job at the start of sim_reconcile_unaccounted_jobs(), then fully undoes that seed plus every incremental charge made during the pass at the end, leaving the function's net effect on the real accounting arrays at exactly zero on return; the caller's own unchanged, immediately-following sched_account_jobs() call remains the sole, non-double-counting authoritative charge for the round (unchanged architecture/ordering). Deliberately NOT implemented via sched_account_jobs()/a new sched_unaccount_jobs() exported pair (tried first, then reverted): that exact by-name function is separately instrumented and call-count-asserted by test_sim_contention_parity.c's g_round_idx mechanism ('sched_account_jobs() called exactly once per round' is itself a protected invariant there) -- doubling its call count broke 2 of that file's tests (masked at first by a coincidental g_round_resinuse[8] array-capacity cap that hid the true +5/+10 call-count delta behind an apparent '+2' pattern; properly diagnosed by reading the array declaration, not guessed). sched_check_host_satisfy()/sched_check_shared_satisfy() were relocated from scheduler.h to core.h (pure declaration move, bodies unchanged in resource.c, following the exact sched_account_jobs()/sched_resolve_job_picker() precedent already established for this exact purpose) since sim_loop.c must never include scheduler.h (pulls in client.h/xce/cluster.h, forbidden by test_sim_isolation.c's Req 5 AC 17 enforcement). New regression test test_reconcile_respects_host_capacity_under_contention (test_sim_reconcile_unaccounted.c) -- two PEND jobs each recorded-DISPATCH to the same 1-slot-capacity host; asserts exactly one is admitted (not both, not neither), the rejected one's entry is not consumed, and post-round resInUse reads exactly 1.0 (never 2.0). Fixing this surfaced 12 pre-existing tests (9 in test_sim_reconcile_unaccounted.c, 3 in test_sim_verify.c) whose fixtures named a host in a recorded allocation without ever registering a real xce_sched_worker_t of that name -- harmless before this fix (nothing checked capacity), now correctly rejected by the new gate; fixed by registering a real (or, for test_sim_verify.c, which never white-box-includes resource.c, a bare link-satisfying) worker for each such fixture, generous capacity since none of those 12 tests are actually about capacity limits. Verified: full scheduler suite 85/85 Ok, 0 Fail (asan profile, xce-build.sh scheduler asan), independently re-confirmed by running sched_sim_reconcile_unaccounted_test (22/22) and sched_sim_verify_test (17/17) directly with ASAN_OPTIONS=detect_leaks=0 (this test-file family's own established convention, unrelated pre-existing leak reports otherwise printed by these binaries when run standalone outside meson's own suite-level ASAN_OPTIONS). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1611, bug-1613, bug-1799, bug-1764, bug-1759, bug-1779 +- Gitea: #65 diff --git a/.wolf/bugs/bug-1837.md b/.wolf/bugs/bug-1837.md new file mode 100644 index 00000000..889cd873 --- /dev/null +++ b/.wolf/bugs/bug-1837.md @@ -0,0 +1,41 @@ +--- +id: "bug-1837" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-20T22:05:00Z" +last_seen: "2026-09-21T05:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["scheduler", "sim", "sim_reconcile_unaccounted_jobs", "picker-plugin", "threading-contract", "jbuf", "xce-audit-invariants", "related-to-bug-1759", "related-to-bug-1779", "tdd", "allocator-thread-relay", "user-directed"] +related: ["bug-1759", "bug-1779", "bug-1836"] +legacy: true +gitea_issue: 67 +gitea_synced_status: "fixed" +--- + +# bug-1837: Found during /xce-audit-invariants scheduler (dimension 3, jbuf capacity-1 rendezvous audit), not… + +## Description +Found during /xce-audit-invariants scheduler (dimension 3, jbuf capacity-1 rendezvous audit), not user-reported and not a live symptom: sim_reconcile_unaccounted_jobs() (sim_loop.c) calls picker->picker_iface->on_dispatch(picker, job->jobid) directly from the sim driver thread (sched_sim_driver_run()'s own thread, between xce_core_begin_session() and xce_core_signal_go()), but plugin.h documents on_dispatch()/on_pend() as 'Post-allocation hooks (MUST) -- allocator thread, after each allocation attempt' -- a MUST-level threading contract this call site violates by construction (driver-thread code calling an allocator-thread-only hook). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +bug-1759/bug-1779's own fix added this on_dispatch() call to sim_reconcile_unaccounted_jobs() specifically to prevent sim_picker_thread_fn() from re-offering (and re-PENDing) an already-reconciled job on a later round -- a real, necessary fix for that narrower symptom (a duplicate PEND report) -- but the fix's own header comment justifies it only in those terms ('the picker never re-decides an already-resolved job'), never addressing thread-safety of calling an allocator-thread-only hook from the driver thread instead. It currently causes no observable corruption only because the sole real (non-no-op) picker plugin, builtin_fcfs.c, happens to guard its entire candidate queue with an internal lock (priv->lock, taken in every hook including end_session()) -- an implementation detail plugin.h's own documented contract does not require picker plugins to provide ("the picker thread holds no lock the allocator thread needs"). A future or alternate picker plugin implemented per that documented (lock-free) contract would race sim_reconcile_unaccounted_jobs()'s driver-thread on_dispatch() call against the picker thread's own end_session()/next_job() tail-end activity from the prior round, unsynchronized. + +## Fix +FIXED (2026-09-21, TDD). Chose the fuller of the two directions the audit finding itself proposed: moved the on_dispatch() call onto the real allocator thread instead of documenting an exception to plugin.h's contract. Since the scheduler-sim-live-allocation spec, sim mode's sched_sim_driver_run() (sim_loop.c) spawns the SAME unmodified sched_job_picker()/sched_allocator() thread pair (sched_election.c) normal mode uses, so a real allocator thread now genuinely exists to route through in production. Mechanism: two new dedicated cbufs on xce_sched_core_t (core.h) -- reconcile_req (jobid) and reconcile_ack (rc) -- deliberately SEPARATE from jbuf, not a widened jbuf item type, since jbuf's item size/capacity are a pinned determinism invariant (test_core_jbuf_capacity_pin.c, Req 5 AC 7) that a byte-for-byte source-text-scanning regression test forbids changing. A new reserved sentinel, XCE_SCHED_RECONCILE_WAKE (== UINT64_MAX, core.h; same reserved-value idiom this codebase already uses for SCHED_SIM_START_TS_LATEST, sim.h, and jbuf's own pre-existing 0 end-of-session marker), is what the driver pushes onto jbuf itself to wake the allocator thread out of its cbuf_get(&core->jbuf) block -- never a real jobid, so it can never be mistaken for one by sched_get_job(). sim_reconcile_unaccounted_jobs()'s on_dispatch call site (sim_loop.c) now goes through a new sim_reconcile_invoke_on_dispatch() helper: pushes jobid onto reconcile_req, THEN pushes XCE_SCHED_RECONCILE_WAKE onto jbuf (ordering matters -- cbuf_put()'s own mutex gives happens-before, so reconcile_req is guaranteed populated by the time the allocator's cbuf_get(&core->jbuf) returns the sentinel), then blocks on reconcile_ack for the result -- one request in flight at a time, matching jbuf's own capacity-1 rendezvous discipline without touching jbuf itself. sched_election.c's sched_allocator() gained a matching branch: on XCE_SCHED_RECONCILE_WAKE, drain reconcile_req, call on_dispatch() (no plugin_job_election()/sched_make_allocation()/decision -- reconciliation already did its own accounting/charging on the driver thread), ack via reconcile_ack, continue -- a true no-op for normal mode, which never populates reconcile_req. A new xce_sched_sim_driver_t.allocator_thread_live bool (sim.h) tells sim_reconcile_invoke_on_dispatch() whether to route through the relay at all: sched_sim_driver_run() sets it true immediately after successfully spawning the allocator thread (same thread, program order -- no atomics needed, no startup race, since both the write and every later read happen on the driver thread only); it stays false for a bare xce_sched_sim_driver_t that never spawned any thread pair at all (test_sim_reconcile_unaccounted.c's own documented Testing Strategy: ~23 existing tests call sim_reconcile_unaccounted_jobs() directly, standalone, with no picker/allocator threads running), in which case a direct on_dispatch() call is safe by construction -- there is provably only one thread touching picker state, so plugin.h's allocator-thread-only contract has nothing to violate. This let all ~23 pre-existing direct-call tests in that file pass unchanged. Blast radius (grepped the full src+testsuite tree for core->jbuf and every hand-rolled white-box sched_allocator()/sched_job_picker() stub before assuming any of this was safe, per this project's own established discipline): 4 other test files drive sim_reconcile_unaccounted_jobs()'s success path through the REAL sched_sim_driver_run() round loop with their own local sched_allocator() stub (a simplified PEND-only stand-in, never the real sched_election.c symbol) -- test_sim_verify.c, test_sim_round_boundary_reconciliation.c, test_sim_contention_parity.c, test_sim_e2e_original_motivation.c. Confirmed empirically, not just by static grep: a first full-suite build+test run with the fix applied but before patching these four hung all four at meson's 30s per-test timeout (SIGTERM) -- exactly the deadlock a stub that never answers reconcile_ack would cause. Added the identical relay branch to each of their local sched_allocator() stubs; re-ran the full suite and all four passed. TDD: new test_reconcile_invokes_on_dispatch_on_allocator_thread_not_caller (test_sim_reconcile_unaccounted.c) uses a minimal test-owned picker plugin (not builtin_fcfs, whose own internal locking was exactly what made the pre-fix violation dormant/unobservable) whose on_dispatch() records pthread_self(), plus a one-shot relay_one_reconcile_request() thread standing in for sched_allocator(); asserts the recorded tid equals the spawned relay thread's tid and NOT the calling (test) thread's own tid. RED confirmed by temporarily forcing sim_reconcile_invoke_on_dispatch() back to an unconditional direct call and rebuilding: exactly this new test hung (the relay thread's pthread_join() never returned, since nothing was ever pushed onto jbuf/reconcile_req) and was killed at meson's 30s timeout, all 84 other tests still passed. GREEN confirmed after restoring the real fix: full scheduler suite 85/85 Ok, zero regressions, matching the exact pre-fix baseline count. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1759, bug-1779, bug-1836 +- Gitea: #67 diff --git a/.wolf/bugs/bug-1838.md b/.wolf/bugs/bug-1838.md new file mode 100644 index 00000000..2d09324b --- /dev/null +++ b/.wolf/bugs/bug-1838.md @@ -0,0 +1,41 @@ +--- +id: "bug-1838" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-21T00:30:00Z" +last_seen: "2026-09-21T02:40:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c"] +tags: ["scheduler", "sim", "replay", "sim_reconcile_unaccounted_jobs", "scheduler-driver-equivalence", "reconciliation-trigger", "design-correction", "architecture", "user-directed", "xce-audit-invariants", "related-to-bug-1836"] +related: ["bug-1836", "bug-1837", "bug-1611", "bug-1613", "bug-1759", "bug-1779", "bug-1804"] +legacy: true +gitea_issue: 70 +gitea_synced_status: "fixed" +--- + +# bug-1838: Follow-up architectural correction after bug-1836 (found during user discussion, not a… + +## Description +Follow-up architectural correction after bug-1836 (found during user discussion, not a separately-reported live symptom): even with bug-1836's capacity gate in place, sim_reconcile_unaccounted_jobs() still unnecessarily PREEMPTED the live picker/allocator for any freshly-PEND job with a matching recorded DISPATCH decision, rather than letting the live path -- now genuinely capacity-checked, per scheduler-sim-live-allocation -- decide it on its own the way normal mode always has. User specifically asked why sim mode diverged from normal mode given they 'share the same core', then proposed the architecturally correct framing directly: the simulator should run the identical decision-making core (minus controller/worker I/O) and use the recording only to verify/reconstruct, not to short-circuit live decisions. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sim_reconcile_unaccounted_jobs()'s trigger condition was widened from JOB_IS_RUN-only to state-independent (PEND-or-RUN) by the scheduler-sim-eager-reconciliation spec (2026-09-11), to fix a real problem at the time: a job's recorded DISPATCH decision only exists in the ONE round's decoded_session it was captured in (freed at round end), so a RUN-only gate meant reconciliation could never find it once the job's SEPARATE, later-replayed RUN-confirmation event finally made it RUN. That widening was the only available fix at the time because sim mode's own live picker/allocator path had no working capacity accounting (bug-1613) and so could not be trusted to independently re-derive the SAME decision the original recording made -- reconciliation's job was to force in the recording's answer since the live path's own answer might diverge or be wrong. scheduler-sim-live-allocation (completed 2026-09-19) closed that gap for real: sim mode now spawns the SAME real, capacity-checked sched_job_picker()/sched_allocator() threads (sched_election.c) normal mode does, with sched_account_jobs() correctly wired into its round sequence -- its own design.md explicitly states a 'Live/Recorded Authority Rule' (live decision wins when the live path reaches a job; reconciliation is a narrow safety net, not a first-resort source of truth) but the actual per-round ORDERING (reconciliation runs unconditionally before the live picker/allocator gets a turn each round) never enforced that rule -- so the PEND-inclusive trigger left in place after scheduler-sim-live-allocation landed kept preempting jobs the live path was now fully capable of deciding correctly and verifiably on its own. This was the deeper root cause underlying bug-1836's exact reproduction (44 jobs force-dispatched via reconciliation before the live picker/allocator ever got a turn); bug-1836's capacity gate made that force-application SAFE (never exceeds real capacity) but did not stop it from happening. Also confirmed (via direct investigation): a snapshot-loaded job (snapshot_load.c's xce_sched_core_load_snapshot()) is registered with the live picker via on_job_push() for every restored job unconditionally, RUN or PEND -- so there is no 'invisible to the picker' case a RUN-only trigger could miss; every legitimate reconciliation candidate today is also, structurally, a job the live picker/allocator can independently resolve. The sole remaining genuine case: a job already RUN before this replay's own observation window (dispatched too far in the past for its DISPATCH decision to still be in any currently-decoded session, and never offered by next_job() since RUN jobs are never picker candidates) -- for that job alone, the recording remains the sole source of truth. + +## Fix +FIXED (2026-09-21). Narrowed sim_reconcile_unaccounted_jobs()'s trigger back to JOB_IS_RUN(job->state)-only (undoing the 2026-09-11 widening), so a freshly-PEND job now always flows through the same real, capacity-checked live picker/allocator normal mode uses -- verified via the pre-existing --sim-verify generate-then-compare mechanism -- rather than being force-applied from history. bug-1836's capacity gate (sim_reconciled_alloc_fits_capacity()/sim_reconcile_toggle_prior_charges()) remains in place unchanged as defense-in-depth for the narrower remaining RUN-only case. Fixing this correctly broke 6 test binaries whose fixtures depended on the old PEND-inclusive behavior; investigation of each was more nuanced than initially guessed: test_sim_verify.c's and test_sim_contention_parity.c's failures were NOT missing-worker-registration issues (the initial hypothesis) but rather that those two files each stub sched_job_picker()/sched_allocator() to a deliberately PEND-only pair (used to keep their OTHER, unrelated tests decoupled from real dispatch logic) -- under the new design a PEND job in those fixtures genuinely can never be dispatched by anything, so the correct fix was inverting each test's assertions to lock in the new, correct behavior (a PEND job stays PEND, and --sim-verify correctly reports the resulting MISMATCH against a history that shows it dispatched, since these minimal harnesses have no real capacity-tracked path to reproduce that decision) rather than trying to make the stub dispatch something it structurally cannot. The 5 directly-affected tests in test_sim_reconcile_unaccounted.c were each individually re-examined for their TRUE intent and fixed accordingly: one inverted/renamed to lock in 'PEND is now always ignored regardless of a matching decision' (test_reconcile_ignores_pend_state_job_even_when_dispatch_recorded), three converted from a PEND-starting to a RUN-starting fixture job (still meaningfully testing the same underlying resInUse-timing/retry/multi-round mechanics, now via the only state that's still eligible), and one (the bug-1759/1779 on_dispatch-retirement regression test) kept its job PEND-built-then-force-set-to-RUN specifically to reproduce the pathological 'RUN job the picker was never told about' state reconciliation's on_dispatch() safety net still needs to guard against, since no real production path can reach that shape anymore post-bug-1838. A sweep for other coincidentally-passing tests found and fixed one more: bug-1836's own new regression test (test_reconcile_respects_host_capacity_under_contention) also used two PEND jobs, converted to RUN with its 'loser' assertion corrected to match. Added a new live-dispatch-vs-recording round-trip test (test_sim_live_allocation.c) to restore the coverage inverting test_sim_verify.c's test removed (that a genuinely live-dispatched decision correctly reproduces a matching historical recording when both the job and the resources it wants are real). scheduler-sim-eager-reconciliation's own requirements.md/design.md got dated 'Correction (2026-09-21, bug-1838)' notes in place, following the exact precedent format its own 2026-09-11 correction already established, superseding (not deleting) the original PEND-inclusive Acceptance Criterion 1 text. Verified: full scheduler suite 85/85 Ok, 0 Fail (asan profile); independently re-confirmed by running sched_sim_reconcile_unaccounted_test (22/22), sched_sim_verify_test (17/17), sched_sim_contention_parity_test (2/2), and sched_sim_live_allocation_test (2/2) directly. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1836, bug-1837, bug-1611, bug-1613, bug-1759, bug-1779, bug-1804 +- Gitea: #70 diff --git a/.wolf/bugs/bug-1839.md b/.wolf/bugs/bug-1839.md new file mode 100644 index 00000000..b5a544ee --- /dev/null +++ b/.wolf/bugs/bug-1839.md @@ -0,0 +1,41 @@ +--- +id: "bug-1839" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-09-22T02:30:00Z" +last_seen: "2026-09-22T02:30:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["controller", "ha", "standby-endpoint", "err-handler", "null-deref", "vtask-pool", "peer-ctrl-dispatch", "vote-ep-teardown", "related-to-bug-1197", "related-to-bug-1298", "live-crash", "user-reported", "tdd"] +related: ["bug-1197", "bug-1298"] +legacy: true +gitea_issue: 71 +gitea_synced_status: "fixed" +--- + +# bug-1839: Live 3-node HA cluster crash: xce_controller SIGSEGV in peer_ctrl_msg_handler_inline (ha.c:7035… + +## Description +Live 3-node HA cluster crash: xce_controller SIGSEGV in peer_ctrl_msg_handler_inline (ha.c:7035, `ha_state_t *ha = vctx->ha;`). gdb analysis of the actual core dump (build cd9ecb41) confirmed vctx == NULL (i.e. ep->data == NULL) while the endpoint itself was fully alive and in active use: ep->refcnt=191964, ep->tearing_down=0, ep->heap_alloc=1, ep->msg_handler=peer_ctrl_msg_handler, ep->err_handler=standby_ep_err_handler_primary, ep->fsm_out.state=FSM_STATE_ERROR. The crash fired on the vtask-pool path (vcpu_entry -> run_task -> peer_ctrl_vtask_entry -> peer_ctrl_msg_handler_inline); g_drive_async_peerctrl_dispatch_enabled defaults to 1 (ON, the production path). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +standby_ep_err_handler_primary (the err_handler for an outbound standby/heartbeat replication channel, registered in ha_connect_prep_peer) synchronously did `free(sctx); ep->data = NULL;` on whichever thread handle_error() called it from (the io_worker thread), with zero coordination with the vtask pool. If a peer-control vtask for the SAME ep was already queued or in flight (entirely plausible -- the endpoint is actively receiving replication ACKs/heartbeats via peer_ctrl_msg_handler -> peer_ctrl_dispatch_to_pool -> the vtask pool, and can independently hit a write/SSL error on that same connection), that vtask later ran peer_ctrl_msg_handler_inline, read ep->data as NULL, and crashed with a NULL dereference (not a use-after-free -- ep->data was explicitly nulled, not freed-and-reused memory). This is exactly the hazard this codebase already identified and fixed for the sibling vote_ep_ctx_t case: vote_ep_err_handler (ha.c:7787) and pre_vote_ep_err_handler (ha.c:7825) both explicitly do NOT free their context, with an inline comment ('Do NOT free vctx here -- peer_ctrl_msg_handler may still reference ep->data.'). standby_ep_err_handler_primary was never updated to the same discipline when the vtask-pool dispatch machinery (raft-drive-async, bug-1197/bug-1298) was introduced and retrofitted onto the other err handlers -- it slipped through because it's a distinct, older handler for the ongoing standby-heartbeat connection rather than the vote-time connection. Audited the other ~6 `ep->data = NULL` sites in ha.c (ha_connect_prep_peer/ha_connect_finish_peer/ha_connect_to_peer) and confirmed none of them share this bug class -- each frees an endpoint that either was never registered with iomgr at all, or whose iomgr_monitor() call itself just failed, so no vtask could ever have been dispatched against it. + +## Fix +FIXED (2026-09-22, TDD, user-reported live crash). Reused the established, already-tested vote_ep_teardown_async(ha, ep) mechanism (ha.c:7232) instead of inventing a new one: it calls endpoint_mark_tearing_down(ep) (sets the flag + drains dispatch_inflight, closing the TOCTOU window against peer_ctrl_msg_handler's late-event gate), iomgr_unmonitor() if monitored, then enqueues a VTASK_IO teardown task bound to this ep via vtask_enq_classified -- the per-endpoint FIFO guarantees this teardown vtask runs strictly after any earlier-queued or in-flight same-ep peer-control vtask. Its body (vote_ep_teardown_vtask_entry) is fully generic over what ep->data holds -- it just endpoint_unref()s the endpoint, and endpoint_fini()'s own FREEUP(ep->data) frees whatever ep->data currently points to once the last reference truly releases. standby_ep_err_handler_primary's `free(sctx); ep->data = NULL;` tail was replaced with a single `vote_ep_teardown_async(ha, ep);` call (its ha->standby_endpoints removal and repl_remove_standby bookkeeping stayed unchanged, since those only read sctx's still-valid fields before the deferred free). Added an `if (endpoint_is_tearing_down(ep)) return 0;` idempotency guard at the top of the handler: the old `if (!sctx) return 0` accidentally doubled as a reentry guard (a second call after the first had already nulled ep->data would bail), but that protection disappears once ep->data is no longer nulled synchronously, and handle_error() firing this handler more than once for the same ep (e.g. a read error and a write error both observed before iomgr_unmonitor takes effect) would otherwise double-drop the endpoint's owner reference. Also found and fixed a second-order regression this exposed: on_standby_connected's stale-duplicate-channel eviction path (ha.c ~9461-9476) called standby_ep_err_handler_primary(stale_same_peer) and THEN did its own iomgr_unmonitor + endpoint_unref(stale_same_peer) afterward -- correct under the OLD contract (the handler used to only free ep->data, leaving ref-dropping to the caller) but now a double-unref/use-after-free once the handler owns the full teardown itself. Removed the caller's now-redundant iomgr_unmonitor/endpoint_unref calls; the err handler's own vote_ep_teardown_async call handles everything. Caught by ASAN (heap-use-after-free in endpoint_unref via on_standby_connected, immediately on the first full-suite rebuild after the primary fix) rather than by design review -- confirms the value of running the full suite, not just the new tests, after this kind of shared-contract change. Also updated 3 pre-existing tests in test_ha_liveness.c (test_reconnect_resolves_dual_channel_same_peer, test_reconnect_resolves_dual_channel_promoted_vote_ep, test_reconnect_preserves_other_peer_channel) that called standby_ep_err_handler_primary_for_test(ep) followed by a manual endpoint_fini(ep)/free(ep) -- with ha.pool == NULL in these unit fixtures, vote_ep_teardown_async now takes its documented no-pool inline fallback (WARN + endpoint_unref, refcnt 1->0, fini+free) inside the handler itself, so the tests' own manual cleanup became a double-free; removed it. TDD: two new tests in test_ha_vep_teardown.c using a real standby_ep_ctx_t (standby_ep_ctx_build_for_test) and the real standby_ep_err_handler_primary_for_test entry point (not the generic vote_ep_ctx_t stand-in test (a) in that file uses): test_standby_err_handler_defers_teardown_until_inflight_completes proves the deferred-teardown ordering (mirrors test_vep_teardown_runs_after_inflight's gate/sequence-ticket technique -- parks an in-flight same-ep vtask, calls the err handler, asserts teardown does NOT run while the vtask is parked, then releases it and asserts teardown runs strictly after); test_standby_err_handler_idempotent_against_double_invocation proves calling the handler twice in a row on the same (still-alive, tearing-down) ep is a safe no-op with exactly one teardown vtask ever running. RED confirmed against the pre-fix handler: both new tests failed with 'ep must be marked tearing_down...' (endpoint_is_tearing_down never became true, since the old code never called vote_ep_teardown_async at all) -- 1219 tests, 2 failures, zero other regressions. GREEN after the fix plus the on_standby_connected correction: full controller suite 1219/1219 Ok, zero failures. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1197, bug-1298 +- Gitea: #71 diff --git a/.wolf/bugs/bug-1840.md b/.wolf/bugs/bug-1840.md new file mode 100644 index 00000000..f043b08d --- /dev/null +++ b/.wolf/bugs/bug-1840.md @@ -0,0 +1,42 @@ +--- +id: "bug-1840" +status: "cannot-reproduce" +status_updated: "2026-09-25" +status_note: "no ASAN repro found. Strongest code-level candidate fixed 2026-09-25: bug-1769 (sched_make_alter_allocation wrote obj1->alloc 136 bytes into a 24-byte heap chunk for any ALTER '{..}||{..}' job -- exactly the kind of out-of-bounds write that corrupts glibc fastbin metadata and surfaces later in an unrelated malloc_consolidate). Reopen to investigating if it recurs, ideally captured on an ASAN build." +component: "scheduler" +severity: "crash" +timestamp: "2026-09-22T02:35:00Z" +last_seen: "2026-09-22T02:35:00Z" +occurrences: 1 +files: ["unknown -- heap corruption detected far from its cause"] +tags: ["scheduler", "heap-corruption", "malloc-consolidate", "netio", "ssl-handshake", "live-crash", "user-reported", "needs-asan-repro", "needs-diagnosis"] +related: ["bug-1839"] +repro_attempts: "0/1 ASAN run, 2026-09-25: 3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' (2400 elements) on the current ASAN build -- no AddressSanitizer report from the scheduler; plus ~10 ASAN HA soaks today (sequential/chaotic, all with scheduler replay-verify) with no scheduler heap error" +legacy: true +gitea_issue: 74 +gitea_synced_status: "cannot-reproduce" +--- + +# bug-1840: Live 3-node HA cluster crash (same soak run as bug-1839): xce_scheduler SIGABRT, glibc… + +## Description +Live 3-node HA cluster crash (same soak run as bug-1839): xce_scheduler SIGABRT, glibc `malloc_consolidate(): unaligned fastbin chunk detected`, surfacing during an OpenSSL TLS handshake's MAC-buffer growth: abort -> malloc_printerr -> malloc_consolidate -> _int_malloc -> __libc_malloc2 -> CRYPTO_malloc -> CRYPTO_clear_realloc -> BUF_MEM_grow_clean -> mem_write -> bwrite_conv -> BIO_write -> ssl3_finish_mac -> tls_get_message_body -> state_machine -> handle_ssl_handshake (netio.c:548) -> io_handler (netio.c:588) -> io_worker_dispatch_batch (netio.c:648) -> io_worker (netio.c:689). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +- Attempts: 0/1 ASAN run, 2026-09-25: 3-node HA + worker + scheduler, 8 waves x 6 concurrent 'jobsub -a 50' (2400 elements) on the current ASAN build -- no AddressSanitizer report from the scheduler; plus ~10 ASAN HA soaks today (sequential/chaotic, all with scheduler replay-verify) with no scheduler heap error + +## Root cause +NOT YET DIAGNOSED -- heap corruption detected far from its cause; a corrupted fastbin chunk was only found during an unrelated later malloc_consolidate() call, inside an OpenSSL TLS handshake on the scheduler's io_worker thread. The actual out-of-bounds write that corrupted the allocator's freelist could have happened in any thread, any amount of time earlier, and is not reconstructable from a non-ASAN build.dbg core dump -- there is no shadow memory, no poisoning, nothing to localize it to a specific allocation or write site. Confirmed directly via gdb against the real core file (/tmp/core.xce_scheduler, build cd9ecb41): the crashing frames' locals (malloc_consolidate's av/fb/p) are all optimized out / no debug symbols available for glibc or libcrypto/libssl in this environment, so there is nothing further to extract from the core itself beyond the symbol-only backtrace already captured. Needs an ASAN reproduction of the same 3-node HA deploy soak scenario to catch the actual out-of-bounds write with a real stack trace. + +## Fix +NOT FIXED -- awaiting an ASAN repro. Recommended next step: `/xce-build scheduler asan` (or `bash src/tools/xce-build.sh scheduler asan`), then rerun the same HA deploy scenario that produced /tmp/core.xce_scheduler until ASAN aborts at the real corruption site, which will pinpoint the actual overflow/write instead of just its downstream symptom. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1839 +- Gitea: #74 diff --git a/.wolf/bugs/bug-1841.md b/.wolf/bugs/bug-1841.md new file mode 100644 index 00000000..6078c86f --- /dev/null +++ b/.wolf/bugs/bug-1841.md @@ -0,0 +1,41 @@ +--- +id: "bug-1841" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-09-22T09:00:00Z" +last_seen: "2026-09-22T09:00:00Z" +occurrences: 1 +files: ["src/controller/resync.c (resync_resolve_round() -- the fix)", "src/controller/persist.c/src/include/persist.h (new persist_notify_worker_status_updated()", "mirroring persist_notify_job_updated_already_persisted()/bug-1679)", "src/controller/testsuite/test_worker_resync.c (new regression test)"] +tags: ["auto-detected", "resource-leak", "resync", "worker-rejoin-resync", "resInUse", "scheduler", "controller", "hook-notification"] +related: ["bug-1679", "bug-1799", "bug-1801", "bug-1815", "bug-1812", "bug-1751"] +legacy: true +gitea_issue: 75 +gitea_synced_status: "fixed" +--- + +# bug-1841: User-reported (xce_scheduler -i -s /tmp/scheduler-replay-20260922-071101.replay, worker pc13v, 22… + +## Description +User-reported (xce_scheduler -i -s /tmp/scheduler-replay-20260922-071101.replay, worker pc13v, 22 real slots), also reported to occur in normal (non-sim) mode: after worker pc13v rejoined mid-replay (session~162, multiple WORKERS_PUSH+JOB_PUSH bursts, several `JOBS_PUSH: refusing resync state 0x1 for jobid ... existing job is RUN with a real allocation on record` WARNs -- bug-1812's guard correctly refusing a stale PEND resync), every one of that worker's jobs eventually legitimately finished (`info jobs` at session=423 shows all of jobids [0-22] DONE, [23-59] PEND, zero RUN), yet `info worker pc13v` still showed `resource slots: value=22.00 used=22.00` for the rest of the run (session 162 through 423, ~18 simulated minutes) -- the worker was permanently starved even though nothing was actually running on it. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The worker-rejoin-resync feature's resync_arm_round() (resync.c) sets XCE_WORKER_RECONCILING on the controller's local worker_t.host.status and (via the HOOK_DETAIL_WORKER_JOIN hook already firing on rejoin) pushes it to the scheduler over WORKERS_PUSH. On the scheduler side, sched_reset_worker_()/sched_update_account() (bug-1799/bug-1801, 2026-09-11) deliberately FREEZE that worker's resInUse -- skip both the per-round zero-reset and the recharge -- while IS_WORKER_RECONCILING is set, by design, so in-flight reconciliation bookkeeping isn't clobbered mid-flight; bug-1799's own fix comment documents that normal reset+resum 'resumes and self-corrects on the very next round' once the status bit clears. But resync_resolve_round() (called from both the report-completion path, resync_apply_report -> resync_maybe_resolve, and the Req-5.3 timeout path, resync_on_timeout) only cleared XCE_WORKER_RECONCILING on the CONTROLLER's own local worker_t -- it never fired HOOK_DETAIL_WORKER_UPDATE or otherwise pushed the corrected status to the scheduler (contrast mark_worker_back()/worker.c:878-888, which does exactly that for the sibling XCE_WORKER_UNAVAIL bit). core_upsert_worker() (scheduler/src/core_events.c) only ever learns a worker's status from whatever WORKERS_PUSH it last received (`worker->status = w->status`, a bare unconditional copy, no other refresh path) -- so the scheduler's copy of the worker's status kept XCE_WORKER_RECONCILING set forever after the controller-side round resolved, permanently freezing resInUse at whatever it was when reconciliation started, even though every job on that worker had long since finished DONE. Reproduces identically in both sim/replay and live/normal mode since both drive the identical xce_core_begin_session() -> sched_reset_worker_()/sched_update_account() freeze logic; sim just replays whatever WORKERS_PUSH events the live controller actually sent, and the live controller never sent the correcting one either. test_worker_resync.c thoroughly tested that RECONCILING clears on the controller's OWN worker_t after resync_resolve_round(), but nothing tested that the scheduler-facing notification actually fires -- a gap noted separately during controller-peers-env-fallback design research (Sep 15 observation: 'full resync path lacks test coverage'). + +## Fix +FIXED (2026-09-22). Added persist_notify_worker_status_updated(cluster_t* c, worker_t* w) (persist.c, declared in persist.h next to persist_worker_hook_fun) -- the worker-side sibling of persist_notify_job_updated_already_persisted() (bug-1679): fires XCE_CONTROLLER_HOOK_WORKER's HOOK_DETAIL_WORKER_UPDATE AFTER_EVENT under the existing t_apply_seam_notifying guard so persist_worker_hook_fun (also subscribed to this hook, unconditionally WAL-appends on AFTER_EVENT) does NOT durably persist a WAL_WORKER_CTRL fact for this fire -- correct, since XCE_WORKER_RECONCILING/resync_state_t is intentionally ephemeral, re-derived fresh per rejoin, never replayed from WAL. resync_resolve_round() (resync.c) now calls this immediately after clearing XCE_WORKER_RECONCILING and detaching w->resync, guarded so it only fires when a round was genuinely outstanding (never on the already-documented idempotent no-op case). Verified safe under all three call sites' shared precondition (caller already holds w's write lock): confirmed neither HOOK_DETAIL_WORKER_UPDATE subscriber (sched_worker_change/apiserver_worker_change) nor persist_worker_hook_fun (guarded out entirely by the flag before touching w) locks w itself, so no deadlock risk. TDD: added test_resync_resolve_round_notifies_worker_update (test_worker_resync.c) -- a HOOK_DETAIL_WORKER_UPDATE spy hook proves resync_resolve_round() fires it exactly once per genuine resolve and not at all for the idempotent already-resolved no-op. RED confirmed first (Expected 1 Was 0) against the unfixed code, then GREEN after the fix. Full controller_unit_test suite: 1220 Tests, 0 Failures, 0 Ignored, OK (debug profile) -- no regressions. Note: the user's original .replay file is a frozen recording of the pre-fix controller's actual live traffic and will still show the frozen state when replayed, since sim mode only replays already-recorded events; the fix takes effect for new live runs/recordings going forward, not retroactively on old captures. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1679, bug-1799, bug-1801, bug-1815, bug-1812, bug-1751 +- Gitea: #75 diff --git a/.wolf/bugs/bug-1842.md b/.wolf/bugs/bug-1842.md new file mode 100644 index 00000000..91e4b66b --- /dev/null +++ b/.wolf/bugs/bug-1842.md @@ -0,0 +1,41 @@ +--- +id: "bug-1842" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-22T19:35:00Z" +last_seen: "2026-09-22T19:35:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_verify.c (sim_verify_alloc_eq", "the fix)", "src/lib/xcealloc.c (make_alloc", "the field's origin)", "src/scheduler/testsuite/test_sim_verify.c (new regression test)", "src/scheduler/testsuite/test_sim_live_allocation.c (stale-comment cleanup)"] +tags: ["scheduler", "sim", "sim-verify", "replay", "from-field", "wall-clock", "make_alloc", "xcealloc", "decision-verification", "false-positive", "user-reported", "tdd"] +related: ["bug-1611", "bug-1613", "bug-1779", "bug-1759", "bug-1838"] +legacy: true +gitea_issue: 77 +gitea_synced_status: "fixed" +--- + +# bug-1842: User-reported: replaying a real captured .replay file (scheduler-replay-20260922-095207.replay)… + +## Description +User-reported: replaying a real captured .replay file (scheduler-replay-20260922-095207.replay) via --sim --sim-verify logged a MISMATCH for every single DISPATCH decision, even though the printed 'expected:'/'actual:' summaries (jobid, nhosts) were byte-for-byte identical on both sides -- e.g. 'expected: DISPATCH jobid=0-13-1790071174-0[44] nhosts=1 -- actual: DISPATCH jobid=0-13-1790071174-0[44] nhosts=1' still reported as MISMATCH. User asked whether the comparison should stop using a timestamp as part of its check condition. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sim_decision_eq()/sim_log_mismatch() (sim_verify.c) never compare the decision's own `ts` field (only log it for diagnostics) -- the user's timestamp hypothesis was right in spirit but aimed at the wrong field. The REAL culprit is buried one level deeper: sim_verify_alloc_eq() compares xce_alloc_t field-by-field including `from`, and make_alloc() (xcealloc.c:30) unconditionally stamps `alloc->from = time(NULL)` -- the real OS wall clock, not core.c's virtualized sim clock -- every time an allocation is built. `expected->from` (decoded from the .replay file) is whatever real second the ORIGINAL recording ran in; `actual->from` (freshly rebuilt by the live seqjob_allocator.c/parjob_allocator.c -> sched_make_allocs() -> make_alloc() chain during THIS replay) is whatever real second THIS replay runs in. Two independent real-world timestamps that essentially never coincide for any replay of a real recording, even though the decision itself (job/host/resource/value/duration) is exactly what was recorded. Confirmed empirically: a clean run of xce_scheduler --sim --sim-verify against the user's own replay file mismatched 120/120 DISPATCH decisions, 100% on this field, zero relation to real-time-vs-batch simulation speed (reproduced identically at --sim-speed=0, 1, and 20). The codebase already had a documented workaround for the same footgun elsewhere: dup_alloc() (xcealloc.c:179) has to re-stamp ->from/->duration from its source AFTER calling make_alloc(), specifically because make_alloc() clobbers them with the current wall time otherwise. A prior session (2026-09-21, see this same discovery independently reached and left unfixed) had already isolated this exact mechanism via test_sim_verify_no_false_mismatch_after_live_dispatch_and_later_confirmation's own header comment, which explicitly (and accurately, until this fix) documented it as accepted 'practically negligible' same-second-window flakiness. + +## Fix +FIXED (2026-09-22). Dropped the `a->from == b->from` clause from sim_verify_alloc_eq() (sim_verify.c) with a doc comment explaining why: `from` is reservation-start bookkeeping (real wall-clock, for expiry/accounting purposes), not part of a DISPATCH decision's semantic identity -- unlike `duration`, which is a genuine request-derived value and stays part of the comparison. TDD: new test test_dispatch_decisions_differing_from_timestamp_still_match (test_sim_verify.c) builds two structurally-identical dispatch allocations with deliberately far-apart `from` values (1790000000 vs 1) and asserts sim_decision_eq() still reports a match. Also updated test_sim_live_allocation.c's test_sim_verify_no_false_mismatch_after_live_dispatch_and_later_confirmation header comment, which previously described this as accepted same-second-window flakiness -- now accurately states the flakiness is eliminated by construction. Full scheduler suite: 85/85 Ok, zero regressions (xce-build.sh scheduler asan --incremental). Live-verified against the user's own real replay file: DISPATCH-vs-DISPATCH false mismatches dropped from 120/120 to 0/120 (--sim-speed=0, --sim-verify, --logfile capture, before/after diff). A separate, distinct issue remains in the same file (61 'expected: -- actual: PEND ... main_reason=0' mismatches) -- see bug-1843, not touched by this fix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1611, bug-1613, bug-1779, bug-1759, bug-1838 +- Gitea: #77 diff --git a/.wolf/bugs/bug-1843.md b/.wolf/bugs/bug-1843.md new file mode 100644 index 00000000..0f84b34e --- /dev/null +++ b/.wolf/bugs/bug-1843.md @@ -0,0 +1,41 @@ +--- +id: "bug-1843" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-22T19:35:00Z" +last_seen: "2026-09-23T08:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_loop.c (sched_sim_driver_run()'s decision-drain loop", "sim_reconcile_unaccounted_jobs())", "src/scheduler/src/sched_election.c (sched_job_picker()/sched_allocator())"] +tags: ["scheduler", "sim", "sim-verify", "replay", "recording", "recording_session.c", "realloc-failure", "silent-data-loss", "tdd", "wrap-realloc", "fault-injection"] +related: ["bug-1842", "bug-1611", "bug-1613", "bug-1779", "bug-1759"] +legacy: true +gitea_issue: 78 +gitea_synced_status: "fixed" +--- + +# bug-1843: Same user report as bug-1842: replaying scheduler-replay-20260922-095207.replay via --sim… + +## Description +Same user report as bug-1842: replaying scheduler-replay-20260922-095207.replay via --sim --sim-verify also logs a small number of 'expected: -- actual: PEND jobid=... main_reason=0' mismatches -- live decisions with no recorded counterpart at all (beyond decoded_session->n_decisions), distinct from bug-1842's DISPATCH-vs-DISPATCH false positives (already fixed). 61 occurrences total across the full 643-session replay file (0.02% of the 3431 live PEND decisions produced). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED (2026-09-23), and the leading hypothesis from the prior session's investigation (event-batching-granularity in sim_loop.c's replay driver -- 'a job's first appearance lands one round earlier during replay than the session it was recorded in') is REFUTED by direct measurement, not just re-reasoned about: added temporary per-round diagnostic logging to sched_sim_driver_run() (sim_loop.c, reverted after use -- not part of the fix) and replayed the exact scheduler-replay-20260922-095207.replay capture this bug was originally filed against with --sim --sim-verify --sim-speed=0. For every one of the 61 mismatches, the LIVE (replay) side deterministically re-offers the full, correct set of already-known PEND candidates every round (e.g. round begin_round=...789016869984 produced 38 PEND decisions for jobs [22]..[59], with ZERO new events pushed into that round -- core state was provably identical to the immediately preceding round, which also produced 38). The RECORDED side, for that exact same round, has only 32 decisions (jobs [22]..[53]) -- missing exactly the newest-added cohort, jobs [54]..[59] -- while the immediately preceding and following recorded rounds both have all 38. This is the true shape of the bug: an isolated, self-healing gap in the ORIGINAL LIVE RECORDING, not a replay-side batching/discretization artifact (sim_loop.c's event-buffer-then-push-before-begin_session mechanism was verified, by direct code reading of both sim_loop.c's replay-side push loop and driver.c's live-side xce_sched_driver_translate_pending_events()/sched_record_write_events() call order, to be a byte-for-byte-faithful reproduction of the recorder's own file ordering -- there is no live-vs-replay discretization gap to close there). True root cause: sched_record_session_add_decision() (recording_session.c) grows its per-round ctx->decisions array via realloc(), doubling from an initial capacity of 8 (8 -> 16 -> 32 -> 64...). If that realloc() call fails, the existing code silently drops the CURRENT decision (free_record_decision(rd); return;) and leaves ctx->n/ctx->cap exactly where they were -- every SUBSEQUENT decision that round hits the same n==cap growth check and is ALSO silently dropped, so the round effectively truncates at whichever doubling boundary (8/16/32/64) the failing realloc() landed on (n=32 here -- exactly one of those boundaries, and exactly matching the empirically observed truncation point). Nothing marks the round as incomplete: sched_record_session_finalize() (recording_session.c) then serializes and writes ctx's now-short decisions array as an ordinary, apparently-complete ReplaySession -- indistinguishable on replay from a genuinely smaller live round. This is the ONE inconsistent failure-handling path in this file: every OTHER failure in the same recording subsystem (write_capnp_message() failures, XCE_DECISION_PREEMPT/NONE cases) is either propagated as a hard error (sched_record_abort_on_error()) or logged loudly by design -- only this one growth failure was silent-and-partial. The live SCHEDULING round itself was never affected either way (on_decision() still fires for every decision drained from core.decisions regardless of recording state) -- only the .replay FILE's own record of that one round silently lost its tail, which --sim-verify's replay comparison then had no way to distinguish from a genuinely smaller live round. + +## Fix +FIXED (2026-09-23, TDD). recording_session.c: added a `bool truncated` field to sched_record_session_ctx_s; sched_record_session_add_decision()'s realloc-failure branch now sets ctx->truncated = true (in addition to its existing ERROR log) before dropping the decision. sched_record_session_finalize() now checks ctx->truncated FIRST and, if set, frees everything and returns -ENOMEM WITHOUT encoding/writing anything -- routing through the exact same sched_record_abort_on_error() cleanup path (via sched_record_session_end(), driver.c) that every other recording failure in this subsystem already uses, instead of silently writing a truncated-looking-complete ReplaySession. sim_loop.c's own --sim-output path (sched_record_session_finalize() called directly) already treated any negative return as fatal, so it is covered for free with zero changes there. recording.h's doc comments for sched_record_session_add_decision()/sched_record_session_end()/sched_record_session_finalize() updated to describe the new contract. TDD: added test_decision_batch_growth_failure_drops_incomplete_session() (test_record_lifecycle.c) -- this codebase had no existing malloc/realloc fault-injection harness, so added `-Wl,--wrap=realloc` linker interposition scoped to ONLY this test executable's meson target (same established `-Wl,--wrap=` idiom already used elsewhere in this test suite for xcecli_*/sched_meet_host_selection, meson.build), with a `__wrap_realloc()` that passes through to `__real_realloc()` unless the one new test arms it. The test forces a REAL, deterministic realloc() failure on the 9th decision's array growth (8 -> 16) and asserts sched_record_session_end() returns an error and recording is cleanly disabled (matching test_write_failure_disables_recording_and_driver_keeps_running()'s existing assertion style). Verified genuine red->green: temporarily reverted just the production fix and reran -- scheduler record lifecycle test FAILED (exit 1) without the fix, confirming the new test actually exercises the bug; restored the fix and reran -- full 85/85 scheduler suite green. NOTE: this fix prevents FUTURE recordings from losing decisions this way -- it cannot retroactively repair the already-captured scheduler-replay-20260922-095207.replay file, which will continue to show the same 61 --sim-verify mismatches when replayed (now fully explained, not a live bug) unless/until it is re-captured. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1842, bug-1611, bug-1613, bug-1779, bug-1759 +- Gitea: #78 diff --git a/.wolf/bugs/bug-1844.md b/.wolf/bugs/bug-1844.md new file mode 100644 index 00000000..95d218b6 --- /dev/null +++ b/.wolf/bugs/bug-1844.md @@ -0,0 +1,41 @@ +--- +id: "bug-1844" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-23T09:15:00Z" +last_seen: "2026-09-23T09:15:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_verify.c (sim_verify_alloc_eq())"] +tags: ["scheduler", "sim", "sim-verify", "replay", "instid", "nextid", "monotonic-counter", "non-deterministic-field", "tdd"] +related: ["bug-1842", "bug-1838-followup", "bug-1843", "bug-1845"] +legacy: true +gitea_issue: 81 +gitea_synced_status: "fixed" +--- + +# bug-1844: User-reported follow-up to bug-1843: replaying a fresh capture… + +## Description +User-reported follow-up to bug-1843: replaying a fresh capture (scheduler-replay-20260923-085356.replay, built after bug-1843's fix) with --sim --sim-verify still shows a huge volume of MISMATCH lines (4881 in one run) on DISPATCH decisions whose printed description is IDENTICAL on both sides -- e.g. 'expected: DISPATCH jobid=...[40] nhosts=1 -- actual: DISPATCH jobid=...[40] nhosts=1' still flagged as a mismatch. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED via temporary field-by-field diagnostic instrumentation in sim_verify_alloc_eq() (added, verified, then reverted -- not part of the fix) replayed against the real captured .replay file: `instid` was the ONLY field ever differing (value/duration/resname/reserve matched exactly every single time, e.g. recorded instid=826 vs replay instid=1, both otherwise byte-identical). `instid` is stamped from `sched->core.nextid++` (scheduler.c's sched_get_id()) at allocation-build time -- a bare per-process monotonic counter tracking how many allocation instances THIS core has built over its own entire execution history, not a semantic property of what got allocated. The ORIGINAL recording's counter (running, and building allocations, for potentially far longer before this capture window started) and a fresh replay's own counter (starting from whatever its loaded snapshot's nextid happens to be) have no reason to reach the same value for 'the same' dispatch. Exact same class of bug as bug-1838-followup/bug-1842's `from` field (also non-deterministic bookkeeping, already excluded) -- this comparator simply had a second field of the same kind left in. + +## Fix +FIXED (2026-09-23, TDD). Dropped the `a->instid == b->instid` clause from sim_verify_alloc_eq() (sim_verify.c), same treatment as bug-1842's `from` exclusion. Updated that function's doc comment to document the exclusion and cite the real captured-file evidence (826 vs 1). Added test_dispatch_decisions_differing_instid_still_match() (test_sim_verify.c), mirroring the existing test_dispatch_decisions_differing_from_timestamp_still_match() pattern -- two allocations identical in every field except instid (826 vs 1, matching the real observed divergence) must still compare equal. Verified genuine red->green: temporarily re-added the instid comparison and reran -- scheduler sim verify test FAILED (exit 1); restored the fix and reran -- full 85/85 scheduler suite green. NOTE: this fix does NOT address the second, separate anomaly found in the same replay file during this investigation (see bug-1845): a genuine one-round-early DISPATCH timing difference for one job (a resource-accounting/capacity divergence between live and replay, not a field-comparison issue) -- open, tracked separately. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1842, bug-1838-followup, bug-1843, bug-1845 +- Gitea: #81 diff --git a/.wolf/bugs/bug-1845.md b/.wolf/bugs/bug-1845.md new file mode 100644 index 00000000..7ec3b8c8 --- /dev/null +++ b/.wolf/bugs/bug-1845.md @@ -0,0 +1,41 @@ +--- +id: "bug-1845" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-23T10:00:00Z" +last_seen: "2026-09-23T10:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/sim_verify.c (sim_verify_job_preason_eq())"] +tags: ["scheduler", "sim", "sim-verify", "replay", "pend-reason", "unsatisfiable-cache", "alloc_cache", "simulation-isolation", "tdd"] +related: ["bug-1843", "bug-1844", "bug-1842", "bug-1846"] +legacy: true +gitea_issue: 83 +gitea_synced_status: "fixed" +--- + +# bug-1845: Follow-up to bug-1843/bug-1844: replaying a fresh capture… + +## Description +Follow-up to bug-1843/bug-1844: replaying a fresh capture (scheduler-replay-20260923-085356.replay) with --sim --sim-verify still showed a large volume of MISMATCH lines on PEND decisions where recorded and replay agreed on jobid/main_reason but the deep host/resource reason-detail comparison disagreed (e.g. recorded {main_reason=0, nhost_reasons=0, nres_reasons=0} vs replay {main_reason=0, nhost_reasons=0, nres_reasons=2}), live-inspected via gdb by the user. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED (2026-09-23) as a documented, intentional architectural asymmetry, not a defect: core.h states outright that 'a simulation-mode session never touches the real cache implementation at all' (simulation isolation). core->unsatisfiable_lookup_cb/unsatisfiable_populate_cb (alloc_cache.c's xce_alloc_cache_lookup_cb/populate_cb) are registered only by main.c's normal-mode branch -- sim_loop.c never registers them. sched_make_primary_alloc_() (scheduler.c) consults that cache FIRST; on a hit it records reason detail via sched_pend_from_cached_reason_(), which sets ONLY main_reason (no host/resource detail at all -- skipping that expensive per-host computation is the entire point of the cache). Replay, having no cache, always falls through to the full per-host computation (sched_try_alloc_on_host_(), resource.c) and therefore always produces full detail. Two independent, architecturally-legitimate detail levels for the exact same semantic outcome (this job is PEND, for this main reason), not a decision correctness signal. Separately verified the cache mechanism itself is SOUND (not a distinct bug): its key (alloc_cache.c's build_key_(): resreq_fp + user + queue + project, no host/capacity dimension) is safe within a round because capacity only decreases monotonically as earlier jobs dispatch and the cache is cleared fresh every round (xce_alloc_cache_clear_cb() at begin_session) -- a cached 'unsatisfiable' verdict from an earlier job this round remains valid for a later job sharing the same key. + +## Fix +FIXED (2026-09-23, TDD). sim_verify_job_preason_eq() (sim_verify.c) now compares ONLY main_reason, dropping the nhost_reasons/nres_reasons/host_based_reasons/res_based_reasons deep comparison entirely (that deep check was added as an optional 'cheap to add' enhancement, never proven necessary by a dedicated test, and is provably wrong given the cache-isolation asymmetry above). Doc comment updated to explain why. Added test_pend_decisions_differing_reason_detail_still_match() (test_sim_verify.c) -- two PEND decisions with the same jobid/main_reason but one with empty (cache-hit-shaped) and one with populated (full-computation-shaped) host/resource reason detail must still compare equal. Verified genuine red->green: temporarily re-added the deep comparison and reran -- scheduler sim verify test FAILED; restored the fix and reran -- full 85/85 scheduler suite green. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1843, bug-1844, bug-1842, bug-1846 +- Gitea: #83 diff --git a/.wolf/bugs/bug-1846.md b/.wolf/bugs/bug-1846.md new file mode 100644 index 00000000..8dbd71b7 --- /dev/null +++ b/.wolf/bugs/bug-1846.md @@ -0,0 +1,42 @@ +--- +id: "bug-1846" +status: "fixed" +status_updated: "2026-09-24" +status_note: "77f047f2, a9feded8, ac033f47" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-23T10:05:00Z" +last_seen: "2026-09-23T10:05:00Z" +occurrences: 1 +files: ["src/scheduler/src/plugins/limit.c (match_job_host_limit_); src/scheduler/src/handler.c + main.c + scheduler.c (config push deferral)"] +tags: ["scheduler", "sim", "sim-verify", "replay", "resource-accounting", "resInUse", "driver-equivalence", "needs-diagnosis", "limit", "hlimits", "host-limit", "accounting", "production"] +related: ["bug-1843", "bug-1844", "bug-1845", "bug-1611", "bug-1613"] +fix_commits: ["77f047f2", "a9feded8", "ac033f47"] +legacy: true +gitea_issue: 84 +gitea_synced_status: "fixed" +--- + +# bug-1846: Same replay session as bug-1844/1845 (scheduler-replay-20260923-085356.replay): after excluding… + +## Description +OPEN, NOT YET ROOT-CAUSED. Same replay session as bug-1844/1845 (scheduler-replay-20260923-085356.replay): after excluding instid (bug-1844) and reason-detail (bug-1845), --sim-verify still shows one job (jobid=0-8-1790154325-0[39] in the observed capture) genuinely DISPATCHING one round earlier in replay than in the original live recording -- not a false-positive field comparison, a real positional divergence: replay's round begin_round=...436582151823 dispatches a 13-job batch (jobs [27]..[39]) while the SAME round in the original recording only dispatched 12 (jobs [27]..[38]); job [39]'s recorded dispatch instead lands in the NEXT round, cascading into ~40+ downstream positional MISMATCH lines for that round pair alone. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED (2026-09-23, via temporary per-decision diagnostics in the --sim-verify drain loop, since reverted) -- two distinct things. (1) ROOT-CAUSED + FIXED: match_job_host_limit_() (limit.c) runs on EVERY per-host allocation attempt (limit_prepare_allocation() via plugin_prepare_allocation()) and unconditionally xlist_store()d the matched vres into obj->hlimits[worker->idx]; obj->hlimits is only rebuilt by limit_job_election() on a configver bump. A job that kept pending on the same host accumulated one extra copy per attempt; sched_fill_limit_requests() staged every copy into ctx->alloc_req.host and seqjob's make_host_alloc() built a host alloc carrying all of them. In the captured replay the LIVE recording itself shows it (pc13v host allocs with mem+slots+70/79/94/112/136/175 identical limit=limits2/u=rshen/q=xce/p= entries, resetting only on a config push) -- i.e. a PRODUCTION bug, not sim-only. make_job_account() makes one res_account per entry and sched_update_res_accounts() charges each, so the host limit was charged (and later released) N-fold. Replay had the same accumulation with different counts, which is what --sim-verify flagged. (2) ROOT-CAUSED (2026-09-23): the 9 live rounds cut off after 11-12 decisions were each ended by a mid-round scheduler REJOIN. sched_publish_retry_loop() (scheduler.c) calls sched_check_pending_rejoin() after every successful publish -- i.e. inside the driver's decision-drain loop -- and after a client-library reconnect that issues xcecli_join(). The controller answers every join (on_scheduler_join(), controller/sched.c) with xcesched_push_configs() + xcesched_push_workers() + xcesched_push_jobs(). handler.c applied the config push synchronously on the handler thread (sched_init_config() -> sched_init_queues() -> builtin_fcfs on_queue_update()), which nulls priv->cursor when it reinserts the queue the cursor rests on; next_job() then returns NULL and the round ends. Evidence from the capture (temporary event-stream diagnostics, reverted): every truncated round is followed by a workersPush + full 50-job jobsPush (ntotal=50, pos=0, the join's resync -- e.g. jobs just dispatched reported back as PEND), every full-length dispatching round has none, and the config push itself never appears because it bypassed the pending-update queue the recorder writes from. Rejoins came roughly every ~100s (the configver bump also explains the periodic reset of part (1)'s duplicate counts). The underlying connection churn is a separate issue (see bug-1812/1813's pc13v reconnect/resync storm). + +## Fix +PARTIAL (2026-09-23, TDD). limit.c: new hlimits_contains_() pointer-identity check (vres is interned per name in privdata->instances) -- match_job_host_limit_() now stores a matched vres into obj->hlimits[worker] only once. New regression test test_limit_host_dedup.c (sched_limit_host_dedup_test, meson suite 'scheduler'): 5 plugin_prepare_allocation() attempts for one job on one worker must leave hlimits[worker]->count==1 and exactly one staged copy of the vres in alloc_req.host -- verified RED before the fix (Expected 1 Was 2 on attempt 2) and GREEN after. Replay re-run after the fix: replay-side host allocs are now always 3 entries; remaining DISPATCH alloc mismatches are the recording's own inflated entries (recorded with the buggy binary) and will disappear only with a fresh capture. PART (2) FIXED (2026-09-23, TDD, user-chosen design 'defer to round edge'): handler.c's configpushmsg case now calls new sched_stash_config() (parks the Conf4Sched in new xce_scheduler_t.pending_config under pending_config_lock, freeing any older unapplied push) instead of sched_init_config(); main.c's loop calls new sched_apply_pending_config() before each xce_sched_driver_run_normal_mode_round(), which runs the unchanged sched_init_config() strictly between sessions; sched_fini() frees a never-applied push. Same deferral direction as bug-1813's array-job fix. Tests: test_handlers.c's test_config_push_is_stashed_not_applied_on_handler_thread (RED: sched_init_config called 1x on the handler thread; GREEN after) and new test_config_push_deferred.c (sched_config_push_deferred_test: real scheduler_req_handler() + real builtin_fcfs mid-session -- RED: next_job() returned NULL right after the config push, job B never considered; GREEN: session walks B and C, configver unchanged until sched_apply_pending_config() applies it once). Known remaining gap (pre-existing, not addressed): a config push is still not written to a recording, so a replay spanning a config CHANGE won't see it. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1843, bug-1844, bug-1845, bug-1611, bug-1613 +- Gitea: #84 diff --git a/.wolf/bugs/bug-1847.md b/.wolf/bugs/bug-1847.md new file mode 100644 index 00000000..68bbc390 --- /dev/null +++ b/.wolf/bugs/bug-1847.md @@ -0,0 +1,38 @@ +--- +id: "bug-1847" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-24T05:30:00Z" +files: ["src/scheduler/src/scheduler.c (sched_apply_pending_config)"] +tags: ["scheduler", "recording", "replay", "config-push", "driver-equivalence"] +related: [] +legacy: true +gitea_issue: 290 +gitea_synced_status: "fixed" +--- + +# bug-1847: bug-1846 follow-up: a scheduler config push (sent by the controller on every scheduler join) was… + +## Description +bug-1846 follow-up: a scheduler config push (sent by the controller on every scheduler join) was never written to a .replay recording, so a replay spanning a rejoin or a genuine config change never saw it. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +handler.c applied config pushes synchronously via sched_init_config(), bypassing the pending-update queue that xce_sched_driver_translate_pending_events() records from. bug-1846's deferral moved application to a round boundary on the driver thread (main.c -> sched_apply_pending_config()), making it recordable at the right point. + +## Fix +FIXED (2026-09-24, TDD). sched_apply_pending_config() now writes a CONFIG_PUSH core event via the existing sched_record_write_events() after sched_init_config() succeeds (no-op when not recording), ahead of the round's other recorded events -- matching the order they take effect. Test: test_config_push_deferred.c's test_applied_config_push_is_recorded (RED: record_event_count stayed 0; GREEN: 1 event written, stashing alone writes nothing). Known fidelity limit: replay applies configPush through core_events.c's CONFIG_PUSH path (queues/resources/resvals/configver), which does not re-run groupmgr_load_configs()/plugin_config_update()/sched_init_sharedInst() like sched_init_config() does -- a replayed config CHANGE to limits/groups still won't take effect in sim (pre-existing driver-equivalence gap, not introduced here). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #290 diff --git a/.wolf/bugs/bug-1848.md b/.wolf/bugs/bug-1848.md new file mode 100644 index 00000000..339319e9 --- /dev/null +++ b/.wolf/bugs/bug-1848.md @@ -0,0 +1,38 @@ +--- +id: "bug-1848" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "leak" +timestamp: "2026-09-24T05:30:00Z" +files: ["src/scheduler/src/schedobj.c", "core_events.c", "snapshot_load.c (hlimits zarray_init)"] +tags: ["scheduler", "limit", "hlimits", "memory-leak"] +related: [] +legacy: true +gitea_issue: 291 +gitea_synced_status: "fixed" +--- + +# bug-1848: Leak: xce_sched_obj_t.hlimits was created with a NULL zarray item killer, so reset_sched_obj()… + +## Description +Leak: xce_sched_obj_t.hlimits was created with a NULL zarray item killer, so reset_sched_obj() (limit_job_election()'s config-version rebuild) and schedobj teardown leaked the per-worker xlist_t that match_job_host_limit_() callocs, one per worker a job had tried. Visible as part of the large LeakSanitizer report on every --sim replay run. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +zarray_fini() only frees items through the zarray's killer; all three schedobj constructors passed NULL. + +## Fix +FIXED (2026-09-24, TDD). New kill_hlimits_list() (schedobj.c, declared in sched/types.h) frees one per-worker list (xlist_fini + free; never its vres entries, owned by the limit plugin's instance table); installed at all three zarray_init(&(obj->hlimits), ...) sites -- schedobj.c uses kill_hlimits_list(); core_events.c and snapshot_load.c get identical static local copies (core_hlimits_list_killer / snapshot_hlimits_list_killer) because both deliberately don't share helpers with schedobj.c and several test binaries (sched_core_events_test, sched_core_full_cycle_test, sched_snapshot_save_load_test, sched_sim_startup_test) link them without schedobj.c -- a shared symbol broke those links. zarray_fini() preserves the killer, so reset_sched_obj() leaves the zarray reusable. Test: test_limit_host_dedup.c's test_hlimits_zarray_releases_its_lists (RED: killer NULL; GREEN). A direct __lsan_do_recoverable_leak_check() test was tried and rejected -- the fixture's own resreq parsing (parse_select strdup) leaks independently, so the baseline is never leak-free. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #291 diff --git a/.wolf/bugs/bug-1849.md b/.wolf/bugs/bug-1849.md new file mode 100644 index 00000000..680820e5 --- /dev/null +++ b/.wolf/bugs/bug-1849.md @@ -0,0 +1,38 @@ +--- +id: "bug-1849" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-24T05:30:00Z" +files: ["src/scheduler/src/snapshot_load.c (xce_sched_core_load_snapshot)"] +tags: ["scheduler", "sim", "snapshot", "logging"] +related: [] +legacy: true +gitea_issue: 292 +gitea_synced_status: "fixed" +--- + +# bug-1849: Every --sim run logged 'WARN xce_sched_core_load_snapshot failed to resolve job picker plugin… + +## Description +Every --sim run logged 'WARN xce_sched_core_load_snapshot failed to resolve job picker plugin (rc=-2); picker rebuild will be skipped for this load' -- misleading; flagged as a suspect during the bug-1846 investigation. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Benign ordering, not a fault: the only caller (sim.c's sim_locate_and_load_snapshot()) loads the snapshot before any picker is registered; sim_bootstrap_picker() (sim_picker.c, bug-1567) then replays the full on_queue_update + on_job_push walk against the restored core (builtin_fcfs has no on_config, and on_job_push filters non-PEND jobs itself), so the picker ends up fully rebuilt. + +## Fix +FIXED (2026-09-24). Downgraded to TRACE with a comment explaining the deferral; header comment updated. No test (log-level-only change). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #292 diff --git a/.wolf/bugs/bug-1850.md b/.wolf/bugs/bug-1850.md new file mode 100644 index 00000000..86a64d02 --- /dev/null +++ b/.wolf/bugs/bug-1850.md @@ -0,0 +1,39 @@ +--- +id: "bug-1850" +status: "fixed" +status_updated: "2026-09-25" +status_note: "6b1862f8; soaks 2026-09-25: 0 NOT_READY-triggered rotations, 1 live same-controller retry (worker, recovered in 1.2s), all PASS" +component: "lib" +severity: "unknown" +timestamp: "2026-09-24T05:30:00Z" +files: ["src/lib/client_failover.c (xcecli_retry_next / xce_failover_classify)", "src/lib/xceapi.c (10s reply deadline)"] +tags: ["libxce", "client", "failover", "reconnect", "ha", "scheduler"] +related: [] +fix_commits: ["6b1862f8"] +legacy: true +gitea_issue: 274 +gitea_synced_status: "fixed" +--- + +# bug-1850: root trigger of bug-1846 part 2's mid-round rejoins: the live scheduler reconnected/re-joined… + +## Description +OPEN, NOT FIXED -- root trigger of bug-1846 part 2's mid-round rejoins: the live scheduler reconnected/re-joined roughly every ~100s in scheduler-replay-20260923-085356.replay (each rejoin = full config/worker/job re-push from on_scheduler_join()). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +UNCONFIRMED (code reading only; the live scheduler.log for the capture window isn't available here). The controller never closes a scheduler connection proactively (only handle_disconnect() on a socket error sets XCE_CTX_ERR). But xcecli_retry_next() calls xce_failover_rotate() -> xce_ctx_reconnect() on EVERY retry, and xce_failover_classify() treats ETIMEDOUT (any request whose reply takes >10s, cbuf_deadline_10s in xceapi.c), E_XCE_NOT_READY and E_XCE_REDIRECT/STANDBY as retryable. So a slow or 'not ready' reply to a dispatch/pend-reason publish from a HEALTHY primary rotates the scheduler to the next master (typically a standby, which redirects back), and each reconnect fires the rejoin callback -> sched_check_pending_rejoin() -> xcecli_join() -> full re-push, which adds load and can self-reinforce. Also: a reply arriving after its 10s deadline lands in the capacity-1 wbuf and can be consumed by the NEXT request (xcecli_ctx_recover() drains only one stale entry). + +## Fix +NOT FIXED. Next step: grep the live scheduler.log for 'publish retry-budget exhausted', 'xcecli failover', 'pending rejoin succeeded' around each truncated round to identify the triggering rc. Any fix changes libxce HA client failover semantics (e.g. not rotating away from a primary on ETIMEDOUT/NOT_READY) -> spec workflow per CLAUDE.md, not a drive-by. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #274 diff --git a/.wolf/bugs/bug-1851.md b/.wolf/bugs/bug-1851.md new file mode 100644 index 00000000..78b3a64e --- /dev/null +++ b/.wolf/bugs/bug-1851.md @@ -0,0 +1,38 @@ +--- +id: "bug-1851" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-09-24T07:40:00Z" +files: ["src/controller/worker.c"] +tags: ["controller", "asan", "use-after-free", "worker-rejoin-resync", "ha-soak", "resvals", "xstring"] +related: [] +legacy: true +gitea_issue: 293 +gitea_synced_status: "fixed" +--- + +# bug-1851: Live sequential HA soak (run-ha-soak.sh --sequential 6, round 5): newly-promoted primary ctrl-3… + +## Description +Live sequential HA soak (run-ha-soak.sh --sequential 6, round 5): newly-promoted primary ctrl-3 (term 10) died ~8s after promotion with ASAN heap-use-after-free READ in strdup <- collect_resval (dataex.c:71) <- apiserver_worker_change (apiserver.c:450) <- apiserver_worker_hook_fun <- xce_run_hooks <- persist_notify_worker_status_updated (persist.c:7997) <- resync_resolve_round (resync.c:418) <- resync_arm_round (resync.c:315) <- apply_join_state (worker.c:351) <- rejoin_worker (worker.c:444) <- handle_worker_join. Freed by the same thread in rejoin_worker (worker.c:434, xstring_dereference of the old hostmodel). Soak surfaced it as 'FAIL: round 5: no new primary elected after kill (no serving primary at term>9)' -- the new primary crashed rather than failing to elect. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +update_val_res() (worker.c:~99) memcpy()s the xce_val_t into w->host.resvals, so the 'type'/'model' string resources seeded by add_hostattrs_resvals() ALIAS the interned xstrings owned by w->host.attrs->hosttype/hostmodel (no reference taken). rejoin_worker() dereferences (frees, at refcount 0 -- even when the new value is the same string, since it dereferences before makeString) the old hosttype/hostmodel, then apply_join_state() calls resync_arm_round() BEFORE add_hostattrs_resvals() re-seeds resvals. When the resync round resolves synchronously inside arm (resync_resolve_round -> persist_notify_worker_status_updated -> worker hooks), apiserver_worker_change -> collect_resval strdup()s the dangling svalue. Latent since the alias was introduced; made reachable by worker-rejoin-resync's synchronous resolve+notify path on rejoin (bug-1841 area). + +## Fix +FIXED (2026-09-24, TDD). update_val_res() (worker.c) now stores a private strdup() for Value_u_svalue (copy taken before freeing the previous one, so val may alias v) and frees an old string when a slot changes type; the owning arrays now free their entries: w->host.resvals gets kill_resval (lib/wobj.c, replacing the FIXME NULL killer) and c->sharedinst's resval_obj_killer (cluster.c, previously a no-op) frees svalue + the xce_val_t -- both previously also leaked every xce_val_t. Also closes the same alias hazard for worker-reported string resvals (update_resval_list aliased the decoded request) and config values (mark_host_res). Regression test test_worker_join.c::test_rejoin_worker_string_resvals_survive_outstanding_round: first join arms a round (non-terminal job on host), second rejoin_worker with new attrs resolves it and fires a registered XCE_CONTROLLER_HOOK_WORKER hook that reads the type/model resvals -- RED reproduced the live stack exactly (heap-use-after-free, freed by rejoin_worker worker.c:432 xstring_dereference); GREEN after. Full meson test 109/109 OK. Note: get_resval() still hands out a borrowed svalue pointer (opget.c), now into the resval's own copy -- same lifetime contract as before, not widened. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #293 diff --git a/.wolf/bugs/bug-1852.md b/.wolf/bugs/bug-1852.md new file mode 100644 index 00000000..279adda7 --- /dev/null +++ b/.wolf/bugs/bug-1852.md @@ -0,0 +1,38 @@ +--- +id: "bug-1852" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-24T08:10:00Z" +files: ["src/scheduler/src/driver.c"] +tags: ["scheduler", "recording", "replay", "sim-verify", "field-completeness", "MOVE_PTR", "driver"] +related: [] +legacy: true +gitea_issue: 294 +gitea_synced_status: "fixed" +--- + +# bug-1852: Scheduler replay-parity soak gate (run-ha-soak.sh --sequential 6, post bug-1851): 1 of 7 clean… + +## Description +Scheduler replay-parity soak gate (run-ha-soak.sh --sequential 6, post bug-1851): 1 of 7 clean recordings failed --sim-verify with 3 MISMATCHes, all in its first decision round: live 'DISPATCH jobid=... nhosts=1' vs sim 'PEND ... main_reason=0' (sim: unsatisfy_ unmet resource 0 (slots) shortfall 1). Sim trace: snapshot at record start restored the worker with 0/0 resvals; the recording's only WORKERS_PUSH 'carries 10 resval(s)' but core_apply_worker_resvals 'applied 0' -- every recorded NameValue.val was NULL. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The recorder encoded each event batch AFTER applying it: driver.c's xce_sched_driver_translate_pending_events() called xce_core_push_events() and only then sched_record_write_events() on the same xce_core_event_t payloads. Applying is destructive -- core_apply_worker_resvals() MOVE_PTRs each resval (nv->val -> worker->resvals), core_build_job MOVE_PTRs a JOBS_PUSH's allocs, JOB_RUSAGE transfers rusages, and core_apply_config_resvals() MOVE_PTRs a CONFIG_PUSH's shared resvals (bug-1847's new config-push recording also recorded after sched_init_config). So recordings silently lost exactly those fields: a worker's resval names were written with NULL values. Masked whenever the worker's values were already in the record-start snapshot; exposed when a recording opened after a scheduler restart, before the worker's first value push. Field-completeness violation (persistence.md). bug-1688's 'a repeat WORKERS_PUSH carries 20 resval names with nv->val==NULL' was traced from a replay file and was very likely this same stripping, not something the controller sends. + +## Fix +FIXED (2026-09-24, TDD). driver.c records the batch BEFORE xce_core_push_events(); the periodic SnapshotEntry checkpoint was split out of sched_record_write_events() into new sched_record_checkpoint_after_apply() (recording.h/recording_events.c), called AFTER the push so a checkpoint still reflects the events ahead of it in the stream. sched_apply_pending_config() (scheduler.c) likewise records the CONFIG_PUSH before sched_init_config() and checkpoints after. Stubs added to the 4 driver test units that stub the recorder. Regression test test_record_lifecycle.c::test_recorded_workers_push_keeps_resval_values (RESOURCES_PUSH 'slots' + WORKERS_PUSH hostA slots=22 through the real driver with recording on; decodes the recorded WORKERS_PUSH) -- RED (recorded val NULL), GREEN after; 3 periodic-checkpoint tests updated to the write-then-checkpoint contract. Full meson test 109/109 OK. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #294 diff --git a/.wolf/bugs/bug-1853.md b/.wolf/bugs/bug-1853.md new file mode 100644 index 00000000..3a2510cf --- /dev/null +++ b/.wolf/bugs/bug-1853.md @@ -0,0 +1,38 @@ +--- +id: "bug-1853" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-09-24T08:20:00Z" +files: [".claude/skills/xce-test-ha/run-ha-soak.sh"] +tags: ["ha-soak", "chaotic", "walcompare", "false-positive", "test-harness"] +related: [] +legacy: true +gitea_issue: 295 +gitea_synced_status: "fixed" +--- + +# bug-1853: Test-harness false positive, not a controller safety bug + +## Description +Test-harness false positive, not a controller safety bug. 10-round chaotic soak (run-ha-soak.sh --chaotic 10): VERDICT committed_divergence=1 committed_job_loss=0 result=FAIL. Single hit, round 3, mid-loop walcompare with boundary=auto: 'WALCMP_DIVERGENCE lsn=58 class=committed compared=3 present=3'. Final settled sweep: committed prefix consistent, all 33 jobs present. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +LSN 58 was never committed. ctrl-3 (term-4 primary) appended lsn=58 at 08:06:22, its only standby NACKed it ('WAL NACK from standby lsn=58 success=0'), acks never passed acked_lsn=57; the soak then killed ctrl-3. New primary ctrl-1 (term 5) correctly appended its own-term noop at lsn=58 (08:06:35). The round-3 walcheck fell back to boundary=auto (live_committed_boundary unavailable right after the election) and auto-derive (min CRC-intact tip across ALL members' on-disk WALs, including the just-killed ex-primary's) put lsn 58 inside the 'committed' window, so the dead ex-primary's uncommitted term-4 tail vs the new term-5 entry was classified as class=committed. ctrl-3 truncated/overwrote it on rejoin; settled sweep consistent. Same family as the transient-artifact false positives documented at live_committed_boundary()/bug-1487. + +## Fix +FIXED (2026-09-24, harness only). walcheck_count() takes an optional 2nd arg 'midloop', passed only by run_chaotic_rounds' per-round check: it takes the serving primary's own committedLsn (probe_primary, up to 3 tries; live_committed_boundary() needs 3-of-3 convergence, which a chaotic round never reaches -- first cut retried it and always fell back to auto) before falling back to auto, and compares only live members guaranteed to hold the committed prefix -- the serving primary plus live standbys whose node_probe appliedLsn >= boundary (all live members under auto); with fewer than 2 such members the mid-loop check is logged as skipped. Every other caller, including the settled final sweep, keeps the unchanged full 3-member comparison, so an excluded member is still checked once it has caught up. Log lines now name members=. SKILL.md chaotic-verdict section updated. Candidate (c), re-check-after-settle, rejected: a real divergence can 'heal' onto wrong content. Validated: 10-round chaotic soak PASS (committed_divergence=0, job_loss=0). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #295 diff --git a/.wolf/bugs/bug-1854.md b/.wolf/bugs/bug-1854.md new file mode 100644 index 00000000..8440c1d1 --- /dev/null +++ b/.wolf/bugs/bug-1854.md @@ -0,0 +1,41 @@ +--- +id: "bug-1854" +status: "fixed" +status_updated: "2026-09-25" +status_note: "663ea63c" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-24T08:35:00Z" +files: ["src/scheduler/src/scheduler.c"] +tags: ["scheduler", "config-push", "rejoin", "resources", "log-noise"] +related: [] +fix_commits: ["663ea63c"] +legacy: true +gitea_issue: 275 +gitea_synced_status: "fixed" +--- + +# bug-1854: Every HA soak run logs 'ERROR [sched_init_resources] sched_init_resources add resource … + +## Description +OPEN, NOT FIXED, low priority. Every HA soak run logs 'ERROR [sched_init_resources] sched_init_resources add resource failed' for EVERY configured resource (~33 lines), once per scheduler process, ~30s after it joined -- i.e. when a primary failover makes the scheduler re-join and the controller re-pushes its config (231/264/429 lines in 3 soak runs today). Scheduling is unaffected in those runs (all soaks PASS, replay parity 0 mismatches). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_init_config() is written as a first-time initializer but runs on every config push (sched_apply_pending_config, including each rejoin's push). sched_init_resources() calls makeResource() for each config->resources[i], which fails when the name is already registered in core.resmgr (key_add of an existing key) and only logs. Consequences: (1) ERROR log noise on every rejoin; (2) latent: a rejoin/upgrade config push that CHANGES a resource definition (flags, valtype, order) never updates the scheduler's copy. Pre-existing -- the same call ran on the handler thread before bug-1846's deferral; not introduced by it. sched_init_queues() re-adding queues on each push is worth checking under the same lens. + +## Fix +NOT FIXED. Candidate: on an existing name, update the registered xce_resource_t in place (or treat identical definitions as a no-op and log at DEBUG), mirroring core_events.c's core_upsert_* upsert posture for RESOURCES_PUSH; decide separately whether a changed definition needs picker/accounting refresh. + +UPDATE (2026-09-25): 2026-09-25: same defect also at the core CONFIG_PUSH path -- core_events.c core_apply_config_resources() calls makeResource() per pushed resource; makeResource_() (lib/resobj.c) returns NULL/EEXIST for any already-registered name and the caller logs ERROR 'core_apply_config_resources add resource failed'. Seen as 33 ERROR lines in data/scheduler-replay-20260925-133708.replay (sim --sim-verify, b63bb581): the scheduler rejoined at 13:39:27 and the controller re-pushed CONFIG (plus WORKERS/JOB_PUSH) onto a core whose resources were already registered. Verify 0 mismatches, all 60 jobs DONE -- noise only in that run. A fix must cover both sites (sched_init_resources and core_apply_config_resources): identical definition -> no-op at DEBUG; changed definition -> decide update-in-place vs loud WARN. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #275 diff --git a/.wolf/bugs/bug-1855.md b/.wolf/bugs/bug-1855.md new file mode 100644 index 00000000..b0cf24e3 --- /dev/null +++ b/.wolf/bugs/bug-1855.md @@ -0,0 +1,43 @@ +--- +id: "bug-1855" +status: "fixed" +status_updated: "2026-09-24" +status_note: "commit 'feat(buglog): status field, buglog skill, status-driven gitea-sync' (2026-09-24)" +component: "tooling" +severity: "unknown" +timestamp: "2026-09-24T09:39:50Z" +last_seen: "2026-09-24T09:39:50Z" +occurrences: 1 +files: ["scripts/gitea-sync.py", ".claude/skills/gitea-sync/SKILL.md"] +tags: ["gitea-sync", "tooling", "buglog"] +related: ["bug-1677"] +legacy: true +gitea_issue: 88 +gitea_synced_status: "fixed" +--- + +# bug-1855: gitea-sync mirrored unfixed bugs as closed + bug:fixed: an issue was closed whenever the buglog… + +## Description +gitea-sync mirrored unfixed bugs as closed + bug:fixed: an issue was closed whenever the buglog entry's fix text was non-empty, including fix texts reading 'NOT FIXED', 'PENDING' or 'PARTIALLY FIXED' (e.g. bug-1450 #25, bug-1451 #35, bug-1744 #47) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +buglog.json had no status field, so scripts/gitea-sync.py and the gitea-sync skill inferred open/closed from fix being non-empty. Same script also: crashed on entries whose fix is a list (.strip() on list); replaced an issue's whole label set with just bug:fixed on close (wiping bug/category/user labels); wrote gitea_issue back only after the whole loop via a full json.dumps rewrite (a mid-run failure lost issue numbers -> duplicate issues next run, and the rewrite churned ~1,200 unrelated lines); read approvals.implementation, which no spec.json has, so spec implementation/validation issues and milestones never closed; never reopened an issue whose bug came back. + +## Fix +Added a status field to every buglog entry (backfilled 2026-09-24) and made status the only open/closed source: STATUS_MAP -> state + one bug: label; entries sharing an issue take the most-open status; only status labels are swapped (other labels kept); reopen supported; gitea_synced_status tracks what was mirrored so unchanged entries make no API calls; per-entry surgical write-back via scripts/buglog.py set_fields; spec issues close on DONE_PHASES (implementation-complete/implemented/superseded). Skill doc updated to match. + +UPDATE (2026-09-24): Found during the first live run: DELETE/204 replies have an empty body (json.loads crashed the run after the first label removal), requests had no timeout (a stalled call hung the run), spec.json write-back reflowed 15 hand-formatted files, and the implementation-issue body showed the tasks approval flag as 'Tasks generated'. All fixed; first live run: bugs 20 created, 2 reopened (#25, #35), 14 relabelled; specs 65 milestones + 195 issues (158 closed at creation). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1677 +- Gitea: #88 diff --git a/.wolf/bugs/bug-1856.md b/.wolf/bugs/bug-1856.md new file mode 100644 index 00000000..f4a492bc --- /dev/null +++ b/.wolf/bugs/bug-1856.md @@ -0,0 +1,41 @@ +--- +id: "bug-1856" +status: "open" +status_updated: "2026-09-24" +status_note: "upstream OpenWolf defect; mitigated locally by openwolf.buglog.auto_detect=false; scripts/buglog.py add uses max id + 1; report upstream" +component: "unknown" +severity: "unknown" +timestamp: "2026-09-24T09:39:50Z" +last_seen: "2026-09-24T09:39:50Z" +occurrences: 1 +files: [".wolf/hooks/post-write.js", ".wolf/config.json"] +tags: ["openwolf", "buglog", "upstream"] +related: ["bug-1630"] +legacy: true +gitea_issue: 89 +gitea_synced_status: "open" +--- + +# bug-1856: OpenWolf 2.5.0 post-write hook would create a duplicate bug id: autoDetectBugFix numbers new… + +## Description +OpenWolf 2.5.0 post-write hook would create a duplicate bug id: autoDetectBugFix numbers new entries bug- (next: bug-1194), but ids run to bug-1854 and bug-1194 already exists + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Upstream OpenWolf hook (.wolf/hooks/post-write.js autoDetectBugFix, nextId = bugLog.bugs.length + 1) assumes ids are dense and never renumbered; this buglog's ids are sparse after renumbering/consolidation (see bug-1630). The hook also never sets the project's status field. + +## Fix +_Not recorded (migrated from buglog.json)._ + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1630 +- Gitea: #89 diff --git a/.wolf/bugs/bug-1857.md b/.wolf/bugs/bug-1857.md new file mode 100644 index 00000000..e78ae1a9 --- /dev/null +++ b/.wolf/bugs/bug-1857.md @@ -0,0 +1,41 @@ +--- +id: "bug-1857" +status: "fixed" +status_updated: "2026-09-24" +status_note: "3cc2f6e3" +component: "config" +severity: "unknown" +timestamp: "2026-09-24T13:02:14Z" +last_seen: "2026-09-24T13:02:14Z" +occurrences: 1 +files: [".claude/skills/xce-test-multihost/run-multihost-verify.sh", "src/tools/deploy-ha.sh", "src/configs/limits.yaml"] +tags: ["test-fixture", "scheduler", "limits", "multihost"] +related: [] +fix_commits: ["3cc2f6e3"] +legacy: true +gitea_issue: 276 +gitea_synced_status: "fixed" +--- + +# bug-1857: xce-test-multihost FAIL: job never reached RUN within 30s (last observed state: PEND)… + +## Description +xce-test-multihost FAIL: job never reached RUN within 30s (last observed state: PEND); scheduler.log: 'unsatisfy_ unmet resource 33 (limit=limit1) shortfall 30.000000' + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +run-multihost-verify.sh deploys with the repo's default src/configs/limits.yaml, whose limit1 caps all users except rshen at a shared 3 slots. xce_test submits as root, so the 33-task job is capped. The fixture passed 2026-09-06 only because global limits did not yet count a job's full task total; the bug-1790 fix (9caf1424, 2026-09-08) correctly started doing so, leaving the fixture stale. Allocator itself is fine: passes with limit1 raised to 100. + +## Fix +UPDATE (2026-09-24): Fix (uncommitted): deploy-ha.sh gains optional XCE_DEPLOY_CONFIG_DIR (default unchanged: src/configs); run-multihost-verify.sh copies src/configs into $TOP/configs, replaces limits.yaml with 'limits: []', and deploys with that override. Verified 2026-09-24: multihost e2e PASS (plain 33-task job + array element both split across 2 hosts, DONE), 0 limit1 rejections; default deploy path still builds from repo configs; bad override fails with 'config dir not found'. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #276 diff --git a/.wolf/bugs/bug-1858.md b/.wolf/bugs/bug-1858.md new file mode 100644 index 00000000..1a2b7999 --- /dev/null +++ b/.wolf/bugs/bug-1858.md @@ -0,0 +1,41 @@ +--- +id: "bug-1858" +status: "fixed" +status_updated: "2026-09-24" +status_note: "9e9f2798" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-24T13:12:53Z" +last_seen: "2026-09-24T13:12:53Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c", "src/scheduler/src/schedobj.c"] +tags: ["scheduler", "asan", "sim", "bad-free", "soak"] +related: [] +fix_commits: ["9e9f2798"] +legacy: true +gitea_issue: 277 +gitea_synced_status: "fixed" +--- + +# bug-1858: AddressSanitizer: attempting free on address which was not malloc()-ed in core_upsert_queue… + +## Description +AddressSanitizer: attempting free on address which was not malloc()-ed in core_upsert_queue (core_events.c:383) <- core_apply_config_queues <- xce_core_push_events <- sched_sim_driver_run; address 56 bytes inside a 57-byte makeString instance. Seen as HA soak (sequential) FAIL: scheduler replay-parity SIMVERIFY errors=1 (simverify-3, rc=1) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +q->resreq is an interned, refcounted xstring from makeString() (pointer past the object header), but core_upsert_queue releases the old value with FREEUP()/free() instead of dereference(). Only fires when a CONFIG_PUSH updates an existing queue whose resreq is non-NULL (e.g. a queue restored from snapshot via snapshot_load.c:313). Same bad free(q->resreq) exists in the normal-mode path, schedobj.c:603 (sched_update_existing_queue). Convention elsewhere: qobj.c:69 dereference(q->queue.resreq). + +## Fix +UPDATE (2026-09-24): Fix (uncommitted): core_events.c core_upsert_queue and schedobj.c sched_update_existing_queue now dereference() the old interned q->resreq (NULL-guarded) instead of FREEUP()/free(). Regression tests added (red first, both hit the exact bad-free): test_core_events.c test_queue_update_replaces_existing_interned_resreq, test_job_update.c test_update_existing_queue_replaces_interned_resreq (the existing queue-update test had deliberately skipped the resreq branch). Verified 2026-09-24: scheduler suite 87/87 (ASAN); the crashing HA-soak recording scheduler-replay-20260924-130653.replay now replays with rc=0, 0 ASAN errors, 0 mismatches. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #277 diff --git a/.wolf/bugs/bug-1859.md b/.wolf/bugs/bug-1859.md new file mode 100644 index 00000000..5513fea3 --- /dev/null +++ b/.wolf/bugs/bug-1859.md @@ -0,0 +1,43 @@ +--- +id: "bug-1859" +status: "fixed" +status_updated: "2026-09-25" +status_note: "7c98c056" +component: "scheduler" +severity: "unknown" +timestamp: "2026-09-24T13:45:31Z" +last_seen: "2026-09-24T13:45:31Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c", "src/controller/job.c"] +tags: ["ha", "chaotic-soak", "resync", "scheduler", "controller", "intermittent"] +related: [] +fix_commits: ["7c98c056"] +legacy: true +gitea_issue: 278 +gitea_synced_status: "fixed" +--- + +# bug-1859: xce-test-ha --chaotic FAIL: chaotic: 1/4 worker-resync job(s) never reconciled after the loop… + +## Description +xce-test-ha --chaotic FAIL: chaotic: 1/4 worker-resync job(s) never reconciled after the loop settled: 0-2-1790256202-546579(last=PEND). scheduler.log: 'JOBS_PUSH: refusing resync state 0x1 for jobid ... -- existing job is RUN with a real allocation on record (existing state 0x2 kept)' + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not yet confirmed. Timeline (2026-09-24 chaotic soak, 6 rounds): job dispatched on primary ctrl-1 (term 6) at 13:23:23.74, commit_advance 80->81 at 13:23:23.79; worker started it and finished it at 13:23:43 (retained outcome exit_status=1); ctrl-1 killed in round 5, overlapping kills follow; after settle the cluster reports the job PEND while the scheduler keeps RUN+alloc and refuses the PEND resync (core_events.c:984), so neither side converges. Also: round 6 'scheduler restart did not confirm join within 20s (bug-1760 workaround)'. Candidates: dispatch/RUN not in the surviving committed prefix while the scheduler already applied it, or worker's retained finish never re-delivered to the new primary (cf. bug-1741 fire-and-forget finish notify). Same chaotic soak passed earlier the same day (21 jobs, 6 resync jobs reconciled), so intermittent. + +## Fix +UPDATE (2026-09-24): Frequency 2026-09-24: 1 failure in 3 chaotic 6-round soaks (pass 12:5x, FAIL 13:2x, pass 13:5x on 9e9f2798). Failing run's logs kept at scratchpad ha-chaos-bug1859-run1 (session-local). + +UPDATE (2026-09-24): ROOT CAUSE (from preserved logs of the failing run): job 546579 dispatched+committed (LSN 81) on ctrl-1 term 6; worker finished it 13:23:43 and sent the fire-and-forget finish notify (xcecli_notify = single-shot send, rc ignored: 'FIXME: add error handling when msg send failure') into a dead connection during overlapping failovers; worker could not reach any primary for 84s; its retained finished outcome expired at finish_time + 60s (default window; xce.yaml sets no worker_resync_window_ms, controller pushes 0 -> default) DURING the outage; at rejoin (13:25:07) ctrl-2's resync query (op 43) got 'unrecognized' -> record_job_reconcile_pending_fact requeued it PEND while the scheduler was disconnected; scheduler rejoined 13:25:11 and its JOBS_PUSH bug-1812 guard refused the PEND (RUN+alloc on record) -> permanent divergence. FIX (uncommitted): worker retention window no longer counts outage time -- xcecd_on_reconnect() records the reconnect time and expiry is measured from max(finish_time, last_reconnect); nothing expires while checking_connection()'s hello probe reports unhealthy. Reported finish_time_ms unchanged. Tests: test_finished_job_retention.c test_reconnect_restarts_window_for_outcome_finished_before_outage (84s outage) + test_unhealthy_connection_suspends_expiry, both red first; worker suite 10/10. The scheduler-side divergence (controller requeue while scheduler disconnected is refused by bug-1812 guard) logged separately. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #278 diff --git a/.wolf/bugs/bug-1860.md b/.wolf/bugs/bug-1860.md new file mode 100644 index 00000000..e90ca0f2 --- /dev/null +++ b/.wolf/bugs/bug-1860.md @@ -0,0 +1,41 @@ +--- +id: "bug-1860" +status: "fixed" +status_updated: "2026-09-27" +status_note: "c69705ae: controller is ground truth on rejoin; controller refuses non-PEND dispatch and re-sends state; scheduler adopts controller allocation on RUN update" +component: "controller" +severity: "unknown" +timestamp: "2026-09-24T14:38:03Z" +last_seen: "2026-09-24T14:38:03Z" +occurrences: 1 +files: ["src/scheduler/src/core_events.c", "src/controller/sched.c", "src/controller/resync.c"] +tags: ["scheduler", "controller", "resync", "ha", "divergence", "design"] +related: [] +fix_commits: ["c69705ae"] +legacy: true +gitea_issue: 279 +gitea_synced_status: "fixed" +--- + +# bug-1860: Scheduler keeps a job RUN (with allocation) forever after the controller requeued it to PEND while… + +## Description +Scheduler keeps a job RUN (with allocation) forever after the controller requeued it to PEND while the scheduler was disconnected: scheduler.log 'JOBS_PUSH: refusing resync state 0x1 for jobid ... -- existing job is RUN with a real allocation on record' on rejoin; job never redispatched (seen in bug-1859's chaotic soak) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +core_events.c JOBS_PUSH bug-1812 guard refuses any non-terminal resync state that would drop a real allocation, to protect against a committed snapshot racing the scheduler's own fresh dispatch. It cannot tell that race apart from a genuine, committed controller-side requeue (e.g. resync 'unrecognized' -> record_job_reconcile_pending_fact) that happened while the scheduler was disconnected, and JOBS_PUSH carries no per-job freshness (commit LSN/version) to decide. Needs a design decision (e.g. per-job last-modified LSN in the push). + +## Fix +_Not recorded (migrated from buglog.json)._ + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #279 diff --git a/.wolf/bugs/bug-1861.md b/.wolf/bugs/bug-1861.md new file mode 100644 index 00000000..c5c8b520 --- /dev/null +++ b/.wolf/bugs/bug-1861.md @@ -0,0 +1,41 @@ +--- +id: "bug-1861" +status: "fixed" +status_updated: "2026-09-25" +status_note: "b5d59253" +component: "controller" +severity: "unknown" +timestamp: "2026-09-24T15:05:56Z" +last_seen: "2026-09-24T15:05:56Z" +occurrences: 1 +files: ["src/controller/config_repl.c"] +tags: ["controller", "security", "tmpfile", "config-upgrade"] +related: [] +fix_commits: ["b5d59253"] +legacy: true +gitea_issue: 280 +gitea_synced_status: "fixed" +--- + +# bug-1861: config_repl.c do_upgrade_config_with_bytes: temp decode file… + +## Description +config_repl.c do_upgrade_config_with_bytes: temp decode file /tmp/xce_upgrade_cfg..tmp -- the name is identical across processes (parallel controller_unit_test instances failed handle_upgrade_config with -EINVAL, 'a majority-acked upgrade must succeed') and is opened O_CREAT|O_TRUNC without O_EXCL in world-writable /tmp + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +pthread_self() is a thread-descriptor address, not a unique id: the main thread's value repeats across processes (esp. with ASLR off under ASAN), so two processes on one host share and clobber the file; a predictable name opened without O_EXCL in /tmp is also a symlink-attack target (write/truncate through an attacker's link). + +## Fix +_Not recorded (migrated from buglog.json)._ + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #280 diff --git a/.wolf/bugs/bug-1862.md b/.wolf/bugs/bug-1862.md new file mode 100644 index 00000000..8825af52 --- /dev/null +++ b/.wolf/bugs/bug-1862.md @@ -0,0 +1,41 @@ +--- +id: "bug-1862" +status: "fixed" +status_updated: "2026-09-25" +status_note: "c29518b2" +component: "worker" +severity: "crash" +timestamp: "2026-09-24T15:24:57Z" +last_seen: "2026-09-24T15:24:57Z" +occurrences: 1 +files: ["src/worker/src/xcecd_resync.c", "src/worker/src/xcecd_worker_utils.c", "src/worker/src/controller_connector.c", "src/worker/src/main.c"] +tags: ["worker", "asan", "race", "use-after-free", "resync", "ha"] +related: [] +fix_commits: ["c29518b2"] +legacy: true +gitea_issue: 281 +gitea_synced_status: "fixed" +--- + +# bug-1862: xcecd_virtual ASAN heap-use-after-free in find_worker_by_jobid (xcecd_worker_utils.c:702) <… + +## Description +xcecd_virtual ASAN heap-use-after-free in find_worker_by_jobid (xcecd_worker_utils.c:702) <- xcecd_resync_classify_job (xcecd_resync.c:33) <- xcecd_resync_build_report <- handle_jobstatequery (controller_connector.c:1662) on a vtask thread; freed by remove_worker (xcecd_worker_utils.c:283) <- cleanup_worker_list <- main loop (main.c:652). Worker aborted mid chaotic HA soak (2026-09-24, run 1 of 3), so later rounds lost worker-resync coverage. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The worker's live job list is mutated by the main loop (add_worker / cleanup_worker_list -> remove_worker) with no lock, while the controller's OPCode_jobStateQuery is answered on a vtask thread (myreq_handler) that walks the same list. Any rejoin resync overlapping a job reap reads a freed node. + +## Fix +UPDATE (2026-09-25): Fix (uncommitted): xcecd_worker_utils.c gains a worker-list mutex (xcecd_worker_list_lock/unlock). add_worker takes it around linking a fully-built entry; remove_worker unlinks under it and frees after. Readers on vtask threads: handle_jobstatequery holds it from reading the list head through the whole query page (report build walks the list several times; the send is a non-blocking xce_request); the job-control handler copies job_status under it and re-finds the entry under it to set killed_signal; remote task start strdup's the five fields it sends under it and frees them after xcecli_runtask. The finished-job retention store (appended by the main loop, read by resync) gets its own internal mutex via public wrappers around *_locked_ bodies. Main-thread-only traversals (uv_ipc.c find_worker_by_pid, cleanup/update) need no lock. Test: test_xcecd_resync.c test_build_report_safe_against_concurrent_list_mutation (mutator thread add/remove/retain vs 20000 locked report builds) -- heap-use-after-free in find_worker_by_jobid 3/3 with no-op lock stubs, clean 5/5 with the fix; worker suite 10/10. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #281 diff --git a/.wolf/bugs/bug-1863.md b/.wolf/bugs/bug-1863.md new file mode 100644 index 00000000..d98f95a1 --- /dev/null +++ b/.wolf/bugs/bug-1863.md @@ -0,0 +1,41 @@ +--- +id: "bug-1863" +status: "fixed" +status_updated: "2026-09-25" +status_note: "aea34738" +component: "scheduler" +severity: "leak" +timestamp: "2026-09-25T13:07:56Z" +last_seen: "2026-09-25T13:07:56Z" +occurrences: 1 +files: ["src/scheduler/src/scheduler.c", "src/scheduler/src/plugins/seqjob_allocator.c", "src/scheduler/src/plugins/parjob_allocator.c"] +tags: ["scheduler", "allocation", "compound", "accounting"] +related: [] +fix_commits: ["aea34738"] +legacy: true +gitea_issue: 282 +gitea_synced_status: "fixed" +--- + +# bug-1863: A COMPOUND resreq with 2+ primaries (e.g + +## Description +A COMPOUND resreq with 2+ primaries (e.g. '1*{A} + 2*{B}') keeps only the LAST primary's allocation: seqjob/parjob my_alloc assign ctx->alloc = make_job_alloc(...) per primary, overwriting (and leaking) the previous primary's job alloc, while every primary's hosts were already debited into ctx->uncomitted -- the dispatched allocation omits earlier primaries' hosts and accounting diverges from the decision. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_make_compound_alloc_() (scheduler.c) loops primaries on one ctx; each allocator's success path replaces ctx->alloc instead of merging host allocs (and shared allocs) across primaries. Latent until bug-1768 (fixed 2026-09-25) because nested primaries previously requested nothing. + +## Fix +_Not recorded (migrated from buglog.json)._ + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #282 diff --git a/.wolf/bugs/bug-1864.md b/.wolf/bugs/bug-1864.md new file mode 100644 index 00000000..9aee99d7 --- /dev/null +++ b/.wolf/bugs/bug-1864.md @@ -0,0 +1,41 @@ +--- +id: "bug-1864" +status: "fixed" +status_updated: "2026-09-25" +status_note: "0d3bb965" +component: "lib" +severity: "unknown" +timestamp: "2026-09-25T14:26:45Z" +last_seen: "2026-09-25T14:26:45Z" +occurrences: 1 +files: ["src/lib/vtaskmgr.c", "src/lib/dataq.c", "src/lib/netio.c"] +tags: ["lib", "vtaskmgr", "ordering", "race", "replication", "ha", "controller"] +related: [] +fix_commits: ["0d3bb965"] +legacy: true +gitea_issue: 283 +gitea_synced_status: "fixed" +--- + +# bug-1864: Standby controllers NACK in-order WAL catch-up entries (ae_reject_logmismatch, prevIdx one ahead… + +## Description +Standby controllers NACK in-order WAL catch-up entries (ae_reject_logmismatch, prevIdx one ahead of their log) ~576 times in a 45-min run (data/logs, 2026-09-25): e.g. ctrl-2 21:37:50.115 handled walCatchupEntry seq 258 (prevIdx=149) BEFORE seq 257 (prevIdx=148). With both standbys NACKing, the primary marks them catch-up, loses its write quorum and answers jobdispatch/join with E_XCE_NOT_READY (-10013) (ctrl-1 21:39:24.494); the scheduler client fails over (bug-1850), rejoins, and gets a JOBS_PUSH built from lagging committed state -> 'refusing resync state 0x1 ... existing job is RUN' WARNs for 14 jobs. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +vtaskmgr's dispatcher (pick_eligible_task -> dataq_deq(taskq, vtask_eligible)) evaluates each queued task's eligibility one item at a time while holding only the taskq lock; the per-endpoint busy flag it reads (endpoint_is_task_blocked) is cleared by the vcpu finishing the previous message WITHOUT that lock. Within one scan, task N (seq 257) is checked while the endpoint is still busy and skipped, the endpoint unblocks, and task N+1 (seq 258) for the SAME endpoint is then found eligible and dequeued first. endpoint_block_tasks gives mutual exclusion per endpoint but not FIFO order. Log proof (ctrl-2 21:37:50.113-.118): '04b650 not eligible' -> 'endpoint not blocked' -> '03cae0 eligible' in one scan; 03cae0=seq 258, 04b650=seq 257; tasks enqueued in arrival order by the single netio thread (vtask_handle_msg). + +## Fix +UPDATE (2026-09-25): Fix (uncommitted): dataq gains dataq_deq_ctx (per-scan matcher context); vtaskmgr's scan is now vtask_deq_eligible(taskq, is_blocked) with a per-scan set of endpoints found busy -- every later task of such an endpoint stays ineligible for the rest of that scan (overflow past 64 distinct busy endpoints makes all later IO tasks ineligible), so per-endpoint FIFO holds without taking any new lock (bug-1282 ABBA). Test lib/testsuite/test_vtask_order.c: is_blocked injected to flip ep busy->free mid-scan; reproduced the reorder (N+1 dequeued) before, NULL after; other-endpoint progress and FIFO/generic cases pass. Suites: lib 1/1, controller 5/5, worker 10/10, scheduler 87/87, harness 5/5. Reorder signature (reject prevIdx=K then accept K-1): user run 26 (-> 576 NACKs incl. cascades), pre-fix chaotic soaks 2 and 5. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #283 diff --git a/.wolf/bugs/bug-1865.md b/.wolf/bugs/bug-1865.md new file mode 100644 index 00000000..212a22a8 --- /dev/null +++ b/.wolf/bugs/bug-1865.md @@ -0,0 +1,41 @@ +--- +id: "bug-1865" +status: "fixed" +status_updated: "2026-09-25" +status_note: "910b43cd; soaks 2026-09-25 (2 chaotic + 1 sequential) fdatasync_failed=0, all PASS" +component: "controller" +severity: "unknown" +timestamp: "2026-09-25T14:43:27Z" +last_seen: "2026-09-25T14:43:27Z" +occurrences: 1 +files: ["src/controller/ha.c", "src/controller/config_repl.c"] +tags: ["controller", "ha", "fd", "double-close", "persist", "wal", "durability"] +related: [] +fix_commits: ["910b43cd"] +legacy: true +gitea_issue: 284 +gitea_synced_status: "fixed" +--- + +# bug-1865: Standby's WAL fd closed out from under it at failover: chaotic HA soak 2026-09-25 (ha-chaos-o1)… + +## Description +Standby's WAL fd closed out from under it at failover: chaotic HA soak 2026-09-25 (ha-chaos-o1), ctrl-3 at 14:39:55.574 'liveness poll: no primary activity for 11s -> failover' / 'primary_ctrl_fd RDHUP', and 3 ms later 'flusher fallback fdatasync FAILED (errno=9 Bad file descriptor) -- durable_lsn held at 90', then errno=22 EINVAL once the number was reused by a socket; 2305 fdatasync failures, durable_lsn stuck; soak still PASSed (counted as excluded bug924) but one round's jobsub failed EIO. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +config_repl.c:2192 stores the netio endpoint's raw fd (c->ha.primary_ctrl_fd = ep->fd) and ha_on_primary_ctrl_fd_rdhup (ha.c:8648, three branches) later close()s it directly. netio owns that socket and closes it on endpoint teardown, so by then the number may have been reused -- here by the standby's WAL segment (lowest free fd) -- and HA's close() closes the WAL instead. Double close / stale fd. Once a socket takes the number, WAL fdatasync gets EINVAL and pwrite could write WAL bytes into a socket. + +## Fix +_Not recorded (migrated from buglog.json)._ + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #284 diff --git a/.wolf/bugs/bug-1866.md b/.wolf/bugs/bug-1866.md new file mode 100644 index 00000000..495856a4 --- /dev/null +++ b/.wolf/bugs/bug-1866.md @@ -0,0 +1,59 @@ +--- +id: "bug-1866" +status: "fixed" +status_updated: "2026-09-27" +status_note: "" +component: "worker" +severity: "leak" +timestamp: "2026-09-25T15:42:02Z" +last_seen: "2026-09-25T15:42:02Z" +occurrences: 1 +files: ["src/worker/src/xcecd_worker_utils.c"] +tags: ["worker", "leak", "asan"] +related: [] +fix_commits: ["e84a8d43"] +gitea_issue: 285 +gitea_synced_status: "fixed" +--- + +# bug-1866: xcecd_virtual leaks up to five strings per finished job (LeakSanitizer at exit) + +## Description +**Observed:** +``` +Direct leak of 12 byte(s) in 12 object(s) allocated from: + strndup add_worker (xcecd_worker_utils.c:140) + <- xcecd_process_command <- handle_message_from_queue +``` +**Expected:** no leaks reported when xcecd_virtual exits. +**Impact:** one leaked string per job that carries a launch type, plus one per queue/job pre- or post-exec command; grows with the number of finished jobs for the life of the daemon. +**Found by:** LeakSanitizer at xcecd_virtual exit after a job burst run, 2026-09-25. + +## Environment +- OS: no /etc/os-release; Docker container; NixOS kernel; kernel Linux 6.18.33 x86_64 +- libc: glibc 2.42-47 (interpreter /nix/store/wb6rhpznjfczwlwx23zmdrrw74bayxw4-glibc-2.42-47/lib/ld-linux-x86-64.so.2) +- Toolchain: gcc (GCC) 15.2.0, meson 1.9.1 +- Build: asan (xce.build), leak detection on at process exit +- Code: scheduler-framework @ d87b78dc +- Deployment: xcecd_virtual worker; the burst run's cluster topology was not recorded + +## Reproduction +1. Check out d87b78dc and add `test_remove_worker_frees_every_startinfo_copy` from e84a8d43 to `src/worker/testsuite/test_finished_job_retention.c`. +2. `/xce-build worker` +3. `ASAN_OPTIONS=detect_leaks=0 src/xce.build/worker/testsuite/finished_job_retention_test` +- Frequency: deterministic (58 bytes leaked per add/remove round trip) +- Evidence: the unit test; the original burst-run log was not kept. + +## Root cause +add_worker() strndup()s launch_type, q_pre_exec, q_post_exec, job_pre_exec and job_post_exec out of the job's start info, but remove_worker() only freed exechome, execcwd, execuser, job_cmd, pids and job_exdata. LeakSanitizer reported only the launch_type copy because the burst's jobs had no pre/post-exec commands. Pre-existing, origin unknown. remove_worker() is the only place a worker entry is freed, so no other path is affected. + +## Fix +remove_worker() also frees launch_type, q_pre_exec, q_post_exec, job_pre_exec and job_post_exec (src/worker/src/xcecd_worker_utils.c). + +## Verification +- Test: test_remove_worker_frees_every_startinfo_copy in src/worker/testsuite/test_finished_job_retention.c compares ASAN's live-heap bytes across an add/remove round trip (the suite runs with detect_leaks=0): RED before the fix leaked 58 B, the five strings; GREEN after, 0 B +- Suites: worker 12/12, asan +- Live: n/a; the leak is in a pure unlink/free path the unit test covers completely + +## References +- Gitea: #285 diff --git a/.wolf/bugs/bug-1867.md b/.wolf/bugs/bug-1867.md new file mode 100644 index 00000000..8158546f --- /dev/null +++ b/.wolf/bugs/bug-1867.md @@ -0,0 +1,42 @@ +--- +id: "bug-1867" +status: "fixed" +status_updated: "2026-09-26" +status_note: "e4bb492a: capnp Job.exitInfo (JobExitInfo) -- checkpoint encode/decode/restore carry exit_info. Live force-snapshot soak blocked by pre-existing bug-1870." +component: "lib" +severity: "unknown" +timestamp: "2026-09-26T03:33:15Z" +last_seen: "2026-09-26T03:33:15Z" +occurrences: 1 +files: ["src/schema/controller.capnp"] +tags: ["persistence", "snapshot", "exit_info", "field-completeness", "controller"] +related: ["bug-1689", "bug-1754"] +fix_commits: ["e4bb492a"] +legacy: true +gitea_issue: 286 +gitea_synced_status: "fixed" +--- + +# bug-1867: Controller job snapshot/checkpoint drops job.exit_info: the capnp Job struct (controller.capnp)… + +## Description +Controller job snapshot/checkpoint drops job.exit_info: the capnp Job struct (controller.capnp) has no exit-info field, so capnp_encode_job() never writes it and a job restored from a snapshot (rather than WAL replay) has exit_info NULL for both DONE/EXIT (FINISH) and failed (FAIL) jobs + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +struct Job in src/schema/controller.capnp has finishtime/runtime/pidinfo/pendreason but no exit code; xce_job_t.exit_info (job_exitinfo_t*) has no capnp mapping. bug-1689 and bug-1754 made WAL_JOB_FINISH/WAL_JOB_FAIL carry the exit code, but anything restored from a checkpoint/entity snapshot after WAL truncation loses it (persistence.md field-completeness). + +## Fix +UPDATE (2026-09-26): live-verified by the bug-1870 force-snapshot soak (real snapshot install, 63fd3d2a) + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1689, bug-1754 +- Gitea: #286 diff --git a/.wolf/bugs/bug-1868.md b/.wolf/bugs/bug-1868.md new file mode 100644 index 00000000..0e93abdf --- /dev/null +++ b/.wolf/bugs/bug-1868.md @@ -0,0 +1,42 @@ +--- +id: "bug-1868" +status: "fixed" +status_updated: "2026-09-26" +status_note: "7864a3da: xcecd_create_job_output() unlinks then creates with O_EXCL|O_NOFOLLOW|O_CLOEXEC." +component: "worker" +severity: "security" +timestamp: "2026-09-26T07:40:17Z" +last_seen: "2026-09-26T07:40:17Z" +occurrences: 1 +files: ["src/worker/src/uv_proc.c"] +tags: ["security", "worker", "symlink", "tmp", "uv_proc"] +related: ["bug-1508"] +fix_commits: ["7864a3da"] +legacy: true +gitea_issue: 288 +gitea_synced_status: "fixed" +--- + +# bug-1868: uv_proc_exec (worker/src/uv_proc.c) creates each task's output file /tmp/job__task.out… + +## Description +uv_proc_exec (worker/src/uv_proc.c) creates each task's output file /tmp/job__task.out as root with O_WRONLY|O_CREAT|O_TRUNC and no O_NOFOLLOW/O_EXCL: a local user who pre-plants a symlink at a predictable future job's output name makes the root worker truncate and write job output into the symlink target + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Predictable name in world-writable /tmp opened by root following symlinks. Only mitigated by the host's fs.protected_symlinks / fs.protected_regular sysctls, not by the code. Sibling of bug-1508 (read side, fixed in 7723a4f4). + +## Fix +_Not recorded (migrated from buglog.json)._ + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1508 +- Gitea: #288 diff --git a/.wolf/bugs/bug-1869.md b/.wolf/bugs/bug-1869.md new file mode 100644 index 00000000..ae3f1d6d --- /dev/null +++ b/.wolf/bugs/bug-1869.md @@ -0,0 +1,42 @@ +--- +id: "bug-1869" +status: "fixed" +status_updated: "2026-09-26" +status_note: "5df33e69: xcecd_open_owned_file()/xcecd_owned_file_exists() (xcecd_tmpfile.h) for the worker log, xcecd_worker.log and restart flag; fchmod instead of chmod(path)." +component: "worker" +severity: "security" +timestamp: "2026-09-26T07:55:23Z" +last_seen: "2026-09-26T07:55:23Z" +occurrences: 1 +files: ["src/worker/src/process_worker_manager.c", "src/worker/src/xcecd.c"] +tags: ["security", "worker", "symlink", "tmp"] +related: ["bug-1868"] +fix_commits: ["5df33e69"] +legacy: true +gitea_issue: 289 +gitea_synced_status: "fixed" +--- + +# bug-1869: Other root-owned opens of fixed /tmp names in the worker daemon follow symlinks… + +## Description +Other root-owned opens of fixed /tmp names in the worker daemon follow symlinks: process_worker_manager.c opens /tmp/xcecd_worker.log O_WRONLY|O_CREAT|O_APPEND (xcecd_worker's stdout/stderr appended through a planted symlink into any file), and xcecd_prepare_restart() (xcecd.c) opens /tmp/.xcecd.restarting O_CREAT|O_WRONLY (creates an empty root-owned file at a dangling symlink's target) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Same class as bug-1868: fixed, predictable names in world-writable /tmp opened by root without O_NOFOLLOW/O_EXCL; only the host's fs.protected_symlinks sysctl mitigates. Found while fixing bug-1868; left out of that fix's scope. + +## Fix +_Not recorded (migrated from buglog.json)._ + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1868 +- Gitea: #289 diff --git a/.wolf/bugs/bug-1870.md b/.wolf/bugs/bug-1870.md new file mode 100644 index 00000000..b72c5e7c --- /dev/null +++ b/.wolf/bugs/bug-1870.md @@ -0,0 +1,41 @@ +--- +id: "bug-1870" +status: "fixed" +status_updated: "2026-09-26" +status_note: "63fd3d2a: checkpoints no longer write the LSN-consuming WAL_CHECKPOINT marker; force-snapshot, config-upgrade-wal-trim, sequential soaks PASS." +component: "controller" +severity: "unknown" +timestamp: "2026-09-26T09:55:09Z" +last_seen: "2026-09-26T09:55:09Z" +occurrences: 1 +files: ["src/controller/persist.c (checkpoint marker append)", "src/controller/repl.c / config_repl.c (broadcast / catch-up)"] +tags: ["ha", "replication", "checkpoint", "wal", "force-snapshot", "soak", "livelock"] +related: [] +fix_commits: ["63fd3d2a"] +legacy: true +gitea_issue: 296 +gitea_synced_status: "fixed" +--- + +# bug-1870: xce-test-ha --force-snapshot FAIL: 'could not submit 110 jobs to trigger a checkpoint' -- writes… + +## Description +xce-test-ha --force-snapshot FAIL: 'could not submit 110 jobs to trigger a checkpoint' -- writes stall right after the first max-wal-records checkpoint. The primary's WAL holds a CHECKPOINT entry (plen 0) at the checkpoint LSN that no standby ever receives (primary ctrl-1: 109 JOB_SUBMIT, 110 CHECKPOINT, 111 JOB_SUBMIT; standbys stop at 109). Every later AppendEntries carries prevIdx=, standbys reply ae_reject_logmismatch, the primary marks catch-up and re-sends the last common entry forever: 'commit: async continuation deadline expired without commit', 'HA quorum lost: connected=0'. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not yet root-caused. The checkpoint marker is appended to the local WAL (and consumes an LSN in the replicated log) without being broadcast to / streamed to standbys, so the log has a hole standbys can never fill. Reproduced on a565779d (baseline, 2026-09-26) and on the bug-1867 working tree -- independent of both. Repro: nix develop ./src --command bash .claude/skills/xce-test-ha/run-ha-soak.sh --force-snapshot (max-wal-records=100, wal-segment-bytes=4096). + +## Fix +UPDATE (2026-09-27): follow-up cb1415d7: load_snapshot seeded current_lsn = checkpoint_lsn (off by one, hidden by the removed marker) -- broke --config-upgrade soak; fixed + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Gitea: #296 diff --git a/.wolf/bugs/bug-1871.md b/.wolf/bugs/bug-1871.md new file mode 100644 index 00000000..90bc0e48 --- /dev/null +++ b/.wolf/bugs/bug-1871.md @@ -0,0 +1,41 @@ +--- +id: "bug-1871" +status: "investigating" +status_updated: "2026-09-27" +status_note: "found while soak-verifying the HA group; logs in /home/claude/tmp/xce-ha-final-force-snapshot" +component: "lib" +severity: "unknown" +timestamp: "2026-09-27T07:03:48Z" +last_seen: "2026-09-27T07:03:48Z" +occurrences: 1 +files: ["src/lib/netio.c (endpoint_block_tasks/unblock", "handle_input)", "src/controller/ha.c ha_handle_snapshot_chunk"] +tags: ["netio", "epoll", "snapshot", "latency", "ha", "soak"] +related: ["bug-1870"] +legacy: true +gitea_issue: 297 +gitea_synced_status: "investigating" +--- + +# bug-1871: HA snapshot transfer to a rejoining standby stalls mid-stream: primary queues all chunks at once… + +## Description +HA snapshot transfer to a rejoining standby stalls mid-stream: primary queues all chunks at once (528KB, 9 chunks); the standby reads a few (1 or 5 of 9 in two runs), its endpoint is unblocked after the last processed chunk, and then no further input event arrives on that connection for 3s+ although the primary has more chunks queued -- --force-snapshot soak's single post-convergence check saw no install (2 of 4 runs, 2026-09-27) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Not root-caused. Suspect: edge-triggered epoll input + per-endpoint task blocking (netio endpoint_block_tasks): data that arrives while the endpoint is blocked is not re-signalled after endpoint_unblock_tasks, so reading resumes only on the next write from the peer (e.g. the next heartbeat). Likely delays, not loses, the transfer; the soak check was changed to poll 30s. + +## Fix +_Not recorded (migrated from buglog.json)._ + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-1870 +- Gitea: #297 diff --git a/.wolf/bugs/bug-206.md b/.wolf/bugs/bug-206.md new file mode 100644 index 00000000..ac11a079 --- /dev/null +++ b/.wolf/bugs/bug-206.md @@ -0,0 +1,38 @@ +--- +id: "bug-206" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-05-31T08:40:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["term-fencing", "raft", "ha", "xce-testing", "test-seam", "vote-reply"] +related: [] +legacy: true +--- + +# bug-206: test_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 — ha->term must be atomically updated… + +## Description +test_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 — ha->term must be atomically updated to peer term when peer term > ours + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +vote_reply_msg_handler delegated term fencing entirely to raft_cb_become_follower (via raft_on_vote_reply). In XCE_TESTING builds, vote_reply_dispatch calls xce_vote_reply_hook and returns early, bypassing raft_on_vote_reply and thus never updating ha->term. + +## Fix +Added explicit CAS loop in vote_reply_msg_handler before vote_reply_dispatch to advance ha->term when vr_term > ha->term. This applies on both the production path and the XCE_TESTING hook path. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-208.md b/.wolf/bugs/bug-208.md new file mode 100644 index 00000000..83e97e75 --- /dev/null +++ b/.wolf/bugs/bug-208.md @@ -0,0 +1,38 @@ +--- +id: "bug-208" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "leak" +timestamp: "2026-05-31" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/testsuite/test.c"] +tags: ["asan", "leak", "scheduler", "test", "host_eval_ctx"] +related: [] +legacy: true +--- + +# bug-208: ASAN LeakSanitizer: detected memory leaks in scheduler testsuite — build exits with error code + +## Description +ASAN LeakSanitizer: detected memory leaks in scheduler testsuite — build exits with error code + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +sched_meet_host_selection() lazily allocates w->host_eval_ctx via build_host_eval_ctx(). Four test functions (test_meet_host_selection_always_true_passes, test_meet_host_selection_always_false_fails, test_meet_host_selection_numeric_match, test_meet_host_selection_string_match) called zarray_fini() without first calling sched_worker_release_host_ctx() to free the lazily-allocated host_eval_ctx. + +## Fix +Added sched_worker_release_host_ctx(&w) calls before zarray_fini() in all four affected test functions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-320.md b/.wolf/bugs/bug-320.md new file mode 100644 index 00000000..fe546728 --- /dev/null +++ b/.wolf/bugs/bug-320.md @@ -0,0 +1,38 @@ +--- +id: "bug-320" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-05T06:50:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["ha", "majority-convergence", "n_peers", "off-by-one", "integration-test"] +related: [] +legacy: true +--- + +# bug-320: 3-node startup: majority convergence never fires; cluster falls through to committing_timeout… + +## Description +3-node startup: majority convergence never fires; cluster falls through to committing_timeout degraded-mode WARN + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha->n_peers was set to candidates_count (3, total including self) in ha_start_election_async. ha_check_majority_convergence loops i=0..n_peers-1 checking standby_converged[], but standby_converged[] is indexed by votes_idx (relative, self excluded: 0..n_other-1). With n_peers=3, it checked standby_converged[2] which is never set, so n_converged (2) < n_peers (3) always — majority convergence path could never succeed. + +## Fix +Changed ha->n_peers = n_peers to ha->n_peers = n_other (= n_peers-1 = other-node count). Moved the assignment after n_other is computed. ha_repl_ack_received quorum formula (ha->n_peers+2)/2 still yields correct majority for both old and new value. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-335.md b/.wolf/bugs/bug-335.md new file mode 100644 index 00000000..467723fa --- /dev/null +++ b/.wolf/bugs/bug-335.md @@ -0,0 +1,38 @@ +--- +id: "bug-335" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-06-09T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/scheduler.c"] +tags: ["scheduler", "resvals", "latent-bug", "gap-analysis", "c"] +related: [] +legacy: true +--- + +# bug-335: UPDATE_RESVALS dequeue writes worker resval at index 0 unconditionally instead of resolving… + +## Description +UPDATE_RESVALS dequeue writes worker resval at index 0 unconditionally instead of resolving resource idx by name + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +zarray_put(&w->resvals, 0, rv->val) hardcodes index 0; worker resvals are indexed by resource idx (see schedobj.c:291) + +## Fix +FIXED 2026-06-10: UPDATE_RESVALS now resolves rv->name via key_find on sched->resmgr and writes to r->idx (matching UPDATE_RESVALS_FULL pattern) instead of hardcoding index 0. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-336.md b/.wolf/bugs/bug-336.md new file mode 100644 index 00000000..6f772ed2 --- /dev/null +++ b/.wolf/bugs/bug-336.md @@ -0,0 +1,38 @@ +--- +id: "bug-336" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-06-09T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/src/scheduler.c"] +tags: ["scheduler", "resources", "latent-bug", "gap-analysis", "silent-error", "c"] +related: [] +legacy: true +--- + +# bug-336: XCE_SCHED_OP_UPDATE_RESOURCES cannot update existing resources; makeResource returns NULL/EEXIST… + +## Description +XCE_SCHED_OP_UPDATE_RESOURCES cannot update existing resources; makeResource returns NULL/EEXIST and return value is silently ignored + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +makeResource_ in src/lib/resobj.c rejects duplicates with EEXIST; dequeue case ignores NULL return, so 'add or update' (EARS-UB-1.1) is only 'add' + +## Fix +FIXED 2026-06-10: UPDATE_RESOURCES now checks makeResource return; on NULL (EEXIST) looks up existing resource by name and updates valtype/ordtype/flags/interval fields in place (add-or-update semantics). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-337.md b/.wolf/bugs/bug-337.md new file mode 100644 index 00000000..eeeda248 --- /dev/null +++ b/.wolf/bugs/bug-337.md @@ -0,0 +1,38 @@ +--- +id: "bug-337" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-09T00:02:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["ha", "heartbeat", "liveness", "vote-gate", "dead-field", "gap-analysis"] +related: [] +legacy: true +--- + +# bug-337: Vote-deny gate inert: ha_heartbeat_is_fresh reads ha->last_heartbeat which has no production… + +## Description +Vote-deny gate inert: ha_heartbeat_is_fresh reads ha->last_heartbeat which has no production writer after the liveness refactor + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha-heartbeat-liveness refactor moved activity tracking to ha->last_activity but ha_heartbeat_is_fresh (ha.c:1762) still reads the orphaned last_heartbeat timespec; only tests write it, so freshness is always stale after ~heartbeat_timeout_ms of uptime and standbys grant votes despite a live primary (gates at ha.c:214 and ha.c:1841) + +## Fix +FIXED 2026-06-09: ha_heartbeat_is_fresh now reads ha->last_activity (returns 0 when last_activity==0); dead last_heartbeat field removed from ha_state_t; test_ha_vote_gate.c helper rewritten as set_last_activity_age and fresh/stale tests now also set primary_ctrl_fd. All tests green. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-338.md b/.wolf/bugs/bug-338.md new file mode 100644 index 00000000..0ca55f43 --- /dev/null +++ b/.wolf/bugs/bug-338.md @@ -0,0 +1,38 @@ +--- +id: "bug-338" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-09T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["ha", "election", "vote-gate", "raft", "unit-test", "uncommitted-drift", "gap-analysis"] +related: [] +legacy: true +--- + +# bug-338: controller_unit_test: test_vote_granted_to_same_candidate_same_term FAIL (Expected 1 Was 0)… + +## Description +controller_unit_test: test_vote_granted_to_same_candidate_same_term FAIL (Expected 1 Was 0); test_standby_fresh_heartbeat_denies_vote_request FAIL (Expected 0 Was 1, Req 6.2) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Uncommitted handle_request_vote changes: (1) duplicate-vote deny made candidate-agnostic (denies same-candidate idempotent re-grant, diverging from raft.c which keeps re-grant); (2) STANDBY vote deny now requires primary_ctrl_fd >= 0, tests use ha_init default fd=-1 + +## Fix +FIXED 2026-06-09: handle_request_vote Req 2.6 check restored to same-candidate idempotent re-grant (deny only when voted_for_name differs from candidate_id), matching raft.c; test_standby_fresh_heartbeat_denies_vote_request updated to set primary_ctrl_fd=999. Also fixed myapp.c leader_known check to treat a vote for myconf->name as a self-vote (production records real name, not the 'self' test placeholder). All tests green. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-339.md b/.wolf/bugs/bug-339.md new file mode 100644 index 00000000..ba48190b --- /dev/null +++ b/.wolf/bugs/bug-339.md @@ -0,0 +1,38 @@ +--- +id: "bug-339" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-09T17:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/ha.c", "src/controller/myapp.c", "src/controller/testsuite/test_ha_rdhup_integration.c"] +tags: ["guard-clause", "c", "raft", "election-safety", "restart", "persistence", "gap-analysis", "ha", "election", "rdhup", "failover", "freshness-gate", "committed_lsn", "applied_lsn", "split-vote", "infinite-loop"] +related: ["bug-338"] +legacy: true +--- + +# bug-339: After killing the primary in a 3-node controller HA cluster, the two surviving controllers enter… + +## Description +After killing the primary in a 3-node controller HA cluster, the two surviving controllers enter an infinite election loop: both only ever raft_cb_become_follower (never become_leader), terms climb in lockstep (4..23), RDHUP -> CTRL_ELECTING -> CTRL_STANDBY repeats forever. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +LSN-source asymmetry between candidate-advertised LSN and voter freshness floor, specific to the RDHUP failover election path. ha_on_primary_ctrl_fd_rdhup advertised c->persist.committed_lsn as the RequestVote last_log_lsn, but a STANDBY never advances persist.committed_lsn (only the primary's commit engine writes it at persist.c:1192). Standbys advance applied_lsn (ha_apply_wal_catchup_entry) and last_known_committed_lsn (heartbeat replies). The voter-side freshness floor (cluster.c OPCode_requestVote) is max(applied_lsn, last_known_committed_lsn) via ha_vote_self_freshness. So once a standby applied any WAL beyond its last snapshot, every surviving standby advertised a stale-low LSN (e.g. 0) while each peer computed a HIGH floor (e.g. 10); the freshness gate cand_lsn < my_lsn tripped on BOTH sides, each denied the other's vote, neither reached quorum (2 of 3), each election timed out -> become_follower -> 10s liveness timeout -> RDHUP -> re-elect at term+1. The boot path (myapp.c:217) already advertised applied_lsn, so cold-boot election worked; only the RDHUP failover path was inconsistent. + +## Fix +In ha_on_primary_ctrl_fd_rdhup, compute candidate freshness ONCE via ha_vote_self_freshness(ha, config_version_tag, applied_lsn, &cand_tag, &cand_lsn) right after ha->raft_c=c, and use cand_tag/cand_lsn for BOTH the elect_ctx (votes[0].my_tag/my_lsn + self_tag/self_lsn) and the XCE_TESTING ha_rdhup_election_hook args. This makes candidate-advertised LSN identical to the voter-side floor, so for any two nodes A,B: A grants B iff floor_B >= floor_A -> the freshest node is always granted by every peer; equal-floor ties are broken by raft priority delay. Regression test added: test_rdhup_candidate_advertises_freshness_floor_not_stale_committed (applied/learned=10, persist.committed=0 -> asserts advertised my_lsn==10). RED (Was 0) -> GREEN verified. Follow-up: boot election path (myapp.c) also routed through ha_vote_self_freshness (was raw applied_lsn) so both election paths are symmetric with the voter floor. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-338 diff --git a/.wolf/bugs/bug-340.md b/.wolf/bugs/bug-340.md new file mode 100644 index 00000000..98e64e07 --- /dev/null +++ b/.wolf/bugs/bug-340.md @@ -0,0 +1,38 @@ +--- +id: "bug-340" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-10T00:23:21.918Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c", "src/controller/testsuite/test_ha_liveness.c"] +tags: ["logic-fix", "c", "ha", "heartbeat", "rejoin", "restart", "standby-endpoints", "reconnect", "epoll", "liveness", "election-loop"] +related: [] +legacy: true +--- + +# bug-340: After kill+RESTART of a controller (ctrl-1), the restarted node loops forever… + +## Description +After kill+RESTART of a controller (ctrl-1), the restarted node loops forever: ha_on_primary_ctrl_fd_rdhup → CTRL_ELECTING → raft_cb_become_follower → CTRL_STANDBY every ~heartbeat_timeout, term climbing unbounded (473+), while ctrl-2 stays PRIMARY at term 3. On ctrl-2: every heartbeat tick logs `xce_encode op 0 (NOP)` then `WARN endpoint_update_epoll rc1=-1 < 0` (netio.c:575) — a wedged outbound standby endpoint. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +primary_heartbeat_fn never evicted a dead/wedged outbound standby endpoint from ha->standby_endpoints. When a peer is killed/restarted, its outbound ep on the primary wedges: handle_error (netio.c:420) runs iomgr_unmonitor (EPOLL_CTL_DEL, does NOT close the fd) and the err handler may leave the ep in standby_endpoints; epoll_ctl(EPOLL_CTL_MOD) on the DEL'd fd then returns ENOENT (rc1=-1) on every heartbeat enqueue. xce_nop_req still returns >=0 (enqueue ok) so the heartbeat layer never detected the wedge. Critically, primary_heartbeat_fn step 2 (reconnect) treats a peer as 'found' (connected) if ANY ep for it is in standby_endpoints — so the wedged ep SUPPRESSED reconnection. ctrl-2 therefore never opened a fresh connection to the restarted ctrl-1, never sent a ctrlhello, so ctrl-1 never set primary_ctrl_fd / refreshed last_activity and its liveness watchdog re-elected every heartbeat_timeout forever. (The earlier suspected 'become_follower doesn't reset last_activity' was NOT a bug: ha_arm_heartbeat_timeout_vtask already stamps last_activity=now, so the ~11s period is exactly one full timeout — confirming the watchdog is correct and the sole cause is the broken channel.) + +## Fix +In primary_heartbeat_fn (ha.c) step 1, added liveness eviction symmetric with the standby's own liveness poll: for each standby ep, if ep->ts_last_in > 0 (was responsive at least once) AND (now_wall - ep->ts_last_in) > heartbeat_timeout_s, remove it from ha->standby_endpoints (zarray_del, under ha->lock) and collect it; after releasing ha->lock, repl_remove_standby() each evicted ep. Removal is the load-bearing action — the reconnect scan (step 2) then re-establishes a fresh ctrlhello channel to the restarted peer on this/next tick, and ctrl-1 settles via ha_discover_primary (no term gate, ha.c:3807) → become_follower → STANDBY. ts_last_in/ts_last_out are existing endpoint_t fields (wall clock, set in handle_input/handle_output), so no libnetio change. The evicted ep object itself is reclaimed by its err handler when the dead connection finally surfaces an epoll event (bounded leak, consistent with codebase). Eviction keys on ts_last_in>0 so a just-connected joiner (ts_last_in==0) gets grace. Regression tests: test_primary_hb_evicts_silent_standby (RED→GREEN), test_primary_hb_keeps_live_standby, test_primary_hb_keeps_unconfirmed_standby in test_ha_liveness.c. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-343.md b/.wolf/bugs/bug-343.md new file mode 100644 index 00000000..0205defa --- /dev/null +++ b/.wolf/bugs/bug-343.md @@ -0,0 +1,38 @@ +--- +id: "bug-343" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-06-14T04:34:32+00:00" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_demotion_convergence.c"] +tags: ["asan", "use-after-free", "ha", "rejoin", "delaymgr", "test-harness", "ha-convergence-recovery", "task-3.2"] +related: [] +legacy: true +--- + +# bug-343: AddressSanitizer: heap-use-after-free in ha_rejoin_reconcile_fn (ha_convergence.c:236) during… + +## Description +AddressSanitizer: heap-use-after-free in ha_rejoin_reconcile_fn (ha_convergence.c:236) during test_rejoin_second_call_no_second_restart + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +dc_run_pending_task peeked tasks.head and ran entry(args) WITHOUT dequeuing. The once-guard correctly makes the 2nd ha_rejoin_reconcile schedule no new task, so head still pointed at the first (already-run) task whose body had freed its heap ctx (args). The 2nd dc_run_pending_task re-ran that task on the freed ctx. The convergence task bodies free their own args (production delaymgr killer frees only the delay_task_t node, never args; production execv never re-runs a one-shot). + +## Fix +Made dc_run_pending_task DEQUEUE via ordq_get(&mgr->tasks, NULL), run the body, then free(t)/free(it) — mirroring the real delaymgr worker (ordq_get -> runner_fn -> FREEUP for runcount==1). A 2nd call now returns DC_NO_TASK as production would. Production ha_rejoin_reconcile/ha_rejoin_reconcile_fn unchanged (the body-frees-ctx pattern is correct). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-351.md b/.wolf/bugs/bug-351.md new file mode 100644 index 00000000..5a3294ee --- /dev/null +++ b/.wolf/bugs/bug-351.md @@ -0,0 +1,38 @@ +--- +id: "bug-351" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-09T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/request.c"] +tags: ["build-failure", "pointer-type", "c", "request.c", "ha-replication"] +related: [] +legacy: true +--- + +# bug-351: request.c: passing argument 'from' of xce_wal_batch_encode_raw / xce_snapshot_chunk_encode_raw… + +## Description +request.c: passing argument 'from' of xce_wal_batch_encode_raw / xce_snapshot_chunk_encode_raw from incompatible pointer type (xce_peerinfo_t** vs xce_peerinfo_t*) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ep->peer is already xce_peerinfo_t* (netio.h endpoint struct); call sites passed &ep->peer producing xce_peerinfo_t** + +## Fix +Dropped the & at both call sites (xce_wal_batch_req_raw line 547, xce_snapshot_chunk_req_raw line 633) to pass ep->peer directly + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-352.md b/.wolf/bugs/bug-352.md new file mode 100644 index 00000000..6d5c2f28 --- /dev/null +++ b/.wolf/bugs/bug-352.md @@ -0,0 +1,38 @@ +--- +id: "bug-352" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-06-10T01:46:28.589Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c"] +tags: ["c", "ha", "election", "asan", "use-after-free", "packet-ownership", "netio", "e2e"] +related: [] +legacy: true +--- + +# bug-352: ASAN heap-use-after-free in free_packet (netio.c:333) via endpoint_fini <- raft_cb_become_follower… + +## Description +ASAN heap-use-after-free in free_packet (netio.c:333) via endpoint_fini <- raft_cb_become_follower (loser path) <- raft_on_vote_reply; election loser crashes on vote-reply processing + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +vote_reply_decode_and_dispatch took pkt = ep->fsm_in.pkt and freed it WITHOUT clearing ep->fsm_in.pkt; vote_reply_dispatch can synchronously reach raft_cb_become_follower -> endpoint_fini(ep), which frees the stale fsm_in.pkt again. repl_ack_decode_and_dispatch already nulls the pointer (correct pattern); the vote-reply path missed it. + +## Fix +FIXED 2026-06-10: ep->fsm_in.pkt = NULL immediately after taking ownership in vote_reply_decode_and_dispatch (ha.c:2413), matching repl_ack_decode_and_dispatch. Found by task 7.7 E2E (3-node deploy, election loser crash). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-353.md b/.wolf/bugs/bug-353.md new file mode 100644 index 00000000..df195793 --- /dev/null +++ b/.wolf/bugs/bug-353.md @@ -0,0 +1,38 @@ +--- +id: "bug-353" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-10T01:46:45.213Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c"] +tags: ["wrong-reference", "c", "ha", "dispatch-gate", "election", "config-replication", "silent-failure"] +related: ["bug-335", "bug-336"] +legacy: true +--- + +# bug-353: Config push silently rejected when standby is in CTRL_ELECTING: ha_dispatch_gate only allows… + +## Description +Config push silently rejected when standby is in CTRL_ELECTING: ha_dispatch_gate only allows requestVote/probeConfigTag, returning -E_XCE_ERR_NOT_READY for all other opcodes. If liveness poll fires during election window, any incoming config push is discarded. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CTRL_ELECTING gate in ha_dispatch_gate (ha.c:1464) allowed only vote/probe opcodes. Config push from primary arriving during the ELECTING window (after liveness-poll-triggered failover, before election result) was rejected with -E_XCE_ERR_NOT_READY (10013). Standby never saved config, leaving conf dir empty. + +## Fix +FIXED 2026-06-10: Added OPCode_hello, OPCode_nop, OPCode_replConfigPush, OPCode_walCatchupEntry, OPCode_snapshotTransfer to CTRL_ELECTING allowlist. A campaigning node must accept replication from the current primary to stay current for both win and loss paths. Controller suite green. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-335, bug-336 diff --git a/.wolf/bugs/bug-378.md b/.wolf/bugs/bug-378.md new file mode 100644 index 00000000..9ab91c80 --- /dev/null +++ b/.wolf/bugs/bug-378.md @@ -0,0 +1,38 @@ +--- +id: "bug-378" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-10T10:02:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_p2_integration.c"] +tags: ["asan", "global-buffer-overflow", "repl_broadcast", "unit-test", "c"] +related: [] +legacy: true +--- + +# bug-378: AddressSanitizer: global-buffer-overflow in dataq_enq via xce_wal_batch_req_raw <- repl_broadcast… + +## Description +AddressSanitizer: global-buffer-overflow in dataq_enq via xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Tests 8 and 9 declared ep_a/ep_b/g_ep9_a/g_ep9_b as 'static int' (4 bytes) and passed them to repl_add_standby without (void*) cast. repl_broadcast then called xce_wal_batch_req_raw(s->ep, ...) using the stored int* as a real network endpoint, causing an 8-byte read past the 4-byte int. Neither test installed repl_broadcast_send_hook to intercept the call. + +## Fix +Added #include typemap.h; added extern repl_broadcast_send_hook declaration guarded by XCE_TESTING; added p2_drop_send_hook no-op; installed hook around persist_wal_sync_append in tests 8 and 9; fixed missing (void*) casts on all four ep variables + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-396.md b/.wolf/bugs/bug-396.md new file mode 100644 index 00000000..51d2efbc --- /dev/null +++ b/.wolf/bugs/bug-396.md @@ -0,0 +1,38 @@ +--- +id: "bug-396" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "worker" +severity: "unknown" +timestamp: "2026-06-10T22:45:00-07:00" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/worker/src/controller_connector.c", "src/lib/xceapi.c", "src/controller/cluster.c"] +tags: ["ha", "notify", "silent-data-loss", "worker", "quorum", "ha-consistency-invariants"] +related: [] +legacy: true +--- + +# bug-396: Worker job state notifications (OPCode_notify) are permanently lost when the controller dispatch… + +## Description +Worker job state notifications (OPCode_notify) are permanently lost when the controller dispatch gate rejects them (quorum loss / COMMITTING window): worker never reads replies, never retries, no rejoin resync + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce_notify is enqueue-and-forget (no cbuf_get); error replies dropped via cbuf_tryput -EAGAIN (xcectx.c:304); xcecd_job_finished return ignored (FIXME at xcecd_worker_utils.c:298); xce_join_req carries no running-job state so rejoin cannot resync + +## Fix +FIXED 2026-06-12 (ha-commit-barrier-rollout tasks 1.2+4.2+4.3): dispatch gate admits fact opcodes (notify/update/setpreason/clrpreason) without write quorum (ha.c CTRL_PRIMARY allowlist); facts append via RETAIN (log-is-the-queue, no timeout) and commit when quorum returns. End-to-end retention proof: test_fact_retention_across_quorum_loss. Worker-side redelivery gap remains (worker-rejoin-resync backlog covers disconnect windows). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-410.md b/.wolf/bugs/bug-410.md new file mode 100644 index 00000000..0d164f43 --- /dev/null +++ b/.wolf/bugs/bug-410.md @@ -0,0 +1,38 @@ +--- +id: "bug-410" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-11T13:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_commit_engine.c"] +tags: ["controller", "testsuite", "repl", "return-convention"] +related: [] +legacy: true +--- + +# bug-410: test_commit_engine fixture: Expected 0 Was 1 from repl_add_standby + +## Description +test_commit_engine fixture: Expected 0 Was 1 from repl_add_standby + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl_add_standby returns the slot index (>=0) on success, not 0 — second add legitimately returns 1 + +## Fix +fixture asserts TEST_ASSERT_EQUAL_INT(i, repl_add_standby(...)) per standby slot + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-411.md b/.wolf/bugs/bug-411.md new file mode 100644 index 00000000..6df4e405 --- /dev/null +++ b/.wolf/bugs/bug-411.md @@ -0,0 +1,39 @@ +--- +id: "bug-411" +status: "fixed" +status_updated: "2026-09-24" +status_note: "95a10ced" +component: "controller" +severity: "unknown" +timestamp: "2026-06-11T13:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/controller_test.c"] +tags: ["controller", "testsuite", "unity", "exit-code", "pre-existing", "ha"] +related: [] +fix_commits: ["95a10ced"] +legacy: true +--- + +# bug-411: 3 pre-existing controller testsuite failures masked by exit code 0… + +## Description +3 pre-existing controller testsuite failures masked by exit code 0: test_committed_lsn_does_not_advance_on_timeout (-110), test_repl_ack_after_promote_gate_allows_jobsub (-10013), test_rdhup_primary_dispatch_gate_opens (-10013) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +each test group calls UNITY_BEGIN/UNITY_END internally, resetting Unity counters — main()'s final UNITY_END returns 0 unless the LAST group fails, so meson reported the suite green despite 3 real failures; Unity also misattributes failure FILE names to whichever file's UNITY_BEGIN ran (e.g. reports test_repl_unit.c for a test living in test_p2_integration.c) + +## Fix +not fixed (out of task 1.1 boundary) — documented; failures exist identically on baseline without ha-commit-engine changes (verified by stash/rebuild). persist_wal_sync_append returns -ETIMEDOUT while the test expects rc==0 with sync_rc==-ETIMEDOUT; the two gate tests fail ha_has_write_quorum (standby_endpoints empty with n_peers=2) + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-435.md b/.wolf/bugs/bug-435.md new file mode 100644 index 00000000..5c2e1a48 --- /dev/null +++ b/.wolf/bugs/bug-435.md @@ -0,0 +1,38 @@ +--- +id: "bug-435" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-11T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_p2_integration.c"] +tags: ["ha-commit-engine", "task-6.1", "stale-expectation", "unity-misattribution"] +related: ["bug-411"] +legacy: true +--- + +# bug-435: test_committed_lsn_does_not_advance_on_timeout: Expected 0 Was -110 (baseline failure, masked by… + +## Description +test_committed_lsn_does_not_advance_on_timeout: Expected 0 Was -110 (baseline failure, masked by bug-411) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Stale test expectation: pinned the old accept-on-timeout contract (rc==0 + sync_rc=-ETIMEDOUT advisory). ha-commit-engine made persist_wal_sync_append reject-on-timeout (returns -ETIMEDOUT, Req 2.1/4.4). Fixture was also engine-less (deprecated ce==NULL legacy path). + +## Fix +Updated expectation to rc==-ETIMEDOUT and wired a commit engine (n_peers=3) with a priming async append so the sync entry lands at LSN 1 on the engine path; kept the committed_lsn-unmoved assertion. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-411 diff --git a/.wolf/bugs/bug-436.md b/.wolf/bugs/bug-436.md new file mode 100644 index 00000000..3b0e9d5a --- /dev/null +++ b/.wolf/bugs/bug-436.md @@ -0,0 +1,38 @@ +--- +id: "bug-436" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-11T00:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_ha_repl_ack.c", "src/controller/testsuite/test_ha_rdhup_integration.c"] +tags: ["ha-commit-engine", "task-6.1", "quorum-gate", "fixture-fix"] +related: ["bug-411"] +legacy: true +--- + +# bug-436: test_repl_ack_after_promote_gate_allows_jobsub / test_rdhup_primary_dispatch_gate_opens: Expected… + +## Description +test_repl_ack_after_promote_gate_allows_jobsub / test_rdhup_primary_dispatch_gate_opens: Expected 0 Was -10013 (NOT_READY; baseline failures, masked by bug-411) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CTRL_PRIMARY dispatch gate now requires write quorum (>= n_peers/2 connected standbys). Fixtures promote via a standby ACK but never register the connected standby; legacy fallback counts ha->standby_endpoints (NULL -> 0). In the rdhup test ha_on_primary_ctrl_fd_rdhup also overwrites ha->n_peers=1 with candidates_count=2 (ha.c:3484). + +## Fix +Populated standby_endpoints with one endpoint (the ACKing standby) in both fixtures so quorum legitimately holds on a healthy primary; gate unchanged. Freed the zarray before ha_fini (ha_fini does not own it). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-411 diff --git a/.wolf/bugs/bug-445.md b/.wolf/bugs/bug-445.md new file mode 100644 index 00000000..8e243a59 --- /dev/null +++ b/.wolf/bugs/bug-445.md @@ -0,0 +1,38 @@ +--- +id: "bug-445" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "crash" +timestamp: "2026-06-12T09:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_commit_barrier_reply.c"] +tags: ["testsuite", "xstring", "fixture", "segv", "ha-commit-barrier-rollout"] +related: [] +legacy: true +--- + +# bug-445: AddressSanitizer SEGV in hash_string via makeResource when running controller_unit_test -t… + +## Description +AddressSanitizer SEGV in hash_string via makeResource when running controller_unit_test -t test_commit_barrier_reply standalone + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Global xstring table uninitialised in standalone runs: makeString() returns NULL, resobj name becomes NULL, key_add hashes NULL. In full-suite runs test_worker_join.c:520 initialises the table unguarded, so the bug only appears with -t selection. + +## Fix +Probe makeString("cbrp-probe") in the group entry; call xstring_table_init(139) only when the probe returns NULL (avoids double init in full-suite order). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-446.md b/.wolf/bugs/bug-446.md new file mode 100644 index 00000000..f229e79a --- /dev/null +++ b/.wolf/bugs/bug-446.md @@ -0,0 +1,38 @@ +--- +id: "bug-446" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-12T09:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_commit_barrier_reply.c"] +tags: ["testsuite", "jobj", "fixture", "ha-commit-barrier-rollout"] +related: [] +legacy: true +--- + +# bug-446: makeJob() returns NULL (Expected Non-NULL) in commit-barrier reply fixture + +## Description +makeJob() returns NULL (Expected Non-NULL) in commit-barrier reply fixture + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +job_init (lib/jobj.c) returns -EINVAL when xce_job_t.submit == NULL, and makeJobsub requires job_obj_init() module tables which no controller test group initialises. + +## Fix +Fixture calls job_obj_init(64, 64) once per process (static guard) and provides a zeroed static job_submit_data_t as jdef.submit. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-447.md b/.wolf/bugs/bug-447.md new file mode 100644 index 00000000..ff1d0ebc --- /dev/null +++ b/.wolf/bugs/bug-447.md @@ -0,0 +1,38 @@ +--- +id: "bug-447" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "leak" +timestamp: "2026-06-12T09:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/queue.c"] +tags: ["production-gap", "queue_ctrl", "worker_ctrl", "job_ctrl", "key_find", "wal", "ha-commit-barrier-rollout", "red-finding"] +related: [] +legacy: true +--- + +# bug-447: queue_ctrl/worker_ctrl append NO WAL entry and reply status 1 on success; job_ctrl also returns 1… + +## Description +queue_ctrl/worker_ctrl append NO WAL entry and reply status 1 on success; job_ctrl also returns 1 (key_find found-rc leak) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +key_find (xkvmgr_find, lib/kvmgr.c:130-151) returns 1 when the key is found. queue_ctrl (queue.c:60), worker_ctrl (worker.c:643) and job_ctrl (job.c:314) store it in rc and never reset it: the 'if (rc == 0)' AFTER_EVENT hook gate is skipped (no WAL_QUEUE_CTRL / WAL_WORKER_CTRL append on the found path) and the requester reply carries status 1 instead of 0. Production gap discovered by task-3.1 RED tests; queue/host control state changes are currently never persisted or replicated. + +## Fix +FIXED (task 3.2, 2026-06-12): normalized the key_find found-rc to 0 in queue_ctrl (queue.c), worker_ctrl (worker.c) and job_ctrl (job.c) after the miss/-ENOENT checks, so the AFTER_EVENT hook gate fires (WAL append + replication) and the handlers' commit-barrier pattern sees 0. Handlers do_jobcontrol_op/do_queuecontrol_op/do_hostcontrol_op now run persist_commit_barrier(&c->persist, 0) on rc==0 before replying; handle_worker_join barriers between AFTER_EVENT and the success reply. All 12 test_commit_barrier_reply tests PASS; full controller suite 0 FAIL. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-448.md b/.wolf/bugs/bug-448.md new file mode 100644 index 00000000..bc7721b2 --- /dev/null +++ b/.wolf/bugs/bug-448.md @@ -0,0 +1,39 @@ +--- +id: "bug-448" +status: "fixed" +status_updated: "2026-09-27" +status_note: "0c1bac44" +component: "controller" +severity: "leak" +timestamp: "2026-06-12T10:05:12.464Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/job.c"] +tags: ["wrong-value", "c", "controller", "job", "refcount", "leak", "pre-existing"] +related: ["bug-447"] +fix_commits: ["0c1bac44"] +legacy: true +--- + +# bug-448: job_ctrl reference leak: reference(j) taken at job.c:317 but normal exit path (job.c:444-446)… + +## Description +job_ctrl reference leak: reference(j) taken at job.c:317 but normal exit path (job.c:444-446) never calls dereference(j); only the workermgr-miss error path releases it + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +asymmetric reference/dereference across exit paths, predates ha-commit-barrier-rollout + +## Fix +NOT YET FIXED - discovered by task 3.2 adversarial review; add dereference(j) on the normal exit path + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-447 diff --git a/.wolf/bugs/bug-451.md b/.wolf/bugs/bug-451.md new file mode 100644 index 00000000..2f679bf3 --- /dev/null +++ b/.wolf/bugs/bug-451.md @@ -0,0 +1,41 @@ +--- +id: "bug-451" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-12T11:05:00-07:00" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/job.c (caller) / src/controller/persist.c (consumer)"] +tags: ["pre-existing", "type-confusion", "va_list", "wal", "rusage", "asan", "oob-read", "ha-commit-barrier-rollout"] +related: ["bug-447", "bug-448"] +legacy: true +--- + +# bug-451: va_list type confusion: WAL_JOB_RUSAGE hook payload reads job_t fields from an xce_job_rusages_t* + +## Description +va_list type confusion: WAL_JOB_RUSAGE hook payload reads job_t fields from an xce_job_rusages_t* + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +update_job_rusages fires AFTER_EVENT HOOK_DETAIL_JOB_RUSAGE_UPDATE with (cluster_t*, xce_job_rusages_t*) at job.c:1015-1016, but persist_job_hook_fun's default arm (persist.c:1354-1359) reads the second vararg as job_t* and dereferences j->job.state at offset 56 == sizeof(xce_job_rusages_t) — an out-of-bounds read on an exactly-sized allocation; the WAL_JOB_RUSAGE payload state field is garbage (jobid at offset 0 lines up by accident) + +## Fix +FIXED (TDD RED->GREEN). Added a dedicated HOOK_DETAIL_JOB_RUSAGE_UPDATE arm to persist_job_hook_fun (persist.c) that reads the real xce_job_rusages_t* payload and looks the job up by jobid via the cluster_t* the hook already carries (c->jobmgr), instead of falling through to the default arm that misread it as a job_t*. Fix candidate (1) from the original investigation was used -- a dedicated RUSAGE arm -- not candidate (2) (firing the hook with (c, j) like other job hooks), because sched_job_rusage_update_fun/apiserver_job_rusage_update_fun (this hook's other two listeners, sched.c/apiserver.c) genuinely need the full rusages payload (pids/pgids/usages), so changing what job.c fires would have broken them. Regression test (test_commit_barrier_reply.c's test_update_op_rusage_receipt_reply_facts_retained, extended): drives the real do_update_op/update_job_rusages call path and scans the resulting WAL_JOB_RUSAGE entry, asserting state == JOB_STAT_PEND (the job's real state) -- RED under the old code (read back 0, the zeroed padding past xce_job_rusages_t's end), GREEN under the fix. Also fixed test_persist_hooks.c's test_job_hook_rusage_writes_wal_entry, which previously masked the bug by passing a job_t* stub instead of the real xce_job_rusages_t* shape -- now builds a minimal real cluster+jobmgr+job and passes the correct payload type; added a sibling test for the job-not-found graceful-skip path (returns 0, appends nothing, no crash). Full controller_unit_test suite: 1096 tests, 2 pre-existing unrelated failures (test_config_repl.c, filed as bug-1561 -- overlayfs fdatasync artifact, zero diff in that file this session), no regressions. + +[Original entry] +NOT FIXED (discovered while writing task 3.3 receipt-ACK verdict pins; production out of boundary). Test test_update_op_rusage_receipt_reply_facts_retained uses a padded union so the pin stays in-bounds under ASAN without masking the bug; fix candidates: dedicated RUSAGE arm in persist_job_hook_fun reading xce_job_rusages_t*, or fire the hook with (c, j) like the other job hooks + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-447, bug-448 diff --git a/.wolf/bugs/bug-459.md b/.wolf/bugs/bug-459.md new file mode 100644 index 00000000..3621a7bb --- /dev/null +++ b/.wolf/bugs/bug-459.md @@ -0,0 +1,38 @@ +--- +id: "bug-459" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-13T00:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/repl.c", "src/controller/config_repl.c"] +tags: ["ha", "config-repl", "quorum", "configAck", "commit-barrier", "upgradeConfig"] +related: ["bug-396"] +legacy: true +--- + +# bug-459: configAck quorum fabricated success: counted acked_config_lsn > 0 (any previous upgrade's ack)… + +## Description +configAck quorum fabricated success: counted acked_config_lsn > 0 (any previous upgrade's ack), denominator n_active with no self-count, and n_active==0 short-circuited to success; upgradeConfig also ACKed the admin tool unconditionally and requested restart even when the majority wait timed out (logged-only result) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +legacy connectivity-based counting on the configAck track predated the commit engine's per-push arithmetic; push_config_to_standbys returned the transport push count and only logged the barrier outcome, so do_upgrade_config_with_bytes could not observe failure + +## Fix +task 5.2: configAck branch counts acked_config_lsn >= push_tag (entry_lsn carries the tag), majority 1+count > n_peers/2 over engine n_peers, n_peers<=1 immediate 0, ce==NULL -EINVAL fail-loud; push_config_to_standbys returns the barrier result (push count via acked_out); do_upgrade_config_with_bytes sends send_upgrade_error and skips success ACK + restart on non-zero; committed_config_tag advance stays success-branch-only + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-396 diff --git a/.wolf/bugs/bug-465.md b/.wolf/bugs/bug-465.md new file mode 100644 index 00000000..395cb12a --- /dev/null +++ b/.wolf/bugs/bug-465.md @@ -0,0 +1,38 @@ +--- +id: "bug-465" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-13T01:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["ha", "config-repl", "barrier", "capnp", "wire-path", "task-5.2"] +related: [] +legacy: true +--- + +# bug-465: Config-ACK signal never produced in production: repl_on_config_ack had zero production callers… + +## Description +Config-ACK signal never produced in production: repl_on_config_ack had zero production callers; standby replconfigpushreplymsg silently dropped in repl_ack_decode_and_dispatch (ha.c) and myreq_handler default case (cluster.c) -> every multi-member config upgrade barrier timed out (upgrades bricked) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha-config-bootstrap task 8.1's production ingestion half was never landed; the 5.2 barrier relied on a signal nothing produced (tests injected repl_on_config_ack directly) + +## Fix +Added replconfigpushreplymsg decode branch in repl_ack_decode_and_dispatch (ha.c) and an OPCode_reply ingestion case in cluster.c myreq_handler, both calling repl_on_config_ack(&c->persist.repl, ep, versionTag); wire-path integration test added (test_config_ack_wire_path_completes_barrier) + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-466.md b/.wolf/bugs/bug-466.md new file mode 100644 index 00000000..daa0a298 --- /dev/null +++ b/.wolf/bugs/bug-466.md @@ -0,0 +1,38 @@ +--- +id: "bug-466" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-06-13T01:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c"] +tags: ["ha", "iomgr", "deadlock", "config-repl", "api-split", "task-5.2"] +related: ["bug-465"] +legacy: true +--- + +# bug-466: Unconditional 500ms config barrier wait in push_config_to_standbys reachable from iomgr epoll… + +## Description +Unconditional 500ms config barrier wait in push_config_to_standbys reachable from iomgr epoll thread (ha.c raft_cb_become_leader, ha_dispatch_repl_action_for_standby, legacy election path; join.c controller join) -> blocks all controller I/O; the awaited ACK arrives on the blocked thread (structural deadlock) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +task 5.2 made the barrier unconditional inside push_config_to_standbys while best-effort catch-up/election pushes on iomgr-reachable paths still called the same entry + +## Fix +Split the API: exported transport-only fan-out push_config_transport_only (no barrier, iomgr-safe) and switched all 5 ha.c sites + join.c:187 to it; push_config_to_standbys (transport+barrier) kept for the vtask-context upgrade path only; doc comments updated in config_repl.h + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-465 diff --git a/.wolf/bugs/bug-477.md b/.wolf/bugs/bug-477.md new file mode 100644 index 00000000..95f952a0 --- /dev/null +++ b/.wolf/bugs/bug-477.md @@ -0,0 +1,38 @@ +--- +id: "bug-477" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-13" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect — OUT OF task 2.2 boundary)"] +tags: ["ha-single-leader-role", "task-2.2", "test-fixture", "asan", "repl_broadcast", "global-buffer-overflow", "out-of-boundary"] +related: [] +legacy: true +--- + +# bug-477: AddressSanitizer: global-buffer-overflow in xce_wal_batch_req_raw <- repl_broadcast <… + +## Description +AddressSanitizer: global-buffer-overflow in xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async <- ha_arm_own_term_gate <- ha_promote_to_primary, during test_promotion_registers_persist_hooks (test_ha_role_lifecycle.c case 3) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +task-2.2's ha_promote_to_primary now appends a WAL_NOOP via persist_wal_async, which broadcasts to registered standbys. The lifecycle fixture (lifecycle_cluster_setup) registers fake `int g_lc_eps[]` endpoints via repl_add_standby and NEVER installs repl_broadcast_send_hook, so repl_broadcast calls xce_wal_batch_req_raw(s->ep) and reads ep->peer (offset 36) past the 4-byte int. IDENTICAL to the prior test_p2_integration tests 8/9 bug already in buglog. + +## Fix +Test-side fix (parent must apply, mirrors the documented p2_integration fix): in test_ha_role_lifecycle.c add `extern int (*repl_broadcast_send_hook)(void*, wal_catchup_entry_t**, int);` and a no-op drop hook, install it in lifecycle_cluster_setup before promotion (reset to NULL in teardown), and cast endpoints to (void*). Production ha.c is correct per design (persist_wal_async appends+broadcasts by design). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-478.md b/.wolf/bugs/bug-478.md new file mode 100644 index 00000000..8dff3f6b --- /dev/null +++ b/.wolf/bugs/bug-478.md @@ -0,0 +1,38 @@ +--- +id: "bug-478" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-13" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect — OUT OF task 2.2 boundary)"] +tags: ["ha-single-leader-role", "task-2.2", "test-fixture", "off-by-one", "lsn", "out-of-boundary"] +related: [] +legacy: true +--- + +# bug-478: task-2.1 lifecycle fixture off-by-one: case 5/6/7 assert writable_lsn==noop_lsn and… + +## Description +task-2.1 lifecycle fixture off-by-one: case 5/6/7 assert writable_lsn==noop_lsn and current_lsn==noop_lsn+1, but lifecycle_cluster_setup primes current_lsn = appended_lsn + 1, so a real persist_wal_async appends the noop at appended_lsn+1 (current_lsn->appended_lsn+2), giving noop_lsn=appended_lsn+1 != asserted appended_lsn. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +persist_wal_async assigns hdr.lsn = current_lsn++ (entry takes the CURRENT value). The fixture comment assumes the next append takes current_lsn-1, which is off by one. To make the noop land at `appended_lsn`, the prime must be `current_lsn = appended_lsn`, not `appended_lsn + 1`. + +## Fix +Test-side fix (parent): set g_lc.persist.current_lsn = appended_lsn in lifecycle_cluster_setup. Production noop_lsn = current_lsn-1 (post-append) is correct per design §Promotion Rewrite / brief. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-503.md b/.wolf/bugs/bug-503.md new file mode 100644 index 00000000..84b62c43 --- /dev/null +++ b/.wolf/bugs/bug-503.md @@ -0,0 +1,38 @@ +--- +id: "bug-503" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-13T13:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_ha_startup.c"] +tags: ["ha-single-leader-role", "dispatch-gate", "write-quorum", "test-retarget", "commit-barrier", "c"] +related: [] +legacy: true +--- + +# bug-503: test_dispatch_gate_primary_no_quorum_denies_job_ops: Expected -10013 Was 0 (jobsub allowed… + +## Description +test_dispatch_gate_primary_no_quorum_denies_job_ops: Expected -10013 Was 0 (jobsub allowed, expected -XCE_ERR_NOT_READY) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Retargeting the dispatch-gate test from CTRL_PRIMARY_COMMITTING to CTRL_PRIMARY (ha-single-leader-role task 5.1) changed gate semantics: the deleted COMMITTING branch denied job ops UNCONDITIONALLY, but the CTRL_PRIMARY branch only denies job ops when ha_has_write_quorum() is false. ha_init leaves n_peers=0, so ha_has_write_quorum returns true (single-node = self-majority, ha.c:1437) and jobsub was ALLOWED (rc=0). A blind role swap does not reproduce the original deny. + +## Fix +Set ha.n_peers = 2 in both retargeted dispatch-gate tests so a multi-node primary with zero connected standbys genuinely lacks write quorum (connected 0 < n_peers/2=1), making the universal commit barrier deny job ops with -XCE_ERR_NOT_READY while still permitting replication ops. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-538.md b/.wolf/bugs/bug-538.md new file mode 100644 index 00000000..246c1ef6 --- /dev/null +++ b/.wolf/bugs/bug-538.md @@ -0,0 +1,38 @@ +--- +id: "bug-538" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-14" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["ha", "election-safety", "rdhup", "config-bootstrap", "Req2.8", "Req10.5", "voter-only-guard"] +related: [] +legacy: true +--- + +# bug-538: config-less controller (my_config_tag==0) became PRIMARY after RDHUP and pushed empty config… + +## Description +config-less controller (my_config_tag==0) became PRIMARY after RDHUP and pushed empty config, breaking replication on a live 3-node cluster + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_on_primary_ctrl_fd_rdhup did not apply the no-config voter-only guard that ha_start_election_async has; after primary RDHUP it set CTRL_ELECTING, allocated elect_ctx and called raft_on_leader_lost even for tag=0 nodes. Two tag=0 nodes granted each other (freshness gate only denies LOWER tag; tag0==tag0 granted), electing a config-less primary. + +## Fix +Added an election-safety guard in ha_on_primary_ctrl_fd_rdhup after the WAL-truncate block and before elect_ctx alloc/hook/raft_on_leader_lost: if my_config_tag==0 && candidates_count>1, log + set role=CTRL_STANDBY and return (stays passive voter). Mirrors ha_start_election_async. Single-node (n_peers==1) unaffected. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-543.md b/.wolf/bugs/bug-543.md new file mode 100644 index 00000000..dc606b9d --- /dev/null +++ b/.wolf/bugs/bug-543.md @@ -0,0 +1,38 @@ +--- +id: "bug-543" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-14T09:36:36.967406Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c"] +tags: ["ha", "replication", "wal", "cold-start", "lsn-0", "catch-up", "correctness", "off-by-one"] +related: [] +legacy: true +--- + +# bug-543: WAL HOLE on cold-start catch-up: a fresh EMPTY standby (standby_lsn=0) catching up to a primary… + +## Description +WAL HOLE on cold-start catch-up: a fresh EMPTY standby (standby_lsn=0) catching up to a primary holding [0,1] receives ONLY lsn 1, missing lsn 0 + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_stream_wal_to_standby filter `if (hdr.lsn > standby_lsn)` excludes lsn 0 when standby_lsn==0 (0>0 is false). standby_lsn=0 is ambiguous: it means BOTH 'empty standby (applied nothing)' AND 'standby has applied lsn 0'. The streamer cannot distinguish, computes start_lsn=standby_lsn+1=1, and drops the genuine lsn-0 entry. Only non-cosmetic because lsn-0 entries can be REAL data (first-ever WAL entry of a fresh cluster is lsn 0 from a real write OR the promotion NOOP). + +## Fix +FIXED via reserve-lsn-0: persist_init() now sets current_lsn=1 on a fresh store (persist.c), so lsn 0 is reserved as the none/empty sentinel and the first-ever WAL entry (promotion NOOP) is lsn 1. The streamer filter hdr.lsn>standby_lsn now yields 1>0=true for an empty standby (standby_lsn=0), streaming the first entry — no hole. No streamer change needed; the reserved base makes the existing filter correct. Also fixes the uncommittable cold-start NOOP (now lsn 1, committable 0->1) and the own-term writable_lsn gate (now 1, effective). Test: test_coldstart_lsn0.c H1-H4 all green incl H4 no-hole. Base-shift fallout fixed in test_commit_engine, test_commit_barrier_rollout, test_commit_barrier_reply, test_p2_integration, test_egress_filter (removed now-redundant priming appends / wired engine). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-561.md b/.wolf/bugs/bug-561.md new file mode 100644 index 00000000..6cb90e94 --- /dev/null +++ b/.wolf/bugs/bug-561.md @@ -0,0 +1,38 @@ +--- +id: "bug-561" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-14T20:05:48.755409" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/include/config.h"] +tags: ["ha", "commit-barrier", "timeout", "replication", "jobsub", "config"] +related: [] +legacy: true +--- + +# bug-561: jobsub intermittent -ETIMEDOUT "Connection timed out": commit: WAIT deadline expired without… + +## Description +jobsub intermittent -ETIMEDOUT "Connection timed out": commit: WAIT deadline expired without commit (timeout_ms=50) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +DEFAULT_SYNC_TIMEOUT_MS=50 too tight for durable sync majority replication (each standby fdatasyncs before ACK); not configurable (controller always used persist_config_defaults). + +## Fix +Raised DEFAULT_SYNC_TIMEOUT_MS 50->500; added --sync-timeout-ms CLI flag plumbed arguments->myconf.sync_timeout_ms->pcfg.sync_timeout_ms (0=use default) in main.c/myconf.h/cluster.c. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-562.md b/.wolf/bugs/bug-562.md new file mode 100644 index 00000000..51bc7973 --- /dev/null +++ b/.wolf/bugs/bug-562.md @@ -0,0 +1,38 @@ +--- +id: "bug-562" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-06-14T23:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/request.c"] +tags: ["use-after-free", "asan", "capnp", "encode_RequestReply", "redirect_host", "init_reply_by_req", "ha", "joblist", "uninitialized-stack"] +related: [] +legacy: true +--- + +# bug-562: AddressSanitizer: heap-use-after-free in encode_RequestReply / strlen… + +## Description +AddressSanitizer: heap-use-after-free in encode_RequestReply / strlen (lib/controller.capnp.c:13168) on ctrl-2 startup, via xce_reply->xce_encode from OPCode_nop handler (cluster.c:307). Also: job submission succeeds but joblist shows no jobs. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +init_reply_by_req() only set seq+opcode, leaving the rest of the stack-declared xce_reply_t uninitialized. The Cap'n-C generated encode_RequestReply reads the common trailer field redirect_host UNCONDITIONALLY (gated only on != NULL), so a garbage stack pointer in reply.redirect_host was strlen'd. The garbage aliased a 4KB capnp segment freed moments earlier by the WAL-ack reply encode on the same thread -> UAF. Handlers that build a reply via init_reply_by_req and only set status/data_which/data (e.g. the NOP/state-report path) never cleared redirect_host. Empty joblist is downstream: ctrl-2 crash-looping on this UAF destabilizes replication/quorum, so a get request that fails over to the not-caught-up standby reads an empty jobmgr (standby serves get from its own committed state). + +## Fix +memset(reply,0,sizeof(*reply)) at the top of init_reply_by_req() (src/lib/request.c) before setting seq/opcode. All 30+ reply sites follow declare->init->populate, so zeroing in the shared initializer fixes the whole class (redirect_host/redirect_port/status/data garbage) at the source. Added regression test test_init_reply_by_req_zeroes_reply in src/lib/testsuite/test_request.c (poisons reply with 0xFF, asserts redirect_host==NULL etc). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-563.md b/.wolf/bugs/bug-563.md new file mode 100644 index 00000000..117924e1 --- /dev/null +++ b/.wolf/bugs/bug-563.md @@ -0,0 +1,38 @@ +--- +id: "bug-563" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-15T09:16:53.679025Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/request.c"] +tags: ["joblist", "workerlist", "jobsub", "ha", "check_controller", "c-running", "cluster_start", "orphaned-function", "xce_reply_get", "SOLVED"] +related: ["bug-562"] +legacy: true +--- + +# bug-563: jobsub returns ok (status 0, jobid) but joblist AND jobinfo show the job absent on a stable single… + +## Description +jobsub returns ok (status 0, jobid) but joblist AND jobinfo show the job absent on a stable single primary (ctrl-1); reslist works. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +SOLVED (reproduced locally via /xce-build + 3-node deploy-ha, ASAN). cluster_start() — the ONLY place that sets c->running=1 — is ORPHANED (zero callers; grep confirms). The HA startup path (myapp_run) starts iomgr/vtaskmgr/delaymgr/persist_start itself but never calls cluster_start, so c->running stays 0 (calloc default) on the primary. check_controller() (the streaming gate passed to xce_reply_get for joblist/workerlist) returns -EINVAL when !c->running; xce_reply_get then sets nc=0 (chunk count), shipping an EMPTY list to the client even though collect_jobs/key_find found the job. reslist passes checkf=NULL so it was immune. DIAG proof: 'reply_get send: table=2 pos=0 nc=0 nrows=1 checkstatus=-22' (pre-fix) -> 'nc=1 ... checkstatus=1' (post-fix). Single-node/legacy cluster_init path called cluster_start, which is why non-replicated worked. + +## Fix +Set c->running=1 at the TOP of cluster_run() (src/controller/cluster.c), symmetric with the existing c->running=0 on shutdown — the orphaned cluster_start() used to set it. c->running is a liveness/shutdown flag: check_controller() is polled PER CHUNK inside xce_reply_get()'s streaming loop so a long multi-chunk fetch (e.g. thousands of jobs) ABORTS mid-stream (nc=0) if a shutdown request arrives during it. The bug: running started 0 and was never set under HA, so the gate fired on every reply, not just during shutdown. Fix restores running=1 for the run-loop lifetime; the shutdown abort path (running=0 when cluster_run's cbuf_get loop breaks on the stop signal) is unchanged. Verified E2E on a local 3-node deploy-ha (ASAN): jobs now appear in joblist; DIAG showed checkstatus/nc flip -22/0 -> 1/1. Final fix is in cluster_run (not myapp_run — moved there so the 1<->0 lifecycle is co-located in the function that receives the shutdown signal). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-562 diff --git a/.wolf/bugs/bug-604.md b/.wolf/bugs/bug-604.md new file mode 100644 index 00000000..d7170d5e --- /dev/null +++ b/.wolf/bugs/bug-604.md @@ -0,0 +1,38 @@ +--- +id: "bug-604" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-16T15:55:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c"] +tags: ["controller", "persist", "fsync", "thread-shutdown", "test-timeout", "pthread_join", "nanosleep", "condvar", "ha"] +related: [] +legacy: true +--- + +# bug-604: controller unit test TIMEOUT 300.01s killed by SIGTERM; suite exceeds 300s meson cap + +## Description +controller unit test TIMEOUT 300.01s killed by SIGTERM; suite exceeds 300s meson cap + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +fsync_worker slept the full wal_sync_ms in one uninterruptible nanosleep() then checked fsync_stop. persist_stop sets the flag and immediately pthread_join()s, so the join blocked until the in-progress sleep returned — up to a full wal_sync_ms per teardown. Tests set wal_sync_ms=5000 ('slow fsync, no races'), so each persist_stop cost ~5s; across ~7 suites with many fixtures the cumulative teardown time blew past the 300s controller-test cap. The 'commit: WAIT deadline expired' warnings in the log were unrelated intentional negative tests (50-300ms). + +## Fix +Replaced the nanosleep with pthread_cond_timedwait on a new fsync_mutex/fsync_cond (added to persist_t, init in persist_init, destroyed in persist_fini). persist_stop now sets fsync_stop and pthread_cond_signal(fsync_cond) under fsync_mutex before join, so the worker wakes immediately. fdatasync still runs under p->lock, released before re-acquiring fsync_mutex (no lock nesting). Full controller suite now completes in ~10s (was >300s); 0 failures. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-605.md b/.wolf/bugs/bug-605.md new file mode 100644 index 00000000..4e9540f3 --- /dev/null +++ b/.wolf/bugs/bug-605.md @@ -0,0 +1,38 @@ +--- +id: "bug-605" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-16T16:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (ha_dispatch_gate); test: src/controller/testsuite/test_ha_role.c"] +tags: ["ha", "prevote", "ha-dispatch-gate", "opcode-38", "cold-start", "bootstrap-deadlock", "FIXED", "XCE_ERR_STANDBY"] +related: ["bug-339"] +legacy: true +--- + +# bug-605: 3-node fresh HA deploy never elects a primary: all nodes stay CTRL_STANDBY, config-holder (ctrl-1)… + +## Description +3-node fresh HA deploy never elects a primary: all nodes stay CTRL_STANDBY, config-holder (ctrl-1) loops PreVote LOST every ~11s forever; config-less peers idle as voter-only. REPRODUCED via /xce-deploy-ha + clean restart 2026-06-16. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha_dispatch_gate() admitted OPCode_requestVote in CTRL_INIT/CTRL_ELECTING/CTRL_STANDBY but NEVER OPCode_preVoteRequest (opcode 38). ha-prevote added the new opcode + handler (cluster.c:440) but missed wiring it into the role-admission gate. So a STANDBY voter rejected an incoming PreVote with -XCE_ERR_STANDBY (-10014) BEFORE the handler ran -> no preVoteReply sent -> candidate's failed/empty exchange tallied as a denial (pre_vote_reply_decode_and_dispatch counts ok=0 as non-grant; vote_ep_err_handler synthesizes a denied vote). In a cold cluster (config-holder + config-less voter-only peers) every PreVote 'lost' with NO voter ever making a liveness decision (zero 'PreVote DENIED' logs), so no primary was ever elected and the config-holder looped PreVote forever. Confirmed via gdb: myreq_handler ha_dispatch_gate rc=-10014 on opcode 38. Design invariant: a PreVote (strictly non-mutating) must be at least as admissible as a real requestVote in every role. + +## Fix +Added OPCode_preVoteRequest alongside OPCode_requestVote in ha_dispatch_gate() for CTRL_INIT, CTRL_ELECTING, and the CTRL_STANDBY switch (src/controller/ha.c). PreVote now reaches the cluster.c handler and a proper preVoteReply is sent. Added TDD test test_dispatch_gate_allows_prevote_in_voter_roles (test_ha_role.c) asserting gate(preVoteRequest)==gate(requestVote)==0 across voter roles. VERIFIED: controller unit suite OK (new test PASS); fresh /xce-deploy-ha 3-node cold start now elects ctrl-1 PRIMARY (term=2), standbys establish primary channel + config committed, prevote-LOST=0, connections 96->8. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-339 diff --git a/.wolf/bugs/bug-606.md b/.wolf/bugs/bug-606.md new file mode 100644 index 00000000..50539a70 --- /dev/null +++ b/.wolf/bugs/bug-606.md @@ -0,0 +1,38 @@ +--- +id: "bug-606" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "hang" +timestamp: "2026-06-16T17:46:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (ha_begin_failover", "ha_election_stagger_ms", "ha_failover_stagger_fn); src/controller/ha.h; test: src/controller/testsuite/test_ha_role.c"] +tags: ["ha", "raft", "election", "split-vote", "dueling-candidates", "livelock", "randomized-timeout", "failover", "stagger", "jitter", "FIXED"] +related: ["bug-605"] +legacy: true +--- + +# bug-606: After killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in… + +## Description +After killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in synchronized elections: PreVote win → CTRL_ELECTING → become_follower (term bumped) → ~10s → repeat, term climbing unbounded (3→9→∞), no new primary ever elected. Reproduce: deploy 3-node HA, elect primary, submit 3-5 jobs, kill ctrl-1. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Dueling-candidates split-vote livelock — Raft §5.2 randomized election timeout is missing on the FAILOVER path. Both survivors lose the primary at the same instant; their liveness polls fire on the identical fixed heartbeat_timeout, so they start PreVote+real-election in perfect lock-step (ms-aligned timestamps). Both win PreVote (neither sees a primary, so each grants the other — PreVote cannot break this tie), then both campaign at the same term, each votes for itself and denies the other (quorum 2, only 2 live nodes) → split vote → election times out (fixed election_timeout_ms, raft.c:105, NO jitter) → become_follower, term+1 → repeat forever. The priority/jitter config (election_jitter_max_ms, election_priority_step_ms) is applied ONLY at startup (ha_wait_for_heartbeat_or_timeout); the failover re-election path (liveness_poll_fn → ha_on_primary_ctrl_fd_rdhup → PreVote → ha_start_election_async → raft_start_election) has no de-synchronization. raft_on_leader_lost DOES have a priority stagger (raft.c:421) but the failover loop does not go through it. + +## Fix +Added ha_election_stagger_ms() (ha.c/ha.h): per-node failover delay = my_priority*election_priority_step_ms + random[0,election_jitter_max_ms], re-randomized each call (mirrors the startup stagger). ha_begin_failover() now schedules the PreVote start via delaymgr after that stagger (production only; c==NULL/no-delaymgr stays synchronous so existing tests are unaffected). The delaymgr callback ha_failover_stagger_fn guards with role==STANDBY && ha_heartbeat_is_fresh → skips PreVote if a primary connected during the stagger. Effect: two survivors no longer campaign in lock-step; the higher-priority survivor wins and its heartbeats reset the other. TDD: test_election_stagger_desyncs_by_priority (bounds + step>jitter ordering). VERIFIED on kill-primary repro: ctrl-2 (stagger 630ms) won → PRIMARY term=1; ctrl-3 (stagger 1072ms) received ctrl-2's ctrlhello first → STANDBY, then its stagger timer hit the guard and skipped PreVote ('a primary is alive — skipping PreVote'). No term inflation, no livelock. NOTE: optional raft.c:105 give-up-watchdog randomization NOT needed — the failover-start stagger alone resolves it; left as future defense-in-depth. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-605 diff --git a/.wolf/bugs/bug-608.md b/.wolf/bugs/bug-608.md new file mode 100644 index 00000000..1ea6941b --- /dev/null +++ b/.wolf/bugs/bug-608.md @@ -0,0 +1,38 @@ +--- +id: "bug-608" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-16T10:36:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/request.c"] +tags: ["libxce", "failover", "ha", "E_XCE_REDIRECT", "retry", "client", "protocol-mismatch"] +related: [] +legacy: true +--- + +# bug-608: libxce client never fails over after primary killed; xce_joblist returns 'Invalid argument'… + +## Description +libxce client never fails over after primary killed; xce_joblist returns 'Invalid argument' (EINVAL) indefinitely + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Two gaps: (1) xce_request_with_retry() (request.c:1956) is implemented+unit-tested but NO production path calls it - every xce_*_req wrapper calls plain xce_request() (request.c:1385 etc). (2) The retry primitive keys on E_XCE_STANDBY + reply.redirect_host/port (request.c:2010), but the real controller emits a dedicated E_XCE_REDIRECT status with data_which=nodata (join.c:264-266) and never populates those trailer fields. Controller non-primary taxonomy is E_XCE_REDIRECT / E_XCE_NOMASTER / E_XCE_NOT_READY / E_XCE_STANDBY (errnox.h:22-25). So the client both bypasses the wrapper AND, even if wired, would not detect the controller's actual redirect signal. + +## Fix +Spec libxce-failover: wire all production request paths through transparent failover; classify on actual controller statuses (REDIRECT=fast-rotate to find existing primary, address-if-present else rotate master list; NOMASTER/NOT_READY=backoff then cluster-not-ready; STANDBY=rotate); realign the existing retry unit tests to the emitted statuses. Implementation pending (requirements phase). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-609.md b/.wolf/bugs/bug-609.md new file mode 100644 index 00000000..7e85079e --- /dev/null +++ b/.wolf/bugs/bug-609.md @@ -0,0 +1,38 @@ +--- +id: "bug-609" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-16T10:52:01.800Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/client_failover.c"] +tags: ["md", "libxce", "failover", "EINVAL", "XCE_CTX_ERR", "reconnect", "E2E"] +related: ["bug-608"] +legacy: true +--- + +# bug-609: After primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on… + +## Description +After primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on every fresh call — transparent failover did not engage + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +handle_disconnect leaves xce_ctx_t.state=XCE_CTX_ERR after the primary's connection drops; the per-op xceapi guard (xceapi.c:14) returns -EINVAL when state!=XCE_CTX_RUN, BEFORE the op reaches a primary. xcecli_retry_decide classified -EINVAL as TERMINAL (non-retriable), so the failover loop bailed without rotating/recovering. The ctx->state ERR->RUN recovery only ran AFTER a rotate, which a terminal classification never triggered. Surfaced by the live 3-node E2E (task 6.2); the unit tests fed status codes directly to decide() and bypassed the real guard, so they missed it. + +## Fix +In xcecli_retry_decide, before classifying *rc, detect ac->ctx.state==XCE_CTX_ERR (known-dead connection) and treat it as a prompt transport-style failover cause (XCE_FO_RETRY_NOW, no backoff) regardless of *rc, so the loop rotates+recovers. On budget exhaustion surface -E_XCE_NOT_READY (cluster-not-ready), never -EINVAL. Regression: test_decide_known_dead_ctx_retries_not_terminal. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-608 diff --git a/.wolf/bugs/bug-610.md b/.wolf/bugs/bug-610.md new file mode 100644 index 00000000..f80706ee --- /dev/null +++ b/.wolf/bugs/bug-610.md @@ -0,0 +1,38 @@ +--- +id: "bug-610" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "crash" +timestamp: "2026-06-16T10:52:36.535Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/client_failover.c"] +tags: ["async-fix", "md", "libxce", "failover", "crash", "SEGV", "endpoint_connect", "iomgr", "reconnect", "E2E"] +related: ["bug-609", "bug-608"] +legacy: true +--- + +# bug-610: ASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate → xcecli_retry_next →… + +## Description +ASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate → xcecli_retry_next → xcecli_joblist during live master failover (NULL ep->iomgr deref) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce_failover_rotate reconnected via endpoint_fini + endpoint_init + endpoint_connect, but endpoint_init (do_endpoint_init) resets the endpoint struct including ep->iomgr, and the rotate never re-bound it. The client's endpoint gets ep->iomgr = &ctx->iomgr only via xce_ctx_set_endpoint (called once in xcecli_start). endpoint_connect derefs iomgr->ssl_ctx when ep->ssl==NULL (true after endpoint_fini frees the SSL) → NULL deref crash. Also the rotate never re-set owner/msg_handler/err_handler nor iomgr_monitor'd the new fd, so even without the crash the reconnected endpoint would not route replies. Pure unit tests could not exercise the live socket/iomgr path; only the live failover E2E hit it. + +## Fix +Added xce_ctx_reconnect(ctx,host,port) in xcectx.c (where static handle_msg/handle_disconnect live) that mirrors the connection-establishment of xcecli_start/xce_ctx_start without restarting iomgr/vtaskmgr or touching ctx->state: endpoint_fini → endpoint_init → endpoint_set_peer → re-bind ep->iomgr=&ctx->iomgr → endpoint_connect → re-set owner+msg/err handlers → iomgr_monitor. xce_failover_rotate now delegates the reconnect to it. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-609, bug-608 diff --git a/.wolf/bugs/bug-646.md b/.wolf/bugs/bug-646.md new file mode 100644 index 00000000..3257c7b6 --- /dev/null +++ b/.wolf/bugs/bug-646.md @@ -0,0 +1,38 @@ +--- +id: "bug-646" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "hang" +timestamp: "2026-06-17T08:21:32.099Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/netio.c"] +tags: ["wrong-reference", "c", "libxce", "failover", "netio", "epoll", "EPOLL_CTL_MOD", "ENOENT", "silent-failure", "hang", "cbuf_get", "reconnect"] +related: ["bug-610", "bug-609"] +legacy: true +--- + +# bug-646: xce_test jobsub hangs forever after a failover; main thread blocked in cbuf_get (xceapi.c:304)… + +## Description +xce_test jobsub hangs forever after a failover; main thread blocked in cbuf_get (xceapi.c:304) while io_worker sits idle in epoll_wait. Logs: 'rotate failed (rc=-111)' then 'endpoint_update_epoll rc1=-1 < 0'. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +endpoint_update_epoll() swallowed iomgr_update_poll() failures: it WARN'd 'rc1=-1' but returned 0 and left ep->flags unchanged, so the poll interest was never armed. iomgr_update_poll uses EPOLL_CTL_MOD, which fails with ENOENT when the fd is no longer registered (e.g. after a failed reconnect: xce_ctx_reconnect endpoint_fini's the old fd but only iomgr_monitor/EPOLL_CTL_ADDs on a SUCCESSFUL connect). The next endpoint_enq() send then MOD-failed silently, request bytes were never flushed, no reply arrived, and the synchronous client blocked forever in cbuf_get (no timeout) with no socket-drop event to wake it (handle_disconnect only fires on a broken socket). Server side was a red herring: DEFAULT_SYNC_TIMEOUT_MS=500 bounds the commit barrier, and ha_dispatch_gate already replies STANDBY/NOT_READY for non-serving roles. + +## Fix +endpoint_update_epoll() captures errno and RETURNS -errno (not 0) on iomgr_update_poll failure. endpoint_enq(): on negative arm result, run handle_error(ep) (iomgr_unmonitor + handle_disconnect -> state=ERR + inject -ECONNRESET into wbuf, waking the blocked cbuf_get) and return the error so the failover loop rotates. handle_output(): propagates the arm failure so io_handler runs the error path. TDD: test_netio_arm_failure.c drives EPOLL_CTL_MOD ENOENT via endpoint_enq on an unregistered fd; asserts loud rc<0 + err_handler fired (verified red without fix, green with). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-610, bug-609 diff --git a/.wolf/bugs/bug-648.md b/.wolf/bugs/bug-648.md new file mode 100644 index 00000000..ac4e5aa7 --- /dev/null +++ b/.wolf/bugs/bug-648.md @@ -0,0 +1,38 @@ +--- +id: "bug-648" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-17T17:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/netio.c"] +tags: ["libxce", "netio", "ssl", "SSL_read", "EOF", "close_notify", "openssl-3.x", "SSL_ERROR_ZERO_RETURN", "SSL_ERROR_SSL", "silent-failure", "hang"] +related: ["bug-646", "bug-610"] +legacy: true +--- + +# bug-648: Peer-close/EOF on an SSL connection was swallowed: io_fsm_do() set fsm->state=FSM_STATE_ERROR but… + +## Description +Peer-close/EOF on an SSL connection was swallowed: io_fsm_do() set fsm->state=FSM_STATE_ERROR but returned rc=0 on a clean EOF, so handle_input() returned >=0, handle_error/handle_disconnect never ran, and a blocked cbuf_get() reply-waiter was not woken on the first close event. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +io_fsm_do() computed rc = -errno when a closed/reset connection was detected. By OpenSSL 3.x contract (verified against docs + a live probe on OpenSSL 3.6.0), a peer close makes SSL_read() return 0 and is reported via SSL_get_error(), NOT via the return value or errno: clean close_notify -> SSL_ERROR_ZERO_RETURN (errno 0); unexpected EOF (FIN, no close_notify) -> SSL_ERROR_SSL since OpenSSL 3.0 (pre-3.0 it was SSL_ERROR_SYSCALL with errno 0 -- documented); RST -> SSL_ERROR_SYSCALL errno=ECONNRESET. errno is only meaningful for SSL_ERROR_SYSCALL; for SSL_ERROR_SSL the error queue is authoritative and any errno (we observed EPIPE=32) is a leaked side effect of OpenSSL's internal alert-write (write() on a broken connection returns EPIPE; read() never does). So -errno is 0 for the documented EOF cases -> the close looked like success. NOTE: on this 3.6 build a process-death FIN surfaced as SSL_ERROR_SSL/errno=32, which the OLD code propagated -- so this swallow is NOT necessarily the cause of the observed controller-kill permanent hang (that still implicates the reconnect/SSL-handshake path); it is a real, separate, errno-dependent non-determinism. + +## Fix +Extracted pure io_classify_io_error(ssl_err, ecode, io_rc): returns 0 only for WANT_READ/WANT_WRITE and EAGAIN/EWOULDBLOCK syscall; otherwise returns a NEGATIVE error ((ecode!=0)?-ecode:-ECONNRESET) so a detected close NEVER returns 0 regardless of the unreliable errno. io_fsm_do() now sets FSM_STATE_ERROR iff the classifier returns <0. TDD: test_netio_close_propagation.c asserts ZERO_RETURN/errno0 and SSL_ERROR_SSL/EOF map to <0 (verified red with old -ecode, green with fix). libxce suite 168/0; controller_unit_test links clean. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-646, bug-610 diff --git a/.wolf/bugs/bug-650.md b/.wolf/bugs/bug-650.md new file mode 100644 index 00000000..2519a9b4 --- /dev/null +++ b/.wolf/bugs/bug-650.md @@ -0,0 +1,38 @@ +--- +id: "bug-650" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "hang" +timestamp: "2026-06-17T18:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/request.c"] +tags: ["libxce", "request", "xce_request", "failover", "silent-failure", "hang", "cbuf_get", "endpoint_enq", "send-propagation"] +related: ["bug-646", "bug-648"] +legacy: true +--- + +# bug-650: xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed + +## Description +xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed. Log shows 'endpoint marked dead' (bug-646 firing) then a permanent hang; target primary (ctrl-2) is fully idle (no vtask in handle_jobsubmit) and the client fd is an unconnected TCP socket — the request was never transmitted yet the client waits for a reply forever. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +xce_request() and the 5 other request.c senders (xce_reply, xce_replconfigpush_req_raw, xce_wal_batch_req_raw, xce_snapshot_chunk_req_raw, xce_replconfig_reply_raw) called endpoint_enq(ep,&pkt->link) and IGNORED its return, always `return 0`. So when the transport was dead, endpoint_enq() correctly returned <0 and ran handle_error() (bug-646 fix) — but xce_request() swallowed that <0 and reported success. xce_jobsubmit() (xceapi.c:299-304) then saw rc>=0 and fell through to the BLOCKING cbuf_get(); the request was never flushed (server idle) and when err_handler was NULL after a failed reconnect, handle_disconnect never injected -ECONNRESET, so nothing woke the waiter. The bug-646 netio fix was defeated one layer up: the request layer reported success on a failed send. This was the missing link making every prior netio fix insufficient for the live hang. + +## Fix +All six request.c senders propagate the enq result: `int enq_rc = endpoint_enq(ep,&pkt->link); return (enq_rc < 0) ? enq_rc : 0;`. A failed send now returns <0 through xce_jobsubmit_req -> xce_jobsubmit, which returns before cbuf_get; the failover loop classifies the transport error and rotates instead of blocking on a reply that never arrives. TDD: test_request_send_propagation.c builds an OPCode_nop request (encodes OK) on a dead endpoint (EPOLL_CTL_MOD ENOENT) and asserts xce_request returns <0 (verified red with old `return 0`, green with fix). libxce 169/0; controller_unit_test links clean. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-646, bug-648 diff --git a/.wolf/bugs/bug-652.md b/.wolf/bugs/bug-652.md new file mode 100644 index 00000000..5d762326 --- /dev/null +++ b/.wolf/bugs/bug-652.md @@ -0,0 +1,38 @@ +--- +id: "bug-652" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "hang" +timestamp: "2026-06-17T19:10:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/netio.c"] +tags: ["libxce", "netio", "endpoint_enq", "deadlock", "self-deadlock", "ha-lock", "reentrancy", "regression", "controller", "vote_ep_err_handler"] +related: ["bug-646", "bug-650"] +legacy: true +--- + +# bug-652: Controller (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but… + +## Description +Controller (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but the controller freezes. gdb shows Thread 3 in primary_heartbeat_fn holding ha->lock (owner=self) and re-acquiring it via xce_nop_req -> xce_request -> endpoint_enq -> handle_error -> vote_ep_err_handler (ha.c:3478); Thread 9 (io_worker) also blocked on the same ha->lock. Whole controller frozen. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +REGRESSION from bug-646: endpoint_enq() invoked handle_error() synchronously on the send path when the poll arm failed. handle_error() calls the endpoint's err_handler, which on a controller vote endpoint is vote_ep_err_handler() — it locks ha->lock. But endpoint_enq runs on the CALLER's thread, and the controller's primary_heartbeat_fn already holds ha->lock when it sends heartbeat NOPs. Non-recursive mutex -> self-deadlock. The err_handler is contractually an io_worker-thread callback (off an epoll error event, holding no caller locks); calling it from the synchronous send path violated that. With bug-650 now propagating the enq error up the return chain, the handle_error call in endpoint_enq was both redundant and harmful. + +## Fix +endpoint_enq() no longer calls handle_error(); on arm failure it just returns a RETRIABLE -ECONNRESET. The caller propagates it (request.c senders return it -> client failover classifies ECONNRESET as RETRY_NOW and rotates; controller heartbeat path simply gets the error, no lock re-entry). The real teardown stays on the io_worker thread (handle_input/handle_output -> io_handler -> handle_error), which holds no caller locks. Updated test_netio_arm_failure.c: endpoint_enq must return <0 AND must NOT invoke the err_handler (g_err_count==0). libxce 169/0; controller_unit_test links clean. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-646, bug-650 diff --git a/.wolf/bugs/bug-660.md b/.wolf/bugs/bug-660.md new file mode 100644 index 00000000..46550dca --- /dev/null +++ b/.wolf/bugs/bug-660.md @@ -0,0 +1,38 @@ +--- +id: "bug-660" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-17T22:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c"] +tags: ["controller", "ha", "election", "prevote", "convergence", "fresh_heartbeat", "raft_cb_vote_eligible", "primary_ctrl_fd", "slow-election"] +related: [] +legacy: true +--- + +# bug-660: Failover election takes minutes (11+ PreVote rounds) to converge + +## Description +Failover election takes minutes (11+ PreVote rounds) to converge. A standby keeps denying a legitimately-higher-term candidate's PreVote with reason=fresh_heartbeat for ~2.5 min AFTER its primary died (ctrl-3 killed 21:44:14, ctrl-2 logs 'PreVote DENIED ... reason=fresh_heartbeat (my_term=3)' until 21:46:58). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The PreVote/vote eligibility gate raft_cb_vote_eligible (ha.c:311) denied on `role==CTRL_STANDBY && ha_heartbeat_is_fresh(ha)` — checking freshness ALONE. ha_heartbeat_is_fresh derives from ha->last_activity, but ha_arm_heartbeat_timeout_vtask resets last_activity=now on EVERY watchdog (re-)arm, and ha_prevote_rearm_liveness re-arms it after every lost PreVote. So a node with NO live primary perpetually refreshes its own 'fresh heartbeat' each failover cycle and denies all other candidates (reason=fresh_heartbeat), blocking convergence until it happens to restart. The sibling gate in handle_request_vote (ha.c:2205) already required `primary_ctrl_fd >= 0 && is_fresh`; raft_cb_vote_eligible was missing the primary_ctrl_fd guard. The existing fresh-deny tests even documented the intent ('the deny gate requires a live connection AND a fresh heartbeat', set primary_ctrl_fd=999), but production didn't enforce it. + +## Fix +Added `ha->primary_ctrl_fd >= 0 &&` to the ha.c:311 eligibility gate (mirrors ha.c:2205). primary_ctrl_fd is -1 on init / after the primary RDHUPs / after the liveness watchdog fires, and a real fd only via config_repl.c:1451 (first ctrlhello from the live primary) — so a dead/absent primary no longer masquerades as fresh and the standby GRANTS, converging in ~1 round. Kept cluster.c:526 denial-reason logging in sync. TDD: test_pre_vote_dead_primary_grants_despite_fresh_activity (fresh last_activity + primary_ctrl_fd=-1 -> GRANT); verified RED (Expected 1 Was 0) without the guard, GREEN with it, and the 3 existing fresh-deny tests (fd=999) unaffected. Full controller suite + libxce 169/0, zero regressions. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-661.md b/.wolf/bugs/bug-661.md new file mode 100644 index 00000000..c3aa0ba5 --- /dev/null +++ b/.wolf/bugs/bug-661.md @@ -0,0 +1,39 @@ +--- +id: "bug-661" +status: "fixed" +status_updated: "2026-09-24" +status_note: "f139e1c0" +component: "controller" +severity: "data-loss" +timestamp: "2026-06-17T23:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/persist.c", "src/controller/commit.c", "src/controller/repl.c"] +tags: ["controller", "ha", "durability", "fsync", "wal", "commit-engine", "data-loss", "failover", "leader-completeness", "promotion-fence", "async-fsync", "deterministic-repro", "wal-sync-ms"] +related: ["bug-339", "bug-447", "bug-396"] +fix_commits: ["f139e1c0"] +legacy: true +--- + +# bug-661: 3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and… + +## Description +3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and survived first failover, permanently gone after a restarted node with near-empty WAL won election. joblist dropped from 10 to 3. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +'committed' does not imply 'durable'. WAL fsync is asynchronous (persist.c:173 fsync_worker, timer wal_sync_ms); persist_append and the standby WalAck path do NOT fdatasync before ACK. commit_on_ack (commit.c:303) advances committed_lsn on majority ACK = OS page cache only, durable on zero disks. On kill+restart, killed nodes lose the not-yet-fsynced tail; persist_recover hits CRC break at zeroed region (prev_crc=00000000, offset 40) and truncates replay to ~0 entries, yet the hollow node stays eligible, wins election under degraded required=1 quorum (PreVote+freshness gate cannot fence it because the committed entries were durable on nobody), self-promotes advertising committed_lsn=21 over an empty job table, and the node holding the real data converges DOWN. Violates Leader-Completeness assumption (majority-ACK != durable-on-majority) in src/controller/CLAUDE.md. + +## Fix +NOT YET FIXED. Discovery -> new spec ha-commit-durability (Path C, brief written). R1 fsync-before-ACK (standby fdatasync before WalAck, primary self-count only after local fdatasync, committed_lsn advances only on durable majority, group-commit to bound latency). R2 promotion fence (WAL CRC/replay failure => ineligible to lead until caught up, FATAL not silent truncate; never lead below known committed high-water even under degraded quorum). Live E2E reproduces via xce_test jobsub + primary kills on /xce-deploy-ha. DETERMINISTIC REPRO (task 1.3, 2026-06-18): `bash src/tools/test-commit-durability.sh --deterministic` launches the cluster with --wal-sync-ms 30000 (plumbed via deploy-ha.sh start.sh `$*` passthrough, EXTRA_CTRL_ARGS) so the primary's fsync_worker never fires in-window -> every committed entry is page-cache-only on the primary (majority_candidate_locked counts cap=highest_appended_lsn=current_lsn-1, NOT durable). It then drives rolling PRIMARY kills (each right after committing to that primary) + restart, so hollow nodes re-lead. RED CONFIRMED: 6-round run lost the 3 EARLIEST committed JobIds permanently (rounds 4 and 6 + final check), Results 5 passed/3 failed, exit=1 — matching the incident signature (earliest jobs gone). NOTE: a single double-kill does NOT repro — this codebase enforces strict majority (lone survivor will NOT self-elect: log shows 'no single-voter primary within 40s'); loss needs the ROLLING/term-accumulating path the incident used. Evidence log: dur-det-run.log in worktree. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-339, bug-447, bug-396 diff --git a/.wolf/bugs/bug-693.md b/.wolf/bugs/bug-693.md new file mode 100644 index 00000000..7a7ad0d5 --- /dev/null +++ b/.wolf/bugs/bug-693.md @@ -0,0 +1,38 @@ +--- +id: "bug-693" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-18T06:05:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/persist.c"] +tags: ["ha-commit-durability", "durability", "fdatasync", "fail-loud", "remediation", "c"] +related: [] +legacy: true +--- + +# bug-693: durable_lsn advanced even when fdatasync FAILED (EIO/ENOSPC) at two of three sites… + +## Description +durable_lsn advanced even when fdatasync FAILED (EIO/ENOSPC) at two of three sites (flusher_flush_once, persist_stop), letting a non-durable LSN count toward the commit majority + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha-commit-durability task 2.1 flush sites called fdatasync(p->wal_fd) without capturing/checking the return code, then advanced durable_lsn unconditionally. On sync error the bytes are not on stable storage, violating Req 1.1/1.2 and the fail-loud preference. + +## Fix +Capture int rc = fdatasync(...) at every durable_lsn advance site (flusher_flush_once gating + fallback paths, persist_stop) and advance durable_lsn ONLY when rc == 0; on error leave durable_lsn unchanged and log loudly via ERROR(). persist_flush already gated on rc==0 (reference). Added XCE_TESTING fault-injection seam (test_force_fsync_fail + persist_test_set_force_fsync_fail + flusher_fsync_forced_fail helper, constant-false in production) and error-path test test_durable_lsn_holds_on_fsync_failure proving durable_lsn holds at baseline under injected failure and recovers when cleared. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-704.md b/.wolf/bugs/bug-704.md new file mode 100644 index 00000000..222df9a7 --- /dev/null +++ b/.wolf/bugs/bug-704.md @@ -0,0 +1,38 @@ +--- +id: "bug-704" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-18" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/{test_repl_majority", "test_commit_barrier_rollout", "test_coldstart_lsn0", "test_demotion_convergence}.c"] +tags: ["durable", "commit", "majority", "self-cap", "test-fixture", "ha-commit-durability", "task-2.2"] +related: ["bug-661", "bug-693"] +legacy: true +--- + +# bug-704: controller suite regressions after capping commit self-contribution at durable_lsn… + +## Description +controller suite regressions after capping commit self-contribution at durable_lsn: test_repl_majority/test_commit_barrier_rollout/test_coldstart_lsn0/test_demotion_convergence got -ETIMEDOUT / committed_lsn=0 (Expected N Was 0) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha-commit-durability 2.2 changed majority_candidate_locked self-cap from highest_appended_lsn to persist_durable_lsn. Existing commit-path fixtures prime current_lsn (appended tip) but never set durable_lsn, so under the new cap self contributed 0 and committed_lsn never advanced. + +## Fix +In every commit-path stub/in-memory-primed fixture, set durable_lsn to the appended tip (appended_lsn / current_lsn-1) right after committed_lsn=0, preserving the pre-durability 'self covers every appended entry' model. The durability precondition itself is exercised by dedicated new tests. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-661, bug-693 diff --git a/.wolf/bugs/bug-760.md b/.wolf/bugs/bug-760.md new file mode 100644 index 00000000..51a2853a --- /dev/null +++ b/.wolf/bugs/bug-760.md @@ -0,0 +1,38 @@ +--- +id: "bug-760" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-18T13:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 6 +files: ["src/controller/testsuite/test_commit_barrier_rollout.c"] +tags: ["ha-commit-durability", "commit-barrier-rollout", "durable_lsn", "group-commit", "test-race", "async-durability", "validation", "NO-GO"] +related: ["bug-661", "bug-704"] +legacy: true +--- + +# bug-760: test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected… + +## Description +test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected 1 Was 0 — a covering ACK must advance committed_lsn over the retained fact (6/6 deterministic) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha-commit-durability task 2.2 capped the primary commit self-contribution at durable_lsn (advanced asynchronously by the demand-driven group-commit flusher) and wired a durable-advance commit-wake callback. This adjacent-spec (ha-commit-barrier-rollout) test uses a REAL persist+flusher and checks committed_lsn SYNCHRONOUSLY the instant after repl_on_ack(fact_lsn); the flusher has not yet fdatasync'd LSN1, so durable_lsn==0, self-cap==0, majority candidate==0, committed_lsn stays 0. Pre-spec the test passed (self counted the appended tip immediately). Task 2.2 primed durable_lsn in the in-memory stub fixtures (barrier_setup_vcpus) but did NOT update this real-flusher test. The implementation-note claim that this is a pre-existing /tmp-overlay fdatasync-EIO artifact is FALSE in this env: fdatasync returns rc=0 on /tmp here, and the test passed pre-spec. + +## Fix +FIXED — test_commit_barrier_rollout.c:445: insert persist_flush(g_fp) + assert persist_durable_lsn>=fact_lsn BEFORE repl_on_ack, so the covering ACK (commit_on_ack runs synchronously inside repl_on_ack) deterministically completes the durable majority. Mirrors task 2.2's durable_lsn priming of the in-memory barrier_setup_vcpus stubs. Verified: was 6/6 deterministic FAIL, now fact_hook PASS across 4 runs + meson controller suite Ok:1 Fail:0, all groups 0 Failures. Production was never affected (RETAIN facts have no waiter; WAIT-mode replies block on commit barrier commit.c:334). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-661, bug-704 diff --git a/.wolf/bugs/bug-765.md b/.wolf/bugs/bug-765.md new file mode 100644 index 00000000..b0903548 --- /dev/null +++ b/.wolf/bugs/bug-765.md @@ -0,0 +1,39 @@ +--- +id: "bug-765" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "scheduler" +severity: "unknown" +timestamp: "2026-06-18T14:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/scheduler/ (sched_update_existing_job dead; schedUpdateJob → CREATE path)"] +tags: ["scheduler", "controller", "msg-gap", "validate-gap", "not-implemented", "HIGH", "job-state-corruption"] +related: [] +fix_commits: ["5c1f715", "93124c5", "080ff78", "276e26c"] +legacy: true +--- + +# bug-765: scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify… + +## Description +scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify path hits the bulk CREATE path (sched_update_existing_job is dead code) → corrupts/duplicates live job state, no -ENOENT; Req 3 broken at dequeue, Req 6 stores nothing (rusage never stored). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Tasks were checked [x] without the dequeue/update path and rusage storage actually being wired; gap surfaced only by cross-task re-audit (per-task review missed it). + +## Fix +FIXED 2026-06-18 (commits 5c1f715,93124c5,080ff78,276e26c): G-8/G-4 (route notify job-update to in-place sched_update_existing_job, all mutable fields, -ENOENT no-create); G-5 (store rusage on xce_sched_job_t via sched_apply_job_rusage + free_JobRusages_ptr); G-3 (free all 5 killer op payloads with generated deep-freers); G-7 (doc reconciliation — sched_op_name stays static in scheduler.c, lib must not depend on scheduler headers); G-6 (real white-box tests: test_handlers.c 6/6 for 9.2/9.3, test_job_update.c 10/10 for 9.4/9.5/9.7, registered as meson tests). Scheduler suite 31/31 GREEN, ASAN-clean, boundary-clean. validate-impl GO. RESIDUAL (documented, non-blocking): 9.6 dequeue worker-resvals APPLY unit test (dormant path R-3 — enqueue tested), 9.8 documented compile-time absence, 9.9 log-capture gated, 9.2 -ENOMEM fault-injection. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-777.md b/.wolf/bugs/bug-777.md new file mode 100644 index 00000000..b57106b5 --- /dev/null +++ b/.wolf/bugs/bug-777.md @@ -0,0 +1,39 @@ +--- +id: "bug-777" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "data-loss" +timestamp: "2026-06-19T13:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/ha.c (raft_cb_become_leader promotion fence release) + persist.c (recovery_truncated verdict stickiness); design ha-commit-durability Req 2.2/2.6 + Boot Policy Decision Table"] +tags: ["ha", "convergence", "promotion-fence", "recovery-truncated", "torn-tail", "SIGKILL", "availability", "election-wedge", "ha-commit-durability", "ha-convergence-recovery", "live-E2E", "false-positive-data-loss"] +related: ["bug-661", "bug-339"] +fix_commits: ["64ab210"] +legacy: true +--- + +# bug-777: 3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS… + +## Description +3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS 5/5 jobs vanished'. Both surviving nodes recovered RECOVER_TRUNCATED after SIGKILL (torn WAL tail) and stay permanently fenced/non-promotable even though durable_floor==cluster_hwm (they hold ALL committed data). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +A SIGKILL tears the WAL tail; on restart replay_wal hits a CRC-break/short-read at the tail -> RECOVER_TRUNCATED (sticky flag). The promotion-fence release condition requires `!recovery_truncated`, but the truncated flag is NEVER cleared by catch-up — so a node whose intact durable prefix (durable_floor) already reaches/exceeds the cluster committed high-water (durable_floor>=hwm, i.e. holds ALL committed entries; the torn tail is entirely UNCOMMITTED) stays permanently non-promotable. The design assumed 'TRUNCATED => provably below HWM', which is FALSE for a torn TAIL above the committed prefix. When all survivors are in this state (after multiple rolling failovers), none can promote => election/availability WEDGE. Observed: ctrl-1 recovery_truncated=1 durable_floor=13 cluster_hwm=13, PreVote-DENY peers reason=catch_up, indefinitely. + +## Fix +DATA-LOSS dimension RESOLVED by the bug-778 fix (commit 64ab210): the cold-restart loss was the CRC-chain clobber; with the chain intact, cold restart recovers all committed jobs (soak 5/5, joblist 6/6 after cold restart). The AVAILABILITY-WEDGE dimension was a CONSEQUENCE of the spurious RECOVER_TRUNCATED from the broken chain — no longer triggered (nodes now recover CLEAN; round-3 re-elects normally). RESIDUAL HARDENING (latent, not currently triggered): the promotion fence keeps recovery_truncated sticky even when durable_floor>=HWM, so a GENUINE torn-tail that leaves a node RECOVER_TRUNCATED while holding all committed data could still over-fence. Optional follow-up: clear the truncated fence when durable_floor>=cluster_hwm (the torn tail is provably uncommitted). Not blocking — the systematic trigger is gone. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-661, bug-339 diff --git a/.wolf/bugs/bug-778.md b/.wolf/bugs/bug-778.md new file mode 100644 index 00000000..39df63d0 --- /dev/null +++ b/.wolf/bugs/bug-778.md @@ -0,0 +1,39 @@ +--- +id: "bug-778" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "data-loss" +timestamp: "2026-06-19T13:40:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/persist.c:1336 (apply_wal_entry) + src/controller/ha.c ha_apply_single_entry (~1570-1588)"] +tags: ["ha", "wal", "crc-chain", "last_crc", "apply_wal_entry", "persist_wal_write_replicated", "cold-restart", "data-loss", "bug-661", "standby-replication", "recovery", "root-caused"] +related: ["bug-777", "bug-661"] +fix_commits: ["64ab210"] +legacy: true +--- + +# bug-778: persist_recover RECOVER_TRUNCATED: 'CRC chain break ... prev_crc=00000000 expected=' at the… + +## Description +persist_recover RECOVER_TRUNCATED: 'CRC chain break ... prev_crc=00000000 expected=' at the first replicated job entry on cold restart; committed jobs unrecoverable (joblist=0). ROOT CAUSE of bug-777's cold-restart data loss. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +On the standby replicated-apply path, ha_apply_single_entry first calls persist_wal_write_replicated() which correctly sets p->last_crc to the standby's SELF-computed chain crc (the stream carries no crc; the on-disk CRC chain is node-local, prev_crc==last_crc). It THEN builds a synthetic wal_entry_hdr with hdr.crc=0 and calls apply_wal_entry(), whose line 1336 `p->last_crc = hdr->crc` OVERWRITES the correct chain link with 0. The next replicated write computes prev_crc=(seg_offset==0?WAL_CRC_NONE:last_crc)=0 and stamps prev_crc=0 on disk. So the on-disk CRC chain breaks at the 2nd replicated entry. Invisible while live (standby serves in-memory entities; never replays its own WAL), detonates on COLD RESTART where replay_wal validates the chain -> RECOVER_TRUNCATED -> recovers only the clean prefix -> committed jobs lost + the truncated verdict drives the promotion-fence availability wedge (the other half of bug-777). NOT an fd-position/offset race (pwrite+p->lock are correct); it is shared CRC-chain state (p->last_crc) written by two paths where the logical-apply step clobbers the physical-writer's link with 0. + +## Fix +FIXED 2026-06-19 (commit 64ab210). apply_wal_entry is now LOGICAL entity-restore only (signature (p,type,lsn,payload,payload_len); no header); it advances the current_lsn high-water but NEVER touches p->last_crc. The CRC chain has a single owner: persist_wal_write_replicated (live) + replay_wal/persist_recover (recovery). ha_apply_single_entry drops the fabricated crc=0 header. Verified: RED->GREEN test test_replicated_apply_chain_survives_cold_restart (real ha_apply_wal_catchup_entry path, cold-replay -> RECOVER_CLEAN); controller suite 0 failures; 3-node rolling-SIGKILL soak on real ext2/3 fs 5/5 (was wedge+loss at round 3); full cold restart recovers all 6 jobs (was 0), CLEAN verdicts, normal election. Test callers updated: test_repl_apply, test_ha_role_lifecycle, test_egress_filter. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-777, bug-661 diff --git a/.wolf/bugs/bug-786.md b/.wolf/bugs/bug-786.md new file mode 100644 index 00000000..7798df4c --- /dev/null +++ b/.wolf/bugs/bug-786.md @@ -0,0 +1,39 @@ +--- +id: "bug-786" +status: "fixed" +status_updated: "2026-09-24" +status_note: "fae0bc0f, 86099763" +component: "controller" +severity: "unknown" +timestamp: "2026-06-19T15:00:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: ["src/controller/testsuite/test.c (cmd_jobsub)"] +tags: ["xce_test", "jobsub", "garbage-jobid", "e2e", "client", "test-tooling", "xce-test-e2e-primitives", "stale-reply-union", "live-soak"] +related: [] +fix_commits: ["fae0bc0f"] +legacy: true +--- + +# bug-786: xce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on… + +## Description +xce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on repeated/mid-failover submits; controller then logs 'handle_jobsubmit fail to lock job 63-15--1-...'. Client claims success with a malformed id it never got a clean ack for. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +On the 2nd+ submit (and during failover windows) the jobsub reply/jobid is not cleanly populated, leaving an uninitialized jobid struct; cmd_jobsub still prints 'jobsubmit ok' with the garbage id instead of verifying the controller reply status and failing. Forces E2E scripts (test-commit-durability.sh) to strict-regex-filter the 4-field id to avoid chasing phantom jobs. + +## Fix +PENDING — jobsub must report success ONLY on a clean ack with a valid jobid; otherwise print a typed error (REDIRECT/NOMASTER/NOT_READY/TIMEOUT) and set non-zero status. Part of the xce-test-e2e-primitives work (exit-code-as-verdict + $jobid auto-var). | STRENGTHENED 2026-06-19 (commit 8609976): `jobid != 0` gate was insufficient — a stale non-zero reply id (63-15--1-1015808) from a non-jobid reply during failover slipped through. New gate requires the echoed jobid == the locally idgen-generated jobsubmit.jobid (controller commits/echoes exactly that id, job.c:616). Surfaced by the live durability soak (task 7.2). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-795.md b/.wolf/bugs/bug-795.md new file mode 100644 index 00000000..c6d70d9f --- /dev/null +++ b/.wolf/bugs/bug-795.md @@ -0,0 +1,42 @@ +--- +id: "bug-795" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-19" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/{persist.c recovery/truncate", "commit.c commit-gate", "ha.c promotion-fence/election} — UPSTREAM ha-commit-durability Phase 6.2"] +tags: ["ha-commit-durability", "commit-durability", "cold-restart", "total-outage", "wal-truncate", "upstream", "xce-test-e2e-primitives", "needs-real-fs", "rolling-failover", "liveness", "non-deterministic"] +related: ["bug-661", "bug-778", "bug-786"] +fix_commits: ["b77d412"] +legacy: true +--- + +# bug-795: Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a… + +## Description +Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job not found). Recovery logs show WAL truncate. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) — must be reconfirmed on real hardware. UPDATE: with the hardened (trustworthy) soak, the loss ALSO reproduces in ROLLING single-node failover with the cluster confirmed serving (~1/3 runs) — broader than cold-restart-all; bug-661 not fully closed for rolling failover. Separate finding: >60s post-kill election (liveness) every run, always recovers. CORRECTION: NOT an overlay-fdatasync artifact — reproduces OUTSIDE the container (user's real host) AND on an in-sandbox real ext2/3 fs with ZERO fdatasync EIO. Real controller durability bug + a coupled liveness stall (>60s / stuck election after killing ctrl-1). Phase 6.1 answered (REAL); now at 6.2 root-cause. + +## Fix +FIXED AND VERIFIED LIVE -- ha-commit-durability was reopened specifically for this bug (Phase 6, tasks 6.1-6.4). Task 6.1 attempted to reproduce the cold-restart-all loss against a REAL fsync-able filesystem (ext4, not this sandbox's overlayfs) and it did not reproduce -- the original loss was an overlayfs fdatasync artifact, not a genuine product defect (tasks 6.2/6.3 root-cause/fix were correctly marked N/A as a direct consequence). Task 6.4 re-validated on real fs and closed bug-795. spec.json's validate_impl (decision: GO, 2026-07-18) independently reconfirms this with a later, fresh live re-validation: a 4-round rolling SIGKILL+restart soak plus a full cold-restart-all cycle, 8/8 PASS, 0 job loss -- all 20 committed jobs survived both rolling failover and cold-restart-all. Distinct root cause from bug-828 (a premature-truncate bug during a live election under rolling single-node failover, fixed separately in ha-term-safety commit b77d412) -- not the same defect, despite early investigation notes grouping them together as 'bug-828/795'. One adjacent, still-open, low-priority item surfaced during the same validation: an extra/duplicate committed job appeared once during the original 2026-07-02 validation run (over-counting, the opposite symptom, not yet root-caused, does not block the GO) -- filed separately, not part of this entry. + +[Original entry] +NOT FIXED HERE — out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-661, bug-778, bug-786 diff --git a/.wolf/bugs/bug-817.md b/.wolf/bugs/bug-817.md new file mode 100644 index 00000000..d2dcf308 --- /dev/null +++ b/.wolf/bugs/bug-817.md @@ -0,0 +1,41 @@ +--- +id: "bug-817" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-19" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/client_failover.c / client.c failover-rotation (libxce) — UPSTREAM", "not controller-status"] +tags: ["libxce", "failover", "master-rotation", "dead-first-master", "EINVAL", "controller-status", "upstream"] +related: ["bug-646"] +legacy: true +--- + +# bug-817: client op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to… + +## Description +client op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to reach the live primary when a DEAD controller is FIRST in the masters list; with a live-only masters list the same op succeeds in ~46ms. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +libxce client failover does not cleanly advance past a dead FIRST master to the live nodes; surfaces a typed EINVAL instead of rotating. Pre-existing (controller-status consumes failover unchanged). Surfaced by controller-status task 5.1 E2E (authoritative wait-primary timed out after a primary kill while a dead node led the masters list, even though a new primary was serving). + +## Fix +FIXED -- bug-817 and bug-854 are the same defect (client-side positional primary-discovery: a client fails to reach the live primary when the dead controller happens to be first in the configured masters list). libxce-failover's own spec.json validation note states this was 'found and fixed in prior follow-up commits, already present in the validated build.' Independently corroborated by component-failover-adoption/research.md (written 2026-08 this same project week): 'xcecli_start (client.c:125-188) already rotates through ALL configured masters on the very first connect (bug-854 fix, comment at line 136)' -- cited there as current, standing production behavior. Three later specs (ha-term-safety, raft-ha-core-safety, raft-adapter-conformance) all isolate/route-around bug-854 in their own soak verdicts rather than re-fixing it, consistent with it already being fixed elsewhere. + +[Original entry] +NOT fixed in controller-status (out of boundary). Route to libxce-failover. Confirmed affects plain joblist identically. Workaround: ensure the masters list does not lead with a known-dead node, or restart killed nodes (as the durability soak does). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-646 diff --git a/.wolf/bugs/bug-827.md b/.wolf/bugs/bug-827.md new file mode 100644 index 00000000..e81ba68d --- /dev/null +++ b/.wolf/bugs/bug-827.md @@ -0,0 +1,39 @@ +--- +id: "bug-827" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-20T18:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/utils.c"] +tags: ["global-buffer-overflow", "asan", "xce_strerror_r", "errnox", "enum-table-drift", "libxce", "controller-status", "ctrlstatus", "E_XCE_NOT_READY"] +related: [] +fix_commits: ["1aa0761"] +legacy: true +--- + +# bug-827: AddressSanitizer: global-buffer-overflow READ in xce_strerror_r (lib/utils.c:335), triggered from… + +## Description +AddressSanitizer: global-buffer-overflow READ in xce_strerror_r (lib/utils.c:335), triggered from cmd_ctrlstatus (test.c:1273) when ctrlstatus failover exhausts and returns E_XCE_NOT_READY (-10013) on a primary-less cluster + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Enum/table drift: errnox.h E_XCE_* enum gained NOMASTER(10011)/REDIRECT(10012)/NOT_READY(10013)/STANDBY(10014), bumping E_XCE_MAX to 10015, but xce_err_str[] still had only 10 strings + a NULL (11 slots, 88 bytes). xce_strerror_r's only upper guard was errnum>=E_XCE_MAX, so 10011-10014 passed the gate and indexed past the table: NOMASTER->xce_err_str[10]==NULL (strncpy NULL deref); NOT_READY->xce_err_str[12] = 8 bytes past the 88-byte array (the ASAN OOB). Pre-existing libxce foundation bug; controller-status ctrlstatus was the first caller to format these HA codes on a failing cluster. + +## Fix +Filled xce_err_str[] with strings for all 14 codes (enum order); hardened xce_strerror_r to bound against the ACTUAL array size (idx>=nstr) + reject NULL slots (degrade to 'unknown error', never OOB) + null-terminate; added a _Static_assert tying sizeof(xce_err_str) to (E_XCE_MAX-XCE_ERRNO_BASE) so future enum additions without a string fail the build. RED: new test_strerror_all_xce_codes SEGV'd at utils.c:335 under ASAN; GREEN: lib suite Ok:1 Fail:0. Commit 1aa0761. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-828-part2.md b/.wolf/bugs/bug-828-part2.md new file mode 100644 index 00000000..2feeda14 --- /dev/null +++ b/.wolf/bugs/bug-828-part2.md @@ -0,0 +1,39 @@ +--- +id: "bug-828-part2" +status: "fixed" +status_updated: "2026-09-24" +status_note: "b77d4122; upstream gap fixed as bug-661 (f139e1c0)" +component: "controller" +severity: "unknown" +timestamp: "2026-06-23T03:05:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_ha_vote_gate.c (test added); ha.c/persist.c advertise path investigated (no change needed)"] +tags: ["safety", "failover", "safe-committed-lsn", "durable-tip", "advertise", "leader-completeness", "bug-795", "bug-661", "commit-durability", "INVESTIGATION", "c"] +related: ["bug-828", "bug-795", "bug-661", "bug-851", "bug-924"] +fix_commits: ["b77d4122", "f139e1c0"] +legacy: true +--- + +# bug-828-part2: Clean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2… + +## Description +Clean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2 granted voters)' and walcompare reports committed-prefix DIVERGENCE at LSN 2; jobs at LSN 2..5 lost. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +INVESTIGATED, NOT an advertisement-staleness bug. The advertise path already reflects the live durable tip: ha_candidate_advertise_lsn -> persist_durable_floor, and the voter floor = max(persist_durable_floor, learned_committed); persist_durable_floor == persist_durable_lsn (both read p->durable_lsn, the fsync watermark), and part-1's secondary fix advances durable_lsn after the standby's inline fdatasync in persist_wal_write_replicated. LIVE evidence (fix828c_r1..r3, STRICT_TERM clean failover x3) proves the surviving voters advertise 1 because they are GENUINELY durable only at LSN 1: their WAL physically holds 1..2 (LSN2 = their own term-3 NOOP), recovered tip=2, durable_floor=1. The original primary ctrl-1 (down at election time, WAL 1..5 tip=5) advanced committed_lsn 1->2->3->4->5 in ~250ms while standbys had only converged applied_lsn=1 and catch-up streaming was still in flight. So committed advanced past what the surviving majority durably held — the UPSTREAM replication / commit-barrier durability gap (bug-795/661), not the vote advertisement. safe_committed_lsn=1 is the CORRECT, SAFE Raft answer given the real durable state; advertising the true durable tip cannot conjure data never replicated. + +## Fix +No production code change required for part-2 — the durable-tip advertisement is already correct & safe after part-1. Added unit test test_live_durable_tip_advertised_over_lagging_learned (test_ha_vote_gate.c): a standby durable at LSN5 (real persist, flush -> durable_lsn=5) with learned_committed lagging at 1 must advertise 5 as BOTH candidate self-LSN (ha_candidate_advertise_lsn) and voter floor (ha_vote_self_freshness over persist_durable_floor, dominating learned=1), and ha_compute_safe_committed_lsn from two such granting voters must return 5 — locking in the in-scope contract. Explicitly did NOT fold learned_committed into the CANDIDATE advertise (the requested max(...,learned_committed)) because that re-opens the truncated-candidate-over-advertises-and-wins hole ha-commit-durability task 4.2 deliberately closed (candidate must advertise only a DURABLE position; learned_committed is volatile/not necessarily durable on this node). The residual live job loss is bug-795/661, out of scope. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-828, bug-795, bug-661, bug-851, bug-924 diff --git a/.wolf/bugs/bug-828.md b/.wolf/bugs/bug-828.md new file mode 100644 index 00000000..abaa4e41 --- /dev/null +++ b/.wolf/bugs/bug-828.md @@ -0,0 +1,39 @@ +--- +id: "bug-828" +status: "fixed" +status_updated: "2026-09-24" +status_note: "91b9a445, b77d4122" +component: "controller" +severity: "unknown" +timestamp: "2026-06-20T10:19:56.678Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/ha.c (vote_reply_dispatch)", "src/controller/persist.c (persist_wal_write_replicated)"] +tags: ["async-fix", "md", "data-loss", "failover", "committed-job-loss", "wal", "catch-up", "snapshot", "ha-commit-durability", "bug-795", "architecture", "NOT-FIXED", "safety", "safe-committed-lsn", "vote-reply", "leader-completeness", "ha-repl-hardening", "PARTIAL-FIX", "c"] +related: ["bug-795", "bug-851"] +fix_commits: ["91b9a445", "b77d4122"] +legacy: true +--- + +# bug-828: Clean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ...… + +## Description +Clean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ... (from 1 granted voters)' (self only) and truncates committed entries down to its own boot floor. Post-election safe-commit-from-votes plumbing was DEAD in production. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +vote_reply_decode_and_dispatch decoded the granting voter's advertised peerLastLogLsn (vr_lsn at ha.c ~4254) but vote_reply_dispatch DISCARDED it ('(void)peer_lsn; raft_on_vote_reply(...)'). raft_on_vote_reply only bumps s->votes_granted; it never records the peer LSN. The only writer of ctx->votes[idx].vote_granted/peer_lsn (handle_request_vote_reply) was, in production, called ONLY from the connect/error failure paths with (granted=0, lsn=0); the real-grant caller was gated behind a test-only hook (xce_vote_reply_hook, never set in production). So ctx->votes[*] stayed calloc'd zero -> ha_compute_safe_committed_lsn saw n_granted=1 (self) -> safe_committed_lsn = self_lsn. A standby promoting with its own durable_floor=1 then persist_truncate_wal_above(1) destroyed committed LSN 2..5. + +## Fix +PRIMARY: in vote_reply_dispatch production branch, route the decoded reply into the election context via a NEW record-only seam ha_record_vote_reply(ha,ctx,idx,term,granted,tag,lsn) BEFORE raft_on_vote_reply. Factored ha_record_vote_reply_locked out of handle_request_vote_reply: it records ctx->votes[idx] under ctx->lock with the idempotency guard but does NOT call ha_elect_evaluate_and_signal -> no second/racing legacy promotion path (raft_on_vote_reply remains the SOLE production promotion driver; the two quorum tallies (raft s->votes_granted / legacy ctx->votes_granted) and their guards (s->election_decided / ctx->done_posted) stay independent and idempotent). idx == vctx->peer_idx == the local ctx->votes[] index (absolute->local map done in raft_cb_send_vote_request), exactly the index ha_compute_safe_committed_lsn + the test hook use. SECONDARY: persist_wal_write_replicated now advances durable_lsn to the synced tip immediately after the inline fdatasync (mirror persist_flush; monotonic max; fail-loud rc-gated) so a standby advertises its own synced durable floor without waiting for the 10ms cadence flusher. VERIFIED: unit test_safe_commit_reflects_granting_voter_lsn RED (safe=1 'from 1 granted voters') -> GREEN (safe=5 'from 3 granted voters'); 177/177 controller_unit_test pass. NOTE: the LIVE clean_failover_preservation harness still FAILS, but the failure is PRE-EXISTING on baseline 14c41cb (verified by stash+rerun: baseline runs also diverge / lose jobs) — it is the SEPARATE bug-795/828 replication+catch-up gap (standbys' WALs physically contain only LSN 1..2, durable_floor=1; committed entries 2..5 never reach the standby WAL; primary advanced committed_lsn to 5 without a standby durable at 5). The fix DID change the live fingerprint from 'from 1 granted voters' to 'from 2 granted voters', proving the plumbing is now alive. safe-commit-from-votes is correct; the residual live job loss is out of scope (needs the catch-up/snapshot-backfill fix tracked under ha-commit-durability). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-795, bug-851 diff --git a/.wolf/bugs/bug-845.md b/.wolf/bugs/bug-845.md new file mode 100644 index 00000000..88e4695f --- /dev/null +++ b/.wolf/bugs/bug-845.md @@ -0,0 +1,38 @@ +--- +id: "bug-845" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-20T14:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/hook.c (xce_add_hook idempotency); root path src/controller/{persist.c persist_start", "ha.c ha_register_persist_hooks}"] +tags: ["double-register", "duplicate", "ha", "hook-dedup", "investigation", "job-submit", "persistence", "wal", "waldump"] +related: ["bug-795", "bug-828"] +legacy: true +--- + +# bug-845: Every committed job appears as TWO consecutive WAL_JOB_SUBMIT entries with the SAME jobid (e.g + +## Description +Every committed job appears as TWO consecutive WAL_JOB_SUBMIT entries with the SAME jobid (e.g. ctrl-1 LSN 2,3 = jobid 1868520205844480). Surfaced by waldump on /tmp/xce-ha-deploy. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +NOT array expansion — job_submit() assigns array members DISTINCT ids (base_id+i, job.c) and the Job capnp schema has a single jobid field, so identical-jobid pairs cannot be two array tasks. A single client submit is producing two WAL_JOB_SUBMIT appends of the same job. EXACTLY 2 entries per job (never 1 or 3), always at ADJACENT LSNs, for every job on every node. CONFIRMED via enhanced waldump (payload-CRC + subtime columns): each pair is BYTE-IDENTICAL (same PCRC, e.g. LSN 2,3 both 6d79074e; same subtime 1781959730) — proving a literal double-append of the SAME encoded job_t in one submit flow; a failover retry would re-stamp subtime=time(NULL) and differ, so retry is DEFINITIVELY ruled out. Remaining candidates: AFTER_EVENT jlist contains the job twice, or the JOB_NEW hook/handler persists twice per submit. So 'ctrl-1 has 12 submits' is really 6 jobs (halve when reconciling failover-test job totals). Doubles LSN consumption; consistent across nodes here (F@14,15 on both ctrl-1 and ctrl-3) so not itself divergence, but a latent cross-node LSN-divergence risk if the duplicate count ever varies by primary. + +## Fix +FIXED 2026-06-23 (src/lib/hook.c). Root cause: persist_job_hook_fun was registered on XCE_CONTROLLER_HOOK_JOB by TWO owners — persist_start() (unconditional, boot) AND ha_register_persist_hooks() (on PRIMARY promotion) — with the SAME persist_t closure. xce_add_hook did not dedup, xce_run_hooks fires ALL slots, so a promoted primary had 2 active slots => persist_job_hook_fun ran twice per submit => 2 byte-identical adjacent WAL_JOB_SUBMIT appends. Fix: made xce_add_hook idempotent on exact (fn,data) — re-registering an active (fn,data) is a no-op; same fn with DISTINCT data still registers (legitimate multi-closure). Preserves the persist_start test contract many unit tests rely on. Verified: WAL now single-entry per job (waldump); 5/5 hook unit tests; controller suite 343 tests 0 new failures; 3/3 post-fix HA soaks zero job loss. NOTE: an earlier post-fix soak hit the KNOWN intermittent failover durability race (bug-795/828 class, LSN-10 cross-node divergence) — confirmed pre-existing (baseline passes/fails on timing too), NOT caused by this dedup. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-795, bug-828 diff --git a/.wolf/bugs/bug-846.md b/.wolf/bugs/bug-846.md new file mode 100644 index 00000000..a79e2181 --- /dev/null +++ b/.wolf/bugs/bug-846.md @@ -0,0 +1,38 @@ +--- +id: "bug-846" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-20T15:20:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/xceapi.c (per-op state guard) + src/lib/client_failover.c (xce_failover_classify / xcecli_retry_decide)"] +tags: ["client", "stale-report", "einval", "blocker", "failover", "ha", "libxce"] +related: ["bug-646"] +legacy: true +--- + +# bug-846: After the primary dies, xce_test client ops (ctrlstatus/joblist/jobsub) fail PERSISTENTLY with… + +## Description +After the primary dies, xce_test client ops (ctrlstatus/joblist/jobsub) fail PERSISTENTLY with 'Invalid argument' (EINVAL) and wait-primary times out, even though a new primary was elected (ctrl-2 term 3). Blocks failover testing from the client. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Client-side, NOT server. ha_dispatch_gate ALLOWS OPCode_ctrlstatus in every role and rejects client writes with the RETRYABLE -XCE_ERR_NOT_READY (never EINVAL), so a reachable controller would answer. The EINVAL is the xceapi per-op guard: `if (ctx->state != XCE_CTX_RUN) return -EINVAL`. The failover loop only AUTO-RECOVERS ctx.state==XCE_CTX_ERR (xcecli_retry_decide rotates+recovers); xce_failover_classify maps EINVAL -> XCE_FO_TERMINAL ('everything else -> terminal'), so a bare EINVAL is surfaced and the sticky master cursor never advances to the live new primary. So once the ctx lands in a non-ERR non-RUN state (INIT/DOWN) — or yields a bare EINVAL while RUN — failover gives up permanently. Exact stuck state needs a live repro to confirm. + +## Fix +FIXED (verified 2026-07-02, no direct code trail to when — landed under libxce-failover). src/lib/client_failover.c xcecli_retry_decide() now checks `ac->ctx.state == XCE_CTX_ERR` BEFORE classifying rc, bypassing the -EINVAL misclassification entirely (comment there literally reads 'the reported bug'). xcecli_ctx_recover() resets ERR->RUN post-reconnect. This is exactly recommendation (B) from this bug's own original fix note. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-646 diff --git a/.wolf/bugs/bug-847.md b/.wolf/bugs/bug-847.md new file mode 100644 index 00000000..9c043112 --- /dev/null +++ b/.wolf/bugs/bug-847.md @@ -0,0 +1,38 @@ +--- +id: "bug-847" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-20T14:37:08.863Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: ["src/lib/request.c (xce_wal_batch_encode_raw / wal_entry_to_wire / xce_decode extract loop) — capnp multi-segment Data pointers"] +tags: ["c", "reproducer-unignored", "corruption", "ha", "waldump", "stale-report", "reverified-open", "catchup", "data-loss", "replication", "capnp", "critical", "wal"] +related: ["bug-828", "bug-795", "bug-661"] +legacy: true +--- + +# bug-847: Catch-up WAL streaming silently DROPS entry payloads at scale: a rejoining controller (ctrl-1… + +## Description +Catch-up WAL streaming silently DROPS entry payloads at scale: a rejoining controller (ctrl-1 restarted) received [1..14] with correct LSN/type/CRC but PLEN=0 / empty job blobs (waldump PCRC=00000000, jobid=?). Structurally-intact but content-empty WAL = silent data corruption. THE committed-job-loss mechanism. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Multi-segment capnp bug. xce_wal_batch_encode_raw builds a WalCatchupEntry struct-list where each element's payload is a capnp Data (List(UInt8)) allocated via capn_new_list8(cs=root_seg,...). When the message outgrows its first segment (large batch), the per-element payload Data pointer is lost across the segment boundary — on decode extract_data_bytes(we.payload) reads CAPN_NULL and returns len 0. write_WalCatchupEntry uses capn_setp (far-pointer-capable) so the fault is in the overflow/segment handling as used here. REPRODUCED deterministically: src/lib/testsuite/test_wal_wire.c test_wal_batch_roundtrip_large_payloads (14 entries x 872B) FAILS 'Expected 872 Was 0'; the 2-entry test_wal_batch_roundtrip_payload_bytes PASSES. Threshold is size, not count. Live replication (repl_broadcast) carries payloads fine — ONLY the catch-up/rejoin path corrupts, which is why loss only appears after a node restarts and catches up. + +## Fix +FIXED — CONFIRMED via un-ignoring its own purpose-built RED reproducer (strongest possible evidence). Removed the TEST_IGNORE_MESSAGE guard from test_wal_batch_roundtrip_large_payloads (src/lib/testsuite/test_wal_wire.c); rebuilt and ran it 2x under ASAN with leak detection ON — clean PASS both times, all 14 entries' 872-byte payloads byte-exact, zero leaks (285/285 lib tests, 0 failures, 0 ignored). Root-cause investigation traced the encode path (xce_wal_batch_encode_raw / wal_entry_to_wire in src/lib/request.c) and the underlying c-capnproto far-pointer machinery (write_ptr/capn_setp in capn.c) — the cross-segment struct->Data far-pointer path is correct by construction (new_object relocates and re-tags .seg on segment overflow; write_ptr emits a proper far/double-far pointer when segments differ). capn-malloc.c's create() confirms genuine multi-segment allocation occurs at this test's scale (4096-byte segments, 12.2KB of payload), so the test IS exercising the real segment-boundary path, not a false negative from staying single-segment. No production code changed — whatever regressed this (library version bump via the documented nix flake repin, or an incidental request.c change) already fixed it; the TEST_IGNORE was simply never removed. This is the 6th bug in this sweep found to be already-fixed-but-untracked (846/855/1096/1147/1198/847) — buglog.json has significant staleness debt from fixes landing without their originating bug entries being closed. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-828, bug-795, bug-661 diff --git a/.wolf/bugs/bug-848.md b/.wolf/bugs/bug-848.md new file mode 100644 index 00000000..6da562ac --- /dev/null +++ b/.wolf/bugs/bug-848.md @@ -0,0 +1,38 @@ +--- +id: "bug-848" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-20T14:40:02.211Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 3 +files: ["src/controller/cluster.c (cluster_init_phase2", "after persist_recover) + src/controller/ha.c (ha_init applied_lsn=0; state report sr->applied_lsn)"] +tags: ["c", "ha", "raft", "recovery", "catchup", "applied_lsn", "data-loss", "root-trigger", "critical"] +related: ["bug-847", "bug-828", "bug-795"] +legacy: true +--- + +# bug-848: On a PRESERVED-DATA restart (no wipe — user confirmed), a controller recovers its WAL [1..7] from… + +## Description +On a PRESERVED-DATA restart (no wipe — user confirmed), a controller recovers its WAL [1..7] from disk but rejoins advertising applied_lsn=0, so the primary re-streams the WHOLE log [1..committed] and O_TRUNC-overwrites the node's already-correct WAL. Logs: 'HA: learned committed_lsn=10 ... (was 0)'. THE root trigger that forces the full re-stream (then bug-847 empties it). + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ha->applied_lsn is NEVER reconstructed from the recovered WAL on boot. ha_init zeros it (ha.c:1367); persist_recover (cluster.c:1352) replays the WAL and advances persist->current_lsn/durable_lsn, but nothing copies that recovered position into ha->applied_lsn (persist_t and ha_state_t are separate; the only setter besides ha_init is the SNAPSHOT receive path ha.c:1771 = checkpoint_lsn). So the standby state report advertises sr->applied_lsn = ha->applied_lsn = 0 (ha.c:1787) and the primary's ha_evaluate_repl_action streams [0+1 .. committed]. ASYMMETRY proving it's a bug: ELECTION advertise uses ha_candidate_advertise_lsn = persist_durable_floor (the RECOVERED position, correct), but CATCH-UP advertise uses the un-recovered applied_lsn=0. The system knows its durable tip for voting but not for catch-up. Defeats Raft incremental catch-up for every restarted node; also the upstream cause of the earlier missing-prefix cases (full re-stream from a trimmed primary -> prefix gap). + +## Fix +FIXED 2026-06-20 (HA-layer). Added ha_sync_applied_lsn_from_recovery(ha,p) (ha.c/ha.h) = monotonic store of persist_durable_floor(p) into ha->applied_lsn; wired into cluster_init_phase2 right after persist_recover, plus a diagnostic INFO log of recovered durable_floor/current_lsn/applied_lsn. TDD: test_recovered_member_advertises_real_applied_lsn (test_persist_recovery.c) write->recover->sync asserts applied_lsn==durable_floor>0 and monotonic. controller_unit_test 6/6 + adjacent HA 8/8, daemon builds. CAVEAT: live ctrl-1 also showed current_lsn=1 (seg O_TRUNC'd from LSN 1) implying persist_recover may not have loaded the WAL on that restart at all; if so this HA-layer fix is necessary but not sufficient. The new INFO log reveals on the next live restart whether persist_recover is effective (durable_floor>0) or not (bug-849 candidate: recovery not running/loading on the rejoin path). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-847, bug-828, bug-795 diff --git a/.wolf/bugs/bug-849.md b/.wolf/bugs/bug-849.md new file mode 100644 index 00000000..a7b0d1f0 --- /dev/null +++ b/.wolf/bugs/bug-849.md @@ -0,0 +1,38 @@ +--- +id: "bug-849" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-20T14:42:09.400Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/myapp.c (standby-rejoin path ~317-320) + src/controller/cluster.c cluster_init_phase2 (holds persist_recover)"] +tags: ["wrong-value", "md", "ha", "raft", "recovery", "boot", "election-gated", "catchup", "data-loss", "critical"] +related: ["bug-848", "bug-847", "bug-828", "bug-795"] +legacy: true +--- + +# bug-849: After bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL… + +## Description +After bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL [15..16] only — missing [1..14] (waldump 'GAP: missing prefix LSN 1..14'). Payloads now full (bug-848 era), but the prefix is gone. Loss mode changed (empty-payload -> missing-prefix), loss remains. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +CONFIRMED from live logs (/tmp/xce-ha-deploy/logs/ctrl-1.log). persist_recover (and the bug-848 applied_lsn sync, and catch-up) run ONLY inside cluster_init_phase2, which is ELECTION-GATED — its real callers are all ha.c election/promotion paths (ha.c:1021,1166,2943,3022) + cluster_init. First boot ran it (ctrl-1 won an election; logged 'persist_recover done — durable_floor=0 current_lsn=1'). The 11:07:20 RESTART came up as a follower ('myapp_run: CTRL_STANDBY — waiting for primary to connect and push config') with NO election → cluster_init_phase2 NEVER ran → NO 'persist_recover done' line for pid 304897. So the rejoining follower never recovered its on-disk [1..7], operated with current_lsn=1/applied_lsn=0, then received LIVE replication of the new job [15..16] (repl_broadcast), O_TRUNC'd seg-001 and wrote [15..16] there. [1..7] never recovered + [1..14] never caught up = permanent prefix gap. Ordering corollary: live replication advances applied past the gap so the primary's catch-up never backfills it. + +## Fix +FIXED 2026-06-21 (design B — boot-path-universal recovery). Refined root cause: phase2 is invoked ONLY from the election-result path; a rejoining follower that SKIPS its election (goto skip_election in myapp_run because a primary already exists) never reaches it. Fix: in myapp.c the skip-election standby branch now calls cluster_init_phase2(&c) (build resources + persist_recover + bug-848 ha_sync_applied_lsn) when !ha.phase2_done, BEFORE cluster_run starts serving — so every boot path recovers its own WAL and advertises its real applied_lsn before accepting replication. Idempotent via phase2_done (same once-per-process contract as the election callers); recovery decoupled from the primary config push (a restart already loaded config from disk pre-election). Daemon builds; controller suite incl. test_persist_recovery/test_ha_startup/test_repl_catchup/test_ha_failover 8/8 green. LIVE VALIDATION PENDING: redeploy + kill/restart ctrl-1; expect a 'HA: persist_recover done — durable_floor=7 current_lsn=8' line for the restarted pid and ctrl-1 keeping [1..7] + catching up only the suffix. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-848, bug-847, bug-828, bug-795 diff --git a/.wolf/bugs/bug-850.md b/.wolf/bugs/bug-850.md new file mode 100644 index 00000000..434bfdb0 --- /dev/null +++ b/.wolf/bugs/bug-850.md @@ -0,0 +1,38 @@ +--- +id: "bug-850" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-20T15:14:36.135Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/repl.c repl_add_standby (line ~119) + repl_broadcast in_catch_up gate"] +tags: ["wrong-value", "c", "ha", "raft", "replication", "catchup", "ordering", "race", "in_catch_up", "data-loss", "critical"] +related: ["bug-849", "bug-848", "bug-847", "bug-828", "bug-795"] +legacy: true +--- + +# bug-850: A rejoining standby receives LIVE replication of new entries before its historical catch-up… + +## Description +A rejoining standby receives LIVE replication of new entries before its historical catch-up, advancing current_lsn past the gap so the idempotency guard skips the catch-up of the prefix. Explains ctrl-1 ending with [15..16] only (gap [1..14]). Distinct from bug-849; persists even after recovery. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +repl_add_standby initializes a freshly-registered standby with in_catch_up=0, so it is IMMEDIATELY eligible for repl_broadcast (which correctly skips in_catch_up standbys at repl.c:27). Mechanism (confirmed from ctrl-1.log 11:07:21): ctrl-1 reconnects -> registered in_catch_up=0 -> the new job [15..16] is live-broadcast to it (not gated) -> persist_wal_write_replicated writes them, current_lsn jumps to 17 -> THEN the catch-up stream of [1..14] arrives but persist_wal_write_replicated's idempotency guard (lsn+1 <= current_lsn) skips ALL of them (1+1<=17 ... 14+1<=17) -> permanent prefix gap [1..14]. Independent of bug-849: even with recovery (current_lsn=8), a live [15..16] arriving first jumps current_lsn to 17 and the catch-up of [8..14] is skipped (gap [8..14]). Root architectural cause: catch-up streaming and live broadcast are SEPARATE unordered paths (unlike Raft AppendEntries which is one ordered per-follower stream). + +## Fix +FIXED 2026-06-21 via a CONTIGUITY GUARD at the persist layer (Raft log-matching), NOT in_catch_up. First tried defaulting in_catch_up=1 in repl_add_standby — REVERTED: it broke the whole controller suite (commit-WAIT tests TIMEOUT 300s because a fresh standby no longer counts toward write quorum until convergence, and the unit tests don't run the heartbeat dispatch that would clear it). Correct fix: persist_wal_write_replicated now REJECTS a forward gap (lsn > current_lsn) with -EAGAIN instead of writing it — so a live broadcast that arrives ahead of the in-order catch-up is DROPPED (no write, no ack, no current_lsn jump), and the ACK-driven catch-up continuation re-streams from current_lsn so the log fills in order. ha_apply_single_entry + ha_apply_wal_catchup_entry treat -EAGAIN as a quiet skip (TRACE, no ack, no ERROR spam). Combined with bug-848 (advertise durable_floor == current_lsn-1) the catch-up always starts AT current_lsn so it is never spuriously rejected; only genuine forward gaps are. Files: src/controller/persist.c (guard), src/controller/ha.c (quiet -EAGAIN). TDD: test_repl_write_rejects_forward_gap (test_repl_apply.c) — contiguous write ok, forward-gap lsn 5 @ current_lsn 2 -> -EAGAIN + current_lsn unchanged, fill 2..4 then 5 ok, recovery clean. FULL controller suite green (129/0/0, no timeout); daemon builds. LIVE VALIDATION PENDING with bug-848+849+850 together. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-849, bug-848, bug-847, bug-828, bug-795 diff --git a/.wolf/bugs/bug-851.md b/.wolf/bugs/bug-851.md new file mode 100644 index 00000000..703f4efd --- /dev/null +++ b/.wolf/bugs/bug-851.md @@ -0,0 +1,45 @@ +--- +id: "bug-851" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "data-loss" +timestamp: "2026-06-21T13:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/controller/persist.c:588/726/1040 (hdr.term=p->term", "p->term never set) + src/controller/ha.c (election term = local+1", "PreVote no-persist; two leaders per term) + src/controller/ha_convergence.c (reconcile cannot (term", "lsn) log-match)"] +tags: ["ha", "failover", "replication", "convergence", "raft-safety", "term-monotonicity", "wal-term", "committed-divergence", "data-loss", "spec-needed"] +related: ["bug-850", "bug-849", "bug-848", "bug-646"] +fix_commits: ["b77d412"] +legacy: true +--- + +# bug-851: CHAOTIC leadership churn (3x kill-the-primary) causes SILENT committed-prefix DIVERGENCE: all 3… + +## Description +CHAOTIC leadership churn (3x kill-the-primary) causes SILENT committed-prefix DIVERGENCE: all 3 nodes report committedLsn=54 with lsn range 1..54 / 0 gaps / CRC intact, but WAL CONTENT diverges at LSN 45-54 (ctrl-1/ctrl-2/ctrl-3 each hold DIFFERENT jobs at the same LSNs; fps bfeb26ad/216af434/865304ec). Violates Raft committed-prefix invariant (a committed entry must be byte-identical on every member). Originally seen as a liveness 'commit stall'; the deeper defect is divergence + acked-job loss under churn. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +ROOT-CAUSED 2026-06-21 (owned-cluster repro, see .kiro/specs/ha-commit-durability/repro-851/). THREE compounding defects: (1) Election term not monotonic across cluster: term = locally-persisted+1, PreVote does NOT persist/propagate ('PreVote round ... no persist'); under chaotic kills a leader wins term N, appends own-term noop+jobs, and is killed BEFORE term N propagates to a majority; a restarted node recovers a STALE term and elects at the SAME N+1 => TWO leaders share a term (logs: term3 led by ctrl-3@08:40 AND ctrl-1@08:47:39; term4 led by ctrl-2@08:47:52 AND ctrl-3@08:48:04), each appending its own-term noop at the SAME lsn 45. (2) WAL entry term field is DEAD (always 0): wal_entry_hdr_t.term exists (wal_types.h:30) but writer does hdr.term=p->term (persist.c:588/726/1040) and p->term is NEVER assigned from ha->term anywhere in the controller (zero matches) => every entry persists term=0; waldump confirms term=0 cluster-wide. (3) No (term,lsn) reconcile: with no usable per-entry term, ha_convergence.c reconcile cannot Raft-log-match, so a divergent same-lsn suffix (45@term3 vs 45@term4) looks already-present and is never truncated; ha_compute_safe_committed_lsn then advances committed 'from 1 granted voter' over content other members never received. NOTE: a CLEAN single failover fully converges (all WALs identical 1..22, 848/849/850 validated) — only CHAOTIC churn triggers it. + +## Fix +FIXED AND VERIFIED LIVE -- three distinct contributing root causes were found and fixed, all owned by ha-term-safety (spec.json validate_impl decision: GO, 2026-07-17): (1) Term propagation -- repl_broadcast (persist.c:747, the live replicator) built its wire entry without setting entry.term, so live traffic wire-encoded term=0 even though the primary's own header had the real term (same root cause as bug-990, fixed together in task 3.4: entry.term=hdr->term now carried through the live send+intake path, term-0 legacy direct-apply retired). 175/175 unit tests, live-confirmed on real fs. (2) Election-Safety violation (bug-1095/bug-1096, the 'bug-851 family', found by the raft-sim-conformance oracle): raft_on_vote_reply in libxce's raft.c double-counted duplicate vote replies and never dropped stale-term replies, allowing two leaders at the same term -- fixed with a per-peer vote-reply dedup + stale-term drop, unit-tested at both the raft-sim and controller level. (3) Premature election-time truncation (bug-828, closely related): ha_enter_real_election was truncating the WAL to a stale last_known_committed_lsn before computing the safe commit point, destroying already-durable committed entries -- fixed (commit b77d412), live-verified: clean failover preserves committed jobs, walcompare committed-prefix identical across all 3 members across multiple runs. ha-term-safety's GO decision explicitly cites a fresh 6-round LIVE chaotic-multi-kill soak (bug-851's exact repro scenario) passing with zero committed divergence/job loss -- superseding an earlier in-task note (task 5.4) that had called this env-blocked before that later live run completed. + +[Original entry] +NOT FIXED (HA-core + persistence => full spec workflow per CLAUDE.md, not a quick patch). Keystone lever: stamp hdr.term from live ha->term on the primary append path + carry term through the repl envelope + persist it, THEN make ha_convergence reconcile truncate any local suffix whose (term,lsn) != leader's (standard Raft log-matching). MUST pair with durably-establishing the election term before a new leader acts (no two leaders per term: persist/propagate the won term to a majority before appending the own-term noop, or bump term beyond any (term,lsn) a voter advertises). Route to ha-consistency-invariants / ha-convergence-recovery as a new sub-spec. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-850, bug-849, bug-848, bug-646 + +## Legacy notes +- **root_caused:** 2026-06-21T09:10:00.000Z diff --git a/.wolf/bugs/bug-854.md b/.wolf/bugs/bug-854.md new file mode 100644 index 00000000..147a24d7 --- /dev/null +++ b/.wolf/bugs/bug-854.md @@ -0,0 +1,38 @@ +--- +id: "bug-854" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-21T09:10:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 2 +files: ["src/lib/client.c:403 xcecli_ctrlstatus_primary + src/controller/ctrlstatus.c:104 (standby primaryName=\"\")"] +tags: ["libxce", "client", "failover", "primary-discovery", "ctrlstatus", "ha", "FIXED", "client", "rotation", "xcecli_ctrlstatus_primary"] +related: ["bug-846", "bug-851", "bug-646"] +legacy: true +--- + +# bug-854: Client primary-discovery is POSITIONAL, not failover-correct: with masters=[m0,m1,...] the client… + +## Description +Client primary-discovery is POSITIONAL, not failover-correct: with masters=[m0,m1,...] the client only finds the primary if m0 IS the primary. wait-primary: 9993(primary-first)=ok 45ms; 9992,9993(standby-first)=no primary; 9991,9993(dead-first)=no primary; 9993,9991=ok. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Two compounding defects in libxce: (1) XCECLI_FAILOVER_CALL classifies a standby's status-0 ctrlstatus reply as SUCCESS and never rotates the endpoint (ctrlstatus is answerable in every role, so ANY reachable standby answers 0 — the inner call never rotates off it). (2) xcecli_ctrlstatus_primary captures a redirect hint from the non-authoritative reply but does not force rotation to consume it — it re-queries the same standby until the retry budget exhausts, never traversing the master ring. No server-side primaryName source existed on a standby (voted_for_name is not the leader, and primary_host is vestigial/never-written). Verified: baseline (no fix) with standby-first masters does NOT find the primary (pins to standby). + +## Fix +FIXED (2026-06-23). xcecli_ctrlstatus_primary: force xce_failover_rotate(ctx) + xcecli_ctx_recover(ctx) under failover.lock on every non-authoritative reply (hint-first then next-master rotation). Loop floor raised to max(max_retries, nmasters+1). Verified: wait-primary 'ctrl-3' serving after 92 ms with standby-first masters list; end-to-end clean-failover discover pass. Deliberately NOT adding server-side primaryName source — it is only a latency optimization; the client rotation fix alone resolves the hang. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-846, bug-851, bug-646 diff --git a/.wolf/bugs/bug-855.md b/.wolf/bugs/bug-855.md new file mode 100644 index 00000000..1298cb08 --- /dev/null +++ b/.wolf/bugs/bug-855.md @@ -0,0 +1,38 @@ +--- +id: "bug-855" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-22T03:10:06.653962Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/testsuite/test_config_repl.c (tests) / src/controller/persist.c:705 (guard)"] +tags: ["controller-test", "stale-report", "ha-term-safety", "wal-catchup", "EAGAIN", "not-a-regression", "contiguity-guard", "bug-850", "pre-existing"] +related: ["bug-850", "bug-851"] +legacy: true +--- + +# bug-855: controller_unit_test: 6 test_wal_catchup_* tests FAIL (Expected ... Was -11/-EAGAIN or wrong… + +## Description +controller_unit_test: 6 test_wal_catchup_* tests FAIL (Expected ... Was -11/-EAGAIN or wrong applied_lsn) on the raft_tla checkpoint branch + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +PRE-EXISTING on checkpoint 20e59d0, NOT caused by ha-term-safety 1.1/1.2. The bug-850 contiguity guard (persist.c:705 rejects lsn>current_lsn with -EAGAIN) is committed at HEAD, but the test_wal_catchup_* tests feed forward-gap LSNs (30, 42) into a FRESH persist (current_lsn~=1) and still expect them to apply -> guard returns -EAGAIN -> applied_lsn does not advance. ha-term-safety changes touch only vote/schema/typemap/stubs; git diff --stat shows ZERO changes to persist.c/config_repl.c/test_config_repl.c. Also a separate run TIMED OUT 300s (the bug-850 DNR /tmp fsync-EIO + commit-WAIT signature; environment artifact on overlayfs). + +## Fix +FIXED (verified 2026-07-02): ran controller_unit_test directly — all test_wal_catchup_* tests (append_term_fields_round_trip, entry_advances_applied_lsn, entry_error_skipped_no_crash, entry_null_entry_no_crash, batch_advances_applied_lsn, partial_batch, batch_error_entry_skipped, batch_zero_entries_fallback, batch_256_entries) PASS. Landed as part of ha-term-safety's WAL term-index work (owner this bug deferred to). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-850, bug-851 diff --git a/.wolf/bugs/bug-857.md b/.wolf/bugs/bug-857.md new file mode 100644 index 00000000..64ee550c --- /dev/null +++ b/.wolf/bugs/bug-857.md @@ -0,0 +1,41 @@ +--- +id: "bug-857" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-21T10:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c do_upgrade_config_with_bytes (~1001-1163)"] +tags: ["config", "snapshot", "ha", "needs-verification", "spec-needed", "descoped", "config-upgrade-snapshot", "entity-bearing-snapshots", "state-divergence", "raft-safety", "upgrade"] +related: ["bug-851", "bug-828"] +legacy: true +--- + +# bug-857: Config upgrade can leave members with DIVERGENT materialized state: do_upgrade_config_with_bytes… + +## Description +Config upgrade can leave members with DIVERGENT materialized state: do_upgrade_config_with_bytes pushes config bytes only then restart_requested; each node re-derives state by replaying its WAL under the NEW config, with no snapshot pinning an identical post-upgrade state. Replay is config-dependent (lost_and_found re-homes orphaned jobs), so nodes can diverge; amplified by bug-851 WAL tail divergence. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Config is a separate consistency domain (committed_config_tag, no term, tag anti-entropy) outside both the WAL log and raft.c. Upgrade activation = config push + restart + per-node WAL replay under new config; no snapshot transfers an identical materialized state, and state is interpreted against config. Identified via TLA+ gap analysis (config should be log-entry/snapshot-anchored). NOT empirically reproduced yet. + +## Fix +FIXED AND VERIFIED LIVE -- config-upgrade-snapshot (the spec this was descoped into) is fully task-complete: every task 1 through 8.9 is done, including task 8.5 ('Reproduce bug-857 on the pre-change behavior and confirm it no longer occurs') and task 8.6 (a live chaotic multi-kill soak, 2026-07-29, confirming zero materialized-state/config divergence after a config upgrade). The fix materializes a full entity-bearing state snapshot at the config-change marker (term,lsn) boundary and installs it on every member, exactly as originally planned. Note: config-upgrade-snapshot/spec.json's own `phase` field was left stale at 'tasks-generated' despite the spec being fully implemented -- corrected separately alongside this entry. + +[Original entry] +NOT FIXED. Descoped 2026-06-21 from ha-term-safety R8 into the independent `config-upgrade-snapshot` spec (.kiro/specs/config-upgrade-snapshot/, requirements generated). Fix = upgrade materializes a FULL entity-bearing state snapshot at the config-change marker (term,lsn) boundary, embeds new config, installs on every member; no per-node replay-under-new-config. HARD dependency: entity-bearing-snapshots (snapshots are a 16-byte LSN sentinel today; COW xzarray_freeze/thaw built but dormant). Verify by reproducing bug-857 (upgrade during chaotic multi-kill, diff materialized state). + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-851, bug-828 diff --git a/.wolf/bugs/bug-884.md b/.wolf/bugs/bug-884.md new file mode 100644 index 00000000..6eaf0f7e --- /dev/null +++ b/.wolf/bugs/bug-884.md @@ -0,0 +1,39 @@ +--- +id: "bug-884" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "lib" +severity: "unknown" +timestamp: "2026-06-21T15:20:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/lib/testsuite/test_config.c"] +tags: ["test", "config", "stale-assertion", "ha-commit-engine"] +related: [] +fix_commits: ["7b6cbb8"] +legacy: true +--- + +# bug-884: test_persistence_constants:FAIL Expected TRUE Was FALSE (DEFAULT_SYNC_TIMEOUT_MS==50) + +## Description +test_persistence_constants:FAIL Expected TRUE Was FALSE (DEFAULT_SYNC_TIMEOUT_MS==50) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +commit ca1d9ae raised DEFAULT_SYNC_TIMEOUT_MS 50->500 in config.h but did not update the test assertion; pre-existing red on HEAD, unrelated to raft + +## Fix +synced assertion to ==500 (commit 7b6cbb8) + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-918.md b/.wolf/bugs/bug-918.md new file mode 100644 index 00000000..ceb2072a --- /dev/null +++ b/.wolf/bugs/bug-918.md @@ -0,0 +1,41 @@ +--- +id: "bug-918" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "build" +severity: "unknown" +timestamp: "2026-06-22T03:30:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: ["src/flake.nix"] +tags: ["nix", "flake", "c-capnproto", "env", "build", "worktree-isolation", "worktree"] +related: [] +legacy: true +--- + +# bug-918: nix develop fails: hash mismatch for c-capnproto fixed-output derivation (expected… + +## Description +nix develop fails: hash mismatch for c-capnproto fixed-output derivation (expected sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM=, got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk=) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +flake pins c-capnproto to the MUTABLE github archive/main.tar.gz; upstream main branch advanced so the tarball content/hash drifted, breaking the fixed-output derivation. Only pre-cached nix stores still build. + +## Fix +RESOLVED: user refreshed src/flake.lock; fresh nix develop builds (capnproto 1.2.0). A clean controller rebuild made the full controller test suite green (Ok:1 Fail:0, 12.7s) — the earlier wal_catchup -EAGAIN/decode failures were a STALE incremental-build artifact (schema compiled against the drifted toolchain), not fsync or a source regression. RECURRENCE 2026-07-23 (config worktree, /xce-build controller build+report task for config-upgrade-snapshot Task 1 RED-phase capture): identical hash mismatch reproduced verbatim (same specified sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM= vs got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk= for capnpc-1.0.1.drv source.drv) on FIRST invocation and again on an identical immediate RETRY (deterministic, not transient -- unlike the unrelated ninja-race bugs, retrying does not help here). Build never reached compilation; no xce_controller, controller_unit_test, or xce_test binaries produced; controller.capnp.c/h were not regenerated. This worktree (config, at .../worktrees/config) still has the pre-fix flake.lock/pin state -- the 2026-06-22 fix (refreshing src/flake.lock) was apparently not carried into this worktree, or upstream c-capnproto main.tar.gz drifted again since. NOT fixed this run (build+report only, no source/flake edits authorized by the task). Next step: refresh src/flake.lock in this worktree (nix flake update / lock the xbuild input) the same way the 2026-06-22 fix did, then retry the controller build. RECURRENCE 3 (2026-07-23, config worktree, config-upgrade-snapshot Task 1 implementer session): identical hash mismatch reproduced again AFTER a prior subagent had already logged occurrence 2 (build+report task, no fix attempted). This session went further: (1) confirmed `nix flake lock --update-input xbuild` in src/ is a NO-OP -- xbuild is already pinned to the latest available shen390s/my-nix-channel@master commit (073f527e), so re-locking flake.lock does NOT fix this recurrence (the previous 2026-06-22 fix's mechanism -- refresh flake.lock -- no longer applies once xbuild is already current); (2) confirmed via direct `nix develop` in the CANONICAL non-worktree repo (/home/rshen/projects/xce/xce/src) that it fails IDENTICALLY (same two sha256 hashes, same capnpc-1.0.1.drv path) -- this is a live, machine-wide, upstream c-capnproto main.tar.gz drift, NOT a worktree-isolation problem, and is not fixable from within this repo at all (the broken pin lives inside the external shen390s/my-nix-channel flake, at whatever commit is currently latest); (3) WORKAROUND discovered and used successfully: this machine's /nix/store already has REALIZED (fully built, not just .drv) output paths for capnpc-1.0.1 and capnproto-1.2.0 from a build that completed before the drift -- `find /nix/store -maxdepth 1 -iname '*capnp*'` surfaces them; invoking their bin/capnp and bin/capnpc-c binaries DIRECTLY (bypassing `nix develop`'s full devShell evaluation entirely) still works and successfully compiled a controller.capnp schema change end-to-end, generating correct bindings. This unblocks schema-level capnp verification even while the full `nix develop` shell is broken, but does NOT unblock a full project build/link -- controller_unit_test additionally needs `librtti` (fetched via a private SSH git flake input), which has NEVER been built/cached on this machine at all (no .drv or realized store path exists for it), so a full build+test run remains genuinely infeasible until either upstream fixes the c-capnproto pin or a machine with a pre-cached librtti+full toolchain is used. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ + +## Legacy notes +- **recurrence_4_RESOLVED:** 2026-07-24, config-upgrade-snapshot task 1 resumption session: RESOLVED (repo-local fix, no longer machine-wide-blocked as recurrence 3 concluded). Added a local override in src/nixSupport/xce.nix's capnpc derivation -- `src = pkgs.fetchFromGitHub { owner = "shen390s"; repo = "c-capnproto"; rev = "e45ff682c4756ac5067cbac6d0f282a8ea5f0155"; sha256 = "sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk="; }` -- pinning capnpc's source to an IMMUTABLE commit (the exact content the drifted `main` tarball happened to resolve to, so the hash matches what was already realized in the local /nix/store from recurrence 3's workaround) instead of upstream xbuild's package.nix pinning the MUTABLE `main` ref. This makes the fix independent of upstream's moving target permanently, not just until the next drift. Verified end-to-end: `nix develop --command meson setup xce.build --buildtype=debug -Db_sanitize=address --reconfigure && ninja -C xce.build controller/controller_unit_test` built clean (librtti and all other deps resolved fine -- recurrence 3's librtti-never-cached concern did not reproduce, this machine already had it), and the full `controller_unit_test` binary ran to completion: 978 Tests 0 Failures 0 Ignored, OK. Do-not-repeat: when a fixed-output-derivation hash mismatch traces to an upstream flake pinning a MUTABLE branch/ref (not a tag/commit), the durable fix is a local override.overrideAttrs pinning `src` to an explicit immutable commit + matching sha256 in the CONSUMING repo's own nix file -- do not wait on upstream or keep re-locking flake.lock (recurrence 3 already proved that path is a dead end once xbuild is already at its latest available commit). diff --git a/.wolf/bugs/bug-924.md b/.wolf/bugs/bug-924.md new file mode 100644 index 00000000..4e5f9a06 --- /dev/null +++ b/.wolf/bugs/bug-924.md @@ -0,0 +1,44 @@ +--- +id: "bug-924" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "unknown" +severity: "unknown" +timestamp: "2026-06-22" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 4 +files: ["environment (/ overlay filesystem)"] +tags: ["disk-space", "enospc", "overlayfs", "fdatasync-eio", "controller-tests", "flaky", "ha-term-safety", "recurring", "hang", "timeout"] +related: [] +legacy: true +--- + +# bug-924: controller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11… + +## Description +controller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11 EAGAIN) during ha-term-safety 1.3 validation + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Disk filled to 100% (7466 leftover /tmp/xce_* WAL dirs from repeated ASAN test runs). make_fresh_persist's persist_init mkdir_p hit ENOSPC -> returned NULL -> cascade of 'Expected Non-NULL'. Also overlayfs returned fdatasync EIO under disk pressure. NOT a code regression: baseline full suite passes; test_config_repl catchup tests are order-dependent and fail in isolation on baseline too. + +[UPDATE 2026-09-01, worker-rejoin-resync task 7.2] Re-observed a FOURTH time, with a NEW manifestation shape not previously documented: 14367 leftover /tmp/xce_* dirs, 876.7G, filesystem at 99% used / 12.4G available. This time it did NOT surface as 'Expected Non-NULL' assertion cascades or a silent exit(1) -- it surfaced as controller_unit_test/`meson test` simply HANGING (TIMEOUT, killed by signal 15 after 300s) with no error output at all, reproducible across multiple independent invocations (both via `meson test` and the raw binary directly), always stalled inside persist_fini()/fsync_worker's pthread_cond_timedwait/pthread_cond_destroy teardown path (confirmed via gdb `thread apply all bt` on the wedged process -- Thread 2 stuck in fsync_worker's pthread_cond_timedwait at persist.c:847, Thread 1 stuck in persist_fini's pthread_cond_destroy at persist.c:1362), at a DIFFERENT test each time (not always the same call site) -- consistent with overlayfs fdatasync degrading under near-full disk rather than erroring outright, causing the flusher thread to sit in a slow/stuck wait instead of the previously-documented fast ENOSPC/EIO failure. `find /tmp -maxdepth 1 -name 'xce_*' -exec rm -rf {} +` (no live xce process holding any of them open, confirmed via `ps aux` first) freed 612G (99% -> 30% used); the exact same test run (`controller_unit_test -t test_worker_resync`) that had twice hung/timed out at 90s then completed cleanly in under a second. General lesson extended: an unexplained TIMEOUT/hang in this suite (not just an assertion-failure cascade) is an equally likely symptom of this same recurring /tmp exhaustion -- check `df -h /tmp` before spending time bisecting a hang as a suspected new concurrency bug in the code under test. + +## Fix +rm -rf /tmp/xce_* freed 327GB; full controller suite then Ok:1 Fail:0. For future runs: clean /tmp/xce_* between heavy ASAN test cycles. + +[UPDATE 2026-08-26, xce-test-rc-file task 1.1] Re-observed at much larger scale (14731 leftover /tmp/xce_* dirs, 653.6G, filesystem at 100% used / 0 available) and a MORE SEVERE manifestation than the original 'Expected Non-NULL' assertion failures: mkdtemp() failing inside test_repl_apply.c's make_test_persist_for_apply()/make_test_persist_in() hit ENOSPC, and that pre-existing test file's own hard exit(1) on mkdtemp failure silently killed the ENTIRE controller_unit_test process mid-run (right after test_repl_unit's last test, well before test_repl_apply's own output ever printed) -- exit code 1, no crash/ASAN report, no visible error in the redirected log until grepped for the literal 'mkdtemp failed' stderr line. A leftover orphaned controller_unit_test process (PID from an earlier invocation, blocked in futex_do_wait, presumably stalled retrying a WAL fsync against the full disk) was also found and killed. Fixed for this session via `find /tmp -maxdepth 1 -name 'xce_*' -exec rm -rf {} +` (630G reclaimed, 100% -> 29% used) -- confirmed no live xce process had any of those dirs open before deleting. General lesson: when controller_unit_test dies silently/instantly with exit code 1 and the log just stops mid-file with no crash signature, check `df -h /tmp` and grep for 'mkdtemp failed'/'calloc failed' before assuming a real regression -- this sandbox's /tmp accumulates leftover unit-test mkdtemp() fixture dirs across EVERY session indefinitely (no test file in this suite cleans up its own /tmp/xce_* dirs), so this will keep recurring until either the testsuite adds its own cleanup or something periodically clears /tmp. + +[UPDATE 2026-08-30, bug-1735 verification build] Re-observed a THIRD time, at the largest scale yet: 16238 leftover /tmp/xce_* dirs, 696.1G, filesystem at 100% used / 0 available. This time it manifested as a `controller_unit_test` run reporting `1220 Tests 282 Failures` (mostly test_dispatch_commit.c/test_dispatch_fail_terminal.c/test_dispatch_fail_retry.c/test_jobdispatch_no_double_notify.c 'Expected Non-NULL' -- the exact same fixture-setup-ENOSPC cascade signature as both prior occurrences, on files wholly unrelated to whatever the run was actually verifying) plus a meson-internal `OSError: [Errno 28] No space left on device` while serializing testlog.json/testlog.junit.xml (corrupting those log artifacts, though the console summary line survived). No live xce process was running at the time (`ps aux` confirmed clean) so `find /tmp -maxdepth 1 -name 'xce_*' -exec rm -rf {} +` was safe; reclaimed 696.1G (100% -> 22% used). Re-running the full suite after cleanup is required before trusting any failure count from a run that coincided with disk exhaustion. Still true three occurrences in: nothing in this test suite cleans up its own /tmp/xce_* dirs, so this WILL keep recurring -- the durable fix is adding cleanup to the shared test fixture helpers, not another one-off `rm -rf`. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-970.md b/.wolf/bugs/bug-970.md new file mode 100644 index 00000000..95412bf4 --- /dev/null +++ b/.wolf/bugs/bug-970.md @@ -0,0 +1,38 @@ +--- +id: "bug-970" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-22T10:24:35.905518Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c", "src/controller/persist.c"] +tags: ["ha-term-safety", "task-4.2", "config-marker", "wal", "checkpoint", "compaction", "review-remediation", "controller"] +related: ["bug-851"] +legacy: true +--- + +# bug-970: ha-term-safety task 4.2 rejected (review r1): live config-change path… + +## Description +ha-term-safety task 4.2 rejected (review r1): live config-change path (do_upgrade_config_with_bytes) never appended WAL_CONFIG_MARKER — config_committed_marker had ZERO production callers (dead primitive); and checkpoint/compaction could trim a committed marker segment, regressing the derive to -ENOENT. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +Finding1: do_upgrade_config_with_bytes still drove the term-blind committed_config_tag push/ack/advance and never called config_append_marker, so the derive primitive was dead code. Finding2: persist_checkpoint->trim_wal_segments unlinks older segments and the snapshot sentinel carried no config reference, so a committed marker in a trimmed segment vanished and config_committed_marker regressed/returned -ENOENT. + +## Fix +Added config_commit_change() wired into do_upgrade_config_with_bytes step 7: appends WAL_CONFIG_MARKER as the consensus record (config_append_marker), then delegates byte delivery + the existing majority barrier to push_config_to_standbys (mechanism unchanged); committed_config_tag is now only a cache, config_committed_marker (committed-prefix derive) is authoritative. Compaction safety: persist_checkpoint scans the committed prefix for the latest marker {version_tag,sha256} (scan_latest_committed_config_ref) BEFORE trim and stamps it into the snapshot sentinel (41-byte optional tail); load_snapshot reads it back into persist.cfgref_*; config_committed_marker falls back to it when no marker survives the live WAL. Added integration test (drives config_commit_change) + checkpoint-trim test + persist_test_force_new_segment seam. cb5 upgrade pins made persist-backed so the marker append succeeds. RED confirmed (Expected 1 Was 0; Expected 0 Was -110); GREEN 0 failures direct-run. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-851 diff --git a/.wolf/bugs/bug-976.md b/.wolf/bugs/bug-976.md new file mode 100644 index 00000000..bfab774a --- /dev/null +++ b/.wolf/bugs/bug-976.md @@ -0,0 +1,38 @@ +--- +id: "bug-976" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-22T11:15:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c", "src/controller/ha.c", "src/controller/myapp.c"] +tags: ["ha-term-safety", "config-replication", "verify-before-activate", "serve-fence", "dispatch-gate", "promotion-gate", "safety", "task-4.3", "remediation"] +related: ["bug-851", "bug-924"] +legacy: true +--- + +# bug-976: ha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path… + +## Description +ha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path (handle_repl_config_push pre-activated via load_primary_config before the hash gate); recovery-activate had NO production call site (boot loaded blindly); serve-fence had NO production reader (a fenced member still served) + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +config_activate_committed (verify-before-activate + fence) was unit-correct but not integrated: (A) handle_repl_config_push called load_primary_config UNCONDITIONALLY before config_activate_committed, so mismatched bytes went live before the gate; (B) myapp.c boot step 7 + on_commit_advanced never called the consumer, so a recovering member loaded config blind against the committed marker; (C) ha_dispatch_gate / pre-promotion gate never read config_serve_fence, so a fenced member still served + could promote + +## Fix +(A) handle_repl_config_push now SAVES bytes (conf_save_config) then calls config_activate_committed ONLY — removed premature load_primary_config; activate is the sole activation point (hash-verify gate). (B) config_activate_committed -ENOENT branch falls back to load_primary_config (absent marker != mismatch, no false fence); myapp.c step 7 calls config_activate_committed (latest committed marker, verified). on_commit_advanced documents why activation is deferred off the iomgr thread (async-I/O invariant: scan/load do disk I/O). (C) ha_dispatch_gate CTRL_PRIMARY now refuses config-dependent state-changing ops when config_serve_fenced (replication/read-only pass), pre-promotion gate in raft_cb_become_leader refuses promotion when fenced via new ha_config_promotion_fenced; member stays ALIVE, never self-terminates; fence clears on correct bytes re-verify + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-851, bug-924 diff --git a/.wolf/bugs/bug-981.md b/.wolf/bugs/bug-981.md new file mode 100644 index 00000000..5252ed04 --- /dev/null +++ b/.wolf/bugs/bug-981.md @@ -0,0 +1,38 @@ +--- +id: "bug-981" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-22T11:35:00Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/config_repl.c", "src/controller/ha.c"] +tags: ["ha-term-safety", "config-repl", "serve-fence", "liveness", "async-io", "delaymgr", "commit-advance", "push-before-commit", "R7.9"] +related: [] +legacy: true +--- + +# bug-981: ha-term-safety 4.3 Finding D: on a standby, config MARKER (WAL) and config BYTES (out-of-band… + +## Description +ha-term-safety 4.3 Finding D: on a standby, config MARKER (WAL) and config BYTES (out-of-band push) arrive on independent transports; on a 2nd+ upgrade the NEW bytes are saved before the NEW marker enters the committed prefix, so config_activate_committed hash-mismatches the OLDER committed marker and sets the serve-fence. When the NEW marker later commits, nothing re-runs activation (on_commit_advanced does no disk I/O on the iomgr ACK thread; primary's config-behind re-push net does not fire because conf_save_config already advanced the advertised config_version_tag) -> standby PERMANENTLY serve-fenced + non-promotable until reboot. Liveness violation of R7.9. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +No post-commit retry trigger for a config serve-fence set under the push-before-commit ordering. config_committed_marker returns the older committed marker until the new marker is in the committed prefix; the two activation call sites (boot, handle_repl_config_push) both run BEFORE the new marker commits, and ha_cb_commit_advanced_stub deliberately did nothing (async-I/O invariant forbids disk reads on the iomgr ACK thread). + +## Fix +Added config_schedule_activate_retry(c) (non-blocking ordq_put via delay_task_ms, no-op unless serve-fenced AND delaymgr running) and config_activate_retry_fn(arg) (delaymgr task body that re-runs config_activate_committed off the iomgr thread). ha_cb_commit_advanced_stub now schedules the retry whenever config_serve_fenced(c) is set after commit advances. Once the NEW marker is in the committed prefix and the saved bytes hash-verify, the retry clears the fence and activates; -ENOENT still falls back without fencing; no busy-spin (no-op once unfenced). RED captured by stubbing config_activate_retry_fn to a no-op (fence stayed set, NEW config not loaded); GREEN with the real off-thread retry. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +_Not recorded (migrated from buglog.json)._ diff --git a/.wolf/bugs/bug-990-fix.md b/.wolf/bugs/bug-990-fix.md new file mode 100644 index 00000000..fc28a21b --- /dev/null +++ b/.wolf/bugs/bug-990-fix.md @@ -0,0 +1,38 @@ +--- +id: "bug-990-fix" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-22T16:00:00.000000" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/repl.c (repl_broadcast)", "src/controller/persist.c (persist_wal_async call site + entry_term_locked)", "src/controller/config_repl.c (ha_stream_wal_to_standby catch-up)", "src/controller/ha.c (ha_apply_wal_catchup_entry receive route)", "src/lib/request.c (wal_entry_to_wire per-entry prev tuple encode)"] +tags: ["ha-term-safety", "task-3.4", "live-send-retirement", "term-0", "repl_broadcast", "bug-990", "FIXED", "catch-up-term"] +related: ["bug-990", "bug-851"] +legacy: true +--- + +# bug-990-fix: ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the… + +## Description +ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term; clean-failover committed prefix not byte-identical in TERM (R5.1 term-identity gap). Original bug-990. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The LIVE replication SEND paths (repl_broadcast live append + ha_stream_wal_to_standby catch-up) built wal_catchup_entry_t via memset(0) and never set entry.term/prev_log_index/prev_log_term, so standbys received term=0 entries and applied them via the LEGACY term-0 direct-apply branch (persist_wal_write_replicated stamps the STANDBY's local p->term, not the leader's). The term-carrying library receive route (raft_on_append_entries_request, task 3.2) only fired for term!=0 entries, which the live send never produced. Additionally wal_entry_to_wire only encoded per-entry term, not the per-entry prev tuple, so multi-entry catch-up batches could not be routed through the library one-at-a-time. + +## Fix +Task 3.4 minimal cutover (chosen over driving raft_send_append_entries live, which would rip out the repl model and risk 848/849/850): (1) repl_broadcast gains a prev_log_term param and sets entry.term=hdr->term, entry.prev_log_index=hdr->lsn-1, entry.prev_log_term (sourced under p->lock via new entry_term_locked accessor in persist_wal_async). (2) ha_stream_wal_to_standby sets each catch-up entry's term + prev tuple (tracking the prior streamed entry's lsn/term in ascending order). (3) wal_entry_to_wire now encodes prevLogIndex/prevLogTerm/leaderCommit PER SUB-ENTRY so a multi-entry batch is library-routable. (4) ha_apply_wal_catchup_entry routes every term-carrying entry (single + each batch sub-entry) through new ha_apply_term_entry_via_library -> ha_consume_append_entries -> raft_on_append_entries_request -> on_persist_entry, which persist_set_term(leader_term) before persist_wal_write_replicated (bug-850 contiguity guard preserved beneath). The term-0 legacy direct-apply is retired for live/catch-up entries. ACK/commit path (3.3) unchanged. + +## Verification +Unit RED->GREEN: test_broadcast_carries_leader_term_and_prev_tuple (repl: neuter term -> Expected 7 Was 0) + test_live_broadcast_receive_persists_leader_term (adapter: neuter receive route -> Expected 7 Was 3, i.e. standby persisted its LOCAL term). Full controller suite 175 Tests 0 Failures (clean /tmp direct binary) + meson Ok:1 Fail:0. Live STRICT_TERM gate (clean_failover_preservation.sh) NOT runnable in this overlayfs sandbox: baseline-confirmed no failover completes (split-vote no-promotion within 20s + fdatasync errno=5) — bug-924 env limitation, requires real-fs. The 7 test_wal_catchup_* (test_config_repl.c) + occasional durability FAILs on the direct binary are the SAME pre-existing bug-924 overlayfs fdatasync artifact (identical on baseline). + +## References +- Related: bug-990, bug-851 diff --git a/.wolf/bugs/bug-990.md b/.wolf/bugs/bug-990.md new file mode 100644 index 00000000..aebb6f4e --- /dev/null +++ b/.wolf/bugs/bug-990.md @@ -0,0 +1,41 @@ +--- +id: "bug-990" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-22T13:24:07.476480" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/repl.c (live send repl_broadcast persist.c:747) + src/controller/ha.c (ha_apply_wal_catchup_entry term!=0 legacy direct-apply)"] +tags: ["ha-term-safety", "task-5.2", "live-send-retirement", "term-0", "repl_broadcast", "committed-prefix-divergence", "bug-851", "BLOCKED"] +related: ["bug-851"] +legacy: true +--- + +# bug-990: ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the… + +## Description +ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term (e.g. primary LSN1..5 term=1, ctrl-2/ctrl-3 term=0); a clean failover's committed prefix is byte-identical in payload (pcrc) but NOT in term, so R5.1 'byte-identical WAL including terms' cannot pass. walcompare passes only via legacy-0 -> WAL_LEGACY_TERM tolerance, which masks it on a fresh cluster. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +The live replication SEND path was deferred from tasks 3.2/3.3 and still uses the legacy term-0 repl_broadcast; the standby applies via the legacy direct-apply path, NOT through raft_on_append_entries_request, so it never receives/persists the leader's term. The raft library send loop (on_send_append_entries / raft_send_append_entries) has ZERO live callers. on_persist_entry stamps the real term on the PRIMARY only. + +## Fix +FIXED AND VERIFIED LIVE -- root-caused and fixed together with bug-851 in ha-term-safety task 3.4: repl_broadcast (persist.c:747, the live replicator) built its wire entry without setting entry.term, so standbys persisted term=0 for every live-replicated entry even though the primary's own WAL header carried the real term. Fix carries entry.term=hdr->term/prev_log_term through the live send+intake path (repl_broadcast, persist_wal_async, ha_stream_wal_to_standby, ha_apply_wal_catchup_entry all updated), retiring the term-0 legacy direct-apply branch. 175/175 unit tests, reviewer-approved. Live-confirmed on a real fs ($HOME/tmp/xce-ha-deploy, ext4, 2026-06-22): 'ctrl-2/ctrl-3 persisting the leader's REAL term... walcompare OK over the committed prefix.' + +[Original entry] +NOT FIXED (out of task 5.2 testsuite-only boundary). Remaining integration work: retire the term-0 repl_broadcast live SEND and drive raft_send_append_entries/on_send_append_entries on the live primary so standbys persist the leader's term via raft_on_append_entries_request. Built clean_failover_preservation.sh (XCE_FAILOVER_STRICT_TERM=1) as the ready-to-run assertion this retirement must satisfy. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-851 diff --git a/.wolf/bugs/bug-991.md b/.wolf/bugs/bug-991.md new file mode 100644 index 00000000..a436f3b8 --- /dev/null +++ b/.wolf/bugs/bug-991.md @@ -0,0 +1,38 @@ +--- +id: "bug-991" +status: "fixed" +status_updated: "2026-09-24" +status_note: "" +component: "controller" +severity: "unknown" +timestamp: "2026-06-22T16:30:00.000Z" +last_seen: "2026-09-19T09:00:00Z" +occurrences: 1 +files: ["src/controller/ha.c (ha_apply_term_entry_via_library ~2115; batch ~2205; single-entry ~2225)"] +tags: ["ha", "controller", "wal-catchup", "applied_lsn", "commit-stall", "off-by-one", "term-safety", "bug-991", "streamer-skip"] +related: ["bug-851", "bug-850", "bug-848", "bug-924", "bug-960"] +legacy: true +--- + +# bug-991: Standby WAL catch-up advanced applied_lsn to lsn+1 (next-to-apply) instead of lsn (last-held) + +## Description +Standby WAL catch-up advanced applied_lsn to lsn+1 (next-to-apply) instead of lsn (last-held). The standby then advertised one LSN too high; the leader's catch-up streamer streamed (lsn+1, committed] and SKIPPED the real next-needed entry lsn+1, wedging the library logOk gate forever (no WalAck -> match_index never reaches -> raft commit_index frozen -> committed_lsn pinned at 1). Catch-up-path-specific; live repl_broadcast (absolute-LSN, no skip) commits fine, so failure was nondeterministic. + +## Environment +_Not recorded (migrated from buglog.json)._ + +## Reproduction +_Not recorded (migrated from buglog.json)._ + +## Root cause +applied_lsn carried TWO conflicting semantics. Recovery (=durable_floor), checkpoint (=checkpoint_lsn), init (=0) and the leader's catch-up streamer + ha_evaluate_repl_action all treat it as 'last contiguously-held LSN' and stream (standby_lsn, committed]. But the three live/catch-up apply setters advanced to entry->lsn + 1 ('next-to-apply'). After a standby applied catch-up entry N it advertised N+1, so the leader's next stream began at N+2 and skipped N+1 -> logOk reject -> no WalAck. + +## Fix +fix-1: the three live-apply setters now advance applied_lsn to the LAST-HELD LSN (e->lsn / last_good_lsn / entry->lsn), not +1, so applied_lsn means 'last contiguously-held LSN' everywhere. Reader audit (all consistent with last-held, no +1 reader needed): state report (advertised LSN, ha.c:2461) now matches streamer contract; convergence reported_lsn>=committed_lsn (2536) correct; snapshot/stream boundary standby_lsn+1 routed through the library, so raft.log_len already == applied_lsn after fix-1; raft.log_len legitimately differs from applied_lsn on the legacy term-0 direct-apply and recovery paths, so re-sourcing would regress bug-848 incremental catch-up -> fix-1 alone delivers the value the streamer+logOk need. Unit RED->GREEN: test_catchup_applied_lsn_is_last_held_no_skip (Expected 1 Was 2 pre-fix, PASS post-fix). 7 legacy test_wal_catchup_* expected values updated lsn+1 -> last-held. NOTE: the LIVE clean_failover_preservation STRICT_TERM commit=5 target was NOT reachable in the sandbox: committed_lsn stalls at 1 IDENTICALLY with this fix stashed (baseline experiment) -> that stall is a SEPARATE upstream blocker (library raft.log_len / commit-advance not advancing past the own-term noop; ha-term-safety task 3.1 area), NOT the bug-991 applied_lsn skip. waldump confirms all 3 members hold LSN 1..5 byte-identical (term 1) -> the replication half (which this fix corrects) is sound. + +## Verification +_Not recorded (migrated from buglog.json)._ + +## References +- Related: bug-851, bug-850, bug-848, bug-924, bug-960 diff --git a/CLAUDE.md b/CLAUDE.md index c96b3c39..9089aba5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,7 +36,7 @@ For details on any topic below, read the corresponding steering file in `.kiro/s - **Middleware chain** (non-negotiable): WhitelistURL → WhitelistIP → RateLimit → JWT → CORS → RBAC - **Error pattern**: `ReturnError(code, msg, nil); return` — early return on every error - **Build**: `/xce-build [component]` — debug + ASAN + tests. Components: `all` (default), `lib`, `controller`, `harness`, `scheduler`, `worker`, `apiserver`, `xcecli`, `xceadmin`. See `build-deployment.md` for full commands -- **Buglog**: log and update bugs only through the `buglog` skill (`scripts/buglog.py add | status | note | recur | list --open | triage`), never by hand-editing or re-serializing `.wolf/buglog.json`. Each entry's `status` is the only source of truth for open/closed (values in `.claude/rules/buglog-status.md`); after a fix commit run `scripts/buglog.py status fixed --note ` +- **Buglog**: each bug is a Markdown record `.wolf/bugs/.md` written to the standard in `.claude/skills/buglog/bug-record.md` (Description, Environment incl. OS/libc/toolchain/commit, Reproduction; Root cause, Fix, Verification and fix commits once fixed). Create and change records only through the `buglog` skill (`scripts/buglog.py new | status | note | recur | check | list --open`), which enforces the standard (so does the pre-commit hook); `.wolf/buglog.json` is a generated index, never edit it. The front-matter `status` is the only source of truth for open/closed (values in `.claude/rules/buglog-status.md`); after a fix commit run `scripts/buglog.py status fixed --commit ` - **Save/restore field-completeness**: any persistence mechanism (WAL fact, scheduler snapshot, checkpoint) must capture every field of the entity it claims to persist — a state that "recovers correctly" while silently missing a field it depends on (e.g. `job.state` correct but `job.allocs` lost) is a bug, not an acceptable simplification, unless the omission is explicitly documented as a scope decision (see `persistence.md`'s "Snapshot Save/Load Field-Completeness" + `APPLY_CLEAN`/bug-1678 sections). When adding or extending save/restore logic, verify the field round-trips through BOTH the write side and every read/apply seam, and run `/xce-audit-invariants` (controller dimension 7 / scheduler dimension 4) rather than assuming a passing test suite already proves it — `APPLY_CLEAN`-style verdicts check row counts and cursor position, not per-field completeness ## Key Files diff --git a/scripts/buglog.py b/scripts/buglog.py index 18ae04c7..8947eb18 100644 --- a/scripts/buglog.py +++ b/scripts/buglog.py @@ -1,48 +1,82 @@ #!/usr/bin/env python3 -"""Read and edit .wolf/buglog.json without re-serializing it. +"""Read, write and check the XCE bug records: one Markdown file per bug. -Re-serializing the whole buglog (json.dumps / jq) rewrites ~1,200 unrelated -lines because existing entries mix raw UTF-8 and \\uXXXX escapes, so every -write here edits only the lines of the entry it touches. Every write is -re-parsed before it is saved and saved atomically (tmp file + rename), the -same way OpenWolf's own hooks write it. +Each bug is `.wolf/bugs/.md`: a front-matter block of short machine +fields (one `key: ` per line, which is also valid YAML), then +`# : ` and the sections of the record standard +(.claude/skills/buglog/bug-record.md). `check` enforces that standard. -The `status` values and when to use each are defined in -.claude/rules/buglog-status.md. +`.wolf/buglog.json` is an index generated from the records for OpenWolf's +hooks and `jq` queries; every write here regenerates it, never edit it by +hand. The retired hook-generated entries live in `.wolf/buglog-archive.json`. + +Status values and when to use each: .claude/rules/buglog-status.md. Commands: - add --error MSG [--root-cause T] [--fix T] [--file F] [--tags a,b] - [--related bug-1,bug-2] [--status S] [--note T] - Append a new entry with the next free id; prints the id. - status ID STATUS [--note T] [--duplicate-of ID] [--repro T] - Set status + status_updated (today) + status_note. --note is - required except for `fixed`; duplicate needs --duplicate-of, - cannot-reproduce needs --repro. - note ID TEXT Append an "UPDATE (date): TEXT" paragraph to `fix`. - recur ID [--note T] The bug was seen again: bump occurrences/last_seen; - cannot-reproduce -> investigating, fixed/wontfix -> open. - set ID key= ... Set arbitrary single-line fields. - show ID Print one entry. + new --summary S --component C --severity V [--binary B] [--tags a,b] + [--files f,g] [--related bug-1,bug-2] + Create a record from the template, Environment filled in. + check [ID ...] [--all] [--staged] Check records against the standard. + status ID STATUS [--note T] [--commit SHA]... [--duplicate-of ID] [--repro T] + Change status; refused if the record does not meet what the + new status requires. + note ID TEXT [--section S] Append "UPDATE (date): TEXT" to a section. + recur ID [--note T] Seen again: bump occurrences/last_seen; + cannot-reproduce -> investigating, fixed/wontfix -> open. + set ID key= ... Set front-matter fields. + show ID Print one record. list [--status S,S | --open] [--tag T] [--grep RE] [--since DATE] - One line per matching entry. - triage List entries with no (or an unknown) status. - next-id Print the id `add` would use. + triage Records with no (or an unknown) status. + env [BINARY] Print an Environment section for this machine. + index Regenerate .wolf/buglog.json. + next-id Print the id `new` would use. -Importable: set_fields(path, bug_id, {key: value}), BuglogEditError. +Importable: load, load_all, set_meta, check, build_index, BuglogEditError. """ import argparse import datetime import json import os +import platform import re +import shutil +import subprocess import sys from pathlib import Path -DEFAULT_PATH = Path(__file__).resolve().parent.parent / ".wolf" / "buglog.json" - STATUSES = ["open", "partial", "investigating", "fixed", "cannot-reproduce", "wontfix", "duplicate"] OPEN_STATUSES = {"open", "partial", "investigating"} +COMPONENTS = ["controller", "scheduler", "worker", "lib", "harness", + "apiserver", "xcecli", "xceadmin", "autoscaler", "build", + "config", "tooling"] +SEVERITIES = ["data-loss", "crash", "hang", "wrong-result", "leak", + "security", "noise"] +SECTIONS = ["Description", "Environment", "Reproduction", "Root cause", + "Fix", "Verification", "References", "Legacy notes"] +META_ORDER = ["id", "status", "status_updated", "status_note", "component", + "severity", "timestamp", "last_seen", "occurrences", "files", + "tags", "related", "fix_commits", "duplicate_of", + "repro_attempts", "legacy", "gitea_issue", + "gitea_synced_status", "gitea_synced_hash"] +ID_RE = re.compile(r"bug-\d+(?:-[a-z0-9-]+)?") +# Text a migrated record carries where the old log had no such field. +NOT_RECORDED = "_Not recorded (migrated from buglog.json)._" +STATUS_WORDS = re.compile(r"\b(OPEN|FIXED|NOT FIXED|PRIORITY)\b") + +ROOT = BUGS_DIR = INDEX_PATH = ARCHIVE_PATH = None + + +def configure(root): + """Point the module at a repository root (tests use a temp dir).""" + global ROOT, BUGS_DIR, INDEX_PATH, ARCHIVE_PATH + ROOT = Path(root) + BUGS_DIR = ROOT / ".wolf" / "bugs" + INDEX_PATH = ROOT / ".wolf" / "buglog.json" + ARCHIVE_PATH = ROOT / ".wolf" / "buglog-archive.json" + + +configure(Path(__file__).resolve().parent.parent) class BuglogEditError(Exception): @@ -57,178 +91,590 @@ def _now(): return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -def _load(path): - return json.loads(Path(path).read_text(encoding="utf-8"))["bugs"] - - -def _save(path, text): - """Refuse to write text that no longer parses; write atomically.""" - json.loads(text) - path = Path(path) - tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") - tmp.write_text(text, encoding="utf-8") - os.replace(tmp, path) - - -def _entry_span(lines, bug_id): - """(start, end, indent) of the entry: start = its "id" line, end = its '}'.""" - id_re = re.compile(r'^(\s+)"id": ' + re.escape(json.dumps(bug_id)) + r",?$") - starts = [i for i, ln in enumerate(lines) if id_re.match(ln)] - if len(starts) != 1: - raise BuglogEditError(f"{bug_id}: expected exactly one entry, found {len(starts)}") - start = starts[0] - indent = id_re.match(lines[start]).group(1) - close_re = re.compile(r"^" + re.escape(indent[:-2]) + r"\},?$") - end = next(i for i in range(start, len(lines)) if close_re.match(lines[i])) - return start, end, indent - - -def set_fields(path, bug_id, fields): - """Set each key in `fields` on entry `bug_id`, touching only its lines. - - An existing single-line field is replaced in place; a new one is inserted - right after the entry's "id" line. Multi-line (array/object) fields are - refused -- edit those by hand. - """ - path = Path(path) - lines = path.read_text(encoding="utf-8").split("\n") - start, end, indent = _entry_span(lines, bug_id) - - for key, value in fields.items(): - rendered = json.dumps(value, ensure_ascii=False) - key_re = re.compile(r"^" + re.escape(indent) + re.escape(json.dumps(key)) + r": (.*?)(,?)$") - hit = next((i for i in range(start, end) if key_re.match(lines[i])), None) - if hit is not None: - old, comma = key_re.match(lines[hit]).groups() - if old in ("[", "{"): - raise BuglogEditError(f"{bug_id}.{key}: multi-line value, edit by hand") - lines[hit] = f"{indent}{json.dumps(key)}: {rendered}{comma}" - else: - lines.insert(start + 1, f"{indent}{json.dumps(key)}: {rendered},") - end += 1 - - text = "\n".join(lines) - entry = next(b for b in json.loads(text)["bugs"] if b.get("id") == bug_id) - for key, value in fields.items(): - if entry[key] != value: - raise BuglogEditError(f"{bug_id}.{key}: wrote {entry[key]!r}, wanted {value!r}") - _save(path, text) - - -def next_id(bugs): - """One past the highest numeric id. (OpenWolf's own hook uses len(bugs)+1, - which collides with existing ids once any were renumbered or skipped.)""" - nums = [int(m.group(1)) for b in bugs - if (m := re.fullmatch(r"bug-(\d+)", str(b.get("id", ""))))] - return f"bug-{max(nums, default=0) + 1:03d}" - - -def append_entry(path, entry): - """Append `entry` as the last element of the `bugs` array.""" - path = Path(path) - lines = path.read_text(encoding="utf-8").split("\n") - close = max(i for i, ln in enumerate(lines) if ln == " ]") - prev = close - 1 - if lines[prev] == " }": - lines[prev] = " }," - elif lines[prev] != " \"bugs\": [": - raise BuglogEditError(f"unexpected line before bugs array end: {lines[prev]!r}") - body = json.dumps(entry, indent=2, ensure_ascii=False).split("\n") - lines[close:close] = [" " + ln for ln in body] - text = "\n".join(lines) - if json.loads(text)["bugs"][-1] != entry: - raise BuglogEditError("appended entry did not round-trip") - _save(path, text) - - def _split(value): return [v.strip() for v in (value or "").split(",") if v.strip()] -def _text(value): - if isinstance(value, list): - return " || ".join(str(v) for v in value) - return value or "" +def _run(cmd, **kw): + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, **kw) + except (OSError, subprocess.TimeoutExpired): + return None + return r.stdout.strip() if r.returncode == 0 else None -def _find(bugs, bug_id): - for b in bugs: - if b.get("id") == bug_id: - return b - raise BuglogEditError(f"{bug_id}: no such entry") +# ── Record format ──────────────────────────────────────────────────────────── + +class Record: + def __init__(self, meta, summary, sections, path=None): + self.meta = meta + self.summary = summary + self.sections = sections + self.path = path + + @property + def id(self): + return self.meta.get("id") -def _line(b): - note = b.get("status_note") or _text(b.get("error_message")) - return f"{b['id']:<10} {b.get('status') or '-':<16} {note[:110]}" +def render_meta(meta): + keys = [k for k in META_ORDER if k in meta] + \ + sorted(k for k in meta if k not in META_ORDER) + lines = ["---"] + [f"{k}: {json.dumps(meta[k], ensure_ascii=False)}" + for k in keys] + ["---"] + return "\n".join(lines) + "\n" -def cmd_add(a, path): +def render(rec): + out = [render_meta(rec.meta), f"# {rec.id}: {rec.summary}", ""] + for name in SECTIONS: + if name in rec.sections: + out += [f"## {name}", rec.sections[name].strip("\n"), ""] + return "\n".join(out) + + +_HEADER_RE = re.compile(r"^## (" + "|".join(map(re.escape, SECTIONS)) + r")\s*$") +_ESCAPED_RE = re.compile(r"^\\(## (" + "|".join(map(re.escape, SECTIONS)) + r")\s*)$", + re.M) + + +def escape_text(text): + """Stop a line of free text from being read as a section header.""" + return re.sub(_HEADER_RE.pattern, r"\\\g<0>", text, flags=re.M) + + +def unescape_text(text): + return _ESCAPED_RE.sub(r"\1", text) + + +def parse(text, path=None): + lines = text.split("\n") + if not lines or lines[0] != "---": + raise BuglogEditError(f"{path}: no front matter") + meta = {} + i = 1 + while i < len(lines) and lines[i] != "---": + key, sep, raw = lines[i].partition(": ") + if not sep: + raise BuglogEditError(f"{path}:{i + 1}: expected 'key: value'") + try: + meta[key] = json.loads(raw) + except json.JSONDecodeError as e: + raise BuglogEditError(f"{path}:{i + 1}: {key}: {e}") from None + i += 1 + if i == len(lines): + raise BuglogEditError(f"{path}: front matter not closed") + + summary, sections, current, buf, fence = None, {}, None, [], None + for ln in lines[i + 1:]: + m = re.match(r"^\s*(`{3,}|~{3,})", ln) + if m: + mark = m.group(1) + if fence is None: + fence = mark + elif mark[0] == fence[0] and len(mark) >= len(fence): + fence = None + if fence is None and not m: + h = _HEADER_RE.match(ln) + if h: + if current: + sections[current] = "\n".join(buf).strip("\n") + current, buf = h.group(1), [] + continue + if summary is None and current is None and ln.startswith("# "): + title = ln[2:].strip() + prefix = f"{meta.get('id')}: " + summary = title[len(prefix):] if title.startswith(prefix) else title + continue + if current: + buf.append(ln) + if current: + sections[current] = "\n".join(buf).strip("\n") + return Record(meta, summary or "", sections, path) + + +def record_path(bug_id): + if not ID_RE.fullmatch(bug_id or ""): + raise BuglogEditError(f"{bug_id!r}: not a bug id") + return BUGS_DIR / f"{bug_id}.md" + + +def load(bug_id): + path = record_path(bug_id) + if not path.exists(): + raise BuglogEditError(f"{bug_id}: no such record") + return parse(path.read_text(encoding="utf-8"), path) + + +def _id_key(bug_id): + m = re.match(r"bug-(\d+)(.*)", bug_id) + return (int(m.group(1)), m.group(2)) if m else (0, bug_id) + + +def load_all(): + recs = [parse(p.read_text(encoding="utf-8"), p) for p in BUGS_DIR.glob("*.md")] + return sorted(recs, key=lambda r: _id_key(r.id or "")) + + +def _write_atomic(path, text): + tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + + +def save(rec, reindex=True): + """Write a record; refuses text that would not parse back identically.""" + text = render(rec) + again = parse(text, rec.path) + if again.meta != rec.meta or again.summary != rec.summary or \ + again.sections != {k: v.strip("\n") for k, v in rec.sections.items()}: + raise BuglogEditError(f"{rec.id}: record would not round-trip") + path = record_path(rec.id) + _write_atomic(path, text) + if reindex: + build_index() + + +def set_meta(bug_id, fields, reindex=True): + """Set front-matter fields only; the body bytes are left untouched.""" + path = record_path(bug_id) + text = path.read_text(encoding="utf-8") + rec = parse(text, path) + rec.meta.update(fields) + head_end = text.index("\n---\n", 3) + len("\n---\n") + new = render_meta(rec.meta) + text[head_end:] + if parse(new, path).meta != rec.meta: + raise BuglogEditError(f"{bug_id}: front matter would not round-trip") + _write_atomic(path, new) + if reindex: + build_index() + + +# Kept for scripts/gitea-sync.py and older callers. +def set_fields(_path_ignored, bug_id, fields): + set_meta(bug_id, fields) + + +# ── Index ──────────────────────────────────────────────────────────────────── + +def index_entry(rec): + s, m = rec.sections, rec.meta + unesc = lambda k: unescape_text(s.get(k, "")) # noqa: E731 + fix = unesc("Fix") + if s.get("Verification") and s["Verification"] != NOT_RECORDED: + fix += "\n\nVerification:\n" + unesc("Verification") + entry = { + "id": rec.id, + "status": m.get("status"), + "status_updated": m.get("status_updated"), + "status_note": m.get("status_note", ""), + "component": m.get("component"), + "severity": m.get("severity"), + "timestamp": m.get("timestamp"), + "last_seen": m.get("last_seen", m.get("timestamp")), + "occurrences": m.get("occurrences", 1), + "error_message": (rec.summary + "\n\n" + unesc("Description")).strip(), + "file": ", ".join(m.get("files") or []), + "root_cause": unesc("Root cause"), + "fix": fix.strip(), + "tags": m.get("tags") or [], + "related_bugs": m.get("related") or [], + "fix_commits": m.get("fix_commits") or [], + "record": str(record_path(rec.id).relative_to(ROOT)), + } + for k in ("duplicate_of", "repro_attempts", "gitea_issue", "gitea_synced_status"): + if k in m: + entry[k] = m[k] + return entry + + +def build_index(): + data = { + "version": 1, + "generated": "by scripts/buglog.py from .wolf/bugs/*.md -- do not edit", + "bugs": [index_entry(r) for r in load_all()], + } + _write_atomic(INDEX_PATH, json.dumps(data, indent=2, ensure_ascii=False) + "\n") + + +def next_id(): + ids = [p.stem for p in BUGS_DIR.glob("*.md")] + if ARCHIVE_PATH.exists(): + ids += [b.get("id", "") for b in + json.loads(ARCHIVE_PATH.read_text(encoding="utf-8")).get("bugs", [])] + nums = [int(m.group(1)) for i in ids if (m := re.match(r"bug-(\d+)", i))] + return f"bug-{max(nums, default=0) + 1:03d}" + + +# ── Checks against the record standard ────────────────────────────────────── + +def _empty(value): + v = (value or "").strip() + return not v or v.upper().startswith("TODO") or v == NOT_RECORDED + + +def _item(section, label): + """Value of a `- Label: value` list item, or None.""" + m = re.search(r"^\s*[-*] " + re.escape(label) + r":[ \t]*(.*)$", section or "", re.M) + return m.group(1).strip() if m else None + + +def _bold(section, label): + """Text after `**Label:**` up to the next bold label.""" + m = re.search(r"\*\*" + re.escape(label) + r":\*\*(.*?)(?=^\*\*[\w ]+:\*\*|\Z)", + section or "", re.S | re.M) + return m.group(1).strip() if m else None + + +def bad_commits(shas): + bad = [] + for sha in shas: + if not re.fullmatch(r"[0-9a-f]{7,40}", str(sha)) or \ + _run(["git", "-C", str(ROOT), "cat-file", "-e", f"{sha}^{{commit}}"]) is None: + bad.append(sha) + return bad + + +def check(rec, target_status=None): + """Problems with `rec` under the record standard, as a list of strings. + + `target_status` checks the record as if it were moved to that status: + that is how `status` refuses an incomplete change. Legacy (migrated) + records are exempt from the reporting sections, but a status change to + fixed/partial still needs the fix-time sections. + """ + m, s, P = rec.meta, rec.sections, [] + status = target_status or m.get("status") + legacy = bool(m.get("legacy")) + strict_fix = not legacy or target_status is not None + + if not ID_RE.fullmatch(str(m.get("id", ""))): + P.append(f"id {m.get('id')!r} is not a bug id") + elif rec.path and Path(rec.path).stem != m["id"]: + P.append(f"id {m['id']} does not match file name {Path(rec.path).name}") + if status not in STATUSES: + P.append(f"status {status!r} is not one of {', '.join(STATUSES)}") + if status not in ("fixed", None) and _empty(m.get("status_note")): + P.append(f"status {status} needs a status_note (what remains / why / what was seen)") + if status == "duplicate" and not m.get("duplicate_of"): + P.append("status duplicate needs duplicate_of") + if status == "cannot-reproduce" and not m.get("repro_attempts"): + P.append("status cannot-reproduce needs repro_attempts (runs tried, date)") + + if not legacy: + if m.get("component") not in COMPONENTS: + P.append(f"component {m.get('component')!r} is not one of {', '.join(COMPONENTS)}") + if m.get("severity") not in SEVERITIES: + P.append(f"severity {m.get('severity')!r} is not one of {', '.join(SEVERITIES)}") + if not re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?Z", str(m.get("timestamp", ""))): + P.append("timestamp must be ISO-8601 UTC, e.g. 2026-09-27T14:05:00Z") + if _empty(rec.summary): + P.append("Summary (the # title) is empty") + elif len(rec.summary) > 100: + P.append(f"Summary is {len(rec.summary)} chars, max 100") + elif STATUS_WORDS.search(rec.summary): + P.append("Summary must not carry status words (OPEN/FIXED/PRIORITY)") + + d = s.get("Description") + for label in ("Observed", "Expected", "Impact", "Found by"): + if _empty(_bold(d, label)): + P.append(f"Description: **{label}:** is missing or empty") + + e = s.get("Environment") + for label, rx, why in ( + ("OS", r"\d+\.\d+", "distribution/version and kernel version"), + ("libc", r"\d+\.\d+", "a version, from the failing binary's interpreter"), + ("Toolchain", r"\d+\.\d+", "compiler and build tool versions"), + ("Build", r"\b(asan|debug|release)\b", "the build profile: asan, debug or release"), + ("Code", r"\b[0-9a-f]{7,40}\b", "the commit the binary was built from"), + ("Deployment", r"\S", "topology, worker kind, hosts")): + v = _item(e, label) + if _empty(v): + P.append(f"Environment: - {label}: is missing or empty ({why})") + elif not re.search(rx, v): + P.append(f"Environment: - {label}: needs {why}") + + r = s.get("Reproduction") or "" + steps = re.search(r"^\s*\d+\.\s+(?!TODO)\S", r, re.M) or "```" in r + if not steps: + P.append("Reproduction: needs numbered steps (1. ) or a code block") + freq = _item(r, "Frequency") + if _empty(freq): + P.append("Reproduction: - Frequency: is missing (deterministic, or N/M runs with the date)") + elif not re.search(r"deterministic|\d+\s*/\s*\d+", freq, re.I): + P.append("Reproduction: - Frequency: must be 'deterministic' or 'N/M runs, '") + if _empty(_item(r, "Evidence")): + P.append("Reproduction: - Evidence: is missing (a durable path, or 'not kept')") + + if strict_fix and status in ("fixed", "partial"): + for name in ("Root cause", "Fix"): + if _empty(s.get(name)): + P.append(f"{name}: required for status {status}") + if strict_fix and status == "fixed": + v = s.get("Verification") + if _empty(v): + P.append("Verification: required for status fixed") + else: + for label in ("Test", "Suites"): + if _empty(_item(v, label)): + P.append(f"Verification: - {label}: is missing or empty") + commits = m.get("fix_commits") or [] + if not commits: + P.append("fix_commits: status fixed needs the fix commit(s) (status ... --commit )") + for sha in bad_commits(commits): + P.append(f"fix_commits: {sha} is not a commit in this repository") + return P + + +# ── Environment collection ─────────────────────────────────────────────────── + +def _os_line(): + kernel = f"{platform.system()} {platform.release()} {platform.machine()}" + distro = None + for f in ("/etc/os-release", "/usr/lib/os-release"): + if os.path.exists(f): + m = re.search(r'^PRETTY_NAME="?([^"\n]*)', Path(f).read_text(), re.M) + distro = m.group(1) if m else None + break + parts = [distro or "no /etc/os-release"] + if os.path.exists("/.dockerenv"): + parts.append("Docker container") + try: + if "NixOS" in Path("/proc/version").read_text(): + parts.append("NixOS kernel") + except OSError: + pass + return "; ".join(parts + [f"kernel {kernel}"]) + + +def _libc_line(binary): + if binary: + out = _run(["readelf", "-l", str(binary)]) or "" + m = re.search(r"interpreter: ([^\]]+)\]", out) + if m: + v = re.search(r"glibc-(\d+\.\d+[\w.-]*)", m.group(1)) + name = f"glibc {v.group(1)}" if v else "unknown libc" + return f"{name} (interpreter {m.group(1)})" + out = _run(["ldd", "--version"]) or "" + v = re.search(r"(\d+\.\d+)", out.split("\n")[0]) if out else None + host = f"glibc {v.group(1)}" if v else "unknown" + return f"TODO: pass the failing binary (`buglog.py env `); host ldd reports {host}" + + +def _toolchain_line(): + probe = "gcc --version | head -1; meson --version" + if os.environ.get("IN_NIX_SHELL") or not shutil.which("nix"): + out = _run(["bash", "-c", probe]) + else: + out = _run(["nix", "develop", str(ROOT / "src"), "--command", "bash", "-c", probe], + cwd=str(ROOT)) + if not out: + return "TODO: gcc and meson versions from the shell that built the binary" + gcc, _, meson = out.partition("\n") + return f"{gcc.strip()}, meson {meson.strip()}" + + +def _build_line(binary): + profile = None + if binary: + for d, p in (("xce.build", "asan"), ("build.dbg", "debug"), ("build.release", "release")): + if d in str(binary): + profile = f"{p} ({d})" + line = profile or "TODO: asan (xce.build) | debug (build.dbg) | release (build.release)" + if os.environ.get("ASAN_OPTIONS"): + line += f", ASAN_OPTIONS={os.environ['ASAN_OPTIONS']}" + return line + + +def _code_line(): + sha = _run(["git", "-C", str(ROOT), "rev-parse", "--short=8", "HEAD"]) + if not sha: + return "TODO: branch @ commit" + branch = _run(["git", "-C", str(ROOT), "rev-parse", "--abbrev-ref", "HEAD"]) or "?" + dirty = _run(["git", "-C", str(ROOT), "status", "--porcelain", "--untracked-files=no"]) + return f"{branch} @ {sha}" + ("-dirty" if dirty else "") + + +def collect_env(binary=None): + return "\n".join([ + f"- OS: {_os_line()}", + f"- libc: {_libc_line(binary)}", + f"- Toolchain: {_toolchain_line()}", + f"- Build: {_build_line(binary)}", + f"- Code: {_code_line()}", + "- Deployment: TODO: topology (single / 3-node HA / multihost), worker kind, hosts", + ]) + + +TEMPLATE = { + "Description": "**Observed:**\nTODO: the exact error, log line or sanitizer text, in a code block\n\n" + "**Expected:** TODO\n**Impact:** TODO\n" + "**Found by:** TODO: user report | soak | ASAN | audit | review | test | spec task", + "Reproduction": "1. TODO: commands someone else can paste\n" + "- Frequency: TODO: deterministic | N/M runs, \n" + "- Evidence: TODO: durable path to logs/cores/replays, or 'not kept'", + "Root cause": "TODO: required for fixed/partial", + "Fix": "TODO: required for fixed/partial", + "Verification": "- Test: TODO: name + file; RED before the fix, GREEN after\n" + "- Suites: TODO: suite results and build profile\n" + "- Live: TODO: soak/deploy run, or n/a and why", + "References": "- Related: none", +} + + +# ── Commands ───────────────────────────────────────────────────────────────── + +def _line(r): + note = r.meta.get("status_note") or r.summary + return f"{r.id:<10} {r.meta.get('status') or '-':<16} {note[:110]}" + + +def _fail(problems, what): + sys.exit(f"{what}:\n" + "\n".join(f" - {p}" for p in problems) + + "\nStandard: .claude/skills/buglog/bug-record.md") + + +def cmd_new(a): + if a.component not in COMPONENTS: + sys.exit(f"--component must be one of {', '.join(COMPONENTS)}") + if a.severity not in SEVERITIES: + sys.exit(f"--severity must be one of {', '.join(SEVERITIES)}") + bid, now = next_id(), _now() + meta = {"id": bid, "status": "open", "status_updated": _today(), + "status_note": "newly logged", "component": a.component, + "severity": a.severity, "timestamp": now, "last_seen": now, + "occurrences": 1, "files": _split(a.files), "tags": _split(a.tags), + "related": _split(a.related)} + sections = dict(TEMPLATE) + sections["Environment"] = collect_env(a.binary) + rec = Record(meta, a.summary, {k: sections[k] for k in SECTIONS if k in sections}) + save(rec) + print(record_path(bid).relative_to(ROOT)) + probs = check(load(bid)) + if probs: + print(f"{len(probs)} item(s) still to fill in; run `buglog.py check {bid}` when done.") + + +def _staged(pathspec, diff_filter="ACMD"): + out = _run(["git", "-C", str(ROOT), "diff", "--cached", "--name-only", + f"--diff-filter={diff_filter}", "--", pathspec]) or "" + return [p for p in out.split("\n") if p] + + +def _staged_records(): + return [Path(p).stem for p in _staged(".wolf/bugs/", "ACM") if p.endswith(".md")] + + +_HEAD_INDEX = None + + +def _head_status(bug_id): + """(found, status) of the bug at HEAD: its record, else its entry in + HEAD's buglog.json (how the migration commit is recognised).""" + global _HEAD_INDEX + text = _run(["git", "-C", str(ROOT), "show", f"HEAD:.wolf/bugs/{bug_id}.md"]) + if text: + try: + return True, parse(text).meta.get("status") + except BuglogEditError: + return True, None + if _HEAD_INDEX is None: + raw = _run(["git", "-C", str(ROOT), "show", "HEAD:.wolf/buglog.json"]) + try: + _HEAD_INDEX = {b.get("id"): b.get("status") + for b in json.loads(raw).get("bugs", [])} if raw else {} + except (json.JSONDecodeError, AttributeError): + _HEAD_INDEX = {} + if bug_id in _HEAD_INDEX: + return True, _HEAD_INDEX[bug_id] + return False, None + + +def cmd_check(a): + global _HEAD_INDEX + _HEAD_INDEX = None # read HEAD afresh on every check + ids = a.ids or [] + if a.staged: + ids = _staged_records() + elif a.all or not ids: + ids = [r.id for r in load_all()] + failed = 0 + for bid in ids: + rec = load(bid) + target, extra = None, [] + if a.staged and rec.meta.get("legacy"): + found, head = _head_status(bid) + if not found: + extra.append("a new record cannot be legacy: true (only migrated records are)") + elif head != rec.meta.get("status"): + # a legacy record whose status changed in this commit must + # meet what the new status requires + target = rec.meta.get("status") + probs = extra + check(rec, target_status=target) + if probs: + failed += 1 + print(f"{bid}:") + for p in probs: + print(f" - {p}") + if a.staged and (_staged(".wolf/bugs/") or _staged(".wolf/buglog.json")): + expect = json.dumps({"version": 1, + "generated": "by scripts/buglog.py from .wolf/bugs/*.md -- do not edit", + "bugs": [index_entry(r) for r in load_all()]}, + indent=2, ensure_ascii=False) + "\n" + if not INDEX_PATH.exists() or INDEX_PATH.read_text(encoding="utf-8") != expect: + failed += 1 + print(".wolf/buglog.json is stale: run `python3 scripts/buglog.py index` and stage it") + if failed: + sys.exit(f"{failed} record(s) do not meet .claude/skills/buglog/bug-record.md") + if not a.staged: + print(f"{len(ids)} record(s) OK") + + +def cmd_status(a): if a.status not in STATUSES: sys.exit(f"unknown status {a.status!r}; one of {', '.join(STATUSES)}") - bugs = _load(path) - bid = next_id(bugs) - now = _now() - entry = {"id": bid, "status": a.status, "status_updated": _today()} - if a.note: - entry["status_note"] = a.note - entry.update({ - "timestamp": now, - "error_message": a.error, - "file": a.file or "", - "root_cause": a.root_cause or "", - "fix": a.fix or "", - "tags": _split(a.tags), - "related_bugs": _split(a.related), - "occurrences": 1, - "last_seen": now, - }) - append_entry(path, entry) - print(bid) - - -def cmd_status(a, path): - if a.status not in STATUSES: - sys.exit(f"unknown status {a.status!r}; one of {', '.join(STATUSES)}") - if a.status == "duplicate" and not a.duplicate_of: - sys.exit("duplicate needs --duplicate-of") - if a.status == "cannot-reproduce" and not a.repro: - sys.exit("cannot-reproduce needs --repro (what was tried, e.g. '0/40 runs, 2026-08-01')") - if a.note is None and a.status != "fixed": - sys.exit(f"{a.status} needs --note (what remains / why / what was seen)") - # Always (re)write the note so a stale one never outlives its status. - fields = {"status": a.status, "status_updated": _today(), - "status_note": a.note or ""} + rec = load(a.id) + rec.meta.update({"status": a.status, "status_updated": _today(), + "status_note": a.note or ""}) + if a.commit: + rec.meta["fix_commits"] = list(dict.fromkeys((rec.meta.get("fix_commits") or []) + + a.commit)) if a.duplicate_of: - fields["duplicate_of"] = a.duplicate_of + rec.meta["duplicate_of"] = a.duplicate_of if a.repro: - fields["repro_attempts"] = a.repro - set_fields(path, a.id, fields) - print(_line(_find(_load(path), a.id))) + rec.meta["repro_attempts"] = a.repro + probs = check(rec, target_status=a.status) + if probs: + _fail(probs, f"{a.id}: not changed to {a.status}") + set_meta(a.id, {k: rec.meta[k] for k in rec.meta}) + print(_line(load(a.id))) -def cmd_note(a, path): - fix = _find(_load(path), a.id).get("fix") or "" - if isinstance(fix, list): - raise BuglogEditError(f"{a.id}.fix is a list, edit by hand") - para = f"UPDATE ({_today()}): {a.text}" - set_fields(path, a.id, {"fix": f"{fix}\n\n{para}" if fix else para}) +def cmd_note(a): + if a.section not in SECTIONS: + sys.exit(f"--section must be one of {', '.join(SECTIONS)}") + rec = load(a.id) + old = rec.sections.get(a.section, "") + old = "" if _empty(old) else old + para = f"UPDATE ({_today()}): {escape_text(a.text)}" + rec.sections[a.section] = f"{old}\n\n{para}" if old else para + save(rec) -def cmd_recur(a, path): - b = _find(_load(path), a.id) - fields = {"occurrences": int(b.get("occurrences") or 1) + 1, "last_seen": _now()} +def cmd_recur(a): + rec = load(a.id) + fields = {"occurrences": int(rec.meta.get("occurrences") or 1) + 1, "last_seen": _now()} reopen = {"cannot-reproduce": "investigating", "fixed": "open", "wontfix": "open"} - if b.get("status") in reopen: - fields.update(status=reopen[b["status"]], status_updated=_today()) - fields["status_note"] = a.note or f"recurred {_today()} (was {b['status']})" + if rec.meta.get("status") in reopen: + fields.update(status=reopen[rec.meta["status"]], status_updated=_today()) + fields["status_note"] = a.note or f"recurred {_today()} (was {rec.meta['status']})" elif a.note: fields["status_note"] = a.note - set_fields(path, a.id, fields) - print(_line(_find(_load(path), a.id))) + set_meta(a.id, fields) + print(_line(load(a.id))) -def cmd_set(a, path): +def cmd_set(a): fields = {} for kv in a.pairs: key, sep, raw = kv.partition("=") @@ -238,55 +684,53 @@ def cmd_set(a, path): fields[key] = json.loads(raw) except json.JSONDecodeError: fields[key] = raw - set_fields(path, a.id, fields) + set_meta(a.id, fields) -def cmd_show(a, path): - print(json.dumps(_find(_load(path), a.id), indent=2, ensure_ascii=False)) - - -def cmd_list(a, path): +def cmd_list(a): want = OPEN_STATUSES if a.open else set(_split(a.status)) or None rx = re.compile(a.grep, re.I) if a.grep else None - for b in _load(path): - if want is not None and b.get("status") not in want: + for r in load_all(): + m = r.meta + if want is not None and m.get("status") not in want: continue - if a.tag and a.tag not in (b.get("tags") or []): + if a.tag and a.tag not in (m.get("tags") or []): continue - if a.since and max(b.get("last_seen") or "", b.get("status_updated") or "") < a.since: + if a.since and max(m.get("last_seen") or "", m.get("status_updated") or "") < a.since: continue - if rx and not rx.search(json.dumps(b, ensure_ascii=False)): + if rx and not rx.search(render(r)): continue - print(_line(b)) - - -def cmd_triage(a, path): - for b in _load(path): - if b.get("status") not in STATUSES: - print(_line(b)) + print(_line(r)) def main(argv=None): p = argparse.ArgumentParser(description=__doc__.split("\n")[0], epilog="Status values: " + ", ".join(STATUSES)) - p.add_argument("--path", default=str(DEFAULT_PATH), help=argparse.SUPPRESS) sub = p.add_subparsers(dest="cmd", required=True) - s = sub.add_parser("add") - s.add_argument("--error", required=True) - s.add_argument("--root-cause") - s.add_argument("--fix") - s.add_argument("--file") + s = sub.add_parser("new") + s.add_argument("--summary", required=True) + s.add_argument("--component", required=True, help=", ".join(COMPONENTS)) + s.add_argument("--severity", required=True, help=", ".join(SEVERITIES)) + s.add_argument("--binary", help="the failing binary, for its libc and build profile") s.add_argument("--tags") + s.add_argument("--files") s.add_argument("--related") - s.add_argument("--status", default="open") - s.add_argument("--note") - s.set_defaults(fn=cmd_add) + s.set_defaults(fn=cmd_new) + + s = sub.add_parser("check") + s.add_argument("ids", nargs="*") + s.add_argument("--all", action="store_true") + s.add_argument("--staged", action="store_true", + help="records staged for commit (the pre-commit hook)") + s.set_defaults(fn=cmd_check) s = sub.add_parser("status") s.add_argument("id") s.add_argument("status") s.add_argument("--note") + s.add_argument("--commit", action="append", default=[], + help="fix commit (repeatable); required for fixed") s.add_argument("--duplicate-of") s.add_argument("--repro") s.set_defaults(fn=cmd_status) @@ -294,6 +738,7 @@ def main(argv=None): s = sub.add_parser("note") s.add_argument("id") s.add_argument("text") + s.add_argument("--section", default="Description") s.set_defaults(fn=cmd_note) s = sub.add_parser("recur") @@ -308,7 +753,7 @@ def main(argv=None): s = sub.add_parser("show") s.add_argument("id") - s.set_defaults(fn=cmd_show) + s.set_defaults(fn=lambda a: print(render(load(a.id)), end="")) s = sub.add_parser("list") s.add_argument("--status") @@ -319,14 +764,22 @@ def main(argv=None): s.set_defaults(fn=cmd_list) s = sub.add_parser("triage") - s.set_defaults(fn=cmd_triage) + s.set_defaults(fn=lambda a: [print(_line(r)) for r in load_all() + if r.meta.get("status") not in STATUSES]) + + s = sub.add_parser("env") + s.add_argument("binary", nargs="?") + s.set_defaults(fn=lambda a: print(collect_env(a.binary))) + + s = sub.add_parser("index") + s.set_defaults(fn=lambda a: build_index()) s = sub.add_parser("next-id") - s.set_defaults(fn=lambda a, path: print(next_id(_load(path)))) + s.set_defaults(fn=lambda a: print(next_id())) a = p.parse_args(argv) try: - a.fn(a, a.path) + a.fn(a) except BuglogEditError as e: sys.exit(str(e)) diff --git a/scripts/git-hooks/pre-commit b/scripts/git-hooks/pre-commit new file mode 100755 index 00000000..e7b8aecb --- /dev/null +++ b/scripts/git-hooks/pre-commit @@ -0,0 +1,7 @@ +#!/bin/sh +# Refuse commits whose bug records (.wolf/bugs/*.md) do not meet +# .claude/skills/buglog/bug-record.md, or whose .wolf/buglog.json index is stale. +# Install: ln -sf ../../scripts/git-hooks/pre-commit .git/hooks/pre-commit +if git diff --cached --name-only -- .wolf/bugs/ .wolf/buglog.json | grep -q .; then + exec python3 "$(git rev-parse --show-toplevel)/scripts/buglog.py" check --staged +fi diff --git a/scripts/gitea-sync.py b/scripts/gitea-sync.py index f00cbdaa..187b7517 100755 --- a/scripts/gitea-sync.py +++ b/scripts/gitea-sync.py @@ -3,7 +3,7 @@ xce-gitea-sync: Sync cc-sdd specs and OpenWolf buglog to Gitea issues. Reads .kiro/specs// (spec.json, brief.md, requirements.md, tasks.md) -and .wolf/buglog.json as ground truth. Creates/updates Gitea milestones, issues, +and the bug records in .wolf/bugs/*.md as ground truth. Creates/updates Gitea milestones, issues, and labels to mirror the local state. Sync is one-way: files -> Gitea. Never Gitea -> files. @@ -20,6 +20,8 @@ Environment: GITEA_TOKEN - API token (required: write:issue, write:milestone scopes) GITEA_OWNER - repo owner (default: nexa) GITEA_REPO - repo name (default: xce) + GITEA_COMMIT_REPO - owner/repo that fix-commit links point at + (default: the `origin` remote's owner/repo) """ import argparse @@ -28,6 +30,7 @@ import hashlib import json import os import re +import subprocess import sys import urllib.error import urllib.parse @@ -35,7 +38,8 @@ import urllib.request from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from buglog import BuglogEditError, set_fields # noqa: E402 +import buglog # noqa: E402 +from buglog import BuglogEditError # noqa: E402 # ── Helpers ────────────────────────────────────────────────────────────────── @@ -99,7 +103,6 @@ REPO_API = f"{API_BASE}/repos/{GITEA_OWNER}/{GITEA_REPO}" SCRIPT_DIR = Path(__file__).resolve().parent PROJECT_ROOT = SCRIPT_DIR.parent SPECS_DIR = PROJECT_ROOT / ".kiro" / "specs" -BUGLOG_PATH = PROJECT_ROOT / ".wolf" / "buglog.json" # ── Label Definitions ──────────────────────────────────────────────────────── @@ -190,14 +193,14 @@ STATUS_PRIORITY = ["open", "investigating", "partial", None, "cannot-reproduce", "duplicate", "wontfix", "fixed"] -def bug_status(bug_entry): - """The entry's status, or None when it has none / an unknown one.""" - status = bug_entry.get("status") +def bug_status(rec): + """The record's status, or None when it has none / an unknown one.""" + status = rec.meta.get("status") return status if status in STATUS_MAP else None -def bug_state(bug_entry): - return STATUS_MAP.get(bug_status(bug_entry), ("open", None))[0] +def bug_state(rec): + return STATUS_MAP.get(bug_status(rec), ("open", None))[0] def _text(value): @@ -342,14 +345,20 @@ def status_label(status): STATUS_LABELS = {lbl for _, lbl in STATUS_MAP.values() if lbl} -def bug_labels(bug_entry): - """Derive Gitea labels from the entry's status and tags.""" +SEVERITY_TO_LABEL = {"crash": "bug:crash", "data-loss": "bug:data-loss"} + + +def bug_labels(rec): + """Derive Gitea labels from the record's status, severity and tags.""" labels = ["bug"] - if status_label(bug_status(bug_entry)): - labels.append(status_label(bug_status(bug_entry))) + if status_label(bug_status(rec)): + labels.append(status_label(bug_status(rec))) + sev = SEVERITY_TO_LABEL.get(rec.meta.get("severity")) + if sev: + labels.append(sev) # Map known tag patterns to labels - for tag in (t.lower() for t in bug_entry.get("tags", [])): + for tag in (t.lower() for t in rec.meta.get("tags") or []): for pattern, label in TAG_TO_LABEL.items(): if pattern.lower() in tag and label not in labels: labels.append(label) @@ -360,88 +369,73 @@ def bug_labels(bug_entry): # ── Body Formatting ────────────────────────────────────────────────────────── -def bug_body(bug_entry): - """Format a buglog entry as a Gitea issue body.""" - parts = [] +def _commit_repo(): + """owner/repo that fix-commit links point at: GITEA_COMMIT_REPO, else the + `origin` remote (fixes land on a fork branch before they reach + GITEA_OWNER/GITEA_REPO, so a bare hash there may not resolve yet).""" + repo = os.environ.get("GITEA_COMMIT_REPO") + if repo: + return repo + try: + url = subprocess.run(["git", "-C", str(PROJECT_ROOT), "remote", "get-url", "origin"], + capture_output=True, text=True, check=True).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return f"{GITEA_OWNER}/{GITEA_REPO}" + m = re.search(r"[:/]([^/:]+/[^/]+?)(?:\.git)?$", url) + return m.group(1) if m else f"{GITEA_OWNER}/{GITEA_REPO}" - status = bug_entry.get("status") or "unknown (needs triage)" - note = _text(bug_entry.get("status_note")) + +def bug_body(rec): + """Format a bug record as a Gitea issue body: a status header, then the + record's own sections (.claude/skills/buglog/bug-record.md).""" + m = rec.meta + parts = [] + status = m.get("status") or "unknown (needs triage)" + note = _text(m.get("status_note")) parts.append(f"**Status:** {status}" + (f" — {note}" if note else "")) - if bug_entry.get("duplicate_of"): - parts.append(f"Duplicate of {bug_entry['duplicate_of']}") - if bug_entry.get("repro_attempts"): - parts.append(f"Repro attempts: {_text(bug_entry['repro_attempts'])}") + parts.append(f"**Component:** {m.get('component', 'unknown')} · " + f"**Severity:** {m.get('severity', 'unknown')}") + commits = m.get("fix_commits") or [] + if commits: + base = f"{GITEA_URL}/{_commit_repo()}/commit" + parts.append("**Fix commits:** " + ", ".join(f"[`{c}`]({base}/{c})" for c in commits)) + if m.get("duplicate_of"): + parts.append(f"**Duplicate of:** {m['duplicate_of']}") + if m.get("repro_attempts"): + parts.append(f"**Repro attempts:** {_text(m['repro_attempts'])}") parts.append("") - error = _text(bug_entry.get("error_message")) - if error: - parts.append("## Error\n") - parts.append(error) + for name in buglog.SECTIONS: + text = buglog.unescape_text(rec.sections.get(name, "")).strip() + if not text or text == buglog.NOT_RECORDED: + continue + parts.append(f"## {name}\n") + parts.append(text) parts.append("") - root_cause = _text(bug_entry.get("root_cause")) - if root_cause: - parts.append("## Root Cause\n") - parts.append(root_cause) - parts.append("") - - fix = _text(bug_entry.get("fix")) - if fix: - parts.append("## Fix\n") - parts.append(fix) - parts.append("") - else: - parts.append("## Fix\n") - parts.append("*Not yet fixed*") - parts.append("") - - files = [f.strip() for f in (bug_entry.get("file") or "").split(",")] - files += [str(f).strip() for f in bug_entry.get("files") or []] - files = [f for f in dict.fromkeys(files) if f] + files = m.get("files") or [] if files: parts.append("## Affected Files\n") - for f in files: - parts.append(f"- `{f}`") + parts.extend(f"- `{f}`" for f in files) parts.append("") - - tags = bug_entry.get("tags", []) + tags = m.get("tags") or [] if tags: parts.append("## Tags\n") parts.append(" ".join(f"`{t}`" for t in tags)) parts.append("") - related = bug_entry.get("related_bugs", []) - if related: - parts.append("## Related Bugs\n") - for rb in related: - parts.append(f"- {rb}") - parts.append("") - parts.append("---") - parts.append(f"*Synced from `.wolf/buglog.json` " - f"({bug_entry.get('id', '?')})*") - + parts.append(f"*Synced from `.wolf/bugs/{rec.id}.md`*") return "\n".join(parts) -def bug_title(bug_entry): - """Build a short title from a buglog entry.""" - bid = bug_entry.get("id", "bug-?") - error = _text(bug_entry.get("error_message")) - # Drop a leading status banner ("OPEN, NOT FIXED -- ...", "PRIORITY: LOW -- - # ...") -- status lives in labels/state, not the title. - error = re.sub(r"^(?:\s*(?:OPEN|NOT (?:YET )?FIXED|NOT A CRASH[^,.]*|" - r"NOT YET ROOT-CAUSED|DEAD-CODE-ONLY|PRIORITY: \w+|" - r"low priority|pre-existing)\b\s*(?:,|--|\.|;)?)+\s*", - "", error, flags=re.I) or error - # Take first sentence or first 100 chars - first_line = error.split("\n")[0].strip() - first_sentence = re.split(r'[.。]\s+', first_line)[0].strip() - if not first_sentence: - first_sentence = first_line - if len(first_sentence) > 120: - first_sentence = first_sentence[:117] + "..." - return f"[{bid}] {first_sentence}" +def bug_title(rec): + title = f"[{rec.id}] {rec.summary}" + return title if len(title) <= 120 else title[:119] + "…" + + +def body_hash(title, body): + return hashlib.sha1(f"{title}\n{body}".encode("utf-8")).hexdigest()[:12] def spec_requirements_body(spec_name, spec_data, brief_path, req_path, design_path): @@ -789,88 +783,90 @@ def _resolve_label_ids(api, label_names): # ── Bug Sync ───────────────────────────────────────────────────────────────── def sync_bugs(api, dry_run=False, quiet=False, since=None, ids=None): - """Sync buglog entries to Gitea issues. + """Sync bug records (.wolf/bugs/*.md) to Gitea issues. - Open/closed and the status label come from each entry's `status` (see - STATUS_MAP). `gitea_synced_status` records the status last mirrored, so an - entry is only touched when its status changed -- including going back to - open (reopen). Sync state is written back per entry, surgically, so a - failure part-way through never loses an already-created issue number. + Open/closed and the status label come from each record's `status` (see + STATUS_MAP). `gitea_synced_status` records the status last mirrored, so a + record is only re-mirrored when its status changed -- including going back + to open (reopen). `gitea_synced_hash` records the title+body last written, + so an issue's body is refreshed whenever the record's text changes (a bug + created open gets its Fix/Verification/commits once it is fixed). Sync + state is written back per record, so a failure part-way through never + loses an already-created issue number. Args: since: ISO date (YYYY-MM-DD). Create issues only for bugs with last_seen >= since; update already-synced ones whose last_seen or status_updated >= since. ids: set of bug ids to sync regardless of dates. Combined with - since, a bug is synced if it matches either -- bulk triage - has bumped last_seen/status_updated across most of the log, - so dates alone cannot pick out recently fixed older bugs. + since, a bug is synced if it matches either. """ - if not BUGLOG_PATH.exists(): - warn(f"buglog not found: {BUGLOG_PATH}") + records = buglog.load_all() + if not records: + warn(f"no bug records in {buglog.BUGS_DIR}") return - - buglog = json.loads(BUGLOG_PATH.read_text())["bugs"] - created = closed = reopened = updated = skipped = 0 + created = closed = reopened = updated = refreshed = skipped = 0 needs_status = [] # Aggregate status per already-synced issue (see STATUS_PRIORITY). issue_status = {} - for bug in buglog: - num = bug.get("gitea_issue") + for rec in records: + num = rec.meta.get("gitea_issue") if num: prev = issue_status.get(num, "fixed") - cur = bug_status(bug) - issue_status[num] = min(prev, cur, key=STATUS_PRIORITY.index) + issue_status[num] = min(prev, bug_status(rec), key=STATUS_PRIORITY.index) done_issues = set() - for bug in buglog: - bid = bug.get("id") - if not bid: - continue + for rec in records: + m, bid = rec.meta, rec.id - # --since: new issues only for recently seen bugs; status changes - # (status_updated) are still mirrored for already-synced ones. if since or ids: recent = False if since: - recent = (bug.get("last_seen") or "") >= since - if bug.get("gitea_issue"): - recent = recent or (bug.get("status_updated") or "") >= since + recent = (m.get("last_seen") or m.get("timestamp") or "") >= since + if m.get("gitea_issue"): + recent = recent or (m.get("status_updated") or "") >= since if ids and bid in ids: recent = True if not recent: skipped += 1 continue - status = bug_status(bug) + status = bug_status(rec) if status is None: needs_status.append(bid) synced_status = status or "unknown" - state = bug_state(bug) - issue_num = bug.get("gitea_issue") + state = bug_state(rec) + issue_num = m.get("gitea_issue") + title, body = bug_title(rec), bug_body(rec) + digest = body_hash(title, body) - if issue_num and bug.get("gitea_synced_status") == synced_status: + if issue_num and m.get("gitea_synced_status") == synced_status \ + and m.get("gitea_synced_hash") == digest: skipped += 1 continue if issue_num: - # Status changed since the last sync (or was never recorded: - # issues synced before the status field existed were closed - # whenever `fix` was non-empty, so re-assert state for them). - # The issue itself follows the aggregate status of every entry - # linked to it; only status labels are touched, so labels a - # person put on the issue survive. + # The issue follows the aggregate status of every record linked + # to it; only status labels are touched, so labels a person put + # on the issue survive. agg = issue_status[issue_num] agg_state = STATUS_MAP.get(agg, ("open", None))[0] + text_changed = m.get("gitea_synced_hash") != digest if dry_run: - if issue_num not in done_issues: - print(f" [DRY-RUN] #{issue_num} ({bid}): -> {agg_state}" - f" ({agg or 'unknown'})") - done_issues.add(issue_num) + if m.get("gitea_synced_status") != synced_status and issue_num not in done_issues: + print(f" [DRY-RUN] #{issue_num} ({bid}): -> {agg_state} ({agg or 'unknown'})") + done_issues.add(issue_num) + if text_changed: + print(f" [DRY-RUN] #{issue_num} ({bid}): refresh title/body") continue try: - if issue_num not in done_issues: + if text_changed: + api.update_issue(issue_num, title=title, body=body) + refreshed += 1 + if not quiet: + print(f" #{issue_num} {bid}: refreshed title/body") + if m.get("gitea_synced_status") != synced_status and issue_num not in done_issues: _, issue = request("GET", f"/issues/{issue_num}") if issue is None: warn(f" {bid}: issue #{issue_num} not found, skipping") @@ -886,9 +882,11 @@ def sync_bugs(api, dry_run=False, quiet=False, since=None, ids=None): api.add_labels(issue_num, [labels[want]["id"]]) if issue.get("state") != agg_state: api.update_issue(issue_num, state=agg_state) - note = _text(bug.get("status_note")) + note = _text(m.get("status_note")) + commits = ", ".join(m.get("fix_commits") or []) api.add_comment(issue_num, f"{bid} status: **{synced_status}**" - + (f" — {note}" if note else "")) + + (f" — {note}" if note else "") + + (f" (fix commits: {commits})" if commits else "")) if agg_state == "closed": closed += 1 else: @@ -899,26 +897,27 @@ def sync_bugs(api, dry_run=False, quiet=False, since=None, ids=None): else: updated += 1 done_issues.add(issue_num) - set_fields(BUGLOG_PATH, bid, {"gitea_synced_status": synced_status}) + buglog.set_meta(bid, {"gitea_synced_status": synced_status, + "gitea_synced_hash": digest}) except (RuntimeError, BuglogEditError) as e: warn(f" {bid}: could not update issue #{issue_num}: {e}") skipped += 1 continue # New bug -- create issue - title = bug_title(bug) if dry_run: print(f" [DRY-RUN] create {state.upper()} issue: {title}") created += 1 continue - label_ids = _resolve_label_ids(api, bug_labels(bug)) - issue = api.create_issue(title, bug_body(bug), labels=label_ids) + label_ids = _resolve_label_ids(api, bug_labels(rec)) + issue = api.create_issue(title, body, labels=label_ids) if state == "closed": # Gitea may ignore `state` on create; close explicitly. api.update_issue(issue["number"], state="closed") - set_fields(BUGLOG_PATH, bid, {"gitea_issue": issue["number"], - "gitea_synced_status": synced_status}) + buglog.set_meta(bid, {"gitea_issue": issue["number"], + "gitea_synced_status": synced_status, + "gitea_synced_hash": digest}) if not quiet: mark = " (closed)" if state == "closed" else "" print(f" created #{issue['number']}: {title}{mark}") @@ -926,7 +925,7 @@ def sync_bugs(api, dry_run=False, quiet=False, since=None, ids=None): if not quiet: print(f"Bugs: {created} created, {closed} closed, {reopened} reopened, " - f"{updated} relabelled, {skipped} skipped") + f"{updated} relabelled, {refreshed} refreshed, {skipped} skipped") if needs_status: warn(f"{len(needs_status)} bug(s) have no status -- triage per " f".claude/rules/buglog-status.md: {' '.join(needs_status)}") diff --git a/scripts/tests/test_buglog.py b/scripts/tests/test_buglog.py new file mode 100644 index 00000000..13ee16e8 --- /dev/null +++ b/scripts/tests/test_buglog.py @@ -0,0 +1,340 @@ +"""Tests for scripts/buglog.py (one Markdown record per bug). + +Run: python3 -m unittest discover -s scripts/tests -v +""" +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import buglog # noqa: E402 + + +FULL_ENV = """\ +- OS: Ubuntu 24.04 LTS; kernel Linux 6.18.33 x86_64 +- libc: glibc 2.42 (from the binary's interpreter) +- Toolchain: gcc 15.2.0, meson 1.9.1 +- Build: asan (xce.build) +- Code: scheduler-framework @ {sha} +- Deployment: single host, xcecd_virtual""" + +FULL_BODY = """\ +# {id}: xcecd_virtual leaks worker strings on every finished job + +## Description +**Observed:** +``` +Direct leak of 12 byte(s) in 12 object(s) +``` +**Expected:** no leak at exit +**Impact:** grows with job count +**Found by:** LeakSanitizer after a job burst + +## Environment +{env} + +## Reproduction +1. `/xce-build worker` +2. run `finished_job_retention_test` +- Frequency: deterministic +- Evidence: the unit test + +## Root cause +{root} + +## Fix +{fix} + +## Verification +{verif} + +## References +- Related: none +""" + + +def git(root, *args): + return subprocess.run(["git", "-C", str(root), *args], check=True, + capture_output=True, text=True).stdout.strip() + + +class BuglogTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + (self.root / ".wolf" / "bugs").mkdir(parents=True) + git(self.root, "init", "-q") + git(self.root, "-c", "user.email=t@t", "-c", "user.name=t", + "commit", "-q", "--allow-empty", "-m", "init") + self.sha = git(self.root, "rev-parse", "--short=8", "HEAD") + buglog.configure(self.root) + + def tearDown(self): + self.tmp.cleanup() + + def write_record(self, bug_id="bug-100", status="open", legacy=False, + root="TODO", fix="TODO", verif="TODO", commits=None, + env=None, extra_meta=None): + meta = {"id": bug_id, "status": status, "status_updated": "2026-09-27", + "status_note": "" if status == "fixed" else "why", + "component": "worker", "severity": "leak", + "timestamp": "2026-09-25T15:42:02Z"} + if legacy: + meta["legacy"] = True + if commits is not None: + meta["fix_commits"] = commits + meta.update(extra_meta or {}) + body = FULL_BODY.format(id=bug_id, env=env if env is not None + else FULL_ENV.format(sha=self.sha), + root=root, fix=fix, verif=verif) + text = buglog.render_meta(meta) + "\n" + body + path = self.root / ".wolf" / "bugs" / f"{bug_id}.md" + path.write_text(text, encoding="utf-8") + return path + + +class ParseRender(BuglogTestCase): + def test_round_trip_keeps_every_field(self): + path = self.write_record(extra_meta={"tags": ["worker", "asan"], + "gitea_issue": 285}) + rec = buglog.load("bug-100") + self.assertEqual(rec.meta["tags"], ["worker", "asan"]) + self.assertEqual(rec.meta["gitea_issue"], 285) + self.assertEqual(rec.summary, + "xcecd_virtual leaks worker strings on every finished job") + self.assertIn("Direct leak", rec.sections["Description"]) + self.assertEqual(buglog.render(rec), path.read_text(encoding="utf-8")) + + def test_section_header_inside_code_fence_is_not_a_section(self): + self.write_record(root="See:\n```\n## Fix\nnot a header\n```") + rec = buglog.load("bug-100") + self.assertIn("not a header", rec.sections["Root cause"]) + self.assertEqual(rec.sections["Fix"], "TODO") + + def test_escaped_section_header_round_trips(self): + text = "a\n## Fix\nb" + rec = buglog.Record({"id": "bug-7", "status": "open"}, "s", + {"Description": buglog.escape_text(text)}) + again = buglog.parse(buglog.render(rec)) + self.assertEqual(buglog.unescape_text(again.sections["Description"]), text) + self.assertNotIn("Fix", again.sections) + + def test_set_meta_leaves_body_bytes_unchanged(self): + path = self.write_record() + body_before = path.read_text(encoding="utf-8").split("\n---\n", 1)[1] + buglog.set_meta("bug-100", {"gitea_issue": 9, "status_note": "x: y"}) + text = path.read_text(encoding="utf-8") + self.assertEqual(text.split("\n---\n", 1)[1], body_before) + self.assertEqual(buglog.load("bug-100").meta["status_note"], "x: y") + + +class Check(BuglogTestCase): + def problems(self, bug_id="bug-100", target=None): + return buglog.check(buglog.load(bug_id), target_status=target) + + def test_complete_open_record_passes(self): + self.write_record() + self.assertEqual(self.problems(), []) + + def test_environment_needs_every_item(self): + env = FULL_ENV.format(sha=self.sha).replace( + "- libc: glibc 2.42 (from the binary's interpreter)\n", "") + self.write_record(env=env) + self.assertTrue(any("libc" in p for p in self.problems())) + + def test_libc_needs_a_version(self): + env = FULL_ENV.format(sha=self.sha).replace("glibc 2.42", "glibc") + self.write_record(env=env) + self.assertTrue(any("libc" in p for p in self.problems())) + + def test_code_needs_a_commit(self): + env = FULL_ENV.format(sha=self.sha).replace(self.sha, "latest") + self.write_record(env=env) + self.assertTrue(any("Code" in p for p in self.problems())) + + def test_todo_counts_as_missing(self): + env = FULL_ENV.format(sha=self.sha).replace( + "single host, xcecd_virtual", "TODO: topology") + self.write_record(env=env) + self.assertTrue(any("Deployment" in p for p in self.problems())) + + def test_fixed_needs_root_cause_fix_verification_and_commit(self): + self.write_record(status="fixed") + probs = " | ".join(self.problems()) + for word in ("Root cause", "Fix", "Verification", "fix_commits"): + self.assertIn(word, probs) + + def test_fixed_with_everything_passes(self): + self.write_record(status="fixed", root="remove_worker skips five strings", + fix="free them", commits=[self.sha], + verif="- Test: test_x: RED before, GREEN after\n" + "- Suites: worker 12/12, asan") + self.assertEqual(self.problems(), []) + + def test_unknown_commit_is_rejected(self): + self.write_record(status="fixed", root="r", fix="f", commits=["deadbeef"], + verif="- Test: t: RED before, GREEN after\n- Suites: s") + self.assertTrue(any("deadbeef" in p for p in self.problems())) + + def test_open_status_needs_note(self): + self.write_record(extra_meta={"status_note": ""}) + self.assertTrue(any("status_note" in p for p in self.problems())) + + def test_legacy_record_is_exempt_until_it_changes_status(self): + self.write_record(legacy=True, env=buglog.NOT_RECORDED) + self.assertEqual(self.problems(), []) + # marking it fixed now demands the fix-time sections + probs = " | ".join(self.problems(target="fixed")) + self.assertIn("Verification", probs) + self.assertIn("fix_commits", probs) + + def test_migrated_placeholder_is_not_verification(self): + self.write_record(legacy=True, root="r", fix="f", commits=[self.sha], + verif=buglog.NOT_RECORDED) + self.assertTrue(any("Verification" in p + for p in self.problems(target="fixed"))) + + +class Commands(BuglogTestCase): + def run_cli(self, *argv): + try: + buglog.main(list(argv)) + return 0 + except SystemExit as e: + return e.code + + def test_status_refuses_incomplete_fixed_and_leaves_file_alone(self): + path = self.write_record() + before = path.read_text(encoding="utf-8") + rc = self.run_cli("status", "bug-100", "fixed", "--commit", self.sha) + self.assertNotEqual(rc, 0) + self.assertEqual(path.read_text(encoding="utf-8"), before) + + def test_status_fixed_records_commits(self): + self.write_record(root="r", fix="f", + verif="- Test: t: RED before, GREEN after\n- Suites: s") + rc = self.run_cli("status", "bug-100", "fixed", "--commit", self.sha) + self.assertIn(rc, (0, None)) + meta = buglog.load("bug-100").meta + self.assertEqual(meta["status"], "fixed") + self.assertEqual(meta["fix_commits"], [self.sha]) + + def test_new_creates_record_with_environment_filled(self): + self.run_cli("new", "--summary", "scheduler hangs on empty queue", + "--component", "scheduler", "--severity", "hang") + rec = buglog.load("bug-001") + self.assertEqual(rec.meta["status"], "open") + env = rec.sections["Environment"] + self.assertRegex(env, r"- OS: .*Linux") + self.assertRegex(env, r"- Code: .*" + self.sha) + # the author still has to write the description + self.assertTrue(buglog.check(rec)) + + def test_next_id_counts_archive(self): + self.write_record(bug_id="bug-100") + (self.root / ".wolf" / "buglog-archive.json").write_text( + json.dumps({"bugs": [{"id": "bug-900"}]}), encoding="utf-8") + self.assertEqual(buglog.next_id(), "bug-901") + + def test_index_keeps_openwolf_fields(self): + self.write_record(extra_meta={"files": ["src/worker/a.c", "src/worker/b.c"], + "tags": ["worker"]}, root="cause text") + buglog.build_index() + idx = json.loads((self.root / ".wolf" / "buglog.json").read_text()) + b = idx["bugs"][0] + self.assertEqual(b["id"], "bug-100") + self.assertTrue(b["error_message"].startswith("xcecd_virtual leaks")) + self.assertIn("Direct leak", b["error_message"]) + self.assertEqual(b["root_cause"], "cause text") + self.assertEqual(b["file"], "src/worker/a.c, src/worker/b.c") + self.assertEqual(b["tags"], ["worker"]) + self.assertEqual(b["status"], "open") + + def test_write_commands_refresh_the_index(self): + self.write_record() + self.run_cli("note", "bug-100", "seen again under load") + idx = json.loads((self.root / ".wolf" / "buglog.json").read_text()) + self.assertIn("seen again under load", idx["bugs"][0]["error_message"]) + + +class Staged(BuglogTestCase): + """`check --staged`, which the pre-commit hook runs.""" + + def commit(self, *paths): + git(self.root, "add", *paths) + git(self.root, "-c", "user.email=t@t", "-c", "user.name=t", + "commit", "-q", "-m", "x") + + def check_staged(self): + try: + buglog.main(["check", "--staged"]) + return 0 + except SystemExit as e: + return e.code or 0 + + def test_complete_record_with_fresh_index_passes(self): + self.write_record() + buglog.build_index() + git(self.root, "add", ".wolf") + self.assertEqual(self.check_staged(), 0) + + def test_incomplete_record_is_refused(self): + self.write_record(env="- OS: TODO") + buglog.build_index() + git(self.root, "add", ".wolf") + self.assertNotEqual(self.check_staged(), 0) + + def test_stale_index_is_refused(self): + self.write_record() + git(self.root, "add", ".wolf") + self.assertNotEqual(self.check_staged(), 0) + + def test_hand_edited_index_alone_is_refused(self): + self.write_record() + buglog.build_index() + self.commit(".wolf") + idx = self.root / ".wolf" / "buglog.json" + idx.write_text(idx.read_text().replace("leaks", "LEAKS")) + git(self.root, "add", ".wolf/buglog.json") + self.assertNotEqual(self.check_staged(), 0) + + def test_legacy_record_marked_fixed_by_hand_needs_fix_sections(self): + self.write_record(legacy=True, env=buglog.NOT_RECORDED) + buglog.build_index() + self.commit(".wolf") + buglog.set_meta("bug-100", {"status": "fixed", "status_note": ""}) + git(self.root, "add", ".wolf") + self.assertNotEqual(self.check_staged(), 0) + + def test_migrated_record_matching_the_old_index_passes(self): + # the migration commit: the record is new, but HEAD's buglog.json + # already had this bug with the same status + (self.root / ".wolf" / "buglog.json").write_text( + json.dumps({"bugs": [{"id": "bug-100", "status": "fixed"}]})) + self.commit(".wolf") + self.write_record(legacy=True, status="fixed", env=buglog.NOT_RECORDED) + buglog.build_index() + git(self.root, "add", ".wolf") + self.assertEqual(self.check_staged(), 0) + + def test_new_record_cannot_claim_legacy(self): + self.write_record(legacy=True, status="fixed", env=buglog.NOT_RECORDED) + buglog.build_index() + git(self.root, "add", ".wolf") + self.assertNotEqual(self.check_staged(), 0) + + def test_untouched_legacy_record_passes(self): + self.write_record(legacy=True, env=buglog.NOT_RECORDED) + buglog.build_index() + self.commit(".wolf") + buglog.set_meta("bug-100", {"gitea_issue": 5}) + git(self.root, "add", ".wolf") + self.assertEqual(self.check_staged(), 0) + + +if __name__ == "__main__": + unittest.main()