From a370d7241771d58f1c0d426e51cc493d6ee4c18f Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Wed, 30 Sep 2026 03:51:44 +0000 Subject: [PATCH] chore(buglog): bug-1721 fixed (f5c887dc) Co-Authored-By: Claude Opus 5.5 (1M context) --- .wolf/buglog.json | 18 ++++++++++-------- .wolf/bugs/bug-1721.md | 33 +++++++++++++++++++++++---------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/.wolf/buglog.json b/.wolf/buglog.json index e718714f..17274854 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -10929,18 +10929,18 @@ }, { "id": "bug-1721", - "status": "open", - "status_updated": "2026-09-24", - "status_note": "clean_failover_preservation.sh default ha/data dir no longer exists", + "status": "fixed", + "status_updated": "2026-09-30", + "status_note": "", "component": "controller", - "severity": "unknown", + "severity": "wrong-result", "timestamp": "2026-08-26T00:00:00Z", "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, - "error_message": "clean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite)\n\nclean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite)", + "error_message": "clean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite)\n\nclean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite)\n\nUPDATE (2026-09-29): once past the ha/data precondition, two more stale assumptions surfaced in the same script: the deploy-ha.sh call used an old 4-argument signature, and both committed-LSN boundaries assumed bug-845's doubled submits (`FAIL: jobs did not commit on a majority within 90s (max committed_lsn=3 < 5)`, then `walcompare: FAIL -- committed-prefix divergence at LSN 5` with all three WALs identical at LSN 1-4).", "file": "src/controller/testsuite/clean_failover_preservation.sh", - "root_cause": "The script hardcodes its DATADIR precondition default to $REPO_ROOT/ha/data (a pre-generated cluster.bin + TLS cert template directory it expects to already exist), but ha/data/ was deliberately deleted from the repo by an unrelated commit (ee8473bd, admin-config-tool: drop ha/data/ entirely, match real client TLS). The script was never updated afterward, so its own documented precondition check now fails unconditionally in any fresh checkout/worktree that has not manually recreated that directory out-of-band. Discovered while adding --norc to the script (xce-test-rc-file spec follow-up) and independently verifying the edit did not break anything -- confirmed via git history that the failure reproduces identically on the pre-edit script too, so it is unrelated to that change.", - "fix": "Not fixed -- out of scope for the xce-test-rc-file spec. Needs either: (a) DATADIR default updated to point at a still-valid TLS/config template location consistent with admin-config-tool's TLS changes, or (b) the script updated to self-generate its own template via deploy-ha.sh/xce-deploy-ha before the precondition check, matching how run-ha-soak.sh and run-resource-verify.sh already self-deploy their own scratch clusters instead of depending on a fixed pre-existing directory.", + "root_cause": "Three stale assumptions in clean_failover_preservation.sh, each hidden behind the previous one. (1) It required /ha/data/{cluster.bin,xce.crt,xce.key} and passed that dir to deploy-ha.sh, but ee8473bd removed ha/data/ and deploy-ha.sh now generates the certs and cluster.bin into the deploy dir itself, taking ` ` -- so the precondition failed in every checkout, and the 4-argument call would have shifted every argument. (2) The pre-kill commit wait's boundary was 1 + 2*N_JOBS because bug-845 once wrote each JOB_SUBMIT twice; since bug-845's fix each job is one WAL entry, so a healthy cluster commits exactly 1 + N_JOBS and the wait timed out. (3) The walcompare --committed boundary repeated the same formula, so it asked for an LSN beyond every member's WAL and reported every member MISSING it.", + "fix": "src/controller/testsuite/clean_failover_preservation.sh: dropped the DATADIR variable and its cluster.bin / TLS-cert preconditions; calls `deploy-ha.sh \"$BINARY\" \"$TOPDIR\" \"$PEERS\"`; both COMMITTED_BOUNDARY defaults are now 1 + N_JOBS (own-term noop + one entry per job), with the bug-845 comments corrected. XCE_FAILOVER_COMMITTED still overrides it.\n\nVerification:\n- Test: the script itself is the test -- before the fix it failed its precondition (`cluster.bin missing under .../ha/data`); after, it passes end to end (deploy, submit, commit wait, kill primary, promotion, rejoin, walcompare, job survival)\n- Suites: n/a -- script-only change; `bash -n` clean\n- Live: 3/3 passing runs, 2026-09-29: N_JOBS=2 twice (walcompare --committed 3) and N_JOBS=5 (--committed 6), each `PASS: clean failover converged; committed prefix identical; no job lost`", "tags": [ "test-harness", "stale-fixture", @@ -10950,7 +10950,9 @@ "admin-config-tool-regression" ], "related_bugs": [], - "fix_commits": [], + "fix_commits": [ + "f5c887dc" + ], "record": ".wolf/bugs/bug-1721.md" }, { diff --git a/.wolf/bugs/bug-1721.md b/.wolf/bugs/bug-1721.md index 852f19f0..f622dd5c 100644 --- a/.wolf/bugs/bug-1721.md +++ b/.wolf/bugs/bug-1721.md @@ -1,16 +1,17 @@ --- id: "bug-1721" -status: "open" -status_updated: "2026-09-24" -status_note: "clean_failover_preservation.sh default ha/data dir no longer exists" +status: "fixed" +status_updated: "2026-09-30" +status_note: "" component: "controller" -severity: "unknown" +severity: "wrong-result" timestamp: "2026-08-26T00:00:00Z" last_seen: "2026-09-19T09:00:00Z" occurrences: 1 files: ["src/controller/testsuite/clean_failover_preservation.sh"] tags: ["test-harness", "stale-fixture", "clean_failover_preservation.sh", "ha-term-safety", "pre-existing", "admin-config-tool-regression"] related: [] +fix_commits: ["f5c887dc"] legacy: true --- @@ -19,20 +20,32 @@ legacy: true ## Description clean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite) +UPDATE (2026-09-29): once past the ha/data precondition, two more stale assumptions surfaced in the same script: the deploy-ha.sh call used an old 4-argument signature, and both committed-LSN boundaries assumed bug-845's doubled submits (`FAIL: jobs did not commit on a majority within 90s (max committed_lsn=3 < 5)`, then `walcompare: FAIL -- committed-prefix divergence at LSN 5` with all three WALs identical at LSN 1-4). + ## Environment -_Not recorded (migrated from buglog.json)._ +- OS: no /etc/os-release; Docker container; NixOS kernel; kernel Linux 6.18.33 x86_64 +- libc: glibc 2.42-47 (nix dev shell; interpreter /nix/store/wb6rhpznjfczwlwx23zmdrrw74bayxw4-glibc-2.42-47/lib/ld-linux-x86-64.so.2) +- Toolchain: bash 5.3, gcc (GCC) 15.2.0, meson 1.9.1 +- Build: asan (controller + harness + xceadmin in one build dir, src/xce.build.bug1871) +- Code: reproduced 2026-09-29 at 980a0054; the script has been broken since ee8473bd (ha/data removal) +- Deployment: local 3-node HA controller cluster the script deploys itself, localhost:29991-29993 ## Reproduction -_Not recorded (migrated from buglog.json)._ +1. `XCE_CONTROLLER_BIN=/controller/xce_controller XCE_FAILOVER_TOP= nix develop ./src --command bash src/controller/testsuite/clean_failover_preservation.sh` +- Frequency: deterministic -- 1/1 runs failed at the precondition, 2026-09-29: `FAIL: cluster.bin missing under /ha/data (deploy prerequisite)` +- Evidence: /home/claude/tmp/xce-1721-green (commit-wait failure), /home/claude/tmp/xce-1721-green2 (walcompare past-the-tip failure, walcompare.out), /home/claude/tmp/xce-1721-green3 (passing run) ## Root cause -The script hardcodes its DATADIR precondition default to $REPO_ROOT/ha/data (a pre-generated cluster.bin + TLS cert template directory it expects to already exist), but ha/data/ was deliberately deleted from the repo by an unrelated commit (ee8473bd, admin-config-tool: drop ha/data/ entirely, match real client TLS). The script was never updated afterward, so its own documented precondition check now fails unconditionally in any fresh checkout/worktree that has not manually recreated that directory out-of-band. Discovered while adding --norc to the script (xce-test-rc-file spec follow-up) and independently verifying the edit did not break anything -- confirmed via git history that the failure reproduces identically on the pre-edit script too, so it is unrelated to that change. +Three stale assumptions in clean_failover_preservation.sh, each hidden behind the previous one. (1) It required /ha/data/{cluster.bin,xce.crt,xce.key} and passed that dir to deploy-ha.sh, but ee8473bd removed ha/data/ and deploy-ha.sh now generates the certs and cluster.bin into the deploy dir itself, taking ` ` -- so the precondition failed in every checkout, and the 4-argument call would have shifted every argument. (2) The pre-kill commit wait's boundary was 1 + 2*N_JOBS because bug-845 once wrote each JOB_SUBMIT twice; since bug-845's fix each job is one WAL entry, so a healthy cluster commits exactly 1 + N_JOBS and the wait timed out. (3) The walcompare --committed boundary repeated the same formula, so it asked for an LSN beyond every member's WAL and reported every member MISSING it. ## Fix -Not fixed -- out of scope for the xce-test-rc-file spec. Needs either: (a) DATADIR default updated to point at a still-valid TLS/config template location consistent with admin-config-tool's TLS changes, or (b) the script updated to self-generate its own template via deploy-ha.sh/xce-deploy-ha before the precondition check, matching how run-ha-soak.sh and run-resource-verify.sh already self-deploy their own scratch clusters instead of depending on a fixed pre-existing directory. +src/controller/testsuite/clean_failover_preservation.sh: dropped the DATADIR variable and its cluster.bin / TLS-cert preconditions; calls `deploy-ha.sh "$BINARY" "$TOPDIR" "$PEERS"`; both COMMITTED_BOUNDARY defaults are now 1 + N_JOBS (own-term noop + one entry per job), with the bug-845 comments corrected. XCE_FAILOVER_COMMITTED still overrides it. ## Verification -_Not recorded (migrated from buglog.json)._ +- Test: the script itself is the test -- before the fix it failed its precondition (`cluster.bin missing under .../ha/data`); after, it passes end to end (deploy, submit, commit wait, kill primary, promotion, rejoin, walcompare, job survival) +- Suites: n/a -- script-only change; `bash -n` clean +- Live: 3/3 passing runs, 2026-09-29: N_JOBS=2 twice (walcompare --committed 3) and N_JOBS=5 (--committed 6), each `PASS: clean failover converged; committed prefix identical; no job lost` ## References -_Not recorded (migrated from buglog.json)._ +- Fix commit: f5c887dc +- Related: bug-845 (doubled submits, fixed), ee8473bd (ha/data removal)