From 9be9e954c7841ed076c232daa1722c47ba53ca03 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 18 Aug 2026 14:10:58 +0000 Subject: [PATCH] test(resource-value-verification): verdict-logic regression selftest (task 8.2) verdict-logic-selftest.sh sources the real aggregate_verdict()/ build_verdict_record() (via run-resource-verify.sh's BASH_SOURCE guard, no live deployment triggered) and exercises them with synthetic per-layer records: all-match/PASS positive control, a single-resource single-layer MISMATCH within a multi-resource batch (asserting both correct attribution of the failure AND that the other resources independently still report PASS), UNREACHABLE distinct from MISMATCH, REJECTED, and a zero-records degenerate case. 53 assertions total. Proven non-tautological: breaking aggregate_verdict()'s comparison logic (two different ways, independently reproduced by review) makes the selftest genuinely fail; restoring makes it pass again. This completes the resource-value-verification spec - all 15 tasks (1.1 through 8.2) implemented, reviewed, and committed. Co-Authored-By: Claude Sonnet 5 --- .../verdict-logic-selftest.sh | 219 ++++++++++++++++++ .../resource-value-verification/tasks.md | 2 +- 2 files changed, 220 insertions(+), 1 deletion(-) create mode 100755 .claude/skills/xce-test-resources/verdict-logic-selftest.sh diff --git a/.claude/skills/xce-test-resources/verdict-logic-selftest.sh b/.claude/skills/xce-test-resources/verdict-logic-selftest.sh new file mode 100755 index 00000000..c6634aa0 --- /dev/null +++ b/.claude/skills/xce-test-resources/verdict-logic-selftest.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# +# verdict-logic-selftest.sh -- deterministic, no-deployment unit test for +# run-resource-verify.sh's verdict-aggregation step: build_verdict_record() +# and aggregate_verdict() (task 7.1), formalizing task 8.2's acceptance +# criteria (Requirement 6.3) as a persistent, re-runnable regression test. +# +# Mirrors xce-test-ha's soak-verdict-selftest.sh convention (source the +# target script, `check DESC EXPECTED ACTUAL`, tally failures, print a +# terminal SELFTEST PASS/FAIL line, exit 0/1) -- this is the equivalent +# harness for run-resource-verify.sh's verdict logic. +# +# WHY THIS EXISTS (not just "re-prove task 7.1 works"): task 7.1's own +# review cycle already ran ~70 ad hoc synthetic assertions against +# aggregate_verdict()/build_verdict_record() across two review rounds +# (see tasks.md's Implementation Notes for task 7.1, and the round-1 +# REJECTED -> round-2 fixed history of bug-1581/bug-1582) -- but those were +# throwaway scratch invocations, never committed. This file makes that +# verification a durable part of the repo: run it any time +# run-resource-verify.sh's verdict logic changes, with no cluster, no +# binaries, no C/Go build -- pure bash, runs in well under a second. +# +# THE CORE PROOF (Requirement 6.3 / task 8.2's acceptance criteria): a +# synthetic batch with ONE mismatched layer on ONE resource must produce an +# overall FAIL that correctly NAMES that specific resource+layer, while +# every OTHER resource/layer in the SAME batch is still reported PASS/MATCH +# -- proving the logic distinguishes pass from fail rather than defaulting +# to one outcome. See scenario 3 below. +# +# Usage: bash verdict-logic-selftest.sh +set -u + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./run-resource-verify.sh +# Sourcing (not executing) run-resource-verify.sh brings aggregate_verdict(), +# build_verdict_record(), controller_values_match(), and VERDICT_FIELD_SEP +# into scope WITHOUT running main()/internal_controller_check() or touching +# any live deployment -- the bottom of that file is deliberately guarded +# (`[[ "${BASH_SOURCE[0]}" == "${0}" ]]`) for exactly this purpose (see its +# own header comment and SKILL.md's "self-contained and independently +# testable" section). +. "$HERE/run-resource-verify.sh" || { + echo "SELFTEST FAIL: cannot source run-resource-verify.sh (verdict library missing)" >&2 + exit 1 +} + +fails=0 +check() { # desc expected actual + if [ "$2" = "$3" ]; then + echo "ok - $1" + else + echo "FAIL - $1: expected [$2] got [$3]" + fails=$((fails + 1)) + fi +} + +check_contains() { # desc needle haystack + case "$3" in + *"$2"*) echo "ok - $1" ;; + *) echo "FAIL - $1: expected to find [$2] in:"$'\n'"$3" + fails=$((fails + 1)) ;; + esac +} + +check_not_contains() { # desc needle haystack + case "$3" in + *"$2"*) echo "FAIL - $1: expected NOT to find [$2] in:"$'\n'"$3" + fails=$((fails + 1)) ;; + *) echo "ok - $1" ;; + esac +} + +# aggregate_verdict() both (a) populates the AGG_* globals in the CALLING +# shell and (b) writes FAILURE/VERDICT/oracle lines via log() (a plain +# echo). Capturing its stdout with `$(...)` would fork a subshell and lose +# (a) -- so redirect to a temp file (plain `>` redirection does NOT fork a +# subshell in bash) to get both the real AGG_* globals AND the printed +# output from the same call. +OUTFILE="$(mktemp)" +trap 'rm -f "$OUTFILE"' EXIT + +run_agg() { # RECORD... + aggregate_verdict "$@" >"$OUTFILE" 2>&1 +} + +# ── Scenario 0: build_verdict_record() field integrity ────────────────────── +# Direct invocation of build_verdict_record() (not aggregate_verdict()) -- +# confirms the 9-field / VERDICT_FIELD_SEP contract itself, and specifically +# guards the bug-1581 regression class: a genuinely-empty field (e.g. an +# UNREACHABLE layer's empty observed value) must round-trip intact, not get +# collapsed/shifted by a naive whitespace-IFS split. +rec0="$(build_verdict_record pg 42 accepted 1 42.000000 1 42.000000 0 "")" +IFS="$VERDICT_FIELD_SEP" read -r f_name f_inj f_status f_ctrl_ok f_ctrl_obs f_sched_ok f_sched_obs f_api_ok f_api_obs <<< "$rec0" +check "scenario0: field 1 (resource name) round-trips" "pg" "$f_name" +check "scenario0: field 3 (injection status) round-trips" "accepted" "$f_status" +check "scenario0: field 8 (api_ok) round-trips as 0, not shifted" "0" "$f_api_ok" +check "scenario0: field 9 (empty api_obs) survives intact, not collapsed" "" "$f_api_obs" + +# ── Scenario 0b: controller_values_match() -- the pure comparison primitive +# aggregate_verdict() itself calls for every layer comparison. +if controller_values_match "1.5" "1.500000"; then r=0; else r=1; fi +check "scenario0b: numeric tolerant match (1.5 vs controller's 1.500000 rendering)" "0" "$r" +if controller_values_match "1.5" "1.600000"; then r=0; else r=1; fi +check "scenario0b: numeric mismatch (1.5 vs 1.6) correctly fails" "1" "$r" +if controller_values_match "type" "type"; then r=0; else r=1; fi +check "scenario0b: non-numeric exact match" "0" "$r" +if controller_values_match "type" "model"; then r=0; else r=1; fi +check "scenario0b: non-numeric mismatch correctly fails" "1" "$r" + +# ── Scenario 1: all-match batch -> overall PASS (positive control) ────────── +# Three resources, every layer matching its injected value. Proves the logic +# does not default to FAIL either. +records=( + "$(build_verdict_record mem 5 accepted 1 5.000000 1 5.000000 1 5.000000)" + "$(build_verdict_record r15m 1.5 accepted 1 1.500000 1 1.500000 1 1.500000)" + "$(build_verdict_record pg 42 accepted 1 42.000000 1 42.000000 1 42.000000)" +) +run_agg "${records[@]}"; rc=$? +out="$(cat "$OUTFILE")" +check "scenario1: exit code 0 (PASS)" "0" "$rc" +check "scenario1: AGG_RESULT=PASS" "PASS" "$AGG_RESULT" +check "scenario1: AGG_TOTAL_RESOURCES=3" "3" "$AGG_TOTAL_RESOURCES" +check "scenario1: AGG_MISMATCHES=0" "0" "$AGG_MISMATCHES" +check "scenario1: AGG_UNREACHABLE=0" "0" "$AGG_UNREACHABLE" +check "scenario1: AGG_REJECTED=0" "0" "$AGG_REJECTED" +check "scenario1: mem reported PASS" "PASS" "${AGG_RESOURCE_STATUS[mem]-}" +check "scenario1: r15m reported PASS" "PASS" "${AGG_RESOURCE_STATUS[r15m]-}" +check "scenario1: pg reported PASS" "PASS" "${AGG_RESOURCE_STATUS[pg]-}" +check "scenario1: zero FAILURE lines emitted" "0" "${#AGG_FAILURE_LINES[@]}" +check_contains "scenario1: VERDICT line reports result=PASS" "result=PASS" "$out" + +# ── Scenario 2 (THE CORE PROOF, Requirement 6.3): one layer mismatched for +# ONE resource -> overall FAIL, correctly attributed to that specific +# resource+layer, while every OTHER resource/layer in the same batch is +# still PASS/MATCH. +# r15m's scheduler-observed value (9.999000) deliberately does not match its +# injected value (1.5); mem and pg match at every layer. +records=( + "$(build_verdict_record mem 5 accepted 1 5.000000 1 5.000000 1 5.000000)" + "$(build_verdict_record r15m 1.5 accepted 1 1.500000 1 9.999000 1 1.500000)" + "$(build_verdict_record pg 42 accepted 1 42.000000 1 42.000000 1 42.000000)" +) +run_agg "${records[@]}"; rc=$? +out="$(cat "$OUTFILE")" +failures="$(printf '%s\n' "${AGG_FAILURE_LINES[@]}")" +check "scenario2: exit code 1 (FAIL)" "1" "$rc" +check "scenario2: AGG_RESULT=FAIL" "FAIL" "$AGG_RESULT" +check "scenario2: AGG_MISMATCHES=1" "1" "$AGG_MISMATCHES" +check "scenario2: AGG_UNREACHABLE=0" "0" "$AGG_UNREACHABLE" +check "scenario2: AGG_REJECTED=0" "0" "$AGG_REJECTED" +check "scenario2: exactly one FAILURE line" "1" "${#AGG_FAILURE_LINES[@]}" +check "scenario2: r15m (mismatched resource) reported FAIL" "FAIL" "${AGG_RESOURCE_STATUS[r15m]-}" +check "scenario2: mem (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[mem]-}" +check "scenario2: pg (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[pg]-}" +check_contains "scenario2: failure names r15m+scheduler+MISMATCH specifically" \ + "resource=r15m layer=scheduler status=MISMATCH" "$failures" +check_not_contains "scenario2: mem is NOT swept into any failure line" "resource=mem" "$failures" +check_not_contains "scenario2: pg is NOT swept into any failure line" "resource=pg" "$failures" +check_contains "scenario2: VERDICT line reports result=FAIL" "result=FAIL" "$out" +check_contains "scenario2: VERDICT line reports mismatches=1" "mismatches=1" "$out" + +# ── Scenario 3: one layer UNREACHABLE (timeout) for ONE resource -> overall +# FAIL, classified distinctly from MISMATCH -- proves the logic +# distinguishes "timed out" from "wrong value" rather than collapsing +# both into one bucket. pg's apiserver layer never came back (api_ok=0, +# empty observed value); mem and r15m match everywhere. +records=( + "$(build_verdict_record mem 5 accepted 1 5.000000 1 5.000000 1 5.000000)" + "$(build_verdict_record r15m 1.5 accepted 1 1.500000 1 1.500000 1 1.500000)" + "$(build_verdict_record pg 42 accepted 1 42.000000 1 42.000000 0 "")" +) +run_agg "${records[@]}"; rc=$? +failures="$(printf '%s\n' "${AGG_FAILURE_LINES[@]}")" +check "scenario3: exit code 1 (FAIL)" "1" "$rc" +check "scenario3: AGG_RESULT=FAIL" "FAIL" "$AGG_RESULT" +check "scenario3: AGG_MISMATCHES=0 (not miscounted as a mismatch)" "0" "$AGG_MISMATCHES" +check "scenario3: AGG_UNREACHABLE=1" "1" "$AGG_UNREACHABLE" +check "scenario3: pg (unreachable resource) reported FAIL" "FAIL" "${AGG_RESOURCE_STATUS[pg]-}" +check "scenario3: mem (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[mem]-}" +check "scenario3: r15m (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[r15m]-}" +check_contains "scenario3: failure names pg+apiserver+UNREACHABLE specifically" \ + "resource=pg layer=apiserver status=UNREACHABLE" "$failures" +check_not_contains "scenario3: mem is NOT swept into any failure line" "resource=mem" "$failures" +check_not_contains "scenario3: r15m is NOT swept into any failure line" "resource=r15m" "$failures" + +# ── Scenario 4 (bonus): a rejected injection name -- distinct failure +# category from MISMATCH/UNREACHABLE, still isolated to its own resource. +records=( + "$(build_verdict_record mem 5 accepted 1 5.000000 1 5.000000 1 5.000000)" + "$(build_verdict_record badname 1 "rejected: unknown resource" 0 "" 0 "" 0 "")" +) +run_agg "${records[@]}"; rc=$? +failures="$(printf '%s\n' "${AGG_FAILURE_LINES[@]}")" +check "scenario4: exit code 1 (FAIL)" "1" "$rc" +check "scenario4: AGG_REJECTED=1" "1" "$AGG_REJECTED" +check "scenario4: AGG_MISMATCHES=0" "0" "$AGG_MISMATCHES" +check "scenario4: AGG_UNREACHABLE=0" "0" "$AGG_UNREACHABLE" +check "scenario4: badname reported FAIL" "FAIL" "${AGG_RESOURCE_STATUS[badname]-}" +check "scenario4: mem (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[mem]-}" +check_contains "scenario4: failure names badname as REJECTED specifically" \ + "resource=badname layer=injection status=REJECTED" "$failures" +check_not_contains "scenario4: mem is NOT swept into any failure line" "resource=mem" "$failures" + +# ── Scenario 5 (bonus): zero records -> FAIL, not a false/default PASS. +# Documents/guards aggregate_verdict()'s own degenerate-input handling +# (AGG_TOTAL_RESOURCES=0 is explicitly classified FAIL, per its header +# comment) -- "nothing was checked" must never read as an overall PASS. +run_agg; rc=$? +check "scenario5: zero records -> exit code 1 (FAIL, not a default PASS)" "1" "$rc" +check "scenario5: zero records -> AGG_RESULT=FAIL" "FAIL" "$AGG_RESULT" + +# ── Verdict ────────────────────────────────────────────────────────────────── +echo +if [ "$fails" -eq 0 ]; then + echo "SELFTEST PASS" + exit 0 +fi +echo "SELFTEST FAIL ($fails failed)" +exit 1 diff --git a/.kiro/specs/resource-value-verification/tasks.md b/.kiro/specs/resource-value-verification/tasks.md index d7bdfe0a..ee06278f 100644 --- a/.kiro/specs/resource-value-verification/tasks.md +++ b/.kiro/specs/resource-value-verification/tasks.md @@ -109,7 +109,7 @@ - _Requirements: 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 3.1, 3.2, 4.1, 4.2, 4.3, 5.1, 5.2, 5.3, 5.4, 6.1, 6.2, 6.4_ - _Depends: 7.1_ -- [ ] 8.2 Verify the verdict logic correctly reports FAIL on a genuine mismatch +- [x] 8.2 Verify the verdict logic correctly reports FAIL on a genuine mismatch - Invoke task 7.1's comparison step directly with a synthetic set of per-layer observed-value records where one layer's value deliberately does not match the corresponding injected value (no live deployment required for this check, since the comparison step is self-contained per task 7.1) - Observable completion: the comparison step reports an overall FAIL verdict that correctly names the specific layer whose synthetic record was mismatched, and reports PASS for every other, matching layer/resource in the same input set — proving the verdict logic distinguishes pass from fail rather than always reporting PASS - _Requirements: 6.3_