Rongsong Shen
9be9e954c7
verdict-logic-selftest.sh sources the real aggregate_verdict()/ build_verdict_record() (via run-resource-verify.sh's BASH_SOURCE guard, no live deployment triggered) and exercises them with synthetic per-layer records: all-match/PASS positive control, a single-resource single-layer MISMATCH within a multi-resource batch (asserting both correct attribution of the failure AND that the other resources independently still report PASS), UNREACHABLE distinct from MISMATCH, REJECTED, and a zero-records degenerate case. 53 assertions total. Proven non-tautological: breaking aggregate_verdict()'s comparison logic (two different ways, independently reproduced by review) makes the selftest genuinely fail; restoring makes it pass again. This completes the resource-value-verification spec - all 15 tasks (1.1 through 8.2) implemented, reviewed, and committed. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
219 lines
12 KiB
Bash
Executable file
219 lines
12 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# verdict-logic-selftest.sh -- deterministic, no-deployment unit test for
|
|
# run-resource-verify.sh's verdict-aggregation step: build_verdict_record()
|
|
# and aggregate_verdict() (task 7.1), formalizing task 8.2's acceptance
|
|
# criteria (Requirement 6.3) as a persistent, re-runnable regression test.
|
|
#
|
|
# Mirrors xce-test-ha's soak-verdict-selftest.sh convention (source the
|
|
# target script, `check DESC EXPECTED ACTUAL`, tally failures, print a
|
|
# terminal SELFTEST PASS/FAIL line, exit 0/1) -- this is the equivalent
|
|
# harness for run-resource-verify.sh's verdict logic.
|
|
#
|
|
# WHY THIS EXISTS (not just "re-prove task 7.1 works"): task 7.1's own
|
|
# review cycle already ran ~70 ad hoc synthetic assertions against
|
|
# aggregate_verdict()/build_verdict_record() across two review rounds
|
|
# (see tasks.md's Implementation Notes for task 7.1, and the round-1
|
|
# REJECTED -> round-2 fixed history of bug-1581/bug-1582) -- but those were
|
|
# throwaway scratch invocations, never committed. This file makes that
|
|
# verification a durable part of the repo: run it any time
|
|
# run-resource-verify.sh's verdict logic changes, with no cluster, no
|
|
# binaries, no C/Go build -- pure bash, runs in well under a second.
|
|
#
|
|
# THE CORE PROOF (Requirement 6.3 / task 8.2's acceptance criteria): a
|
|
# synthetic batch with ONE mismatched layer on ONE resource must produce an
|
|
# overall FAIL that correctly NAMES that specific resource+layer, while
|
|
# every OTHER resource/layer in the SAME batch is still reported PASS/MATCH
|
|
# -- proving the logic distinguishes pass from fail rather than defaulting
|
|
# to one outcome. See scenario 3 below.
|
|
#
|
|
# Usage: bash verdict-logic-selftest.sh
|
|
set -u
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=./run-resource-verify.sh
|
|
# Sourcing (not executing) run-resource-verify.sh brings aggregate_verdict(),
|
|
# build_verdict_record(), controller_values_match(), and VERDICT_FIELD_SEP
|
|
# into scope WITHOUT running main()/internal_controller_check() or touching
|
|
# any live deployment -- the bottom of that file is deliberately guarded
|
|
# (`[[ "${BASH_SOURCE[0]}" == "${0}" ]]`) for exactly this purpose (see its
|
|
# own header comment and SKILL.md's "self-contained and independently
|
|
# testable" section).
|
|
. "$HERE/run-resource-verify.sh" || {
|
|
echo "SELFTEST FAIL: cannot source run-resource-verify.sh (verdict library missing)" >&2
|
|
exit 1
|
|
}
|
|
|
|
fails=0
|
|
check() { # desc expected actual
|
|
if [ "$2" = "$3" ]; then
|
|
echo "ok - $1"
|
|
else
|
|
echo "FAIL - $1: expected [$2] got [$3]"
|
|
fails=$((fails + 1))
|
|
fi
|
|
}
|
|
|
|
check_contains() { # desc needle haystack
|
|
case "$3" in
|
|
*"$2"*) echo "ok - $1" ;;
|
|
*) echo "FAIL - $1: expected to find [$2] in:"$'\n'"$3"
|
|
fails=$((fails + 1)) ;;
|
|
esac
|
|
}
|
|
|
|
check_not_contains() { # desc needle haystack
|
|
case "$3" in
|
|
*"$2"*) echo "FAIL - $1: expected NOT to find [$2] in:"$'\n'"$3"
|
|
fails=$((fails + 1)) ;;
|
|
*) echo "ok - $1" ;;
|
|
esac
|
|
}
|
|
|
|
# aggregate_verdict() both (a) populates the AGG_* globals in the CALLING
|
|
# shell and (b) writes FAILURE/VERDICT/oracle lines via log() (a plain
|
|
# echo). Capturing its stdout with `$(...)` would fork a subshell and lose
|
|
# (a) -- so redirect to a temp file (plain `>` redirection does NOT fork a
|
|
# subshell in bash) to get both the real AGG_* globals AND the printed
|
|
# output from the same call.
|
|
OUTFILE="$(mktemp)"
|
|
trap 'rm -f "$OUTFILE"' EXIT
|
|
|
|
run_agg() { # RECORD...
|
|
aggregate_verdict "$@" >"$OUTFILE" 2>&1
|
|
}
|
|
|
|
# ── Scenario 0: build_verdict_record() field integrity ──────────────────────
|
|
# Direct invocation of build_verdict_record() (not aggregate_verdict()) --
|
|
# confirms the 9-field / VERDICT_FIELD_SEP contract itself, and specifically
|
|
# guards the bug-1581 regression class: a genuinely-empty field (e.g. an
|
|
# UNREACHABLE layer's empty observed value) must round-trip intact, not get
|
|
# collapsed/shifted by a naive whitespace-IFS split.
|
|
rec0="$(build_verdict_record pg 42 accepted 1 42.000000 1 42.000000 0 "")"
|
|
IFS="$VERDICT_FIELD_SEP" read -r f_name f_inj f_status f_ctrl_ok f_ctrl_obs f_sched_ok f_sched_obs f_api_ok f_api_obs <<< "$rec0"
|
|
check "scenario0: field 1 (resource name) round-trips" "pg" "$f_name"
|
|
check "scenario0: field 3 (injection status) round-trips" "accepted" "$f_status"
|
|
check "scenario0: field 8 (api_ok) round-trips as 0, not shifted" "0" "$f_api_ok"
|
|
check "scenario0: field 9 (empty api_obs) survives intact, not collapsed" "" "$f_api_obs"
|
|
|
|
# ── Scenario 0b: controller_values_match() -- the pure comparison primitive
|
|
# aggregate_verdict() itself calls for every layer comparison.
|
|
if controller_values_match "1.5" "1.500000"; then r=0; else r=1; fi
|
|
check "scenario0b: numeric tolerant match (1.5 vs controller's 1.500000 rendering)" "0" "$r"
|
|
if controller_values_match "1.5" "1.600000"; then r=0; else r=1; fi
|
|
check "scenario0b: numeric mismatch (1.5 vs 1.6) correctly fails" "1" "$r"
|
|
if controller_values_match "type" "type"; then r=0; else r=1; fi
|
|
check "scenario0b: non-numeric exact match" "0" "$r"
|
|
if controller_values_match "type" "model"; then r=0; else r=1; fi
|
|
check "scenario0b: non-numeric mismatch correctly fails" "1" "$r"
|
|
|
|
# ── Scenario 1: all-match batch -> overall PASS (positive control) ──────────
|
|
# Three resources, every layer matching its injected value. Proves the logic
|
|
# does not default to FAIL either.
|
|
records=(
|
|
"$(build_verdict_record mem 5 accepted 1 5.000000 1 5.000000 1 5.000000)"
|
|
"$(build_verdict_record r15m 1.5 accepted 1 1.500000 1 1.500000 1 1.500000)"
|
|
"$(build_verdict_record pg 42 accepted 1 42.000000 1 42.000000 1 42.000000)"
|
|
)
|
|
run_agg "${records[@]}"; rc=$?
|
|
out="$(cat "$OUTFILE")"
|
|
check "scenario1: exit code 0 (PASS)" "0" "$rc"
|
|
check "scenario1: AGG_RESULT=PASS" "PASS" "$AGG_RESULT"
|
|
check "scenario1: AGG_TOTAL_RESOURCES=3" "3" "$AGG_TOTAL_RESOURCES"
|
|
check "scenario1: AGG_MISMATCHES=0" "0" "$AGG_MISMATCHES"
|
|
check "scenario1: AGG_UNREACHABLE=0" "0" "$AGG_UNREACHABLE"
|
|
check "scenario1: AGG_REJECTED=0" "0" "$AGG_REJECTED"
|
|
check "scenario1: mem reported PASS" "PASS" "${AGG_RESOURCE_STATUS[mem]-<unset>}"
|
|
check "scenario1: r15m reported PASS" "PASS" "${AGG_RESOURCE_STATUS[r15m]-<unset>}"
|
|
check "scenario1: pg reported PASS" "PASS" "${AGG_RESOURCE_STATUS[pg]-<unset>}"
|
|
check "scenario1: zero FAILURE lines emitted" "0" "${#AGG_FAILURE_LINES[@]}"
|
|
check_contains "scenario1: VERDICT line reports result=PASS" "result=PASS" "$out"
|
|
|
|
# ── Scenario 2 (THE CORE PROOF, Requirement 6.3): one layer mismatched for
|
|
# ONE resource -> overall FAIL, correctly attributed to that specific
|
|
# resource+layer, while every OTHER resource/layer in the same batch is
|
|
# still PASS/MATCH.
|
|
# r15m's scheduler-observed value (9.999000) deliberately does not match its
|
|
# injected value (1.5); mem and pg match at every layer.
|
|
records=(
|
|
"$(build_verdict_record mem 5 accepted 1 5.000000 1 5.000000 1 5.000000)"
|
|
"$(build_verdict_record r15m 1.5 accepted 1 1.500000 1 9.999000 1 1.500000)"
|
|
"$(build_verdict_record pg 42 accepted 1 42.000000 1 42.000000 1 42.000000)"
|
|
)
|
|
run_agg "${records[@]}"; rc=$?
|
|
out="$(cat "$OUTFILE")"
|
|
failures="$(printf '%s\n' "${AGG_FAILURE_LINES[@]}")"
|
|
check "scenario2: exit code 1 (FAIL)" "1" "$rc"
|
|
check "scenario2: AGG_RESULT=FAIL" "FAIL" "$AGG_RESULT"
|
|
check "scenario2: AGG_MISMATCHES=1" "1" "$AGG_MISMATCHES"
|
|
check "scenario2: AGG_UNREACHABLE=0" "0" "$AGG_UNREACHABLE"
|
|
check "scenario2: AGG_REJECTED=0" "0" "$AGG_REJECTED"
|
|
check "scenario2: exactly one FAILURE line" "1" "${#AGG_FAILURE_LINES[@]}"
|
|
check "scenario2: r15m (mismatched resource) reported FAIL" "FAIL" "${AGG_RESOURCE_STATUS[r15m]-<unset>}"
|
|
check "scenario2: mem (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[mem]-<unset>}"
|
|
check "scenario2: pg (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[pg]-<unset>}"
|
|
check_contains "scenario2: failure names r15m+scheduler+MISMATCH specifically" \
|
|
"resource=r15m layer=scheduler status=MISMATCH" "$failures"
|
|
check_not_contains "scenario2: mem is NOT swept into any failure line" "resource=mem" "$failures"
|
|
check_not_contains "scenario2: pg is NOT swept into any failure line" "resource=pg" "$failures"
|
|
check_contains "scenario2: VERDICT line reports result=FAIL" "result=FAIL" "$out"
|
|
check_contains "scenario2: VERDICT line reports mismatches=1" "mismatches=1" "$out"
|
|
|
|
# ── Scenario 3: one layer UNREACHABLE (timeout) for ONE resource -> overall
|
|
# FAIL, classified distinctly from MISMATCH -- proves the logic
|
|
# distinguishes "timed out" from "wrong value" rather than collapsing
|
|
# both into one bucket. pg's apiserver layer never came back (api_ok=0,
|
|
# empty observed value); mem and r15m match everywhere.
|
|
records=(
|
|
"$(build_verdict_record mem 5 accepted 1 5.000000 1 5.000000 1 5.000000)"
|
|
"$(build_verdict_record r15m 1.5 accepted 1 1.500000 1 1.500000 1 1.500000)"
|
|
"$(build_verdict_record pg 42 accepted 1 42.000000 1 42.000000 0 "")"
|
|
)
|
|
run_agg "${records[@]}"; rc=$?
|
|
failures="$(printf '%s\n' "${AGG_FAILURE_LINES[@]}")"
|
|
check "scenario3: exit code 1 (FAIL)" "1" "$rc"
|
|
check "scenario3: AGG_RESULT=FAIL" "FAIL" "$AGG_RESULT"
|
|
check "scenario3: AGG_MISMATCHES=0 (not miscounted as a mismatch)" "0" "$AGG_MISMATCHES"
|
|
check "scenario3: AGG_UNREACHABLE=1" "1" "$AGG_UNREACHABLE"
|
|
check "scenario3: pg (unreachable resource) reported FAIL" "FAIL" "${AGG_RESOURCE_STATUS[pg]-<unset>}"
|
|
check "scenario3: mem (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[mem]-<unset>}"
|
|
check "scenario3: r15m (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[r15m]-<unset>}"
|
|
check_contains "scenario3: failure names pg+apiserver+UNREACHABLE specifically" \
|
|
"resource=pg layer=apiserver status=UNREACHABLE" "$failures"
|
|
check_not_contains "scenario3: mem is NOT swept into any failure line" "resource=mem" "$failures"
|
|
check_not_contains "scenario3: r15m is NOT swept into any failure line" "resource=r15m" "$failures"
|
|
|
|
# ── Scenario 4 (bonus): a rejected injection name -- distinct failure
|
|
# category from MISMATCH/UNREACHABLE, still isolated to its own resource.
|
|
records=(
|
|
"$(build_verdict_record mem 5 accepted 1 5.000000 1 5.000000 1 5.000000)"
|
|
"$(build_verdict_record badname 1 "rejected: unknown resource" 0 "" 0 "" 0 "")"
|
|
)
|
|
run_agg "${records[@]}"; rc=$?
|
|
failures="$(printf '%s\n' "${AGG_FAILURE_LINES[@]}")"
|
|
check "scenario4: exit code 1 (FAIL)" "1" "$rc"
|
|
check "scenario4: AGG_REJECTED=1" "1" "$AGG_REJECTED"
|
|
check "scenario4: AGG_MISMATCHES=0" "0" "$AGG_MISMATCHES"
|
|
check "scenario4: AGG_UNREACHABLE=0" "0" "$AGG_UNREACHABLE"
|
|
check "scenario4: badname reported FAIL" "FAIL" "${AGG_RESOURCE_STATUS[badname]-<unset>}"
|
|
check "scenario4: mem (unaffected resource) still reported PASS" "PASS" "${AGG_RESOURCE_STATUS[mem]-<unset>}"
|
|
check_contains "scenario4: failure names badname as REJECTED specifically" \
|
|
"resource=badname layer=injection status=REJECTED" "$failures"
|
|
check_not_contains "scenario4: mem is NOT swept into any failure line" "resource=mem" "$failures"
|
|
|
|
# ── Scenario 5 (bonus): zero records -> FAIL, not a false/default PASS.
|
|
# Documents/guards aggregate_verdict()'s own degenerate-input handling
|
|
# (AGG_TOTAL_RESOURCES=0 is explicitly classified FAIL, per its header
|
|
# comment) -- "nothing was checked" must never read as an overall PASS.
|
|
run_agg; rc=$?
|
|
check "scenario5: zero records -> exit code 1 (FAIL, not a default PASS)" "1" "$rc"
|
|
check "scenario5: zero records -> AGG_RESULT=FAIL" "FAIL" "$AGG_RESULT"
|
|
|
|
# ── Verdict ──────────────────────────────────────────────────────────────────
|
|
echo
|
|
if [ "$fails" -eq 0 ]; then
|
|
echo "SELFTEST PASS"
|
|
exit 0
|
|
fi
|
|
echo "SELFTEST FAIL ($fails failed)"
|
|
exit 1
|