diff --git a/scripts/prove-guard.sh b/scripts/prove-guard.sh index ec0a3d4..aa188f5 100755 --- a/scripts/prove-guard.sh +++ b/scripts/prove-guard.sh @@ -32,6 +32,16 @@ # Everything after the third argument is the command that runs the guard, so any # runner works. `$PROVE_GUARD_CMD` is used when no command is given. # +# ## It edits the working tree, so mind what else reads it +# +# The mutation is written to the real file and restored by a `trap`. On a host +# where something else renders or deploys FROM this checkout on a schedule — +# Gridiron Central renders every fifteen minutes — a mutated template is briefly +# the one that would be published. Measured 2026-09-10: two agents ran proofs +# concurrently in one checkout and a render fell between them; nothing shipped, +# but only because the windows missed. Two proofs at once in one tree is not +# safe, and a long proof beside a short cron is a race worth knowing about. +# # ## Counting the failures # # "Exactly one" is a claim about test *cases*, and counting matching log lines @@ -88,9 +98,39 @@ fi BACKUP="$(mktemp)" cp "$FILE" "$BACKUP" + +# ── PURGE THE BYTECODE, OR THE RESTORE IS A LIE ───────────────────────────── +# +# **A SAME-LENGTH MUTATION SURVIVES THE RESTORE, IN `__pycache__`.** CPython +# decides a `.pyc` is still valid by comparing the source's mtime — stored as +# WHOLE SECONDS — and its size. Mutating `"auto"` to `"wiki"` changes neither: +# same four bytes, and both writes land in the same second. So the interpreter +# keeps serving the MUTATED bytecode after this script has faithfully put the +# original source back, and every later run in that second asserts against code +# that is not on disk. +# +# Found 2026-09-08, and it had already happened: `tests/run_all.py` reported +# `test_digest.py` failing on a guard that passed standalone minutes earlier, +# because `scripts/__pycache__/gridironlib.cpython-312.pyc` still held `wiki` +# while `scripts/gridironlib.py` said `auto`. That is the exact failure this +# whole script exists to prevent — a mutation left in place, looking green — +# arriving through the door nobody was watching. +# +# Purged BEFORE as well as after: a stale `.pyc` from an earlier run would +# otherwise make the mutation appear to do nothing, and the guard would be +# reported as failing to catch it. +purge_pycache() { + find "$(dirname "$FILE")" -name '__pycache__' -type d -prune -exec rm -rf {} + 2>/dev/null || true +} +purge_pycache + restore() { + # Idempotent, because the success path now restores explicitly and the trap + # still fires afterwards. + [ -f "$BACKUP" ] || return 0 cp "$BACKUP" "$FILE" rm -f "$BACKUP" + purge_pycache echo "prove-guard: restored $FILE" } trap restore EXIT INT TERM @@ -163,7 +203,53 @@ echo # fire was tried once and rejected; it was still reachable through the fallback. # The message compounded it, reporting "this runner printed no summary" about a # runner that printed one this script could not read. -COUNT="$(grep -oiE '[0-9]+ (tests? )?failed' "$LOG" | tail -1 | grep -oE '^[0-9]+' || true)" +# The third order, and it is this household's OWN harness: `tests/harness.py` +# prints `: 1 of 196 checks FAILED` and then one ` X ` line per failed +# check. Neither pattern above reads it — the first needs the number adjacent to +# the word, the second needs it on the right — so every run fell through to the +# approximate count, which matched the summary line AND the ✗ line and returned +# 2 for a single failure. **A correct guard reddening exactly one check +# therefore always exited 3** with "narrow the guard, or narrow the mutation", +# and told its reader "this runner printed no summary" about a runner that +# printed one. That is the same compounding false fire the comment above +# describes, arriving through a different door: the fallback is not +# conservative, and any project using this harness got the wrong verdict on +# every proof it ever ran. +# THE UNIT IS A TEST, NOT AN ASSERTION, and reading the check count was the +# third way this script got the verdict wrong on its own household's harness. +# GUARDS.md §1 says "if breaking the guard's target fails three TESTS, two of +# them are coincidental", and §7 wants "exactly one failing test". A test +# function is one claim with a docstring stating it; asserting that claim three +# ways is style. Counting checks called that three coincidental failures and +# told the author to narrow a guard that was already narrow — pressure to +# assert LESS about a claim, which is how a guard becomes decoration. +# +# Measured 2026-09-09: one mutation of one call site in a wiki renderer +# reddened four checks across two tests and was refused with "narrow the guard, +# or narrow the mutation". Both reds were the same claim. +# +# So `tests/harness.py` now states both numbers — `2 of 477 checks FAILED in 1 +# of 34 tests` — and the test figure is read first. The checks pattern stays +# beneath it for a harness that has not been updated: over-counting is the +# conservative direction, and a stale harness must not silently become +# permissive. +COUNT="$(grep -oiE 'in [0-9]+ of [0-9]+ tests?\b' "$LOG" | tail -1 | grep -oE '[0-9]+' | head -1 || true)" +if [ -z "$COUNT" ]; then + COUNT="$(grep -oiE '[0-9]+ of [0-9]+ checks? FAILED' "$LOG" | tail -1 | grep -oE '^[0-9]+' || true)" +fi +if [ -z "$COUNT" ]; then + # The OTHER shape this household prints. `cronwrap`'s own suite ends on + # `1 check(s) FAILED: ['name']` — no "of N", so the pattern above misses + # it and the run fell through to counting lines, which matches that + # summary AND the per-check line. A correct guard therefore came back + # "2 failures" and was told to narrow itself, while saying "this runner + # printed no summary" about a runner that printed one. Same false fire as + # the two above, third spelling. + COUNT="$(grep -oiE '[0-9]+ check\(s\)? FAILED' "$LOG" | tail -1 | grep -oE '^[0-9]+' || true)" +fi +if [ -z "$COUNT" ]; then + COUNT="$(grep -oiE '[0-9]+ (tests? )?failed' "$LOG" | tail -1 | grep -oE '^[0-9]+' || true)" +fi if [ -z "$COUNT" ]; then COUNT="$(grep -oiE '\bfail(ure)?s?[:= ]+[0-9]+' "$LOG" | tail -1 | grep -oE '[0-9]+$' || true)" fi @@ -182,4 +268,35 @@ if [ "$COUNT" -gt 1 ]; then exit 3 fi +# ── AND GREEN AGAIN ───────────────────────────────────────────────────────── +# +# **THIS SCRIPT NEVER CHECKED THE BASELINE, SO IT CERTIFIED GUARDS THAT DO NOT +# EXIST.** It mutated, ran the command once, and called any red a catch. A +# suite already red for an unrelated reason therefore produced +# "prove-guard: good — the guard caught it, and only it" for an assertion +# nobody had written. Found 2026-09-10 by an audit that hit exactly that state: +# the honest answer was exit 2 and the script said exit 0. +# +# GUARDS.md §1's own recipe has this step and the script had dropped it — +# the recipe ends `cp /tmp/thing.bak src/lib/thing.ts` then +# `npx vitest run ... # expect: green again`. Restoring and re-running proves +# two things at once: the red came FROM the mutation, and the restore actually +# worked. A mutation that leaves the file broken is otherwise reported as a +# successful proof. +# +# It costs a second run of the command. That is the price of the difference +# between a proof and a coincidence. +restore +echo +echo "prove-guard: re-running with $FILE restored — expecting green again" +if ! "${CMD[@]}" >"$LOG" 2>&1; then + echo >&2 + echo "prove-guard: NOTHING WAS PROVEN — the command is red with the file" >&2 + echo "restored, so the red above was not caused by the mutation." >&2 + echo >&2 + echo "Fix the unrelated failure first, then prove the guard. Log: $LOG" >&2 + tail -20 "$LOG" >&2 + exit 2 +fi + echo "prove-guard: good — the guard caught it, and only it ($COUNTED_BY)."