From 7cd556c4b5477ceccdc9d1b60b57f96f51d0dcdc Mon Sep 17 00:00:00 2001 From: null Date: Thu, 10 Sep 2026 04:53:51 -0500 Subject: [PATCH] fix(qa): the browser check measured 5 of 18 pages and heard no errors Two gaps in the tool that exists to find defects. Its default path list was five pages typed by hand: home, about, services, contact and support. That is 5 of the 18 this site serves, and it included no service and no industry page, so a defect on any of them was invisible to it. The default is now whatever the target's own sitemap says it serves, and a sitemap it cannot read exits 2 rather than quietly checking nothing. It also listened for nothing the page said. A page that throws still paints, so every measurement it takes can look healthy while the page is broken: React reported a hydration mismatch on every page of this site for months and this tool never noticed, because nothing here was listening. It now captures page errors and console errors and reports them as findings. Proven: a console.error added to the About page produced "/about @390: the page logged an error", and a clean build reports nothing across all 18 sitemap pages. docs/qa/ClaudeQAPlan.md said "five pages, five widths" and it is now every page in the sitemap, at four widths. Closes #227. Co-Authored-By: Claude Opus 5 (1M context) --- docs/qa/ClaudeQAPlan.md | 2 +- scripts/qa-browser.mjs | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/qa/ClaudeQAPlan.md b/docs/qa/ClaudeQAPlan.md index 4005abb..ccbb5e0 100644 --- a/docs/qa/ClaudeQAPlan.md +++ b/docs/qa/ClaudeQAPlan.md @@ -82,7 +82,7 @@ confidence about *how much*, without measuring. So, before filing a UI defect and before acting on one: ```bash -node scripts/qa-browser.mjs # production, five pages, five widths +node scripts/qa-browser.mjs # production, every page in its sitemap, four widths node scripts/qa-browser.mjs --url http://localhost:3099 --viewports 320,768 ``` diff --git a/scripts/qa-browser.mjs b/scripts/qa-browser.mjs index 0e424ea..4e46978 100755 --- a/scripts/qa-browser.mjs +++ b/scripts/qa-browser.mjs @@ -51,6 +51,7 @@ import { createRequire } from 'node:module' import { execSync } from 'node:child_process' import path from 'node:path' +import { parseSitemap } from './lib/html-audit.js' const args = process.argv.slice(2) const opt = (name, dflt) => { @@ -70,9 +71,30 @@ const collect = (name, dflt) => { for (let j = i + 1; j < args.length && !args[j].startsWith('--'); j++) out.push(args[j]) return out.length ? out : dflt } -const PATHS = collect('paths', ['/', '/about', '/services', '/contact', '/support']) const SHOTS = opt('shots', null) +// The default path list used to be five hand-typed pages: the home page, about, +// services, contact and support. That is 5 of the 18 this site serves, and it +// included none of the service or industry pages, so a defect on any of them was +// invisible to the tool that exists to find defects. The default is now whatever +// the target itself says it serves. +// +// Failing to read the sitemap exits 2. "I checked nothing" must never look like +// "I found nothing". +let PATHS = collect('paths', null) +if (!PATHS) { + try { + const response = await fetch(`${URL_BASE}/sitemap.xml`, { signal: AbortSignal.timeout(20000) }) + if (!response.ok) throw new Error(`HTTP ${response.status}`) + PATHS = parseSitemap(await response.text()).map((entry) => entry.path) + if (!PATHS.length) throw new Error('it lists no pages') + } catch (e) { + console.error(`qa-browser: could not read ${URL_BASE}/sitemap.xml (${e.message}), so NOTHING was checked.`) + console.error(' Pass --paths to check specific pages instead.') + process.exit(2) + } +} + // playwright is global here. Resolve it explicitly rather than failing with a // bare MODULE_NOT_FOUND, which reads as "the script is broken" rather than // "install this". @@ -115,6 +137,14 @@ for (const p of PATHS) { if (r.request().resourceType() === 'image' && r.status() >= 400) httpFailed.push(`${r.status()} ${r.url()}`) }) + // A page that throws still paints, so every measurement below can look + // healthy while the page is broken. React reported a hydration mismatch on + // every page of this site for months and nothing here noticed, because + // nothing here was listening. + const consoleErrors = [] + page.on('pageerror', e => consoleErrors.push(String(e).split('\n')[0])) + page.on('console', msg => { if (msg.type() === 'error') consoleErrors.push(msg.text().split('\n')[0]) }) + let resp try { resp = await page.goto(URL_BASE + p, { waitUntil: 'networkidle', timeout: 45000 }) @@ -179,6 +209,7 @@ for (const p of PATHS) { if (m.cls > 0.1) { bits.push(`CLS ${m.cls}`); findings.push(`${p} @${width}: CLS ${m.cls} (>0.1)`) } if (m.lcp > 2500) { bits.push(`LCP ${m.lcp}ms`); findings.push(`${p} @${width}: LCP ${m.lcp}ms (>2500)`) } if (m.over.length) { bits.push(`OVERFLOW ${m.over.length}`); findings.push(`${p} @${width}: past the right edge — ${m.over.join('; ')}`) } + if (consoleErrors.length) { bits.push(`JS-ERROR ${consoleErrors.length}`); findings.push(`${p} @${width}: the page logged an error — ${[...new Set(consoleErrors)].slice(0, 2).join(' | ')}`) } say(` ${p.padEnd(12)} @${String(width).padEnd(5)} imgs=${String(m.images).padEnd(3)} cls=${String(m.cls).padEnd(7)} lcp=${String(m.lcp + 'ms').padEnd(8)} ${bits.length ? '‼ ' + bits.join(' ') : 'ok'}`) @@ -196,6 +227,6 @@ if (findings.length) { process.exit(1) } say(`qa-browser: nothing found across ${PATHS.length} path(s) x ${VIEWPORTS.length} viewport(s).`) -say(' That is not "the UI is correct" — it is these five measurements,') +say(' That is not "the UI is correct" — it is these six measurements,') say(' on these pages, at these widths. Nothing here opens a menu or') say(' uses a keyboard.')