diff --git a/scripts/audit-html.js b/scripts/audit-html.js index f7cbbe6..a31b665 100644 --- a/scripts/audit-html.js +++ b/scripts/audit-html.js @@ -161,7 +161,23 @@ const auditOrigin = async () => { const distinct = new Set(bodies.values()) if (distinct.size > 1) { - findings.push(`${entry.path}: crawlers were served different bytes (${bodies.size} agents, ${distinct.size} versions)`) + // Two different bodies can mean two different things, and only one of them + // is a problem worth chasing: the origin choosing what to serve by agent, + // or something in front of it varying every response. Cloudflare's email + // obfuscation does the second, with a token that changes each time. Ask + // the same agent twice before blaming the agents. + let varies = false + try { + const again = await fetchAs(url, agents[0]) + varies = again.body !== bodies.get(agents[0]) + } catch { + varies = false + } + findings.push( + varies + ? `${entry.path}: the response body changes between identical requests, so something in front of the origin is rewriting it (Cloudflare email obfuscation does this). Crawlers do not all receive the same page.` + : `${entry.path}: crawlers were served different bytes (${bodies.size} agents, ${distinct.size} versions)`, + ) } const body = bodies.values().next().value if (body) {