From 1f6073823be932c2c8d249a19c771deea580bb4d Mon Sep 17 00:00:00 2001 From: null Date: Sat, 11 Jul 2026 07:14:05 -0500 Subject: [PATCH] test(e2e): harden control census against count-driven label churn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two robustness fixes after the census flaked under full parallel probe load (not a product regression — the failing diff was '# transaction' vs '# transactions', pure pluralization): - normalize collapses count-driven plurals so a data-count of 1 vs N no longer churns the snapshot (the census tracks controls, not the data in their labels) - settle waits for the nav shell + 3 stable count samples (was 2), and the project retries 2x — deterministic snapshots mean a real control change still fails every retry while a load-flake passes Re-baselined from a clean seed. Probe 33/33 stable across runs. Co-Authored-By: Claude Fable 5 --- e2e/control-census.spec.js | 31 +++++++++++++++---- .../census-analytics-census-linux.txt | 4 +-- .../census-profile-census-linux.txt | 2 +- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/e2e/control-census.spec.js b/e2e/control-census.spec.js index 61bf83e..78e439b 100644 --- a/e2e/control-census.spec.js +++ b/e2e/control-census.spec.js @@ -43,17 +43,32 @@ const CONTROL_SELECTOR = [ '[role="combobox"]', ].join(', '); +// Deterministic snapshots, but page-load timing flakes under the parallel probe +// load (16 workers, one dev server) — so retry: a real control change still +// fails every retry, a load-flake passes on the next. +test.describe.configure({ retries: 2 }); + for (const [name, path] of PAGES) { test(`control census: ${name}`, async ({ page }) => { await page.goto(path); await page.waitForLoadState('domcontentloaded'); - // Self-stabilizing settle: sample the control count until it is non-trivial - // and unchanged across two consecutive samples (fixed timeouts race the - // data queries; networkidle is flaky with background refetches). + // Wait for the app shell (nav) to mount, then self-stabilize: sample the + // control count until it is non-trivial and unchanged across THREE + // consecutive samples (two was enough in isolation but raced under load). + await page + .getByRole('navigation') + .first() + .waitFor({ timeout: 15_000 }) + .catch(() => {}); let prev = -1; - for (let i = 0; i < 20; i++) { + let stable = 0; + for (let i = 0; i < 30; i++) { const count = await page.locator(CONTROL_SELECTOR).count(); - if (count > 3 && count === prev) break; + if (count > 3 && count === prev) { + if (++stable >= 2) break; // 3 equal samples total + } else { + stable = 0; + } prev = count; await page.waitForTimeout(500); } @@ -87,7 +102,11 @@ for (const [name, path] of PAGES) { controls.map((c) => c .replace(/\d[\d,.]*/g, '#') // dates, amounts, counts - .replace(/#\s*(am|pm)/gi, '# $1'), + .replace(/#\s*(am|pm)/gi, '# $1') + // Collapse count-driven pluralization (# transaction vs # transactions) + // so a data-count of 1 vs N doesn't churn the snapshot — the census + // tracks *controls*, not the data rendered inside their labels. + .replace(/(# [a-z]+?)s\b/gi, '$1'), ), ), ] diff --git a/e2e/control-census.spec.js-snapshots/census-analytics-census-linux.txt b/e2e/control-census.spec.js-snapshots/census-analytics-census-linux.txt index 4dca16c..ce816e6 100644 --- a/e2e/control-census.spec.js-snapshots/census-analytics-census-linux.txt +++ b/e2e/control-census.spec.js-snapshots/census-analytics-census-linux.txt @@ -15,8 +15,8 @@ link | Bill Tracker link | Calendar link | Release Notes link | Skip to main content -select | # months# months# months -select | # months# months# months# months +select | # month# month# month +select | # month# month# month# month select | All billsAdobe Creative CloudAmazon PrimeAMC A-ListApple iCl select | All categoriesBeautyCoffeeCredit CardsDining OutEntertainmen select | JanuaryFebruaryMarchAprilMayJuneJulyAugustSeptemberOctoberNo diff --git a/e2e/control-census.spec.js-snapshots/census-profile-census-linux.txt b/e2e/control-census.spec.js-snapshots/census-profile-census-linux.txt index 6b43b33..92803d4 100644 --- a/e2e/control-census.spec.js-snapshots/census-profile-census-linux.txt +++ b/e2e/control-census.spec.js-snapshots/census-profile-census-linux.txt @@ -1,7 +1,7 @@ button | Add key button | Change Password button | Current theme: Dark. Click to change. -button | Last LoginJul # # #:# AMDesktop · Chrome on Linux# +button | Last LoginJul # # #:# PMDesktop · Chrome on Linux# button | Open Tanstack query devtools button | Open user menu button | Save Privacy Settings