test(e2e): harden control census against count-driven label churn
Two robustness fixes after the census flaked under full parallel probe load (not a product regression — the failing diff was '# transaction' vs '# transactions', pure pluralization): - normalize collapses count-driven plurals so a data-count of 1 vs N no longer churns the snapshot (the census tracks controls, not the data in their labels) - settle waits for the nav shell + 3 stable count samples (was 2), and the project retries 2x — deterministic snapshots mean a real control change still fails every retry while a load-flake passes Re-baselined from a clean seed. Probe 33/33 stable across runs. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
e0bc82b229
commit
1f6073823b
|
|
@ -43,17 +43,32 @@ const CONTROL_SELECTOR = [
|
||||||
'[role="combobox"]',
|
'[role="combobox"]',
|
||||||
].join(', ');
|
].join(', ');
|
||||||
|
|
||||||
|
// Deterministic snapshots, but page-load timing flakes under the parallel probe
|
||||||
|
// load (16 workers, one dev server) — so retry: a real control change still
|
||||||
|
// fails every retry, a load-flake passes on the next.
|
||||||
|
test.describe.configure({ retries: 2 });
|
||||||
|
|
||||||
for (const [name, path] of PAGES) {
|
for (const [name, path] of PAGES) {
|
||||||
test(`control census: ${name}`, async ({ page }) => {
|
test(`control census: ${name}`, async ({ page }) => {
|
||||||
await page.goto(path);
|
await page.goto(path);
|
||||||
await page.waitForLoadState('domcontentloaded');
|
await page.waitForLoadState('domcontentloaded');
|
||||||
// Self-stabilizing settle: sample the control count until it is non-trivial
|
// Wait for the app shell (nav) to mount, then self-stabilize: sample the
|
||||||
// and unchanged across two consecutive samples (fixed timeouts race the
|
// control count until it is non-trivial and unchanged across THREE
|
||||||
// data queries; networkidle is flaky with background refetches).
|
// consecutive samples (two was enough in isolation but raced under load).
|
||||||
|
await page
|
||||||
|
.getByRole('navigation')
|
||||||
|
.first()
|
||||||
|
.waitFor({ timeout: 15_000 })
|
||||||
|
.catch(() => {});
|
||||||
let prev = -1;
|
let prev = -1;
|
||||||
for (let i = 0; i < 20; i++) {
|
let stable = 0;
|
||||||
|
for (let i = 0; i < 30; i++) {
|
||||||
const count = await page.locator(CONTROL_SELECTOR).count();
|
const count = await page.locator(CONTROL_SELECTOR).count();
|
||||||
if (count > 3 && count === prev) break;
|
if (count > 3 && count === prev) {
|
||||||
|
if (++stable >= 2) break; // 3 equal samples total
|
||||||
|
} else {
|
||||||
|
stable = 0;
|
||||||
|
}
|
||||||
prev = count;
|
prev = count;
|
||||||
await page.waitForTimeout(500);
|
await page.waitForTimeout(500);
|
||||||
}
|
}
|
||||||
|
|
@ -87,7 +102,11 @@ for (const [name, path] of PAGES) {
|
||||||
controls.map((c) =>
|
controls.map((c) =>
|
||||||
c
|
c
|
||||||
.replace(/\d[\d,.]*/g, '#') // dates, amounts, counts
|
.replace(/\d[\d,.]*/g, '#') // dates, amounts, counts
|
||||||
.replace(/#\s*(am|pm)/gi, '# $1'),
|
.replace(/#\s*(am|pm)/gi, '# $1')
|
||||||
|
// Collapse count-driven pluralization (# transaction vs # transactions)
|
||||||
|
// so a data-count of 1 vs N doesn't churn the snapshot — the census
|
||||||
|
// tracks *controls*, not the data rendered inside their labels.
|
||||||
|
.replace(/(# [a-z]+?)s\b/gi, '$1'),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -15,8 +15,8 @@ link | Bill Tracker
|
||||||
link | Calendar
|
link | Calendar
|
||||||
link | Release Notes
|
link | Release Notes
|
||||||
link | Skip to main content
|
link | Skip to main content
|
||||||
select | # months# months# months
|
select | # month# month# month
|
||||||
select | # months# months# months# months
|
select | # month# month# month# month
|
||||||
select | All billsAdobe Creative CloudAmazon PrimeAMC A-ListApple iCl
|
select | All billsAdobe Creative CloudAmazon PrimeAMC A-ListApple iCl
|
||||||
select | All categoriesBeautyCoffeeCredit CardsDining OutEntertainmen
|
select | All categoriesBeautyCoffeeCredit CardsDining OutEntertainmen
|
||||||
select | JanuaryFebruaryMarchAprilMayJuneJulyAugustSeptemberOctoberNo
|
select | JanuaryFebruaryMarchAprilMayJuneJulyAugustSeptemberOctoberNo
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
button | Add key
|
button | Add key
|
||||||
button | Change Password
|
button | Change Password
|
||||||
button | Current theme: Dark. Click to change.
|
button | Current theme: Dark. Click to change.
|
||||||
button | Last LoginJul # # #:# AMDesktop · Chrome on Linux#
|
button | Last LoginJul # # #:# PMDesktop · Chrome on Linux#
|
||||||
button | Open Tanstack query devtools
|
button | Open Tanstack query devtools
|
||||||
button | Open user menu
|
button | Open user menu
|
||||||
button | Save Privacy Settings
|
button | Save Privacy Settings
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue