From 2af22fb54f2caaab3b1225529a3185a4b25b40bc Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 20 Jun 2025 09:59:28 +0100 Subject: [PATCH 1/6] Add stale-screenshot-reporter.ts --- .../src/stale-screenshot-reporter.ts | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 packages/element-web-playwright-common/src/stale-screenshot-reporter.ts diff --git a/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts b/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts new file mode 100644 index 0000000000..7e2ab5b438 --- /dev/null +++ b/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts @@ -0,0 +1,88 @@ +/* +Copyright 2024 - 2025 New Vector Ltd. +Copyright 2024 The Matrix.org Foundation C.I.C. + +SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE files in the repository root for full details. +*/ + +/** + * Test reporter which compares the reported screenshots vs those on disk to find stale screenshots + * Only intended to run from within GitHub Actions + */ + +import { glob } from "glob"; +import { type Reporter, type TestCase } from "@playwright/test/reporter"; +import { type FullConfig } from "@playwright/test"; + +class StaleScreenshotReporter implements Reporter { + private readonly snapshotRoots = new Set(); + private readonly screenshots = new Set(); + private failing = false; + private success = true; + + public onBegin(config: FullConfig): void { + for (const project of config.projects) { + console.log("@@ found snapshotDir", project.snapshotDir); + this.snapshotRoots.add(project.snapshotDir); + } + } + + public onTestEnd(test: TestCase): void { + if (!test.ok()) { + this.failing = true; + } + for (const annotation of test.annotations) { + if (annotation.type === "_screenshot" && annotation.description) { + this.screenshots.add(annotation.description); + } + } + } + + private error(msg: string, file: string) { + if (process.env.GITHUB_ACTIONS) { + console.log(`::error file=${file}::${msg}`); + } + console.error(msg, file); + this.success = false; + } + + public async onExit(): Promise { + if (this.failing) return; + if (!this.snapshotRoots.size) { + this.error("No snapshot directories found, did you set the snapshotDir in your Playwright config?", ""); + return; + } + + const screenshotFiles = new Set(); + for (const snapshotRoot of this.snapshotRoots) { + const files = await glob(`**/*.png`, { cwd: snapshotRoot }); + for (const file of files) { + screenshotFiles.add(file); + } + } + + for (const screenshot of screenshotFiles) { + if (screenshot.split("-").at(-1) !== "linux.png") { + this.error( + "Found screenshot belonging to different platform, this should not be checked in", + screenshot, + ); + } + } + for (const screenshot of this.screenshots) { + screenshotFiles.delete(screenshot); + } + if (screenshotFiles.size > 0) { + for (const screenshot of screenshotFiles) { + this.error("Stale screenshot file", screenshot); + } + } + + if (!this.success) { + process.exit(1); + } + } +} + +export default StaleScreenshotReporter; From 683dfaef6947e2d06d09078b940c5fbdc05f807f Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 20 Jun 2025 10:03:32 +0100 Subject: [PATCH 2/6] Add entrypoint --- packages/element-web-playwright-common/package.json | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/element-web-playwright-common/package.json b/packages/element-web-playwright-common/package.json index 50e38f11cd..d4b0230d63 100644 --- a/packages/element-web-playwright-common/package.json +++ b/packages/element-web-playwright-common/package.json @@ -12,7 +12,16 @@ "engines": { "node": ">=20.0.0" }, - "main": "lib/index.js", + "exports": { + ".": { + "import": "./lib/index.js", + "types": "./lib/index.d.ts" + }, + "./stale-screenshots-reporter": { + "import": "./lib/stale-screenshots-reporter.js", + "types": "./lib/stale-screenshots-reporter.d.ts" + } + }, "bin": { "playwright-screenshots": "playwright-screenshots.sh" }, From a6f851bbdfccb878919e78cc7644381609213aca Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 20 Jun 2025 10:14:18 +0100 Subject: [PATCH 3/6] Iterate --- packages/element-web-playwright-common/package.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/element-web-playwright-common/package.json b/packages/element-web-playwright-common/package.json index d4b0230d63..8cca3237f1 100644 --- a/packages/element-web-playwright-common/package.json +++ b/packages/element-web-playwright-common/package.json @@ -15,11 +15,13 @@ "exports": { ".": { "import": "./lib/index.js", + "require": "./lib/index.js", "types": "./lib/index.d.ts" }, - "./stale-screenshots-reporter": { - "import": "./lib/stale-screenshots-reporter.js", - "types": "./lib/stale-screenshots-reporter.d.ts" + "./stale-screenshot-reporter": { + "import": "./lib/stale-screenshot-reporter.js", + "require": "./lib/stale-screenshot-reporter.js", + "types": "./lib/stale-screenshot-reporter.d.ts" } }, "bin": { From 78bc53ef0a499f27de49cf6cd50fe70f3b1e4b50 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 20 Jun 2025 10:21:26 +0100 Subject: [PATCH 4/6] Iterate --- .../element-web-playwright-common/src/expect/screenshot.ts | 3 +-- .../src/stale-screenshot-reporter.ts | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/element-web-playwright-common/src/expect/screenshot.ts b/packages/element-web-playwright-common/src/expect/screenshot.ts index 37726fd958..6a4fadf0dc 100644 --- a/packages/element-web-playwright-common/src/expect/screenshot.ts +++ b/packages/element-web-playwright-common/src/expect/screenshot.ts @@ -70,8 +70,7 @@ export const expect = baseExpect.extend({ testInfo.annotations.push({ // `_` prefix hides it from the HTML reporter type: "_screenshot", - // include a path relative to `playwright/snapshots/` - description: testInfo.snapshotPath(screenshotName).split("/playwright/snapshots/", 2)[1], + description: testInfo.snapshotPath(screenshotName), }); return { pass: true, message: (): string => "", name: "toMatchScreenshot" }; diff --git a/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts b/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts index 7e2ab5b438..3814daa322 100644 --- a/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts +++ b/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts @@ -12,6 +12,7 @@ Please see LICENSE files in the repository root for full details. */ import { glob } from "glob"; +import path from "node:path"; import { type Reporter, type TestCase } from "@playwright/test/reporter"; import { type FullConfig } from "@playwright/test"; @@ -58,7 +59,7 @@ class StaleScreenshotReporter implements Reporter { for (const snapshotRoot of this.snapshotRoots) { const files = await glob(`**/*.png`, { cwd: snapshotRoot }); for (const file of files) { - screenshotFiles.add(file); + screenshotFiles.add(path.join(snapshotRoot, file)); } } From 75cf1ee738d44e86d8bdcce8b136e6cae62de611 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 20 Jun 2025 10:28:04 +0100 Subject: [PATCH 5/6] Reuse annotation var --- .../src/expect/screenshot.ts | 5 +++-- .../src/stale-screenshot-reporter.ts | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/element-web-playwright-common/src/expect/screenshot.ts b/packages/element-web-playwright-common/src/expect/screenshot.ts index 6a4fadf0dc..86687b28e4 100644 --- a/packages/element-web-playwright-common/src/expect/screenshot.ts +++ b/packages/element-web-playwright-common/src/expect/screenshot.ts @@ -19,6 +19,8 @@ import { import { sanitizeForFilePath } from "playwright-core/lib/utils"; import { extname } from "node:path"; +import { ANNOTATION } from "../stale-screenshot-reporter.js"; + // Based on https://github.com/microsoft/playwright/blob/2b77ed4d7aafa85a600caa0b0d101b72c8437eeb/packages/playwright/src/util.ts#L206C8-L210C2 function sanitizeFilePathBeforeExtension(filePath: string): string { const ext = extname(filePath); @@ -68,8 +70,7 @@ export const expect = baseExpect.extend({ await style?.evaluate((tag) => tag.remove()); testInfo.annotations.push({ - // `_` prefix hides it from the HTML reporter - type: "_screenshot", + type: ANNOTATION, description: testInfo.snapshotPath(screenshotName), }); diff --git a/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts b/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts index 3814daa322..f2e0d39de7 100644 --- a/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts +++ b/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts @@ -16,6 +16,12 @@ import path from "node:path"; import { type Reporter, type TestCase } from "@playwright/test/reporter"; import { type FullConfig } from "@playwright/test"; +/** + * The annotation type used to mark screenshots in tests. + * `_` prefix hides it from the HTML reporter + */ +export const ANNOTATION = "_screenshot"; + class StaleScreenshotReporter implements Reporter { private readonly snapshotRoots = new Set(); private readonly screenshots = new Set(); @@ -34,7 +40,7 @@ class StaleScreenshotReporter implements Reporter { this.failing = true; } for (const annotation of test.annotations) { - if (annotation.type === "_screenshot" && annotation.description) { + if (annotation.type === ANNOTATION && annotation.description) { this.screenshots.add(annotation.description); } } From a440678ffd32aa7fb161ecb88f2decb17f06b3bf Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 20 Jun 2025 10:28:38 +0100 Subject: [PATCH 6/6] Remove console.log --- .../src/stale-screenshot-reporter.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts b/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts index f2e0d39de7..a6ac4f5967 100644 --- a/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts +++ b/packages/element-web-playwright-common/src/stale-screenshot-reporter.ts @@ -30,7 +30,6 @@ class StaleScreenshotReporter implements Reporter { public onBegin(config: FullConfig): void { for (const project of config.projects) { - console.log("@@ found snapshotDir", project.snapshotDir); this.snapshotRoots.add(project.snapshotDir); } }