Merge pull request #36 from element-hq/t3chguy/stale-screenshots-reporter

packages/element-web-playwright-common/package.json

@@ -12,7 +12,18 @@
     "engines": {
         "node": ">=20.0.0"
     },
-    "main": "lib/index.js",
+    "exports": {
+        ".": {
+            "import": "./lib/index.js",
+            "require": "./lib/index.js",
+            "types": "./lib/index.d.ts"
+        },
+        "./stale-screenshot-reporter": {
+            "import": "./lib/stale-screenshot-reporter.js",
+            "require": "./lib/stale-screenshot-reporter.js",
+            "types": "./lib/stale-screenshot-reporter.d.ts"
+        }
+    },
     "bin": {
         "playwright-screenshots": "playwright-screenshots.sh"
     },

packages/element-web-playwright-common/src/expect/screenshot.ts

@@ -19,6 +19,8 @@ import {
 import { sanitizeForFilePath } from "playwright-core/lib/utils";
 import { extname } from "node:path";
 
+import { ANNOTATION } from "../stale-screenshot-reporter.js";
+
 // Based on https://github.com/microsoft/playwright/blob/2b77ed4d7aafa85a600caa0b0d101b72c8437eeb/packages/playwright/src/util.ts#L206C8-L210C2
 function sanitizeFilePathBeforeExtension(filePath: string): string {
     const ext = extname(filePath);
@@ -68,10 +70,8 @@ export const expect = baseExpect.extend<Expectations>({
         await style?.evaluate((tag) => tag.remove());
 
         testInfo.annotations.push({
-            // `_` prefix hides it from the HTML reporter
-            type: "_screenshot",
-            // include a path relative to `playwright/snapshots/`
-            description: testInfo.snapshotPath(screenshotName).split("/playwright/snapshots/", 2)[1],
+            type: ANNOTATION,
+            description: testInfo.snapshotPath(screenshotName),
         });
 
         return { pass: true, message: (): string => "", name: "toMatchScreenshot" };

packages/element-web-playwright-common/src/stale-screenshot-reporter.ts

@@ -0,0 +1,94 @@
+/*
+Copyright 2024 - 2025 New Vector Ltd.
+Copyright 2024 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+/**
+ * Test reporter which compares the reported screenshots vs those on disk to find stale screenshots
+ * Only intended to run from within GitHub Actions
+ */
+
+import { glob } from "glob";
+import path from "node:path";
+import { type Reporter, type TestCase } from "@playwright/test/reporter";
+import { type FullConfig } from "@playwright/test";
+
+/**
+ * The annotation type used to mark screenshots in tests.
+ * `_` prefix hides it from the HTML reporter
+ */
+export const ANNOTATION = "_screenshot";
+
+class StaleScreenshotReporter implements Reporter {
+    private readonly snapshotRoots = new Set<string>();
+    private readonly screenshots = new Set<string>();
+    private failing = false;
+    private success = true;
+
+    public onBegin(config: FullConfig): void {
+        for (const project of config.projects) {
+            this.snapshotRoots.add(project.snapshotDir);
+        }
+    }
+
+    public onTestEnd(test: TestCase): void {
+        if (!test.ok()) {
+            this.failing = true;
+        }
+        for (const annotation of test.annotations) {
+            if (annotation.type === ANNOTATION && annotation.description) {
+                this.screenshots.add(annotation.description);
+            }
+        }
+    }
+
+    private error(msg: string, file: string) {
+        if (process.env.GITHUB_ACTIONS) {
+            console.log(`::error file=${file}::${msg}`);
+        }
+        console.error(msg, file);
+        this.success = false;
+    }
+
+    public async onExit(): Promise<void> {
+        if (this.failing) return;
+        if (!this.snapshotRoots.size) {
+            this.error("No snapshot directories found, did you set the snapshotDir in your Playwright config?", "");
+            return;
+        }
+
+        const screenshotFiles = new Set<string>();
+        for (const snapshotRoot of this.snapshotRoots) {
+            const files = await glob(`**/*.png`, { cwd: snapshotRoot });
+            for (const file of files) {
+                screenshotFiles.add(path.join(snapshotRoot, file));
+            }
+        }
+
+        for (const screenshot of screenshotFiles) {
+            if (screenshot.split("-").at(-1) !== "linux.png") {
+                this.error(
+                    "Found screenshot belonging to different platform, this should not be checked in",
+                    screenshot,
+                );
+            }
+        }
+        for (const screenshot of this.screenshots) {
+            screenshotFiles.delete(screenshot);
+        }
+        if (screenshotFiles.size > 0) {
+            for (const screenshot of screenshotFiles) {
+                this.error("Stale screenshot file", screenshot);
+            }
+        }
+
+        if (!this.success) {
+            process.exit(1);
+        }
+    }
+}
+
+export default StaleScreenshotReporter;