From 8705efec4035b8cebd03db6bf69f622f40b67a18 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 20 Jun 2025 13:27:40 +0100 Subject: [PATCH] Use stale-screenshot-reporter from playwright-common (#30175) * Use stale-screenshot-reporter from playwright-common Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --------- Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 2 +- package.json | 3 +- playwright/stale-screenshot-reporter.ts | 71 ------------------------- yarn.lock | 53 +++++++++++++++--- 4 files changed, 48 insertions(+), 81 deletions(-) delete mode 100644 playwright/stale-screenshot-reporter.ts diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index b2d49d7703..5901d7a700 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -227,7 +227,7 @@ jobs: - name: Merge into HTML Report if: inputs.skip != true - run: yarn playwright merge-reports --reporter=html,./playwright/flaky-reporter.ts,./playwright/stale-screenshot-reporter.ts ./all-blob-reports + run: yarn playwright merge-reports --reporter=html,./playwright/flaky-reporter.ts,@element-hq/element-web-playwright-common/lib/stale-screenshot-reporter.js ./all-blob-reports env: # Only pass creds to the flaky-reporter on main branch runs GITHUB_TOKEN: ${{ github.ref_name == 'develop' && secrets.ELEMENT_BOT_TOKEN || '' }} diff --git a/package.json b/package.json index 36c501e13d..fe08074cae 100644 --- a/package.json +++ b/package.json @@ -181,7 +181,7 @@ "@babel/runtime": "^7.12.5", "@casualbot/jest-sonar-reporter": "2.2.7", "@element-hq/element-call-embedded": "0.12.2", - "@element-hq/element-web-playwright-common": "^1.1.5", + "@element-hq/element-web-playwright-common": "^1.4.2", "@peculiar/webcrypto": "^1.4.3", "@playwright/test": "^1.50.1", "@principalstudio/html-webpack-inject-preload": "^1.2.7", @@ -252,7 +252,6 @@ "fetch-mock": "9.11.0", "fetch-mock-jest": "^1.5.1", "file-loader": "^6.0.0", - "glob": "^11.0.0", "html-webpack-plugin": "^5.5.3", "husky": "^9.0.0", "jest": "^29.6.2", diff --git a/playwright/stale-screenshot-reporter.ts b/playwright/stale-screenshot-reporter.ts deleted file mode 100644 index 36aba56a07..0000000000 --- a/playwright/stale-screenshot-reporter.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2024 New Vector Ltd. -Copyright 2024 The Matrix.org Foundation C.I.C. - -SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE files in the repository root for full details. -*/ - -/** - * Test reporter which compares the reported screenshots vs those on disk to find stale screenshots - * Only intended to run from within GitHub Actions - */ - -import path from "node:path"; -import { glob } from "glob"; - -import type { Reporter, TestCase } from "@playwright/test/reporter"; - -const snapshotRoot = path.join(__dirname, "snapshots"); - -class StaleScreenshotReporter implements Reporter { - private screenshots = new Set(); - private failing = false; - private success = true; - - public onTestEnd(test: TestCase): void { - if (!test.ok()) { - this.failing = true; - } - for (const annotation of test.annotations) { - if (annotation.type === "_screenshot") { - this.screenshots.add(annotation.description); - } - } - } - - private error(msg: string, file: string) { - if (process.env.GITHUB_ACTIONS) { - console.log(`::error file=${file}::${msg}`); - } - console.error(msg, file); - this.success = false; - } - - public async onExit(): Promise { - if (this.failing) return; - const screenshotFiles = new Set(await glob(`**/*.png`, { cwd: snapshotRoot })); - for (const screenshot of screenshotFiles) { - if (screenshot.split("-").at(-1) !== "linux.png") { - this.error( - "Found screenshot belonging to different platform, this should not be checked in", - screenshot, - ); - } - } - for (const screenshot of this.screenshots) { - screenshotFiles.delete(screenshot); - } - if (screenshotFiles.size > 0) { - for (const screenshot of screenshotFiles) { - this.error("Stale screenshot file", screenshot); - } - } - - if (!this.success) { - process.exit(1); - } - } -} - -export default StaleScreenshotReporter; diff --git a/yarn.lock b/yarn.lock index 807587985b..c8d095309b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1677,13 +1677,14 @@ resolved "https://registry.yarnpkg.com/@element-hq/element-web-module-api/-/element-web-module-api-1.0.0.tgz#df09108b0346a44ad2898c603d1a6cda5f50d80b" integrity sha512-FYId5tYgaKvpqAXRXqs0pY4+7/A09bEl1mCxFqlS9jlZOCjlMZVvZuv8spbY8ZN9HaMvuVmx9J00Fn2gCJd0TQ== -"@element-hq/element-web-playwright-common@^1.1.5": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@element-hq/element-web-playwright-common/-/element-web-playwright-common-1.3.0.tgz#5f57eb2ee25e9733c92185ef57431db808deb603" - integrity sha512-tq3lN77f0KnTYtPVkRrwvsWpq/JAC6JqDkC2f6dxuvX3a+OmyjGnuoZh83kB2WNObfrrx7gxmsZpS7ov2mrw4Q== +"@element-hq/element-web-playwright-common@^1.4.2": + version "1.4.2" + resolved "https://registry.yarnpkg.com/@element-hq/element-web-playwright-common/-/element-web-playwright-common-1.4.2.tgz#3c05673ddbdc042cb34fa1063e0b8651a74c3f0e" + integrity sha512-p/P0J1TWlfZgJjUCXYWOL6HqpEg63Rm6sKQjBm/9EJD1xWqJY+xMrZu97cuXSKL60NOGF6vpBhW/gdNQT/Nz1w== dependencies: "@axe-core/playwright" "^4.10.1" "@testcontainers/postgresql" "^11.0.0" + glob "^11.0.3" lodash-es "^4.17.21" mailpit-api "^1.2.0" strip-ansi "^7.1.0" @@ -1879,6 +1880,18 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz#4a2868d75d6d6963e423bcf90b7fd1be343409d3" integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA== +"@isaacs/balanced-match@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz#3081dadbc3460661b751e7591d7faea5df39dd29" + integrity sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ== + +"@isaacs/brace-expansion@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz#4b3dabab7d8e75a429414a96bd67bf4c1d13e0f3" + integrity sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA== + dependencies: + "@isaacs/balanced-match" "^4.0.1" + "@isaacs/cliui@^8.0.2": version "8.0.2" resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" @@ -3846,7 +3859,7 @@ classnames "^2.5.1" vaul "^1.0.0" -"@vector-im/matrix-wysiwyg-wasm@link:../../../.cache/yarn/v6/npm-@vector-im-matrix-wysiwyg-2.38.3-cc54d8b3e9472bcd8e622126ba364ee31952cd8a-integrity/node_modules/bindings/wysiwyg-wasm": +"@vector-im/matrix-wysiwyg-wasm@link:../../Library/Caches/Yarn/v6/npm-@vector-im-matrix-wysiwyg-2.38.3-cc54d8b3e9472bcd8e622126ba364ee31952cd8a-integrity/node_modules/bindings/wysiwyg-wasm": version "0.0.0" uid "" @@ -3855,7 +3868,7 @@ resolved "https://registry.yarnpkg.com/@vector-im/matrix-wysiwyg/-/matrix-wysiwyg-2.38.3.tgz#cc54d8b3e9472bcd8e622126ba364ee31952cd8a" integrity sha512-fqo8P55Vc/t0vxpFar9RDJN5gKEjJmzrLo+O4piDbFda6VrRoqrWAtiu0Au0g6B4hRDPKIuFupk8v9Ja7q8Hvg== dependencies: - "@vector-im/matrix-wysiwyg-wasm" "link:../../../.cache/yarn/v6/npm-@vector-im-matrix-wysiwyg-2.38.3-cc54d8b3e9472bcd8e622126ba364ee31952cd8a-integrity/node_modules/bindings/wysiwyg-wasm" + "@vector-im/matrix-wysiwyg-wasm" "link:../../Library/Caches/Yarn/v6/npm-@vector-im-matrix-wysiwyg-2.38.3-cc54d8b3e9472bcd8e622126ba364ee31952cd8a-integrity/node_modules/bindings/wysiwyg-wasm" "@webassemblyjs/ast@1.14.1", "@webassemblyjs/ast@^1.14.1": version "1.14.1" @@ -6997,7 +7010,7 @@ foreachasync@^3.0.0: resolved "https://registry.yarnpkg.com/foreachasync/-/foreachasync-3.0.0.tgz#5502987dc8714be3392097f32e0071c9dee07cf6" integrity sha512-J+ler7Ta54FwwNcx6wQRDhTIbNeyDcARMkOcguEqnEdtm0jKvN3Li3PDAb2Du3ubJYEWfYL83XMROXdsXAXycw== -foreground-child@^3.1.0: +foreground-child@^3.1.0, foreground-child@^3.3.1: version "3.3.1" resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.1.tgz#32e8e9ed1b68a3497befb9ac2b6adf92a638576f" integrity sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw== @@ -7242,6 +7255,18 @@ glob@^11.0.0: package-json-from-dist "^1.0.0" path-scurry "^2.0.0" +glob@^11.0.3: + version "11.0.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-11.0.3.tgz#9d8087e6d72ddb3c4707b1d2778f80ea3eaefcd6" + integrity sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA== + dependencies: + foreground-child "^3.3.1" + jackspeak "^4.1.1" + minimatch "^10.0.3" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^2.0.0" + glob@^7.1.3, glob@^7.1.4: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" @@ -8222,6 +8247,13 @@ jackspeak@^4.0.1: dependencies: "@isaacs/cliui" "^8.0.2" +jackspeak@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-4.1.1.tgz#96876030f450502047fc7e8c7fcf8ce8124e43ae" + integrity sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ== + dependencies: + "@isaacs/cliui" "^8.0.2" + jake@^10.8.5: version "10.9.2" resolved "https://registry.yarnpkg.com/jake/-/jake-10.9.2.tgz#6ae487e6a69afec3a5e167628996b59f35ae2b7f" @@ -9420,6 +9452,13 @@ minimatch@^10.0.0: dependencies: brace-expansion "^2.0.1" +minimatch@^10.0.3: + version "10.0.3" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.0.3.tgz#cf7a0314a16c4d9ab73a7730a0e8e3c3502d47aa" + integrity sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw== + dependencies: + "@isaacs/brace-expansion" "^5.0.0" + minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"