mirrors/element-web
1
0
Fork 0
mirror of https://github.com/vector-im/element-web.git synced 2026-05-05 12:16:53 +02:00
Code Issues Projects Releases Wiki Activity

Fix some flaky playwright tests (#33085)

Browse Source 
* Tweak flaky test reporter to identify setup failures

* Fix some flaky playwright tests

* Iterate
This commit is contained in:
Michael Telatynski 2026-04-09 15:34:48 +01:00 committed by GitHub
parent a5e09ebb53
commit b6b0b0009c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 26 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/web/playwright/e2e/crypto/toasts.spec.ts
View File

@ -43,11 +43,7 @@ test.describe("Key storage out of sync toast", () => {
});
test("should prompt for recovery key if 'enter recovery key' pressed", { tag: "@screenshot" }, async ({ page }) => {
// We need to wait for there to be two toasts as the wait below won't work in isolation:
// playwright only evaluates the 'first()' call initially, not subsequent times it checks, so
// it would always be checking the same toast, even if another one is now the first.
await expect(page.getByRole("alert")).toHaveCount(2);
await expect(page.getByRole("alert").first()).toMatchScreenshot(
await expect(page.getByRole("alert").filter({ hasText: "Your key storage is out of sync." })).toMatchScreenshot(
"key-storage-out-of-sync-toast.png",
screenshotOptions,
);

2
apps/web/playwright/e2e/left-panel/room-list-panel/room-list.spec.ts
View File

@ -328,11 +328,11 @@ test.describe("Room list", () => {
const roomListView = getRoomList(page);
const videoRoom = roomListView.getByRole("option", { name: "video room" });
await expect(videoRoom).toHaveAttribute("aria-selected", "true"); // wait for room list update
// focus the user menu to avoid to have hover decoration
await page.getByRole("button", { name: "User menu" }).focus();
await expect(videoRoom).toBeVisible();
await expect(videoRoom).toMatchScreenshot("room-list-item-video.png");
});
});

6
apps/web/playwright/e2e/room-directory/room-directory.spec.ts
View File

@ -48,9 +48,9 @@ test.describe("Room Directory", () => {
await app.closeDialog();
const resp = await bot.publicRooms({});
expect(resp.total_room_count_estimate).toEqual(1);
expect(resp.chunk).toHaveLength(1);
expect(resp.chunk[0].room_id).toEqual(roomId);
expect(resp.total_room_count_estimate).toBeGreaterThanOrEqual(1);
expect(resp.chunk).toHaveLength(resp.total_room_count_estimate);
expect(resp.chunk.find((r) => r.room_id === roomId)).toBeTruthy();
},
);

23
packages/playwright-common/flaky-reporter.ts
View File

@ -24,6 +24,8 @@ type PaginationLinks = {
first?: string;
};
const ANSI_COLOUR_REGEX = /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
// We see quite a few test flakes which are caused by the app exploding
// so we have some magic strings we check the logs for to better track the flake with its cause
const SPECIAL_CASES: Record<string, string> = {
@ -38,18 +40,35 @@ class FlakyReporter implements Reporter {
public onTestEnd(test: TestCase): void {
// Ignores flakes on Dendrite and Pinecone as they have their own flakes we do not track
if (["Dendrite", "Pinecone"].includes(test.parent.project()!.name!)) return;
let failures = [`${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`];
if (test.outcome() === "flaky") {
const failures: string[] = [];
const timedOutRuns = test.results.filter((result) => result.status === "timedOut");
const pageLogs = timedOutRuns.flatMap((result) =>
result.attachments.filter((attachment) => attachment.name.startsWith("page-")),
);
// If a test failed due to a systemic fault then the test is not flaky, the app is, record it as such.
const specialCases = Object.keys(SPECIAL_CASES).filter((log) =>
pageLogs.some((attachment) => attachment.name.startsWith("page-") && attachment.body?.includes(log)),
);
if (specialCases.length > 0) {
failures = specialCases.map((specialCase) => SPECIAL_CASES[specialCase]);
failures.push(...specialCases.map((specialCase) => SPECIAL_CASES[specialCase]));
}
// Check for fixtures failing to set up
const errorMessages = timedOutRuns
.map((r) => r.error?.message?.replace(ANSI_COLOUR_REGEX, ""))
.filter(Boolean) as string[];
for (const error of errorMessages) {
if (error.startsWith("Fixture") && error.endsWith("exceeded during setup.")) {
failures.push(error);
}
}
if (failures.length < 1) {
failures.push(`${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`);
}
for (const title of failures) {