diff --git a/apps/web/playwright/e2e/crypto/toasts.spec.ts b/apps/web/playwright/e2e/crypto/toasts.spec.ts
index 876000009b..9ce1b8a5ae 100644
--- a/apps/web/playwright/e2e/crypto/toasts.spec.ts
+++ b/apps/web/playwright/e2e/crypto/toasts.spec.ts
@@ -43,11 +43,7 @@ test.describe("Key storage out of sync toast", () => {
     });
 
     test("should prompt for recovery key if 'enter recovery key' pressed", { tag: "@screenshot" }, async ({ page }) => {
-        // We need to wait for there to be two toasts as the wait below won't work in isolation:
-        // playwright only evaluates the 'first()' call initially, not subsequent times it checks, so
-        // it would always be checking the same toast, even if another one is now the first.
-        await expect(page.getByRole("alert")).toHaveCount(2);
-        await expect(page.getByRole("alert").first()).toMatchScreenshot(
+        await expect(page.getByRole("alert").filter({ hasText: "Your key storage is out of sync." })).toMatchScreenshot(
             "key-storage-out-of-sync-toast.png",
             screenshotOptions,
         );
diff --git a/apps/web/playwright/e2e/left-panel/room-list-panel/room-list.spec.ts b/apps/web/playwright/e2e/left-panel/room-list-panel/room-list.spec.ts
index 5907ad6d97..79cc0b4cf3 100644
--- a/apps/web/playwright/e2e/left-panel/room-list-panel/room-list.spec.ts
+++ b/apps/web/playwright/e2e/left-panel/room-list-panel/room-list.spec.ts
@@ -328,11 +328,11 @@ test.describe("Room list", () => {
 
             const roomListView = getRoomList(page);
             const videoRoom = roomListView.getByRole("option", { name: "video room" });
+            await expect(videoRoom).toHaveAttribute("aria-selected", "true"); // wait for room list update
 
             // focus the user menu to avoid to have hover decoration
             await page.getByRole("button", { name: "User menu" }).focus();
 
-            await expect(videoRoom).toBeVisible();
             await expect(videoRoom).toMatchScreenshot("room-list-item-video.png");
         });
     });
diff --git a/apps/web/playwright/e2e/room-directory/room-directory.spec.ts b/apps/web/playwright/e2e/room-directory/room-directory.spec.ts
index 741fde3505..6eea5abce7 100644
--- a/apps/web/playwright/e2e/room-directory/room-directory.spec.ts
+++ b/apps/web/playwright/e2e/room-directory/room-directory.spec.ts
@@ -48,9 +48,9 @@ test.describe("Room Directory", () => {
             await app.closeDialog();
 
             const resp = await bot.publicRooms({});
-            expect(resp.total_room_count_estimate).toEqual(1);
-            expect(resp.chunk).toHaveLength(1);
-            expect(resp.chunk[0].room_id).toEqual(roomId);
+            expect(resp.total_room_count_estimate).toBeGreaterThanOrEqual(1);
+            expect(resp.chunk).toHaveLength(resp.total_room_count_estimate);
+            expect(resp.chunk.find((r) => r.room_id === roomId)).toBeTruthy();
         },
     );
 
diff --git a/packages/playwright-common/flaky-reporter.ts b/packages/playwright-common/flaky-reporter.ts
index 520c2553a2..cebfee7702 100644
--- a/packages/playwright-common/flaky-reporter.ts
+++ b/packages/playwright-common/flaky-reporter.ts
@@ -24,6 +24,8 @@ type PaginationLinks = {
     first?: string;
 };
 
+const ANSI_COLOUR_REGEX = /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
+
 // We see quite a few test flakes which are caused by the app exploding
 // so we have some magic strings we check the logs for to better track the flake with its cause
 const SPECIAL_CASES: Record<string, string> = {
@@ -38,18 +40,35 @@ class FlakyReporter implements Reporter {
     public onTestEnd(test: TestCase): void {
         // Ignores flakes on Dendrite and Pinecone as they have their own flakes we do not track
         if (["Dendrite", "Pinecone"].includes(test.parent.project()!.name!)) return;
-        let failures = [`${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`];
+
         if (test.outcome() === "flaky") {
+            const failures: string[] = [];
+
             const timedOutRuns = test.results.filter((result) => result.status === "timedOut");
             const pageLogs = timedOutRuns.flatMap((result) =>
                 result.attachments.filter((attachment) => attachment.name.startsWith("page-")),
             );
+
             // If a test failed due to a systemic fault then the test is not flaky, the app is, record it as such.
             const specialCases = Object.keys(SPECIAL_CASES).filter((log) =>
                 pageLogs.some((attachment) => attachment.name.startsWith("page-") && attachment.body?.includes(log)),
             );
             if (specialCases.length > 0) {
-                failures = specialCases.map((specialCase) => SPECIAL_CASES[specialCase]);
+                failures.push(...specialCases.map((specialCase) => SPECIAL_CASES[specialCase]));
+            }
+
+            // Check for fixtures failing to set up
+            const errorMessages = timedOutRuns
+                .map((r) => r.error?.message?.replace(ANSI_COLOUR_REGEX, ""))
+                .filter(Boolean) as string[];
+            for (const error of errorMessages) {
+                if (error.startsWith("Fixture") && error.endsWith("exceeded during setup.")) {
+                    failures.push(error);
+                }
+            }
+
+            if (failures.length < 1) {
+                failures.push(`${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`);
             }
 
             for (const title of failures) {