Run playwright tests on more browsers async

.github/workflows/end-to-end-tests.yaml

@@ -19,6 +19,14 @@ on:
                 required: false
                 default: false
                 description: "A boolean to skip the playwright check itself while still creating the passing check. Useful when only running in Merge Queues."
+            parallelism:
+                type: number
+                required: false
+                description: "The number of runners to spread the tests across. Defaults to 6."
+            browsers:
+                type: string
+                required: false
+                description: "The browser to run the tests in. Some of 'chromium', 'chrome', 'firefox', 'webkit', 'edge'. Defaults to chromium."
 
             react-sdk-repository:
                 type: string
@@ -43,10 +51,13 @@ env:
     PR_NUMBER: ${{ github.event.pull_request.number }}
 
 jobs:
-    build:
+    prepare:
         name: "Build Element-Web"
         runs-on: ubuntu-latest
         if: inputs.skip != true
+        outputs:
+            parallelism: ${{ steps.parallelism.outputs.parallelism }}
+            parallelism_matrix: ${{ steps.parallelism.outputs.parallelism_matrix }}
         steps:
             - name: Checkout code
               uses: actions/checkout@v4
@@ -91,9 +102,17 @@ jobs:
                   path: element-web/webapp
                   retention-days: 1
 
+            - name: Calculate parallelism
+              id: parallelism
+              run: |
+                  echo "parallelism=$PARALLELISM" >> $GITHUB_OUTPUT
+                  echo "parallelism_matrix=$(python3 -c 'print([*range(1, 1 + ${{ env.PARALLELISM }})])')" >> $GITHUB_OUTPUT
+              env:
+                  PARALLELISM: ${{ inputs.parallelism || 6 }}
+
     playwright:
-        name: "Run Tests ${{ matrix.runner }}/${{ strategy.job-total }}"
-        needs: build
+        name: "Run Tests ${{ matrix.runner }}/${{ needs.prepare.outputs.parallelism }} [${{ matrix.browser }}]"
+        needs: prepare
         if: inputs.skip != true
         runs-on: ubuntu-latest
         permissions:
@@ -104,7 +123,8 @@ jobs:
             fail-fast: false
             matrix:
                 # Run multiple instances in parallel to speed up the tests
-                runner: [1, 2, 3, 4, 5, 6]
+                runner: ${{ fromJson(needs.prepare.outputs.parallelism_matrix) }}
+                browser: ${{ fromJson(inputs.browsers || '["chromium", "firefox"]') }}
         steps:
             - uses: actions/checkout@v4
               with:
@@ -139,22 +159,25 @@ jobs:
               with:
                   path: |
                       ~/.cache/ms-playwright
-                  key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}
+                  key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}-${{ matrix.browser }}
 
             - name: Install Playwright browsers
               if: steps.playwright-cache.outputs.cache-hit != 'true'
               working-directory: matrix-react-sdk
               run: yarn playwright install --with-deps
 
             - name: Run Playwright tests
-              run: yarn playwright test --shard ${{ matrix.runner }}/${{ strategy.job-total }}
+              run: yarn playwright test --project "$BROWSER" --shard ${{ matrix.runner }}/${{ needs.prepare.outputs.parallelism }} $IGNORE_SNAPSHOTS
               working-directory: matrix-react-sdk
+              env:
+                  BROWSER: ${{ matrix.browser }}
+                  IGNORE_SNAPSHOTS: ${{ matrix.browser != 'chromium' && '--ignore-snapshots' || '' }}
 
             - name: Upload blob report to GitHub Actions Artifacts
               if: always()
               uses: actions/upload-artifact@v4
               with:
-                  name: all-blob-reports-${{ matrix.runner }}
+                  name: all-blob-reports-${{ matrix.browser }}-${{ matrix.runner }}
                   path: matrix-react-sdk/blob-report
                   retention-days: 1
 

package.json

@@ -54,11 +54,11 @@
         "lint:types": "tsc --noEmit --jsx react && tsc --noEmit --jsx react -p playwright",
         "lint:style": "stylelint \"res/css/**/*.pcss\"",
         "test": "jest",
-        "test:playwright": "playwright test",
-        "test:playwright:open": "yarn test:playwright --ui",
+        "test:playwright": "playwright test --project chromium",
+        "test:playwright:open": "yarn test:playwright --ui --project chromium",
         "test:playwright:screenshots": "yarn test:playwright:screenshots:build && yarn test:playwright:screenshots:run",
         "test:playwright:screenshots:build": "docker build playwright -t matrix-react-sdk-playwright",
-        "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -v $(pwd)/../:/work/ -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it matrix-react-sdk-playwright",
+        "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -v $(pwd)/../:/work/ -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it matrix-react-sdk-playwright --project chromium",
         "coverage": "yarn test --coverage",
         "lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) | xargs -I {} sh -c 'echo \"Linting {}\"; action-validator \"{}\"'"
     },

playwright.config.ts

@@ -14,27 +14,73 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { defineConfig } from "@playwright/test";
+import { defineConfig, devices } from "@playwright/test";
 
 const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
 
+const chromeOptions = {
+    permissions: ["clipboard-write", "clipboard-read", "microphone"],
+    launchOptions: {
+        args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
+    },
+};
+
 export default defineConfig({
     use: {
         viewport: { width: 1280, height: 720 },
         ignoreHTTPSErrors: true,
         video: "retain-on-failure",
         baseURL,
-        permissions: ["clipboard-write", "clipboard-read", "microphone"],
-        launchOptions: {
-            args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
-        },
         trace: "on-first-retry",
     },
     webServer: {
         command: process.env.CI ? "npx serve -p 8080 -L ../webapp" : "yarn --cwd ../element-web start",
         url: `${baseURL}/config.json`,
         reuseExistingServer: true,
     },
+    projects: [
+        /* Test against desktop browsers */
+        {
+            name: "chromium",
+            use: {
+                ...devices["Desktop Chrome"],
+                contextOptions: chromeOptions,
+            },
+        },
+        {
+            name: "firefox",
+            use: {
+                ...devices["Desktop Firefox"],
+                launchOptions: {
+                    firefoxUserPrefs: {
+                        "dom.events.asyncClipboard.writeText": true,
+                        "dom.events.asyncClipboard.readText": true,
+                        "dom.events.testing.asyncClipboard": true,
+                    },
+                },
+            },
+        },
+        {
+            name: "webkit",
+            use: { ...devices["Desktop Safari"] },
+        },
+        {
+            name: "chrome",
+            use: {
+                ...devices["Desktop Chrome"],
+                channel: "chrome",
+                contextOptions: chromeOptions,
+            },
+        },
+        {
+            name: "edge",
+            use: {
+                ...devices["Desktop Edge"],
+                channel: "msedge",
+                contextOptions: chromeOptions,
+            },
+        },
+    ],
     testDir: "playwright/e2e",
     outputDir: "playwright/test-results",
     workers: 1,

playwright/e2e/audio-player/audio-player.spec.ts

@@ -130,7 +130,9 @@ test.describe("Audio player", () => {
         );
     };
 
-    test.beforeEach(async ({ page, app, user }) => {
+    test.beforeEach(async ({ page, app, user, browserName }) => {
+        test.skip(browserName === "firefox", "Firefox doesn't like our ogg files");
+
         await app.client.createRoom({ name: "Test Room" });
         await app.viewRoomByName("Test Room");
 

playwright/e2e/crypto/backups.spec.ts

@@ -31,7 +31,9 @@ test.describe("Backups", () => {
         displayName: "Hanako",
     });
 
-    test("Create, delete and recreate a keys backup", async ({ page, user, app }, workerInfo) => {
+    test("Create, delete and recreate a keys backup", async ({ page, user, app, browserName }) => {
+        test.skip(browserName === "webkit", "This test is broken on Webkit");
+
         // Create a backup
         const securityTab = await app.settings.openUserSettings("Security & Privacy");
 

playwright/e2e/crypto/decryption-failure-messages.spec.ts

@@ -43,7 +43,8 @@ test.describe("Cryptography", function () {
             credentials,
             user,
         }) => {
-            test.setTimeout(60000);
+            test.slow();
+            test.setTimeout(90000);
 
             // Start with a logged-in session, without key backup, and send a message.
             await createRoom(page, "Test room", true);

playwright/e2e/crypto/dehydration.spec.ts

@@ -58,8 +58,6 @@ test.describe("Dehydration", () => {
     });
 
     test("Create dehydrated device", async ({ page, user, app }, workerInfo) => {
-        test.skip(workerInfo.project.name === "Legacy Crypto", "This test only works with Rust crypto.");
-
         // Create a backup (which will create SSSS, and dehydrated device)
 
         const securityTab = await app.settings.openUserSettings("Security & Privacy");

playwright/e2e/crypto/device-verification.spec.ts

@@ -35,7 +35,9 @@ test.describe("Device verification", () => {
     /** The backup version that was set up by the bot client. */
     let expectedBackupVersion: string;
 
-    test.beforeEach(async ({ page, homeserver, credentials }) => {
+    test.beforeEach(async ({ page, homeserver, credentials, browserName }) => {
+        test.skip(browserName === "webkit", "This test is broken on Webkit");
+
         // Visit the login page of the app, to load the matrix sdk
         await page.goto("/#/login");
 

playwright/e2e/crypto/migration.spec.ts

@@ -36,8 +36,8 @@ const test = base.extend({
 test.describe("migration", function () {
     test.use({ displayName: "Alice" });
 
-    test("Should support migration from legacy crypto", async ({ context, user, page }, workerInfo) => {
-        test.skip(workerInfo.project.name === "Legacy Crypto", "This test only works with Rust crypto.");
+    test("Should support migration from legacy crypto", async ({ context, user, page, browserName }, workerInfo) => {
+        test.skip(browserName === "webkit", "This test is broken on Webkit");
         test.slow();
 
         // We should see a migration progress bar

playwright/e2e/editing/editing.spec.ts

@@ -269,7 +269,10 @@ test.describe("Editing", () => {
         room,
         axe,
         checkA11y,
+        browserName,
     }) => {
+        test.skip(browserName === "firefox", "Additional a11y faults are found on Firefox, skip for now");
+
         axe.disableRules("color-contrast"); // XXX: We have some known contrast issues here
         axe.exclude(".mx_Tooltip_visible"); // XXX: this is fine but would be good to fix