E2e reliability and performance

.github/workflows/RunE2ENightly.yml

@@ -2,56 +2,66 @@ name: Run E2E test nightly
 # This workflow is used to test our widgets nightly.
 
 on:
-  schedule:
-    # At 02:00 on every day-of-week.
-    - cron: "0 02 * * 1-5"
+    schedule:
+        # At 02:00 on every day-of-week.
+        - cron: "0 02 * * 1-5"
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
 
 jobs:
-  e2e:
-    name: Run automated end-to-end tests nightly
-    runs-on: ubuntu-latest
-
-    permissions:
-      packages: read
-      contents: read
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-      - name: Setup pnpm
-        uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0
-
-      - name: Setup node
-        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
-        with:
-          node-version-file: ".nvmrc"
-          cache: "pnpm"
-
-      - name: Install dependencies
-        run: pnpm install
-
-      - name: Install Playwright Browsers
-        run: pnpm exec playwright install --with-deps chromium
-
-      - name: Executing E2E tests
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: pnpm -r --workspace-concurrency=1 --no-bail run e2e
-
-      - name: Fixing files permissions
-        if: failure()
-        run: |
-          sudo find ${{ github.workspace }}/packages/* -type d -exec chmod 755 {} \;
-          sudo find ${{ github.workspace }}/packages/* -type f -exec chmod 644 {} \;
-
-      - name: Archive test screenshot diff results
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
-        if: failure()
-        with:
-          name: test-screenshot-results
-          path: |
-            ${{ github.workspace }}/packages/**/**/test-results/**/*.png
-            ${{ github.workspace }}/packages/**/**/test-results/**/*.zip
-          if-no-files-found: error
+    e2e:
+        name: Run automated end-to-end tests nightly
+        runs-on: ubuntu-latest
+
+        permissions:
+            packages: read
+            contents: read
+
+        strategy:
+            fail-fast: false
+            matrix:
+                index: [0, 1, 2, 3]
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+              with:
+                  fetch-depth: 0
+            - name: Setup pnpm
+              uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0
+
+            - name: Setup node
+              uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
+              with:
+                  node-version-file: ".nvmrc"
+                  cache: "pnpm"
+
+            - name: Install dependencies
+              run: pnpm install
+
+            - name: Install Playwright Browsers
+              run: pnpm exec playwright install --with-deps chromium
+
+            - name: Executing E2E tests
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: >-
+                  node ./automation/run-e2e/bin/run-e2e-in-chunks.mjs --chunks 4 --index ${{ matrix.index }} --event-name ${{ github.event_name }}
+
+            - name: Fixing files permissions
+              if: failure()
+              run: |
+                  sudo find ${{ github.workspace }}/packages/* -type d -exec chmod 755 {} \;
+                  sudo find ${{ github.workspace }}/packages/* -type f -exec chmod 644 {} \;
+
+            - name: Archive test screenshot diff results
+              uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+              if: failure()
+              with:
+                  name: test-screenshot-results-${{ matrix.index }}
+                  path: |
+                      ${{ github.workspace }}/packages/**/**/test-results/**/*.png
+                      ${{ github.workspace }}/packages/**/**/test-results/**/*.zip
+                  if-no-files-found: error

automation/run-e2e/bin/migrate-spec.mjs

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+
+/**
+ * Migrates E2E spec files to use shared fixtures and helpers.
+ *
+ * Usage: node migrate-spec.mjs <spec-file-path> [--dry-run]
+ *
+ * Transforms:
+ * 1. Replaces `import { test, expect } from "@playwright/test"` with shared fixtures
+ * 2. Removes afterEach logout blocks (fixture handles session cleanup)
+ * 3. Replaces `waitForLoadState("networkidle")` with `waitForMendixApp(page)`
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+const args = process.argv.slice(2);
+const dryRun = args.includes("--dry-run");
+const filePath = args.find(a => !a.startsWith("--"));
+
+if (!filePath) {
+    console.error("Usage: migrate-spec.mjs <spec-file-path> [--dry-run]");
+    process.exit(1);
+}
+
+const absPath = resolve(filePath);
+let content = readFileSync(absPath, "utf-8");
+const original = content;
+const changes = [];
+
+// 1. Replace import from @playwright/test with shared fixtures (handles both orderings)
+const importPattern =
+    /import\s*\{\s*(?:test\s*,\s*expect|expect\s*,\s*test)\s*\}\s*from\s*["']@playwright\/test["'];?/g;
+if (importPattern.test(content)) {
+    content = content.replace(importPattern, 'import { expect, test } from "@mendix/run-e2e/fixtures";');
+    changes.push("Replaced @playwright/test import with shared fixtures");
+}
+
+// 2. Remove afterEach logout block (multiple patterns observed)
+const afterEachPattern =
+    /\s*test\.afterEach\s*\(\s*["']Cleanup session["']\s*,\s*async\s*\(\s*\{\s*page\s*\}\s*\)\s*=>\s*\{[^}]*(?:window\.mx\.session\.logout|window\.mx\?\.session\?\.logout)[^}]*\}\s*\)\s*;?\n?/g;
+if (afterEachPattern.test(content)) {
+    content = content.replace(afterEachPattern, "\n");
+    changes.push("Removed afterEach session logout block (fixture handles this)");
+}
+
+// 3. Replace waitForLoadState("networkidle") with waitForMendixApp
+const networkIdlePattern = /await\s+page\.waitForLoadState\s*\(\s*["']networkidle["']\s*\)\s*;?/g;
+if (networkIdlePattern.test(content)) {
+    // Add helper import if not already present
+    if (!content.includes("@mendix/run-e2e/mendix-helpers")) {
+        const insertAfterImport = content.indexOf("\n", content.indexOf("import"));
+        if (insertAfterImport !== -1) {
+            content =
+                content.slice(0, insertAfterImport + 1) +
+                'import { waitForMendixApp } from "@mendix/run-e2e/mendix-helpers";\n' +
+                content.slice(insertAfterImport + 1);
+        }
+    }
+    content = content.replace(networkIdlePattern, "await waitForMendixApp(page);");
+    changes.push("Replaced waitForLoadState('networkidle') with waitForMendixApp(page)");
+}
+
+if (content === original) {
+    console.log(`No changes needed: ${absPath}`);
+    process.exit(0);
+}
+
+if (dryRun) {
+    console.log(`[DRY RUN] Would apply to: ${absPath}`);
+    changes.forEach(c => console.log(`  - ${c}`));
+} else {
+    writeFileSync(absPath, content, "utf-8");
+    console.log(`Migrated: ${absPath}`);
+    changes.forEach(c => console.log(`  - ${c}`));
+}

automation/run-e2e/eslint.config.mjs

@@ -1,6 +1,7 @@
 import { defineConfig } from "eslint/config";
 import globals from "globals";
 import js from "@eslint/js";
+import playwright from "eslint-plugin-playwright";
 
 export default defineConfig([
     {
@@ -21,5 +22,14 @@ export default defineConfig([
         rules: {
             "no-unused-vars": "warn"
         }
+    },
+    {
+        files: ["**/e2e/**/*.spec.{,m,c}js"],
+        plugins: { playwright },
+        rules: {
+            "playwright/no-wait-for-timeout": "error",
+            "playwright/no-networkidle": "warn",
+            "playwright/prefer-web-first-assertions": "warn"
+        }
     }
 ]);

automation/run-e2e/lib/fixtures.mjs

@@ -0,0 +1,38 @@
+/* eslint-disable no-undef */
+import { test as base, expect } from "@playwright/test";
+
+async function waitForMendixApp(page, timeout = 60_000) {
+    await page.waitForLoadState("domcontentloaded");
+    await page.waitForFunction(
+        () =>
+            Boolean(window.mx?.session) &&
+            !document.querySelector(".mx-progress-indicator") &&
+            document.querySelector(".mx-page") !== null,
+        undefined,
+        { timeout }
+    );
+}
+
+export { expect };
+
+export const test = base.extend({
+    mendixSession: [
+        async ({ browser }, use) => {
+            const context = await browser.newContext();
+            const page = await context.newPage();
+            const originalGoto = page.goto.bind(page);
+            page.goto = async (url, options) => {
+                const response = await originalGoto(url, options);
+                await waitForMendixApp(page);
+                return response;
+            };
+            await use({ context, page });
+            await page.evaluate(() => window.mx?.session?.logout?.()).catch(() => {});
+            await context.close();
+        },
+        { scope: "worker" }
+    ],
+    page: async ({ mendixSession }, use) => {
+        await use(mendixSession.page);
+    }
+});

automation/run-e2e/lib/mendix-helpers.mjs

@@ -0,0 +1,57 @@
+/* eslint-disable no-undef */
+import { expect } from "@playwright/test";
+
+export async function waitForMendixApp(page, timeout = 60_000) {
+    await page.waitForLoadState("domcontentloaded");
+    await page.waitForFunction(
+        () =>
+            Boolean(window.mx?.session) &&
+            !document.querySelector(".mx-progress-indicator") &&
+            document.querySelector(".mx-page") !== null,
+        undefined,
+        { timeout }
+    );
+}
+
+export async function waitForDataReady(page, timeout = 60_000) {
+    await waitForMendixApp(page, timeout);
+    await page.waitForLoadState("networkidle");
+}
+
+export async function waitForWidget(page, mxName, timeout = 15_000) {
+    const locator = page.locator(`.mx-name-${mxName}`);
+    await expect(locator).toBeVisible({ timeout });
+    return locator;
+}
+
+export async function waitForListData(page, mxName, minRows = 1, timeout = 15_000) {
+    const container = page.locator(`.mx-name-${mxName}`);
+    await expect(container).toBeVisible({ timeout });
+    const rows = container.locator("[class*='item'], tr[class*='row'], [class*='gallery-item']");
+    await expect(rows).toHaveCount(minRows, { timeout });
+    return rows;
+}
+
+export async function safeLogout(page) {
+    await page.evaluate(() => window.mx?.session?.logout?.()).catch(() => {});
+}
+
+export async function navigateToPage(page, path, timeout = 30_000) {
+    await page.goto(path);
+    await waitForMendixApp(page, timeout);
+}
+
+export async function checkAccessibility(page, selector, options = {}) {
+    const AxeBuilder = (await import("@axe-core/playwright")).default;
+    let builder = new AxeBuilder({ page }).withTags(options.tags || ["wcag21aa"]);
+    if (selector) {
+        builder = builder.include(selector);
+    }
+    if (options.exclude) {
+        for (const sel of [].concat(options.exclude)) {
+            builder = builder.exclude(sel);
+        }
+    }
+    const results = await builder.analyze();
+    expect(results.violations).toEqual([]);
+}

automation/run-e2e/package.json

@@ -12,6 +12,11 @@
         "run-e2e": "bin/run-e2e.mjs"
     },
     "type": "module",
+    "exports": {
+        "./fixtures": "./lib/fixtures.mjs",
+        "./mendix-helpers": "./lib/mendix-helpers.mjs",
+        "./playwright.config.cjs": "./playwright.config.cjs"
+    },
     "scripts": {
         "format": "prettier --ignore-path ./node_modules/@mendix/prettier-config-web-widgets/global-prettierignore --write .",
         "lint": "eslint .",

automation/run-e2e/playwright.config.cjs

@@ -9,11 +9,13 @@ module.exports = defineConfig({
     fullyParallel: true,
     /* Fail the build on CI if you accidentally left test.only in the source code. */
     forbidOnly: !!process.env.CI,
+    /* Filter tests by tag: E2E_SUITE=smoke runs only @smoke-tagged tests */
+    grep: process.env.E2E_SUITE === "smoke" ? /@smoke/ : undefined,
     /* Retry on CI only */
     retries: process.env.CI ? 2 : 0,
-    /* Use 4 workers on CI – the runner has multiple cores and each widget's tests
-     * are independent, so parallel execution cuts per-widget runtime significantly. */
-    workers: process.env.CI ? 4 : undefined,
+    /* Worker-scoped session: each worker holds 1 Mendix session. Safe up to 4 workers
+     * against the 5-session developer license (leaves 1 headroom). */
+    workers: process.env.CI ? 4 : 4,
     /* Reporter to use. See https://playwright.dev/docs/test-reporters */
     reporter: [
         ["list"],
@@ -35,11 +37,20 @@ module.exports = defineConfig({
             reuseExistingServer: !process.env.CI
         }
     ], */
+    expect: {
+        toHaveScreenshot: {
+            animations: "disabled",
+            threshold: 0.1
+        }
+    },
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
         /* Base URL to use in actions like `await page.goto('/')`. */
         baseURL: process.env.URL ? process.env.URL : "http://127.0.0.1:8080",
 
+        actionTimeout: 10_000,
+        navigationTimeout: 30_000,
+
         /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
         trace: "on-first-retry",