Automatic tests

.github/workflows/test-in-app-tutorials.yml

@@ -0,0 +1,142 @@
+# Plays every in-app tutorial against the GDevelop editor (web build, master
+# branch) and fails if a tutorial step cannot be completed. A video of each
+# tutorial being played is uploaded as an artifact, for passing and failing
+# runs alike.
+#
+# Runs on every push to main (a tutorial change should not break the
+# tutorials) and every 6 hours (a change in GDevelop master should not
+# either). The schedule only runs from the default branch (main).
+# On pull requests, only the modified tutorials are played (all of them if the
+# test harness itself was modified).
+name: Test in-app tutorials
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  schedule:
+    # Every 6 hours: GDevelop pushes do not trigger this workflow, the
+    # schedule catches editor changes that break tutorials.
+    - cron: '17 */6 * * *'
+  workflow_dispatch:
+
+# A new push cancels the still-running workflow of the same pull request or
+# branch (scheduled runs are grouped separately and never cancel them).
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-in-app-tutorials:
+    # One job per editor layout (see the projects in e2e/playwright.config.js):
+    # they run in parallel and each uploads its own videos.
+    strategy:
+      fail-fast: false
+      matrix:
+        project: [desktop, tablet, mobile]
+    name: test-in-app-tutorials (${{ matrix.project }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout tutorials
+        uses: actions/checkout@v4
+        with:
+          # Full history, to diff against the base branch on pull requests.
+          fetch-depth: 0
+
+      - name: Checkout GDevelop
+        uses: actions/checkout@v4
+        with:
+          repository: 4ian/GDevelop
+          ref: master
+          path: GDevelop
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: |
+            package-lock.json
+            GDevelop/newIDE/app/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install GDevelop editor dependencies
+        run: npm ci
+        working-directory: GDevelop/newIDE/app
+
+      # Fast static check: every selector used by the tutorials must still
+      # exist in the editor sources. Known broken tutorials are skipped —
+      # remove them from --ignore (and from KNOWN_BROKEN_TUTORIAL_IDS in
+      # e2e/in-app-tutorials.spec.js) once fixed.
+      - name: Check tutorial selectors against the editor sources
+        if: matrix.project == 'desktop'
+        run: npm run check-in-app-tutorial-selectors -- --gdevelop-root-path ./GDevelop --ignore flingGame
+
+      - name: Install Playwright browser
+        run: npx playwright install --with-deps chromium
+
+      # On pull requests, only play the tutorials whose json was modified —
+      # or all of them if the test harness itself was modified, and none if
+      # the changes concern neither.
+      - name: Determine which tutorials to play
+        id: scope
+        if: github.event_name == 'pull_request'
+        run: |
+          changed=$(git diff --name-only "origin/$GITHUB_BASE_REF"...HEAD)
+          echo "Changed files:"; echo "$changed"
+          ids=$(echo "$changed" | grep -E '^tutorials/in-app/[^/]+\.json$' | xargs -rn1 basename | sed 's/\.json$//' | paste -sd, -)
+          if echo "$changed" | grep -qE '^(e2e/|scripts/|package(-lock)?\.json|\.github/workflows/)'; then
+            # The harness itself changed: play everything.
+            echo "mode=all" >> "$GITHUB_OUTPUT"
+          elif [ -n "$ids" ]; then
+            echo "ids=$ids" >> "$GITHUB_OUTPUT"
+            echo "mode=modified-tutorials" >> "$GITHUB_OUTPUT"
+          else
+            echo "mode=none" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Starts the editor dev server (from ./GDevelop) and plays the
+      # tutorials. Known broken tutorials are expected to fail.
+      - name: Play in-app tutorials
+        if: steps.scope.outputs.mode != 'none'
+        env:
+          TUTORIAL_IDS: ${{ steps.scope.outputs.ids }}
+        run: npm run test-in-app-tutorials -- --project=${{ matrix.project }}
+
+      # Playwright records webm, which does not play natively on macOS:
+      # convert to mp4 (H.264) so the videos open with QuickTime/Quick Look.
+      - name: Convert videos to mp4
+        if: always()
+        run: |
+          [ -d test-results ] || exit 0
+          sudo apt-get update -qq && sudo apt-get install -y -qq ffmpeg
+          find test-results -name "*.webm" -print0 | while IFS= read -r -d '' file; do
+            ffmpeg -nostdin -y -loglevel error -i "$file" -c:v libx264 -preset veryfast -crf 28 -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -movflags +faststart "${file%.webm}.mp4" && rm "$file"
+          done
+
+      - name: Upload videos and screenshots
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: tutorial-videos-${{ matrix.project }}
+          path: |
+            test-results/**/*.mp4
+            test-results/**/*.png
+            test-results/**/error-context.md
+          # Hourly runs would accumulate a lot of videos: keep them shorter.
+          retention-days: ${{ github.event_name == 'schedule' && 3 || 14 }}
+          if-no-files-found: warn
+
+      # Traces are heavy (~50MB per failed test, with a DOM snapshot of every
+      # action): uploaded separately so downloading the videos stays fast.
+      # Inspect with: npx playwright show-trace trace.zip
+      - name: Upload traces (failures only)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-traces-${{ matrix.project }}
+          path: test-results/**/trace.zip
+          retention-days: ${{ github.event_name == 'schedule' && 3 || 14 }}
+          if-no-files-found: ignore

.gitignore

@@ -2,3 +2,5 @@
 .DS_Store
 /dist
 *.autosave
+/test-results
+/playwright-report

e2e/README.md

@@ -0,0 +1,99 @@
+# In-app tutorials end-to-end tests
+
+These tests play every in-app tutorial of this repository against a real
+GDevelop editor (web build), like a user would: for each step of a tutorial
+flow, the expected action is performed (click the highlighted element, fill the
+input, drag an object to the scene, launch a preview...), and the test checks
+that the tutorial orchestrator advances to the next step. If a step cannot be
+completed, the tutorial is broken and the test fails with the step index, the
+selector involved and a screenshot/trace.
+
+They complement the static selector check
+(`npm run check-in-app-tutorial-selectors`), which quickly catches selectors
+referencing elements that no longer exist in the editor, without running it.
+
+## How it works
+
+- The editor is opened with the `in-app-tutorial-test-mode` URL parameter: the
+  orchestrator (see `InAppTutorialOrchestrator.js` in GDevelop) then exposes
+  its current state (step index, interpolated element to highlight, trigger)
+  on `window.inAppTutorialTestModeState`.
+- Requests to the in-app tutorials API/CDN are intercepted so the **local**
+  tutorial files (with meta steps expanded) are used instead of the deployed
+  ones. Initial project templates are still fetched from the real CDN, so a
+  broken template URL is caught too.
+- The tutorial is started through the regular
+  `?initial-dialog=guided-lesson&tutorial-id=<id>` flow.
+
+## CI
+
+The `.github/workflows/test-in-app-tutorials.yml` workflow runs the static
+selector check and plays every tutorial against GDevelop `master`, on each
+push to `main` and every 6 hours (the schedule catches GDevelop changes that
+break tutorials, since GDevelop pushes do not trigger this workflow). On pull
+requests, only the modified tutorials are played (all of them if the test
+harness itself was modified).
+The tutorials are played on the three editor layouts in parallel jobs —
+desktop (1600×900), tablet (1024×768) and mobile (844×390 landscape), matching
+the editor's responsive thresholds. A video of each tutorial being played
+(converted to mp4, playable natively on macOS) is uploaded per layout as the
+`tutorial-videos-<project>` artifacts, for passing and failing runs alike.
+Playwright traces of failed tests are uploaded separately as
+`test-traces-<project>` (they are heavy — ~50MB per failed test); inspect one
+with `npx playwright show-trace trace.zip`.
+
+Run a single layout locally with `--project=desktop|tablet|mobile`.
+
+Untested tutorials are listed in `SKIPPED_TUTORIAL_IDS`
+(`e2e/in-app-tutorials.spec.js`, currently flingGame — no plan to fix it) and
+in the `--ignore` flag of the selector check step. Tutorials broken on a
+single layout are in `KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS`: they are still
+played and recorded, but expected to fail there.
+
+## Known dev-environment behaviors
+
+- When opening a template that uses leaderboards (plinkoMultiplier) with a
+  logged-in user, the leaderboard auto-creation fails with a 404: template
+  leaderboards only exist in production. This is expected on dev — the player
+  dismisses the error dialog ("Abandon") and the tutorial continues.
+- The test account (see `TEST_ACCOUNT` in `e2e/lib/gdevelopEditor.js`) is a
+  dev-environment account: it contains no secret.
+
+## Running locally
+
+Requirements: a GDevelop checkout with `npm install` ran in `newIDE/app`,
+located next to this repository (or set `GDEVELOP_ROOT_PATH`), and
+`npx playwright install chromium` ran once in this repository.
+
+```bash
+# Run all tutorials (starts the editor dev server if not already running):
+npm run test-in-app-tutorials
+
+# Against an already-running editor, for a single tutorial:
+GDEVELOP_EDITOR_URL=http://localhost:3000 TUTORIAL_IDS=timer npm run test-in-app-tutorials
+
+# Inspect a failure:
+npx playwright show-trace test-results/<test-name>/trace.zip
+```
+
+## Environment variables
+
+| Variable             | Description                                                        |
+| -------------------- | ------------------------------------------------------------------ |
+| `GDEVELOP_EDITOR_URL`| URL of a running editor (default `http://localhost:3000`).         |
+| `GDEVELOP_ROOT_PATH` | Path to the GDevelop checkout (default: `./GDevelop` or `../GDevelop`). |
+| `TUTORIAL_IDS`       | Comma-separated tutorial ids to test (default: all).               |
+
+## How steps are played
+
+| Trigger                | Action performed                                                       |
+| ---------------------- | ---------------------------------------------------------------------- |
+| `presenceOfElement`    | Click the highlighted element, or type the text in bold/quotes from the tooltip if the element is a text input (search bars). |
+| `absenceOfElement`     | Click the highlighted element (e.g. an Apply/OK button).               |
+| `editorIsActive`       | Click the highlighted editor tab.                                      |
+| `valueEquals`          | Fill the highlighted field with the expected value.                    |
+| `valueHasChanged`      | Fill the highlighted field with a test value.                          |
+| `instanceAddedOnScene` | Drag the highlighted object list item onto the scene canvas.           |
+| `objectAddedInLayout`  | Click the highlighted element (e.g. the asset store add button).       |
+| `previewLaunched`      | Click the preview button and wait for the preview window.              |
+| `clickOnTooltipButton` | Click the tooltip button with the expected label.                      |

e2e/in-app-tutorials.spec.js

@@ -0,0 +1,64 @@
+// @ts-check
+/**
+ * Plays every in-app tutorial of this repository against a running GDevelop
+ * editor, and fails if a tutorial step cannot be completed.
+ *
+ * Run with: npx playwright test --config e2e/playwright.config.js
+ * Filter tutorials with: TUTORIAL_IDS=timer,healthBar
+ */
+const { test } = require('@playwright/test');
+const {
+  loadAllTutorials,
+  serveLocalTutorials,
+  startTutorial,
+} = require('./lib/gdevelopEditor');
+const { playTutorial } = require('./lib/tutorialPlayer');
+
+/**
+ * Tutorials that are not tested at all.
+ * - flingGame: references removed editor elements (#layer-name,
+ *   [data-default]) and there is no plan to fix it.
+ */
+const SKIPPED_TUTORIAL_IDS = ['flingGame'];
+
+/**
+ * Tutorials known to be broken on the mobile layout only.
+ * - tilemapPlatformer: after selecting the terrain, the instance properties
+ *   panel (a drawer on mobile) closes again, #freehandBrush disappears and
+ *   the tutorial tooltip is not displayed anymore: a mobile user is left
+ *   without any guidance.
+ */
+const KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS = ['tilemapPlatformer'];
+
+const allTutorials = loadAllTutorials();
+const tutorialIdsFilter = process.env.TUTORIAL_IDS
+  ? process.env.TUTORIAL_IDS.split(',').map((id) => id.trim())
+  : null;
+const tutorials = tutorialIdsFilter
+  ? allTutorials.filter((tutorial) => tutorialIdsFilter.includes(tutorial.id))
+  : allTutorials;
+
+for (const tutorial of tutorials) {
+  test(`in-app tutorial: ${tutorial.id}`, async ({
+    page,
+    context,
+  }, testInfo) => {
+    test.skip(
+      SKIPPED_TUTORIAL_IDS.includes(tutorial.id),
+      'This tutorial is not tested.'
+    );
+    test.fail(
+      testInfo.project.name === 'mobile' &&
+        KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS.includes(tutorial.id),
+      'This tutorial is known to be broken on this layout.'
+    );
+    await serveLocalTutorials(context, allTutorials);
+    await startTutorial(page, tutorial.id);
+    await playTutorial({
+      page,
+      context,
+      tutorial: tutorial.content,
+      log: (message) => console.log(`[${tutorial.id}] ${message}`),
+    });
+  });
+}