Skip to content

Commit cf0bca3

Browse files
ClementPasteauClementPasteau
authored
Add daily automatic tests for in-app tutorials (#114)
1 parent 983a28c commit cf0bca3

13 files changed

Lines changed: 2129 additions & 1 deletion

‎.github/workflows/test-in-app-tutorials.yml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# Plays every in-app tutorial against the GDevelop editor (web build, master
2+
# branch) and fails if a tutorial step cannot be completed. A video of each
3+
# tutorial being played is uploaded as an artifact, for passing and failing
4+
# runs alike.
5+
#
6+
# Runs on every push to main (a tutorial change should not break the
7+
# tutorials) and every 6 hours (a change in GDevelop master should not
8+
# either). The schedule only runs from the default branch (main).
9+
# On pull requests, only the modified tutorials are played (all of them if the
10+
# test harness itself was modified).
11+
name: Test in-app tutorials
12+
13+
on:
14+
push:
15+
branches: [main]
16+
pull_request:
17+
schedule:
18+
# Every 6 hours: GDevelop pushes do not trigger this workflow, the
19+
# schedule catches editor changes that break tutorials.
20+
- cron: '17 */6 * * *'
21+
workflow_dispatch:
22+
23+
# A new push cancels the still-running workflow of the same pull request or
24+
# branch (scheduled runs are grouped separately and never cancel them).
25+
concurrency:
26+
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
test-in-app-tutorials:
31+
# One job per editor layout (see the projects in e2e/playwright.config.js):
32+
# they run in parallel and each uploads its own videos.
33+
strategy:
34+
fail-fast: false
35+
matrix:
36+
project: [desktop, tablet, mobile]
37+
name: test-in-app-tutorials (${{ matrix.project }})
38+
runs-on: ubuntu-latest
39+
timeout-minutes: 90
40+
steps:
41+
- name: Checkout tutorials
42+
uses: actions/checkout@v4
43+
with:
44+
# Full history, to diff against the base branch on pull requests.
45+
fetch-depth: 0
46+
47+
- name: Checkout GDevelop
48+
uses: actions/checkout@v4
49+
with:
50+
repository: 4ian/GDevelop
51+
ref: master
52+
path: GDevelop
53+
54+
- uses: actions/setup-node@v4
55+
with:
56+
node-version: 22
57+
cache: npm
58+
cache-dependency-path: |
59+
package-lock.json
60+
GDevelop/newIDE/app/package-lock.json
61+
62+
- name: Install dependencies
63+
run: npm ci
64+
65+
- name: Install GDevelop editor dependencies
66+
run: npm ci
67+
working-directory: GDevelop/newIDE/app
68+
69+
# Fast static check: every selector used by the tutorials must still
70+
# exist in the editor sources. Known broken tutorials are skipped —
71+
# remove them from --ignore (and from KNOWN_BROKEN_TUTORIAL_IDS in
72+
# e2e/in-app-tutorials.spec.js) once fixed.
73+
- name: Check tutorial selectors against the editor sources
74+
if: matrix.project == 'desktop'
75+
run: npm run check-in-app-tutorial-selectors -- --gdevelop-root-path ./GDevelop --ignore flingGame
76+
77+
- name: Install Playwright browser
78+
run: npx playwright install --with-deps chromium
79+
80+
# On pull requests, only play the tutorials whose json was modified —
81+
# or all of them if the test harness itself was modified, and none if
82+
# the changes concern neither.
83+
- name: Determine which tutorials to play
84+
id: scope
85+
if: github.event_name == 'pull_request'
86+
run: |
87+
changed=$(git diff --name-only "origin/$GITHUB_BASE_REF"...HEAD)
88+
echo "Changed files:"; echo "$changed"
89+
ids=$(echo "$changed" | grep -E '^tutorials/in-app/[^/]+\.json$' | xargs -rn1 basename | sed 's/\.json$//' | paste -sd, -)
90+
if echo "$changed" | grep -qE '^(e2e/|scripts/|package(-lock)?\.json|\.github/workflows/)'; then
91+
# The harness itself changed: play everything.
92+
echo "mode=all" >> "$GITHUB_OUTPUT"
93+
elif [ -n "$ids" ]; then
94+
echo "ids=$ids" >> "$GITHUB_OUTPUT"
95+
echo "mode=modified-tutorials" >> "$GITHUB_OUTPUT"
96+
else
97+
echo "mode=none" >> "$GITHUB_OUTPUT"
98+
fi
99+
100+
# Starts the editor dev server (from ./GDevelop) and plays the
101+
# tutorials. Known broken tutorials are expected to fail.
102+
- name: Play in-app tutorials
103+
if: steps.scope.outputs.mode != 'none'
104+
env:
105+
TUTORIAL_IDS: ${{ steps.scope.outputs.ids }}
106+
run: npm run test-in-app-tutorials -- --project=${{ matrix.project }}
107+
108+
# Playwright records webm, which does not play natively on macOS:
109+
# convert to mp4 (H.264) so the videos open with QuickTime/Quick Look.
110+
- name: Convert videos to mp4
111+
if: always()
112+
run: |
113+
[ -d test-results ] || exit 0
114+
sudo apt-get update -qq && sudo apt-get install -y -qq ffmpeg
115+
find test-results -name "*.webm" -print0 | while IFS= read -r -d '' file; do
116+
ffmpeg -nostdin -y -loglevel error -i "$file" -c:v libx264 -preset veryfast -crf 28 -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -movflags +faststart "${file%.webm}.mp4" && rm "$file"
117+
done
118+
119+
- name: Upload videos and screenshots
120+
if: always()
121+
uses: actions/upload-artifact@v4
122+
with:
123+
name: tutorial-videos-${{ matrix.project }}
124+
path: |
125+
test-results/**/*.mp4
126+
test-results/**/*.png
127+
test-results/**/error-context.md
128+
# Hourly runs would accumulate a lot of videos: keep them shorter.
129+
retention-days: ${{ github.event_name == 'schedule' && 3 || 14 }}
130+
if-no-files-found: warn
131+
132+
# Traces are heavy (~50MB per failed test, with a DOM snapshot of every
133+
# action): uploaded separately so downloading the videos stays fast.
134+
# Inspect with: npx playwright show-trace trace.zip
135+
- name: Upload traces (failures only)
136+
if: always()
137+
uses: actions/upload-artifact@v4
138+
with:
139+
name: test-traces-${{ matrix.project }}
140+
path: test-results/**/trace.zip
141+
retention-days: ${{ github.event_name == 'schedule' && 3 || 14 }}
142+
if-no-files-found: ignore

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,5 @@
22
.DS_Store
33
/dist
44
*.autosave
5+
/test-results
6+
/playwright-report

‎e2e/README.md‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# In-app tutorials end-to-end tests
2+
3+
These tests play every in-app tutorial of this repository against a real
4+
GDevelop editor (web build), like a user would: for each step of a tutorial
5+
flow, the expected action is performed (click the highlighted element, fill the
6+
input, drag an object to the scene, launch a preview...), and the test checks
7+
that the tutorial orchestrator advances to the next step. If a step cannot be
8+
completed, the tutorial is broken and the test fails with the step index, the
9+
selector involved and a screenshot/trace.
10+
11+
They complement the static selector check
12+
(`npm run check-in-app-tutorial-selectors`), which quickly catches selectors
13+
referencing elements that no longer exist in the editor, without running it.
14+
15+
## How it works
16+
17+
- The editor is opened with the `in-app-tutorial-test-mode` URL parameter: the
18+
orchestrator (see `InAppTutorialOrchestrator.js` in GDevelop) then exposes
19+
its current state (step index, interpolated element to highlight, trigger)
20+
on `window.inAppTutorialTestModeState`.
21+
- Requests to the in-app tutorials API/CDN are intercepted so the **local**
22+
tutorial files (with meta steps expanded) are used instead of the deployed
23+
ones. Initial project templates are still fetched from the real CDN, so a
24+
broken template URL is caught too.
25+
- The tutorial is started through the regular
26+
`?initial-dialog=guided-lesson&tutorial-id=<id>` flow.
27+
28+
## CI
29+
30+
The `.github/workflows/test-in-app-tutorials.yml` workflow runs the static
31+
selector check and plays every tutorial against GDevelop `master`, on each
32+
push to `main` and every 6 hours (the schedule catches GDevelop changes that
33+
break tutorials, since GDevelop pushes do not trigger this workflow). On pull
34+
requests, only the modified tutorials are played (all of them if the test
35+
harness itself was modified).
36+
The tutorials are played on the three editor layouts in parallel jobs —
37+
desktop (1600×900), tablet (1024×768) and mobile (844×390 landscape), matching
38+
the editor's responsive thresholds. A video of each tutorial being played
39+
(converted to mp4, playable natively on macOS) is uploaded per layout as the
40+
`tutorial-videos-<project>` artifacts, for passing and failing runs alike.
41+
Playwright traces of failed tests are uploaded separately as
42+
`test-traces-<project>` (they are heavy — ~50MB per failed test); inspect one
43+
with `npx playwright show-trace trace.zip`.
44+
45+
Run a single layout locally with `--project=desktop|tablet|mobile`.
46+
47+
Untested tutorials are listed in `SKIPPED_TUTORIAL_IDS`
48+
(`e2e/in-app-tutorials.spec.js`, currently flingGame — no plan to fix it) and
49+
in the `--ignore` flag of the selector check step. Tutorials broken on a
50+
single layout are in `KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS`: they are still
51+
played and recorded, but expected to fail there.
52+
53+
## Known dev-environment behaviors
54+
55+
- When opening a template that uses leaderboards (plinkoMultiplier) with a
56+
logged-in user, the leaderboard auto-creation fails with a 404: template
57+
leaderboards only exist in production. This is expected on dev — the player
58+
dismisses the error dialog ("Abandon") and the tutorial continues.
59+
- The test account (see `TEST_ACCOUNT` in `e2e/lib/gdevelopEditor.js`) is a
60+
dev-environment account: it contains no secret.
61+
62+
## Running locally
63+
64+
Requirements: a GDevelop checkout with `npm install` ran in `newIDE/app`,
65+
located next to this repository (or set `GDEVELOP_ROOT_PATH`), and
66+
`npx playwright install chromium` ran once in this repository.
67+
68+
```bash
69+
# Run all tutorials (starts the editor dev server if not already running):
70+
npm run test-in-app-tutorials
71+
72+
# Against an already-running editor, for a single tutorial:
73+
GDEVELOP_EDITOR_URL=http://localhost:3000 TUTORIAL_IDS=timer npm run test-in-app-tutorials
74+
75+
# Inspect a failure:
76+
npx playwright show-trace test-results/<test-name>/trace.zip
77+
```
78+
79+
## Environment variables
80+
81+
| Variable | Description |
82+
| -------------------- | ------------------------------------------------------------------ |
83+
| `GDEVELOP_EDITOR_URL`| URL of a running editor (default `http://localhost:3000`). |
84+
| `GDEVELOP_ROOT_PATH` | Path to the GDevelop checkout (default: `./GDevelop` or `../GDevelop`). |
85+
| `TUTORIAL_IDS` | Comma-separated tutorial ids to test (default: all). |
86+
87+
## How steps are played
88+
89+
| Trigger | Action performed |
90+
| ---------------------- | ---------------------------------------------------------------------- |
91+
| `presenceOfElement` | Click the highlighted element, or type the text in bold/quotes from the tooltip if the element is a text input (search bars). |
92+
| `absenceOfElement` | Click the highlighted element (e.g. an Apply/OK button). |
93+
| `editorIsActive` | Click the highlighted editor tab. |
94+
| `valueEquals` | Fill the highlighted field with the expected value. |
95+
| `valueHasChanged` | Fill the highlighted field with a test value. |
96+
| `instanceAddedOnScene` | Drag the highlighted object list item onto the scene canvas. |
97+
| `objectAddedInLayout` | Click the highlighted element (e.g. the asset store add button). |
98+
| `previewLaunched` | Click the preview button and wait for the preview window. |
99+
| `clickOnTooltipButton` | Click the tooltip button with the expected label. |

‎e2e/in-app-tutorials.spec.js‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// @ts-check
2+
/**
3+
* Plays every in-app tutorial of this repository against a running GDevelop
4+
* editor, and fails if a tutorial step cannot be completed.
5+
*
6+
* Run with: npx playwright test --config e2e/playwright.config.js
7+
* Filter tutorials with: TUTORIAL_IDS=timer,healthBar
8+
*/
9+
const { test } = require('@playwright/test');
10+
const {
11+
loadAllTutorials,
12+
serveLocalTutorials,
13+
startTutorial,
14+
} = require('./lib/gdevelopEditor');
15+
const { playTutorial } = require('./lib/tutorialPlayer');
16+
17+
/**
18+
* Tutorials that are not tested at all.
19+
* - flingGame: references removed editor elements (#layer-name,
20+
* [data-default]) and there is no plan to fix it.
21+
*/
22+
const SKIPPED_TUTORIAL_IDS = ['flingGame'];
23+
24+
/**
25+
* Tutorials known to be broken on the mobile layout only.
26+
* - tilemapPlatformer: after selecting the terrain, the instance properties
27+
* panel (a drawer on mobile) closes again, #freehandBrush disappears and
28+
* the tutorial tooltip is not displayed anymore: a mobile user is left
29+
* without any guidance.
30+
*/
31+
const KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS = ['tilemapPlatformer'];
32+
33+
const allTutorials = loadAllTutorials();
34+
const tutorialIdsFilter = process.env.TUTORIAL_IDS
35+
? process.env.TUTORIAL_IDS.split(',').map((id) => id.trim())
36+
: null;
37+
const tutorials = tutorialIdsFilter
38+
? allTutorials.filter((tutorial) => tutorialIdsFilter.includes(tutorial.id))
39+
: allTutorials;
40+
41+
for (const tutorial of tutorials) {
42+
test(`in-app tutorial: ${tutorial.id}`, async ({
43+
page,
44+
context,
45+
}, testInfo) => {
46+
test.skip(
47+
SKIPPED_TUTORIAL_IDS.includes(tutorial.id),
48+
'This tutorial is not tested.'
49+
);
50+
test.fail(
51+
testInfo.project.name === 'mobile' &&
52+
KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS.includes(tutorial.id),
53+
'This tutorial is known to be broken on this layout.'
54+
);
55+
await serveLocalTutorials(context, allTutorials);
56+
await startTutorial(page, tutorial.id);
57+
await playTutorial({
58+
page,
59+
context,
60+
tutorial: tutorial.content,
61+
log: (message) => console.log(`[${tutorial.id}] ${message}`),
62+
});
63+
});
64+
}

0 commit comments

Comments
 (0)