diff --git a/.github/workflows/e2e-ios.yml b/.github/workflows/e2e-ios.yml index cb79b65..5119e93 100644 --- a/.github/workflows/e2e-ios.yml +++ b/.github/workflows/e2e-ios.yml @@ -20,6 +20,13 @@ jobs: # macOS Sequoia+ and a full Xcode 26 install, neither available on 14. runs-on: macos-15 name: e2e-ios (${{ matrix.scenario }}) + env: + # Pinned rather than "newest" and "first iPhone the image lists", so a + # Maestro release or a runner image update can't change the job + # between two runs of the same commit. + MAESTRO_VERSION: '2.11.0' + SIM_DEVICE: 'iPhone 16 Pro' + SIM_RUNTIME: 'com.apple.CoreSimulator.SimRuntime.iOS-18-5' strategy: fail-fast: false matrix: @@ -27,23 +34,54 @@ jobs: # setup and build, so every flow that runs against the example's # default config shares the first one, and only flows that need Metro # started with other EXPO_PUBLIC_* values get a job of their own. + # + # layer_page_load_timeout_ms gives the layer's page longer to load + # from live staging than the SDK's 8s default; cmp-timeout-fail-open + # forces every timeout down instead, and the CMP-failure flow never + # gets as far as the layer. include: - scenario: default-config flows: happy-path cancelled-login deny-all consent-survives-relaunch onetrust_app_id: '' gate_timeout_ms: '' + layer_page_load_timeout_ms: '30000' - scenario: onetrust-adapter-failure flows: onetrust-adapter-failure onetrust_app_id: 'e2e-invalid-app-id' gate_timeout_ms: '' + layer_page_load_timeout_ms: '' - scenario: cmp-timeout-fail-open flows: cmp-timeout-fail-open onetrust_app_id: '' gate_timeout_ms: '1' + layer_page_load_timeout_ms: '' steps: - name: Checkout uses: actions/checkout@v4 + # macos-15 defaults xcode-select to 16.4; idb-companion needs a full + # Xcode 26 install (CLT alone isn't enough), which is present on the + # image but not selected by default. Selected first, since the + # simulator boots against it. + - name: Select Xcode 26 + run: sudo xcode-select -s /Applications/Xcode_26.2.app + + # Booted now, without waiting, so the simulator gets through its first + # boot (data migration included) while dependencies install and the + # app builds, rather than inside the flow step. + - name: Boot simulator + run: | + udid=$(xcrun simctl list devices available -j \ + | jq -r --arg rt "$SIM_RUNTIME" --arg name "$SIM_DEVICE" \ + '(.devices[$rt] // []) | map(select(.name == $name)) | .[0].udid // empty') + if [ -z "$udid" ]; then + echo "::error::No available $SIM_DEVICE simulator on $SIM_RUNTIME" + xcrun simctl list devices available + exit 1 + fi + echo "SIM_UDID=$udid" >> "$GITHUB_ENV" + xcrun simctl boot "$udid" + - name: Setup uses: ./.github/actions/setup @@ -53,12 +91,7 @@ jobs: distribution: 'zulu' java-version: '17' - # macos-15 defaults xcode-select to 16.4; idb-companion needs a full - # Xcode 26 install (CLT alone isn't enough), which is present on the - # image but not selected by default. - - name: Select Xcode 26 - run: sudo xcode-select -s /Applications/Xcode_26.2.app - + # Installs MAESTRO_VERSION from the job's env. - name: Install Maestro and idb-companion run: | curl -Ls "https://get.maestro.mobile.dev" | bash @@ -109,6 +142,7 @@ jobs: env: EXPO_PUBLIC_ONETRUST_APP_ID: ${{ matrix.onetrust_app_id }} EXPO_PUBLIC_GATE_TIMEOUT_MS: ${{ matrix.gate_timeout_ms }} + EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS: ${{ matrix.layer_page_load_timeout_ms }} # A Debug build fetches its JS bundle from Metro at app-launch time, # so the env vars that pick the per-flow scenario have to be set on # the Metro process, not on the (now scenario-independent) build @@ -138,35 +172,102 @@ jobs: # Maestro's own XCTest-runner driver bootstrap ("iOS driver not # ready in time") can take longer than its default on a # GitHub-hosted runner than on a local machine. - MAESTRO_DRIVER_STARTUP_TIMEOUT: '240000' + MAESTRO_DRIVER_STARTUP_TIMEOUT: '300000' FLOWS: ${{ matrix.flows }} + E2E_STAGING_EMAIL: ${{ secrets.E2E_STAGING_EMAIL }} + E2E_STAGING_PASSWORD: ${{ secrets.E2E_STAGING_PASSWORD }} run: | - DEVICE_UDID=$(xcrun simctl list devices available -j | jq -r '[.devices[] | .[] | select(.name | test("iPhone"))][0].udid') - xcrun simctl boot "$DEVICE_UDID" - xcrun simctl bootstatus "$DEVICE_UDID" -b - xcrun simctl install "$DEVICE_UDID" ios/build/Build/Products/Debug-iphonesimulator/ContentpassOnetrustExample.app - flow_files=() - for flow in $FLOWS; do - flow_files+=("e2e/flows/$flow.yaml") - done - # One invocation runs every flow even when an earlier one fails, - # and fails the step if any did. - maestro test "${flow_files[@]}" \ - -e E2E_STAGING_EMAIL="${{ secrets.E2E_STAGING_EMAIL }}" \ - -e E2E_STAGING_PASSWORD="${{ secrets.E2E_STAGING_PASSWORD }}" + xcrun simctl bootstatus "$SIM_UDID" -b + xcrun simctl install "$SIM_UDID" ios/build/Build/Products/Debug-iphonesimulator/ContentpassOnetrustExample.app + + # One invocation runs every flow it's given, even after one fails, + # and records in a JUnit report which ones did. + run_flows() { + local report=$1 + shift + local files=() + for flow in "$@"; do + files+=("e2e/flows/$flow.yaml") + done + maestro test --udid "$SIM_UDID" --format junit --output "$report" \ + "${files[@]}" \ + -e E2E_STAGING_EMAIL="$E2E_STAGING_EMAIL" \ + -e E2E_STAGING_PASSWORD="$E2E_STAGING_PASSWORD" + } + + failed_flows() { + python3 - "$1" <<'PY' + import sys + import xml.etree.ElementTree as ET + + for case in ET.parse(sys.argv[1]).getroot().iter("testcase"): + if case.find("failure") is not None or case.find("error") is not None: + print(case.get("name")) + PY + } + + if run_flows /tmp/maestro-attempt-1.xml $FLOWS; then + exit 0 + fi + + # Every flow starts from a clean slate, so a failed one can simply + # run again. Without a report, Maestro never got as far as a flow + # (most often its iOS driver not starting), so all of them do. + retry="" + if [ -s /tmp/maestro-attempt-1.xml ]; then + retry=$(failed_flows /tmp/maestro-attempt-1.xml | xargs) + fi + if [ -n "$retry" ]; then + echo "::warning title=E2E retry::Retrying failed flows: $retry" + else + retry=$FLOWS + echo "::warning title=E2E retry::Maestro stopped before reporting any flow; retrying all of them" + fi + echo "E2E_RETRIED=true" >> "$GITHUB_ENV" + + run_flows /tmp/maestro-attempt-2.xml $retry + echo "::warning title=Flaky E2E run::Passed on retry: $retry" # Metro's output is the app's JS console, including the SDK's # console.debug lines (every message from the layer's WebView, the # gate's and the CMP adapter's progress), none of which reach the # simulator log that Maestro collects. + # + # Also collected when a retry passed, so a flake still leaves its + # evidence behind instead of disappearing into a green run. - name: Add Metro log to debug output - if: failure() + if: failure() || env.E2E_RETRIED == 'true' run: | mkdir -p ~/.maestro/tests cp /tmp/metro.log ~/.maestro/tests/ 2>/dev/null || true + # Maestro writes the staging account's credentials in plain text into + # its debug output (commands.json, maestro.log). GitHub masks secrets + # in job logs but not inside uploaded files, and this public repo's + # artifacts can be downloaded by anyone, so both are scrubbed before + # the upload. A file that still holds one afterwards (a screenshot, + # say) is dropped rather than uploaded. + - name: Redact credentials from debug output + if: failure() || env.E2E_RETRIED == 'true' + env: + E2E_STAGING_EMAIL: ${{ secrets.E2E_STAGING_EMAIL }} + E2E_STAGING_PASSWORD: ${{ secrets.E2E_STAGING_PASSWORD }} + run: | + dir=~/.maestro/tests + [ -d "$dir" ] || exit 0 + for var in E2E_STAGING_EMAIL E2E_STAGING_PASSWORD; do + secret=${!var} + [ -n "$secret" ] || continue + SECRET=$secret find "$dir" -type f ! -name '*.png' \ + -exec perl -pi -e 's/\Q$ENV{SECRET}\E/[redacted]/g' {} + + grep -rlF -- "$secret" "$dir" | while read -r f; do + echo "Dropping $f, which still contains $var" + rm -f "$f" + done + done + - name: Upload Maestro debug output - if: failure() + if: failure() || env.E2E_RETRIED == 'true' uses: actions/upload-artifact@v4 with: name: maestro-debug-${{ matrix.scenario }} diff --git a/examples/onetrust/e2e/README.md b/examples/onetrust/e2e/README.md index d89ee2e..af5b930 100644 --- a/examples/onetrust/e2e/README.md +++ b/examples/onetrust/e2e/README.md @@ -69,6 +69,24 @@ Every flow starts with `clearKeychain` and a `clearState` launch, since a job runs its flows one after another on the same simulator, and the login tokens live in the keychain, which `clearState` leaves alone. +## In CI + +The flows run against live staging on hosted runners, so the job is built +to tolerate what fails there without the code being at fault: + +- A failed flow runs once more, and so does every flow if Maestro stops + before reporting any (its iOS driver not starting, most often). A pass on + retry turns the job green with a warning annotation naming the flows, and + still uploads the debug output, Metro's log included, so the flake can be + looked into. +- The `default-config` job starts Metro with + `EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS=30000`, so a slow page from + staging doesn't fail the gate open halfway through a flow; the SDK's + default is 8s. +- The simulator boots at the start of the job and settles while the app + builds. It is pinned (`SIM_DEVICE`, `SIM_RUNTIME`), as is Maestro + (`MAESTRO_VERSION`), in the workflow's job `env`. + ## Running a flow ```sh diff --git a/examples/onetrust/src/Config.ts b/examples/onetrust/src/Config.ts index d46975c..21da959 100644 --- a/examples/onetrust/src/Config.ts +++ b/examples/onetrust/src/Config.ts @@ -25,6 +25,14 @@ const forcedTimeoutMs = process.env.EXPO_PUBLIC_GATE_TIMEOUT_MS ? Number(process.env.EXPO_PUBLIC_GATE_TIMEOUT_MS) : undefined; +// Lets an E2E run against live staging give the layer's page longer to load +// than the SDK's default, so a slow response doesn't fail the gate open +// halfway through a flow that is testing something else. +const layerPageLoadTimeoutMs = process.env + .EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS + ? Number(process.env.EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS) + : undefined; + export const GATE_TIMEOUTS: ContentpassGateTimeouts | undefined = forcedTimeoutMs ? { @@ -37,4 +45,6 @@ export const GATE_TIMEOUTS: ContentpassGateTimeouts | undefined = layerPageLoadTimeoutMs: forcedTimeoutMs, layerReadyTimeoutMs: forcedTimeoutMs, } - : undefined; + : layerPageLoadTimeoutMs + ? { layerPageLoadTimeoutMs } + : undefined;