Make the iOS E2E job tolerate staging and simulator flakiness

.github/workflows/e2e-ios.yml

@@ -20,30 +20,68 @@ jobs:
     # macOS Sequoia+ and a full Xcode 26 install, neither available on 14.
     runs-on: macos-15
     name: e2e-ios (${{ matrix.scenario }})
+    env:
+      # Pinned rather than "newest" and "first iPhone the image lists", so a
+      # Maestro release or a runner image update can't change the job
+      # between two runs of the same commit.
+      MAESTRO_VERSION: '2.11.0'
+      SIM_DEVICE: 'iPhone 16 Pro'
+      SIM_RUNTIME: 'com.apple.CoreSimulator.SimRuntime.iOS-18-5'
     strategy:
       fail-fast: false
       matrix:
         # One job per Metro configuration, not per flow: a job is mostly
         # setup and build, so every flow that runs against the example's
         # default config shares the first one, and only flows that need Metro
         # started with other EXPO_PUBLIC_* values get a job of their own.
+        #
+        # layer_page_load_timeout_ms gives the layer's page longer to load
+        # from live staging than the SDK's 8s default; cmp-timeout-fail-open
+        # forces every timeout down instead, and the CMP-failure flow never
+        # gets as far as the layer.
         include:
           - scenario: default-config
             flows: happy-path cancelled-login deny-all consent-survives-relaunch
             onetrust_app_id: ''
             gate_timeout_ms: ''
+            layer_page_load_timeout_ms: '30000'
           - scenario: onetrust-adapter-failure
             flows: onetrust-adapter-failure
             onetrust_app_id: 'e2e-invalid-app-id'
             gate_timeout_ms: ''
+            layer_page_load_timeout_ms: ''
           - scenario: cmp-timeout-fail-open
             flows: cmp-timeout-fail-open
             onetrust_app_id: ''
             gate_timeout_ms: '1'
+            layer_page_load_timeout_ms: ''
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
+      # macos-15 defaults xcode-select to 16.4; idb-companion needs a full
+      # Xcode 26 install (CLT alone isn't enough), which is present on the
+      # image but not selected by default. Selected first, since the
+      # simulator boots against it.
+      - name: Select Xcode 26
+        run: sudo xcode-select -s /Applications/Xcode_26.2.app
+
+      # Booted now, without waiting, so the simulator gets through its first
+      # boot (data migration included) while dependencies install and the
+      # app builds, rather than inside the flow step.
+      - name: Boot simulator
+        run: |
+          udid=$(xcrun simctl list devices available -j \
+            | jq -r --arg rt "$SIM_RUNTIME" --arg name "$SIM_DEVICE" \
+              '(.devices[$rt] // []) | map(select(.name == $name)) | .[0].udid // empty')
+          if [ -z "$udid" ]; then
+            echo "::error::No available $SIM_DEVICE simulator on $SIM_RUNTIME"
+            xcrun simctl list devices available
+            exit 1
+          fi
+          echo "SIM_UDID=$udid" >> "$GITHUB_ENV"
+          xcrun simctl boot "$udid"
+
       - name: Setup
         uses: ./.github/actions/setup
 
@@ -53,12 +91,7 @@ jobs:
           distribution: 'zulu'
           java-version: '17'
 
-      # macos-15 defaults xcode-select to 16.4; idb-companion needs a full
-      # Xcode 26 install (CLT alone isn't enough), which is present on the
-      # image but not selected by default.
-      - name: Select Xcode 26
-        run: sudo xcode-select -s /Applications/Xcode_26.2.app
-
+      # Installs MAESTRO_VERSION from the job's env.
       - name: Install Maestro and idb-companion
         run: |
           curl -Ls "https://get.maestro.mobile.dev" | bash
@@ -109,6 +142,7 @@ jobs:
         env:
           EXPO_PUBLIC_ONETRUST_APP_ID: ${{ matrix.onetrust_app_id }}
           EXPO_PUBLIC_GATE_TIMEOUT_MS: ${{ matrix.gate_timeout_ms }}
+          EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS: ${{ matrix.layer_page_load_timeout_ms }}
         # A Debug build fetches its JS bundle from Metro at app-launch time,
         # so the env vars that pick the per-flow scenario have to be set on
         # the Metro process, not on the (now scenario-independent) build
@@ -138,35 +172,102 @@ jobs:
           # Maestro's own XCTest-runner driver bootstrap ("iOS driver not
           # ready in time") can take longer than its default on a
           # GitHub-hosted runner than on a local machine.
-          MAESTRO_DRIVER_STARTUP_TIMEOUT: '240000'
+          MAESTRO_DRIVER_STARTUP_TIMEOUT: '300000'
           FLOWS: ${{ matrix.flows }}
+          E2E_STAGING_EMAIL: ${{ secrets.E2E_STAGING_EMAIL }}
+          E2E_STAGING_PASSWORD: ${{ secrets.E2E_STAGING_PASSWORD }}
         run: |
-          DEVICE_UDID=$(xcrun simctl list devices available -j | jq -r '[.devices[] | .[] | select(.name | test("iPhone"))][0].udid')
-          xcrun simctl boot "$DEVICE_UDID"
-          xcrun simctl bootstatus "$DEVICE_UDID" -b
-          xcrun simctl install "$DEVICE_UDID" ios/build/Build/Products/Debug-iphonesimulator/ContentpassOnetrustExample.app
-          flow_files=()
-          for flow in $FLOWS; do
-            flow_files+=("e2e/flows/$flow.yaml")
-          done
-          # One invocation runs every flow even when an earlier one fails,
-          # and fails the step if any did.
-          maestro test "${flow_files[@]}" \
-            -e E2E_STAGING_EMAIL="${{ secrets.E2E_STAGING_EMAIL }}" \
-            -e E2E_STAGING_PASSWORD="${{ secrets.E2E_STAGING_PASSWORD }}"
+          xcrun simctl bootstatus "$SIM_UDID" -b
+          xcrun simctl install "$SIM_UDID" ios/build/Build/Products/Debug-iphonesimulator/ContentpassOnetrustExample.app
+
+          # One invocation runs every flow it's given, even after one fails,
+          # and records in a JUnit report which ones did.
+          run_flows() {
+            local report=$1
+            shift
+            local files=()
+            for flow in "$@"; do
+              files+=("e2e/flows/$flow.yaml")
+            done
+            maestro test --udid "$SIM_UDID" --format junit --output "$report" \
+              "${files[@]}" \
+              -e E2E_STAGING_EMAIL="$E2E_STAGING_EMAIL" \
+              -e E2E_STAGING_PASSWORD="$E2E_STAGING_PASSWORD"
+          }
+
+          failed_flows() {
+            python3 - "$1" <<'PY'
+          import sys
+          import xml.etree.ElementTree as ET
+
+          for case in ET.parse(sys.argv[1]).getroot().iter("testcase"):
+              if case.find("failure") is not None or case.find("error") is not None:
+                  print(case.get("name"))
+          PY
+          }
+
+          if run_flows /tmp/maestro-attempt-1.xml $FLOWS; then
+            exit 0
+          fi
+
+          # Every flow starts from a clean slate, so a failed one can simply
+          # run again. Without a report, Maestro never got as far as a flow
+          # (most often its iOS driver not starting), so all of them do.
+          retry=""
+          if [ -s /tmp/maestro-attempt-1.xml ]; then
+            retry=$(failed_flows /tmp/maestro-attempt-1.xml | xargs)
+          fi
+          if [ -n "$retry" ]; then
+            echo "::warning title=E2E retry::Retrying failed flows: $retry"
+          else
+            retry=$FLOWS
+            echo "::warning title=E2E retry::Maestro stopped before reporting any flow; retrying all of them"
+          fi
+          echo "E2E_RETRIED=true" >> "$GITHUB_ENV"
+
+          run_flows /tmp/maestro-attempt-2.xml $retry
+          echo "::warning title=Flaky E2E run::Passed on retry: $retry"
 
       # Metro's output is the app's JS console, including the SDK's
       # console.debug lines (every message from the layer's WebView, the
       # gate's and the CMP adapter's progress), none of which reach the
       # simulator log that Maestro collects.
+      #
+      # Also collected when a retry passed, so a flake still leaves its
+      # evidence behind instead of disappearing into a green run.
       - name: Add Metro log to debug output
-        if: failure()
+        if: failure() || env.E2E_RETRIED == 'true'
         run: |
           mkdir -p ~/.maestro/tests
           cp /tmp/metro.log ~/.maestro/tests/ 2>/dev/null || true
 
+      # Maestro writes the staging account's credentials in plain text into
+      # its debug output (commands.json, maestro.log). GitHub masks secrets
+      # in job logs but not inside uploaded files, and this public repo's
+      # artifacts can be downloaded by anyone, so both are scrubbed before
+      # the upload. A file that still holds one afterwards (a screenshot,
+      # say) is dropped rather than uploaded.
+      - name: Redact credentials from debug output
+        if: failure() || env.E2E_RETRIED == 'true'
+        env:
+          E2E_STAGING_EMAIL: ${{ secrets.E2E_STAGING_EMAIL }}
+          E2E_STAGING_PASSWORD: ${{ secrets.E2E_STAGING_PASSWORD }}
+        run: |
+          dir=~/.maestro/tests
+          [ -d "$dir" ] || exit 0
+          for var in E2E_STAGING_EMAIL E2E_STAGING_PASSWORD; do
+            secret=${!var}
+            [ -n "$secret" ] || continue
+            SECRET=$secret find "$dir" -type f ! -name '*.png' \
+              -exec perl -pi -e 's/\Q$ENV{SECRET}\E/[redacted]/g' {} +
+            grep -rlF -- "$secret" "$dir" | while read -r f; do
+              echo "Dropping $f, which still contains $var"
+              rm -f "$f"
+            done
+          done
+
       - name: Upload Maestro debug output
-        if: failure()
+        if: failure() || env.E2E_RETRIED == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: maestro-debug-${{ matrix.scenario }}

examples/onetrust/e2e/README.md

@@ -69,6 +69,24 @@ Every flow starts with `clearKeychain` and a `clearState` launch, since a
 job runs its flows one after another on the same simulator, and the login
 tokens live in the keychain, which `clearState` leaves alone.
 
+## In CI
+
+The flows run against live staging on hosted runners, so the job is built
+to tolerate what fails there without the code being at fault:
+
+- A failed flow runs once more, and so does every flow if Maestro stops
+  before reporting any (its iOS driver not starting, most often). A pass on
+  retry turns the job green with a warning annotation naming the flows, and
+  still uploads the debug output, Metro's log included, so the flake can be
+  looked into.
+- The `default-config` job starts Metro with
+  `EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS=30000`, so a slow page from
+  staging doesn't fail the gate open halfway through a flow; the SDK's
+  default is 8s.
+- The simulator boots at the start of the job and settles while the app
+  builds. It is pinned (`SIM_DEVICE`, `SIM_RUNTIME`), as is Maestro
+  (`MAESTRO_VERSION`), in the workflow's job `env`.
+
 ## Running a flow
 
 ```sh

examples/onetrust/src/Config.ts

@@ -25,6 +25,14 @@ const forcedTimeoutMs = process.env.EXPO_PUBLIC_GATE_TIMEOUT_MS
   ? Number(process.env.EXPO_PUBLIC_GATE_TIMEOUT_MS)
   : undefined;
 
+// Lets an E2E run against live staging give the layer's page longer to load
+// than the SDK's default, so a slow response doesn't fail the gate open
+// halfway through a flow that is testing something else.
+const layerPageLoadTimeoutMs = process.env
+  .EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS
+  ? Number(process.env.EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS)
+  : undefined;
+
 export const GATE_TIMEOUTS: ContentpassGateTimeouts | undefined =
   forcedTimeoutMs
     ? {
@@ -37,4 +45,6 @@ export const GATE_TIMEOUTS: ContentpassGateTimeouts | undefined =
         layerPageLoadTimeoutMs: forcedTimeoutMs,
         layerReadyTimeoutMs: forcedTimeoutMs,
       }
-    : undefined;
+    : layerPageLoadTimeoutMs
+      ? { layerPageLoadTimeoutMs }
+      : undefined;