diff --git a/.gitignore b/.gitignore index 132e45778..c4f208bd8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ starter/ios/Runner.xcodeproj/project.pbxproj starter/node_modules starter/package-lock.json +# IntelliJ / Android Studio module files (generated by Melos / IDE) +*.iml + # FVM Version Cache .fvm/ @@ -30,4 +33,10 @@ starter/package-lock.json # Starter app generated files starter/android/app/src/main/java/io/ starter/ios/Runner/GeneratedPluginRegistrant.h -starter/ios/Runner/GeneratedPluginRegistrant.m \ No newline at end of file +starter/ios/Runner/GeneratedPluginRegistrant.m + +# GetStorage local persistence (runtime-generated when running starter) +starter/GetStorage.gs +starter/GetStorage.bak +starter/system.gs +starter/system.bak \ No newline at end of file diff --git a/melos.yaml b/melos.yaml index f773fc720..331f4505d 100644 --- a/melos.yaml +++ b/melos.yaml @@ -15,6 +15,21 @@ ignore: - modules/ensemble_ts_interpreter - "**/build/**" +scripts: + ensemble:test: + description: Run declarative Ensemble YAML tests in starter. + run: cd starter && dart run ensemble_test_runner:ensemble_test + + ensemble:generate-test-schema: + description: Regenerate ensemble_test.schema.json from the step vocabulary. + run: | + cd packages/ensemble_test_runner && dart run tool/generate_schema.dart + + ensemble:generate-step-registry: + description: Regenerate test_step_registry.dart from tool/generate_step_registry.dart. + run: | + cd packages/ensemble_test_runner && dart run tool/generate_step_registry.dart + command: version: updateGitTagRefs: true diff --git a/modules/ensemble/lib/ensemble.dart b/modules/ensemble/lib/ensemble.dart index c4e767dda..87ec01269 100644 --- a/modules/ensemble/lib/ensemble.dart +++ b/modules/ensemble/lib/ensemble.dart @@ -98,6 +98,11 @@ class Ensemble extends WithEnsemble with EnsembleRouteObserver { late FirebaseApp ensembleFirebaseApp; static final Map externalDataContext = {}; + /// Clears [initManagers] singleton state between Flutter widget tests. + static void resetInitManagersForTest() { + _instance._completer = null; + } + /// initialize all the singleton/managers. Note that this function can be /// called multiple times since it's being called inside a widget. /// The actual code block to initialize the managers is guaranteed to run diff --git a/modules/ensemble/lib/ensemble_app.dart b/modules/ensemble/lib/ensemble_app.dart index faff5234c..bd63c7023 100644 --- a/modules/ensemble/lib/ensemble_app.dart +++ b/modules/ensemble/lib/ensemble_app.dart @@ -158,6 +158,7 @@ class EnsembleAppState extends State with WidgetsBindingObserver { bool _hasInternet = true; late final StreamSubscription> _connectivitySubscription; + SemanticsHandle? _testSemanticsHandle; @override void initState() { @@ -194,7 +195,7 @@ class EnsembleAppState extends State with WidgetsBindingObserver { } }); if (EnvConfig().isTestMode) { - SemanticsBinding.instance.ensureSemantics(); + _testSemanticsHandle = SemanticsBinding.instance.ensureSemantics(); } } @@ -244,6 +245,7 @@ class EnsembleAppState extends State with WidgetsBindingObserver { @override void dispose() { _connectivitySubscription.cancel(); + _testSemanticsHandle?.dispose(); WidgetsBinding.instance.removeObserver(this); super.dispose(); } diff --git a/packages/ensemble_test_runner/README.md b/packages/ensemble_test_runner/README.md new file mode 100644 index 000000000..4db178a4d --- /dev/null +++ b/packages/ensemble_test_runner/README.md @@ -0,0 +1,100 @@ +# ensemble_test_runner + +Standalone declarative YAML test runner for Ensemble apps. Wraps the **real** Ensemble runtime (`EnsembleApp`), injects mocks via runtime override hooks, and asserts on rendered UI, navigation, APIs, and storage. + +This package is **not** a dependency of `modules/ensemble`. It is a **dev-only** dependency — not shipped in release builds. + +## Write tests (YAML only) + +Add `*.test.yaml` files under `ensemble/tests/` in your app: + +```yaml +id: hello_home_renders +startScreen: Hello Home +initialState: + storage: + helloApp: + name: + first: John + last: Doe +steps: + - expectVisible: + id: greeting_text +``` + +Each `*.test.yaml` file is **one** test — `id`, `steps`, and **either** `startScreen` **or** `prerequisite` are at the root (no `tests:` array). A test with `prerequisite: ` runs after that test on the **same** app session, applying only `initialState`/`mocks` in-place before executing its steps. + +Widget YAML must set `testId` (or `id`, which maps to the same `ValueKey`). + +### Step vocabulary + +The full official catalog (lifecycle, gestures, API mocks, fixtures, debug, etc.) is in **[STEP_VOCABULARY.md](STEP_VOCABULARY.md)**. + +Machine-readable registry (single source): `lib/vocabulary/test_step_registry.dart`. + +### JSON Schema (editor validation) + +A JSON Schema for `*.test.yaml` lives at [`assets/schema/ensemble_test.schema.json`](assets/schema/ensemble_test.schema.json). It is generated from the step registry: + +```bash +cd packages/ensemble_test_runner && dart run tool/generate_schema.dart +``` + +Or per file at the top of a test: + +```yaml +# yaml-language-server: $schema=... +``` + +## App setup + +1. Add `*.test.yaml` files under `ensemble/tests/`. +2. Configure `definitions.local` in `ensemble/ensemble-config.yaml` (`path`, `appHome`, `i18n.path`). +3. Add `ensemble_test_runner` to `dev_dependencies` (same git `url`/`ref` as your `ensemble:` dependency). +4. Run `flutter pub get`. + +## Run + +From your app directory (e.g. `starter/`): + +```bash +dart run ensemble_test_runner:ensemble_test +``` + +The CLI temporarily bundles `ensemble/tests/` as an asset (if needed), writes `test/ensemble_tests.dart`, runs `flutter test`, then restores your `pubspec.yaml` and removes the generated test file. + +By default, output is quiet: no `pub get` package list, no Flutter test progress lines — `SCREEN TRACKER` navigation logs plus the boxed suite report. Use `--verbose` for full subprocess output (useful when debugging). + +Optional: `--app-dir=` when not running from the app root. + +On success the console prints one consolidated boxed report for the suite: each test id (with YAML path), timing, **start screen** or **prerequisite**, **navigation flow**, and a numbered **step outline**. + +## Package layout + +``` +lib/ + entry/ Flutter test entry (`runEnsembleYamlTests`) + cli/ `dart run ensemble_test_runner:ensemble_test` subprocess runner + runner/ Runtime boot, orchestration, session state + actions/ Step execution + assertions/ expect* handlers + discovery/ Find and plan `*.test.yaml` files + parser/ YAML → models + reporters/ Console report formatting + vocabulary/ Step registry + JSON Schema shapes + models/ Shared data types + mocks/ Mock HTTP provider + test logger +bin/ensemble_test.dart CLI executable +tool/ Schema/registry generators +``` + +## Runtime hooks (in `ensemble` core) + +The runner uses small, optional hooks in the core module — not a package dependency: + +- Test harness applies `EnsembleTestSetup` (storage seeds, env overrides) before `EnsembleApp` mounts +- Test harness installs `MockAPIProvider` on `EnsembleConfig.apiProviders['http']` +- Test mode via `--dart-define=testmode=true` (added automatically by the CLI) +- Navigation flow for `expectVisited` is recorded in the test runner via `ScreenTracker.onScreenChange` + +`EnsembleTestHarness` runs storage init inside `tester.runAsync()` so `GetStorage` can finish under the widget test binding. diff --git a/packages/ensemble_test_runner/STEP_VOCABULARY.md b/packages/ensemble_test_runner/STEP_VOCABULARY.md new file mode 100644 index 000000000..2a3643612 --- /dev/null +++ b/packages/ensemble_test_runner/STEP_VOCABULARY.md @@ -0,0 +1,87 @@ +# Ensemble declarative test step vocabulary + +Official step catalog for `ensemble/tests/*.test.yaml`. Every step in [`TestStepRegistry.entries`](lib/vocabulary/test_step_registry.dart) is **implemented** (tier `core` or `extended`). + +| Tier | Meaning | +|------|---------| +| **core** | Primary step name | +| **extended** | Alias or secondary API (`wait` → `pump`, `expectScreen` → `expectNavigateTo`) | + +## Quick reference + +### Lifecycle +`openScreen`, `reloadScreen`, `restartApp`, `resetAppState`, `trigger`, `launchApp` + +### Interactions +`tap`, `doubleTap`, `longPress`, `enterText`, `clearText`, `replaceText`, `submitText`, `focus`, `unfocus` + +### Form controls +`select`, `selectIndex`, `check`, `uncheck`, `toggle`, `setSlider`, `chooseDate`, `chooseTime` + +### Gestures +`scroll`, `scrollUntilVisible`, `swipe`, `drag`, `pullToRefresh` + +### Wait / sync +`wait` (alias `pump`), `pump`, `settle`, `waitFor`, `waitForText`, `waitForGone`, `waitForApi`, `waitForNavigation`, `waitUntil` + +### UI assertions +`expectVisible`, `expectNotVisible`, `expectExists`, `expectNotExists`, `expectText`, `expectNoText`, `expectTextContains`, `expectEnabled`, `expectDisabled` + +### Value / list +`expectValue`, `expectChecked`, `expectSelected`, `expectProperty`, `expectStyle`, `expectCount`, `expectListCount`, `expectListContains`, `expectListItem`, `expectEmpty`, `expectNotEmpty` + +### Navigation +`expectScreen` (alias), `expectNavigateTo`, `expectVisited`, `expectNotVisited`, `expectBackStack`, `expectCanGoBack`, `goBack` + +### API mock / assert +`mockApi`, `mockApiError`, `mockApiFromFixture`, `mockApiException`, `mockTimeout`, `mockNetworkOffline`, `mockNetworkOnline`, `resetApiCalls`, `clearApiMocks`, `expectApiCalled`, `expectApiNotCalled`, `expectApiRequest`, `expectApiRequestContains`, `expectApiHeader`, `expectApiCallOrder`, `expectLastApiCall`, `logApiCalls` + +### State / storage / runtime +`setState`, `expectState`, `expectStateContains`, `expectStateExists`, `expectStateNotExists`, `resetState`, `setStorage`, `expectStorage`, `removeStorage`, `clearStorage`, `setEnv`, `setAuth`, `clearAuth`, `setPermission`, `setDevice`, `setLocale`, `setTheme` + +### Scripts / fixtures / debug / quality +`runScript`, `expectScript`, `expectScriptResult`, `expectConsoleLog`, `loadFixture`, `setStateFromFixture`, `expectMatchesFixture`, `logState`, `logStorage`, `screenshot`, `dumpTree`, `expectNoConsoleErrors`, `expectNoRenderErrors`, `expectError`, `expectNoErrors`, `expectAccessible`, `expectSemanticsLabel`, `expectNoOverflow` + +### Control flow +`group`, `repeat`, `optional`, `ifVisible` + +## Example + +```yaml +id: login_flow +startScreen: Login +steps: + - mockApi: + name: login + response: + statusCode: 200 + body: + token: test + - enterText: + id: email_field + value: user@test.com + - tap: + id: login_button + - waitForNavigation: + screen: Home + - expectVisible: + id: welcome_text +``` + +## JSON Schema + +Editor validation: [`assets/schema/ensemble_test.schema.json`](assets/schema/ensemble_test.schema.json) (regenerate with `dart run tool/generate_schema.dart`). Arg shapes come from [`TestStepArgKind`](lib/vocabulary/test_step_arg_kind.dart) on each [`TestStepRegistryEntry`](lib/vocabulary/test_step_registry.dart). + +Each `*.test.yaml` file is a single test case and must provide **exactly one** of: + +- `startScreen` — cold-starts the app on the given screen and runs steps +- `prerequisite` — ID of another test that must run first; the runner reuses the same app session, applies `initialState`/`mocks` in-place, and then runs this test's steps only + +When multiple tests declare `prerequisite` chains, the runner discovers all YAML files, builds a dependency graph by `id`/`prerequisite`, and executes tests once each in topological order. + +## Adding a step + +1. Add one row to [`tool/generate_step_registry.dart`](tool/generate_step_registry.dart) (`desc` + optional `example` override; defaults come from `defaultExampleForArg`) and run `dart run tool/generate_step_registry.dart`. +2. If needed, add a variant to [`TestStepArgKind`](lib/vocabulary/test_step_arg_kind.dart) and its `jsonSchema` switch. +3. Implement in [`TestStepExecutor`](lib/actions/test_step_executor.dart) and/or [`ExtendedStepHandlers`](lib/actions/extended_step_handlers.dart). +4. Run `dart run tool/generate_schema.dart` and document here. diff --git a/packages/ensemble_test_runner/assets/schema/ensemble_test.schema.json b/packages/ensemble_test_runner/assets/schema/ensemble_test.schema.json new file mode 100644 index 000000000..68c3ff372 --- /dev/null +++ b/packages/ensemble_test_runner/assets/schema/ensemble_test.schema.json @@ -0,0 +1,6125 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://ensembleui.com/ensemble_test.schema.json", + "title": "Ensemble declarative test file", + "description": "Schema for ensemble/tests/*.test.yaml — one test per file; see packages/ensemble_test_runner/STEP_VOCABULARY.md", + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique test identifier" + }, + "type": { + "type": "string" + }, + "startScreen": { + "type": "string", + "minLength": 1, + "description": "Ensemble screen name or id to load first" + }, + "prerequisite": { + "type": "string", + "minLength": 1, + "description": "ID of another test that must run before this one in the same app session" + }, + "initialState": { + "$ref": "#/$defs/initialState" + }, + "mocks": { + "$ref": "#/$defs/mocks" + }, + "steps": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/step" + } + } + }, + "required": [ + "id", + "steps" + ], + "oneOf": [ + { + "required": [ + "startScreen" + ], + "not": { + "required": [ + "prerequisite" + ] + } + }, + { + "required": [ + "prerequisite" + ], + "not": { + "required": [ + "startScreen" + ] + } + } + ], + "$defs": { + "mockResponse": { + "type": "object", + "additionalProperties": false, + "properties": { + "statusCode": { + "type": "integer" + }, + "body": true, + "headers": { + "type": "object", + "additionalProperties": true + } + } + }, + "mockApiEntry": { + "type": "object", + "additionalProperties": false, + "properties": { + "response": { + "$ref": "#/$defs/mockResponse" + }, + "delayMs": { + "type": "integer" + } + }, + "required": [ + "response" + ] + }, + "initialState": { + "type": "object", + "additionalProperties": false, + "properties": { + "storage": { + "type": "object", + "additionalProperties": true + }, + "env": { + "type": "object", + "additionalProperties": true + } + } + }, + "mocks": { + "type": "object", + "additionalProperties": false, + "properties": { + "apis": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/mockApiEntry" + } + } + } + }, + "args_openScreen": { + "type": "object", + "properties": { + "screen": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "openScreen", + "description": "Navigate to another screen by name or id mid-test", + "examples": [ + { + "screen": "Home" + } + ] + }, + "args_reloadScreen": { + "type": "object", + "additionalProperties": false, + "title": "reloadScreen", + "description": "Reload the current screen (same as re-opening it)", + "examples": [ + {} + ] + }, + "args_restartApp": { + "type": "object", + "additionalProperties": false, + "title": "restartApp", + "description": "Reset runtime and reopen the test case start screen", + "examples": [ + {} + ] + }, + "args_resetAppState": { + "type": "object", + "additionalProperties": false, + "title": "resetAppState", + "description": "Clear screen tracker, API call log, and public storage", + "examples": [ + {} + ] + }, + "args_trigger": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "onLoad", + "onTap", + "onLongPress" + ] + }, + "id": { + "type": "string" + } + }, + "required": [ + "action" + ], + "additionalProperties": false, + "title": "trigger", + "description": "Fire a widget action (onLoad, onTap, onLongPress) by testId", + "examples": [ + { + "action": "onTap", + "id": "submit_button" + } + ] + }, + "args_launchApp": { + "type": "object", + "additionalProperties": false, + "title": "launchApp", + "description": "Alias for restartApp — bootstrap from startScreen again", + "examples": [ + {} + ] + }, + "args_tap": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "tap", + "description": "Tap a widget by testId (ValueKey)", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_doubleTap": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "doubleTap", + "description": "Double-tap a widget by testId", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_longPress": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "longPress", + "description": "Long-press a widget by testId", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_enterText": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "value": true, + "submit": { + "type": "boolean" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "enterText", + "description": "Type text into an input field by testId", + "examples": [ + { + "id": "email_field", + "value": "user@test.com" + } + ] + }, + "args_clearText": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "clearText", + "description": "Clear text in an input field by testId", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_replaceText": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "value": true, + "submit": { + "type": "boolean" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "replaceText", + "description": "Replace the full contents of an input field", + "examples": [ + { + "id": "email_field", + "value": "user@test.com" + } + ] + }, + "args_submitText": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "submitText", + "description": "Submit an input field (TextInputAction.done)", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_focus": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "focus", + "description": "Focus an input field by testId", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_unfocus": { + "type": "object", + "additionalProperties": false, + "title": "unfocus", + "description": "Remove focus from the current field", + "examples": [ + {} + ] + }, + "args_select": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "id", + "value" + ], + "additionalProperties": false, + "title": "select", + "description": "Open a dropdown and choose an option by visible label", + "examples": [ + { + "id": "country_dropdown", + "value": "USA" + } + ] + }, + "args_selectIndex": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "index": { + "type": "integer" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "selectIndex", + "description": "Open a dropdown and choose the option at index", + "examples": [ + { + "id": "country_dropdown", + "index": 0 + } + ] + }, + "args_check": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "check", + "description": "Check a checkbox or toggle by testId", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_uncheck": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "uncheck", + "description": "Uncheck a checkbox by testId if currently checked", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_toggle": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "toggle", + "description": "Tap to toggle a switch or checkbox by testId", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_setSlider": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "value": { + "type": "number" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "setSlider", + "description": "Move a slider under testId to a normalized value (0–1)", + "examples": [ + { + "id": "volume_slider", + "value": 0.5 + } + ] + }, + "args_chooseDate": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "id", + "value" + ], + "additionalProperties": false, + "title": "chooseDate", + "description": "Set a date field by testId to the given value string", + "examples": [ + { + "id": "birth_date", + "value": "2024-01-15" + } + ] + }, + "args_chooseTime": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "id", + "value" + ], + "additionalProperties": false, + "title": "chooseTime", + "description": "Set a time field by testId to the given value string", + "examples": [ + { + "id": "birth_date", + "value": "2024-01-15" + } + ] + }, + "args_scroll": { + "type": "object", + "properties": { + "delta": { + "type": "integer" + } + }, + "additionalProperties": false, + "title": "scroll", + "description": "Drag the first Scrollable by delta pixels", + "examples": [ + { + "delta": 300 + } + ] + }, + "args_scrollUntilVisible": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "scrollUntilVisible", + "description": "Scroll until a widget with testId is visible", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_swipe": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": [ + "left", + "right", + "up", + "down" + ] + }, + "id": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "swipe", + "description": "Swipe on a scrollable or widget (direction: left/right/up/down)", + "examples": [ + { + "direction": "left", + "id": "carousel" + } + ] + }, + "args_drag": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "dx": { + "type": "number" + }, + "dy": { + "type": "number" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "drag", + "description": "Drag a widget by testId by dx/dy offset", + "examples": [ + { + "id": "handle", + "dx": 50, + "dy": 0 + } + ] + }, + "args_pullToRefresh": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "pullToRefresh", + "description": "Pull down on a scrollable to trigger refresh", + "examples": [ + { + "id": "scroll_view" + } + ] + }, + "args_wait": { + "type": "object", + "properties": { + "durationMs": { + "type": "integer" + } + }, + "additionalProperties": false, + "title": "wait", + "description": "Alias for pump — advance frame clock by durationMs", + "examples": [ + { + "durationMs": 100 + } + ] + }, + "args_pump": { + "type": "object", + "properties": { + "durationMs": { + "type": "integer" + } + }, + "additionalProperties": false, + "title": "pump", + "description": "Advance the Flutter frame clock by durationMs", + "examples": [ + { + "durationMs": 100 + } + ] + }, + "args_settle": { + "type": "object", + "properties": { + "timeoutMs": { + "type": "integer" + } + }, + "additionalProperties": false, + "title": "settle", + "description": "Run pumpAndSettle until idle or timeout", + "examples": [ + { + "timeoutMs": 5000 + } + ] + }, + "args_waitFor": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "text": { + "type": "string" + }, + "timeoutMs": { + "type": "integer" + } + }, + "additionalProperties": false, + "title": "waitFor", + "description": "Poll until a widget id and/or text appears", + "examples": [ + { + "id": "loading_spinner", + "timeoutMs": 5000 + } + ] + }, + "args_waitForText": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "text": { + "type": "string" + }, + "timeoutMs": { + "type": "integer" + } + }, + "additionalProperties": false, + "title": "waitForText", + "description": "Poll until the given text appears on screen", + "examples": [ + { + "id": "loading_spinner", + "timeoutMs": 5000 + } + ] + }, + "args_waitForGone": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "timeoutMs": { + "type": "integer" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "waitForGone", + "description": "Poll until a widget with testId is removed from the tree", + "examples": [ + { + "id": "loading_spinner", + "timeoutMs": 5000 + } + ] + }, + "args_waitForApi": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "times": { + "type": "integer" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "waitForApi", + "description": "Poll until a mocked API is called N times", + "examples": [ + { + "name": "login", + "times": 1 + } + ] + }, + "args_waitForNavigation": { + "type": "object", + "properties": { + "screen": { + "type": "string" + }, + "timeoutMs": { + "type": "integer" + } + }, + "required": [ + "screen" + ], + "additionalProperties": false, + "title": "waitForNavigation", + "description": "Poll until the given screen is visible", + "examples": [ + { + "screen": "Home", + "timeoutMs": 5000 + } + ] + }, + "args_waitUntil": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "equals": true, + "state": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "equals": true + }, + "additionalProperties": false + }, + "timeoutMs": { + "type": "integer" + } + }, + "additionalProperties": false, + "title": "waitUntil", + "description": "Poll until app state at path equals expected value", + "examples": [ + { + "path": "user.name", + "equals": "Jane" + } + ] + }, + "args_expectVisible": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectVisible", + "description": "Assert a widget with testId is visible", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_expectNotVisible": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectNotVisible", + "description": "Assert a widget with testId is not visible", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_expectExists": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectExists", + "description": "Assert a widget with testId exists in the tree", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_expectNotExists": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectNotExists", + "description": "Assert no widget with testId exists", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_expectText": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ], + "additionalProperties": false, + "title": "expectText", + "description": "Assert exact text is shown", + "examples": [ + { + "text": "Welcome" + } + ] + }, + "args_expectNoText": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ], + "additionalProperties": false, + "title": "expectNoText", + "description": "Assert text is not shown", + "examples": [ + { + "text": "Welcome" + } + ] + }, + "args_expectTextContains": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ], + "additionalProperties": false, + "title": "expectTextContains", + "description": "Assert some text containing the given substring", + "examples": [ + { + "text": "Welcome" + } + ] + }, + "args_expectEnabled": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectEnabled", + "description": "Assert widget semantics report enabled", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_expectDisabled": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectDisabled", + "description": "Assert widget semantics report disabled", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_expectValue": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "equals": true + }, + "required": [ + "id", + "equals" + ], + "additionalProperties": false, + "title": "expectValue", + "description": "Assert input value equals expected (EditableText/TextField)", + "examples": [ + { + "id": "email_field", + "equals": "user@test.com" + } + ] + }, + "args_expectChecked": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "equals": { + "type": "boolean" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectChecked", + "description": "Assert checkbox checked state matches equals", + "examples": [ + { + "id": "terms_checkbox", + "equals": true + } + ] + }, + "args_expectProperty": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "property": { + "type": "string" + }, + "equals": true + }, + "required": [ + "id", + "equals" + ], + "additionalProperties": false, + "title": "expectProperty", + "description": "Assert a widget property (e.g. label) equals expected", + "examples": [ + { + "id": "title", + "property": "label", + "equals": "Hello" + } + ] + }, + "args_expectStyle": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "property": { + "type": "string" + }, + "equals": true + }, + "required": [ + "id", + "equals" + ], + "additionalProperties": false, + "title": "expectStyle", + "description": "Assert style-related property equals expected", + "examples": [ + { + "id": "title", + "property": "label", + "equals": "Hello" + } + ] + }, + "args_expectSelected": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "equals": { + "type": "boolean" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectSelected", + "description": "Assert selected/checked state matches equals", + "examples": [ + { + "id": "terms_checkbox", + "equals": true + } + ] + }, + "args_expectCount": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "equals": { + "type": "integer" + } + }, + "required": [ + "id", + "equals" + ], + "additionalProperties": false, + "title": "expectCount", + "description": "Assert count of widgets with the same testId", + "examples": [ + { + "id": "badge", + "equals": 2 + } + ] + }, + "args_expectListCount": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "itemId": { + "type": "string" + }, + "equals": { + "type": "integer" + } + }, + "required": [ + "equals" + ], + "additionalProperties": false, + "title": "expectListCount", + "description": "Assert number of list items under a list testId", + "examples": [ + { + "id": "items_list", + "equals": 3 + } + ] + }, + "args_expectListContains": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "text": { + "type": "string" + } + }, + "required": [ + "id", + "text" + ], + "additionalProperties": false, + "title": "expectListContains", + "description": "Assert list contains text", + "examples": [ + { + "id": "items_list", + "text": "Item 1" + } + ] + }, + "args_expectListItem": { + "type": "object", + "properties": { + "itemId": { + "type": "string" + } + }, + "required": [ + "itemId" + ], + "additionalProperties": false, + "title": "expectListItem", + "description": "Assert a list item widget with itemId is visible", + "examples": [ + { + "itemId": "row_0" + } + ] + }, + "args_expectEmpty": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectEmpty", + "description": "Assert a list has zero items", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_expectNotEmpty": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectNotEmpty", + "description": "Assert a list has at least one item", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_expectScreen": { + "type": "object", + "properties": { + "screen": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "required": [ + "screen" + ], + "additionalProperties": false, + "title": "expectScreen", + "description": "Alias for expectNavigateTo — assert current screen", + "examples": [ + { + "screen": "Home" + } + ] + }, + "args_expectNavigateTo": { + "type": "object", + "properties": { + "screen": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "required": [ + "screen" + ], + "additionalProperties": false, + "title": "expectNavigateTo", + "description": "Assert the current visible screen name/id", + "examples": [ + { + "screen": "Home" + } + ] + }, + "args_expectVisited": { + "type": "object", + "properties": { + "screen": { + "type": "string" + } + }, + "required": [ + "screen" + ], + "additionalProperties": false, + "title": "expectVisited", + "description": "Assert a screen appears in navigation history", + "examples": [ + { + "screen": "Login" + } + ] + }, + "args_expectNotVisited": { + "type": "object", + "properties": { + "screen": { + "type": "string" + } + }, + "required": [ + "screen" + ], + "additionalProperties": false, + "title": "expectNotVisited", + "description": "Assert a screen was never visited", + "examples": [ + { + "screen": "Login" + } + ] + }, + "args_expectBackStack": { + "type": "object", + "properties": { + "screens": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + }, + "required": [ + "screens" + ], + "additionalProperties": false, + "title": "expectBackStack", + "description": "Assert navigation history suffix matches screens", + "examples": [ + { + "screens": [ + "Home", + "Details" + ] + } + ] + }, + "args_expectCanGoBack": { + "type": "object", + "properties": { + "equals": { + "type": "boolean" + } + }, + "additionalProperties": false, + "title": "expectCanGoBack", + "description": "Assert whether back navigation is possible", + "examples": [ + { + "equals": true + } + ] + }, + "args_goBack": { + "type": "object", + "additionalProperties": false, + "title": "goBack", + "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", + "examples": [ + {} + ] + }, + "args_mockApi": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "response": { + "$ref": "#/$defs/mockResponse" + }, + "delayMs": { + "type": "integer" + } + }, + "required": [ + "name", + "response" + ], + "additionalProperties": false, + "title": "mockApi", + "description": "Register a mock HTTP API response by API name", + "examples": [ + { + "name": "login", + "response": { + "statusCode": 200, + "body": { + "token": "test-token" + } + } + } + ] + }, + "args_mockApiError": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "statusCode": { + "type": "integer" + }, + "body": true, + "delayMs": { + "type": "integer" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "mockApiError", + "description": "Mock an API to return an error status/body", + "examples": [ + { + "name": "login", + "statusCode": 401, + "body": { + "error": "Unauthorized" + } + } + ] + }, + "args_mockApiFromFixture": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "fixture": { + "type": "string" + }, + "statusCode": { + "type": "integer" + } + }, + "required": [ + "name", + "fixture" + ], + "additionalProperties": false, + "title": "mockApiFromFixture", + "description": "Load mock response body from a JSON fixture asset", + "examples": [ + { + "name": "users", + "fixture": "fixtures/users.json" + } + ] + }, + "args_mockApiException": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "mockApiException", + "description": "Force an API call to throw an exception", + "examples": [ + { + "name": "login", + "message": "Network error" + } + ] + }, + "args_mockTimeout": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "delayMs": { + "type": "integer" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "mockTimeout", + "description": "Mock an API with a long delay (simulate timeout)", + "examples": [ + { + "name": "slow_api", + "delayMs": 60000 + } + ] + }, + "args_mockNetworkOffline": { + "type": "object", + "additionalProperties": false, + "title": "mockNetworkOffline", + "description": "Simulate offline network for API calls", + "examples": [ + {} + ] + }, + "args_mockNetworkOnline": { + "type": "object", + "additionalProperties": false, + "title": "mockNetworkOnline", + "description": "Restore online network for API calls", + "examples": [ + {} + ] + }, + "args_resetApiCalls": { + "type": "object", + "additionalProperties": false, + "title": "resetApiCalls", + "description": "Clear recorded API call history", + "examples": [ + {} + ] + }, + "args_clearApiMocks": { + "type": "object", + "additionalProperties": false, + "title": "clearApiMocks", + "description": "Remove all registered API mocks", + "examples": [ + {} + ] + }, + "args_expectApiCalled": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "times": { + "type": "integer" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "expectApiCalled", + "description": "Assert an API was called an exact number of times", + "examples": [ + { + "name": "login", + "times": 1 + } + ] + }, + "args_expectApiNotCalled": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "times": { + "type": "integer" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "expectApiNotCalled", + "description": "Assert an API was never called", + "examples": [ + { + "name": "login", + "times": 1 + } + ] + }, + "args_expectApiRequest": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "body": true, + "query": true, + "headers": true, + "times": { + "type": "integer" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "expectApiRequest", + "description": "Assert last API request body/query/headers match", + "examples": [ + { + "name": "login", + "body": { + "email": "user@test.com", + "password": "secret" + } + } + ] + }, + "args_expectApiRequestContains": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "body": true, + "query": true, + "headers": true, + "times": { + "type": "integer" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "expectApiRequestContains", + "description": "Assert API request contains partial body/query", + "examples": [ + { + "name": "login", + "body": { + "email": "user@test.com", + "password": "secret" + } + } + ] + }, + "args_expectApiHeader": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "header": { + "type": "string" + }, + "equals": true, + "times": { + "type": "integer" + } + }, + "required": [ + "name", + "header", + "equals" + ], + "additionalProperties": false, + "title": "expectApiHeader", + "description": "Assert an API request header equals expected", + "examples": [ + { + "name": "login", + "header": "Authorization", + "equals": "Bearer test-token" + } + ] + }, + "args_expectApiCallOrder": { + "type": "object", + "properties": { + "names": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + }, + "required": [ + "names" + ], + "additionalProperties": false, + "title": "expectApiCallOrder", + "description": "Assert APIs were called in order", + "examples": [ + { + "names": [ + "auth", + "profile" + ] + } + ] + }, + "args_expectLastApiCall": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "times": { + "type": "integer" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "expectLastApiCall", + "description": "Assert the most recent API call name", + "examples": [ + { + "name": "login", + "times": 1 + } + ] + }, + "args_setState": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "value": true + }, + "required": [ + "path" + ], + "additionalProperties": false, + "title": "setState", + "description": "Set app data-context state at path to value", + "examples": [ + { + "path": "user.name", + "value": "Jane" + } + ] + }, + "args_expectState": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "equals": true, + "contains": true + }, + "required": [ + "path" + ], + "additionalProperties": false, + "title": "expectState", + "description": "Assert app state at path equals expected", + "examples": [ + { + "path": "user.name", + "equals": "Jane" + } + ] + }, + "args_expectStateContains": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "equals": true, + "contains": true + }, + "required": [ + "path" + ], + "additionalProperties": false, + "title": "expectStateContains", + "description": "Assert app state at path contains subset", + "examples": [ + { + "path": "user.name", + "equals": "Jane" + } + ] + }, + "args_expectStateExists": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + }, + "required": [ + "path" + ], + "additionalProperties": false, + "title": "expectStateExists", + "description": "Assert state path resolves without error", + "examples": [ + { + "path": "user.id" + } + ] + }, + "args_expectStateNotExists": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + }, + "required": [ + "path" + ], + "additionalProperties": false, + "title": "expectStateNotExists", + "description": "Assert state path is null or absent", + "examples": [ + { + "path": "user.id" + } + ] + }, + "args_resetState": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "resetState", + "description": "Clear state at path (set to null)", + "examples": [ + { + "path": "cart" + } + ] + }, + "args_setStorage": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "equals": true, + "value": true + }, + "required": [ + "key" + ], + "additionalProperties": false, + "title": "setStorage", + "description": "Write a value to public GetStorage by key", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + }, + "args_expectStorage": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "equals": true, + "value": true + }, + "required": [ + "key" + ], + "additionalProperties": false, + "title": "expectStorage", + "description": "Assert public storage key equals expected", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + }, + "args_removeStorage": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "equals": true, + "value": true + }, + "required": [ + "key" + ], + "additionalProperties": false, + "title": "removeStorage", + "description": "Remove a key from public storage", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + }, + "args_clearStorage": { + "type": "object", + "additionalProperties": false, + "title": "clearStorage", + "description": "Clear all non-encrypted public storage keys", + "examples": [ + {} + ] + }, + "args_setEnv": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "equals": true, + "value": true + }, + "required": [ + "key" + ], + "additionalProperties": false, + "title": "setEnv", + "description": "Override an environment variable for the test", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + }, + "args_setAuth": { + "type": "object", + "properties": { + "user": { + "type": "object", + "additionalProperties": true + } + }, + "required": [ + "user" + ], + "additionalProperties": false, + "title": "setAuth", + "description": "Simulate a signed-in user", + "examples": [ + { + "user": { + "id": "1", + "email": "user@test.com" + } + } + ] + }, + "args_clearAuth": { + "type": "object", + "additionalProperties": false, + "title": "clearAuth", + "description": "Clear the signed-in user", + "examples": [ + {} + ] + }, + "args_setPermission": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false, + "title": "setPermission", + "description": "Set a permission flag for the test runtime", + "examples": [ + { + "name": "camera", + "value": "granted" + } + ] + }, + "args_setDevice": { + "type": "object", + "properties": { + "width": { + "type": "number" + }, + "height": { + "type": "number" + } + }, + "additionalProperties": false, + "title": "setDevice", + "description": "Override viewport physical size (width/height)", + "examples": [ + { + "width": 390, + "height": 844 + } + ] + }, + "args_setLocale": { + "type": "object", + "properties": { + "locale": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "setLocale", + "description": "Set APP_LOCALE environment override", + "examples": [ + { + "locale": "en_US" + } + ] + }, + "args_setTheme": { + "type": "object", + "properties": { + "mode": { + "type": "string" + }, + "theme": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "setTheme", + "description": "Set APP_THEME / theme mode override", + "examples": [ + { + "mode": "dark" + } + ] + }, + "args_runScript": { + "type": "object", + "properties": { + "script": { + "type": "string" + }, + "path": { + "type": "string" + }, + "equals": true + }, + "additionalProperties": false, + "title": "runScript", + "description": "Evaluate a script expression in the data context", + "examples": [ + { + "script": "1 + 1", + "equals": 2 + } + ] + }, + "args_expectScriptResult": { + "type": "object", + "properties": { + "script": { + "type": "string" + }, + "path": { + "type": "string" + }, + "equals": true + }, + "additionalProperties": false, + "title": "expectScriptResult", + "description": "Evaluate script and assert result equals expected", + "examples": [ + { + "script": "1 + 1", + "equals": 2 + } + ] + }, + "args_expectConsoleLog": { + "type": "object", + "properties": { + "contains": { + "type": "string" + } + }, + "required": [ + "contains" + ], + "additionalProperties": false, + "title": "expectConsoleLog", + "description": "Assert a console log line contains text", + "examples": [ + { + "contains": "Screen loaded" + } + ] + }, + "args_group": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/$defs/step" + }, + "minItems": 1 + } + }, + "required": [ + "steps" + ], + "additionalProperties": false, + "title": "group", + "description": "Run nested steps as a named group", + "examples": [ + { + "name": "login_flow", + "steps": [ + { + "tap": { + "id": "login_button" + } + } + ] + } + ] + }, + "args_repeat": { + "type": "object", + "properties": { + "times": { + "type": "integer" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/$defs/step" + }, + "minItems": 1 + } + }, + "required": [ + "times", + "steps" + ], + "additionalProperties": false, + "title": "repeat", + "description": "Repeat nested steps N times", + "examples": [ + { + "times": 3, + "steps": [ + { + "tap": { + "id": "next_button" + } + } + ] + } + ] + }, + "args_optional": { + "type": "object", + "properties": { + "step": { + "$ref": "#/$defs/step" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/$defs/step" + }, + "minItems": 1 + } + }, + "additionalProperties": false, + "title": "optional", + "description": "Run nested steps; swallow failures", + "examples": [ + { + "steps": [ + { + "tap": { + "id": "dismiss_banner" + } + } + ] + } + ] + }, + "args_ifVisible": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "step": { + "$ref": "#/$defs/step" + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/$defs/step" + }, + "minItems": 1 + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "ifVisible", + "description": "Run nested steps only if testId is visible", + "examples": [ + { + "id": "promo_banner", + "steps": [ + { + "tap": { + "id": "close_banner" + } + } + ] + } + ] + }, + "args_logApiCalls": { + "type": "object", + "additionalProperties": false, + "title": "logApiCalls", + "description": "Log all recorded API calls to the test log", + "examples": [ + {} + ] + }, + "args_screenshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "screenshot", + "description": "Capture golden or dump widget tree for debugging", + "examples": [ + { + "name": "home_screen" + } + ] + }, + "args_dumpTree": { + "type": "object", + "additionalProperties": false, + "title": "dumpTree", + "description": "Print the widget tree to the debug console", + "examples": [ + {} + ] + }, + "args_logState": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + }, + "required": [ + "path" + ], + "additionalProperties": false, + "title": "logState", + "description": "Log resolved state at path", + "examples": [ + { + "path": "user.id" + } + ] + }, + "args_logStorage": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "equals": true, + "value": true + }, + "required": [ + "key" + ], + "additionalProperties": false, + "title": "logStorage", + "description": "Log public storage value for key", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + }, + "args_expectNoConsoleErrors": { + "type": "object", + "additionalProperties": false, + "title": "expectNoConsoleErrors", + "description": "Assert no console errors were recorded", + "examples": [ + {} + ] + }, + "args_expectNoRenderErrors": { + "type": "object", + "additionalProperties": false, + "title": "expectNoRenderErrors", + "description": "Assert no Flutter render errors were recorded", + "examples": [ + {} + ] + }, + "args_expectError": { + "type": "object", + "properties": { + "contains": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "expectError", + "description": "Assert a Flutter error was recorded (optional filter)", + "examples": [ + { + "contains": "overflow" + } + ] + }, + "args_expectNoErrors": { + "type": "object", + "additionalProperties": false, + "title": "expectNoErrors", + "description": "Alias for expectNoRenderErrors", + "examples": [ + {} + ] + }, + "args_expectAccessible": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectAccessible", + "description": "Assert widget has accessibility label or value", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_expectSemanticsLabel": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "required": [ + "id", + "label" + ], + "additionalProperties": false, + "title": "expectSemanticsLabel", + "description": "Assert semantics label equals expected", + "examples": [ + { + "id": "submit_button", + "label": "Submit" + } + ] + }, + "args_expectNoOverflow": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "additionalProperties": false, + "title": "expectNoOverflow", + "description": "Assert widget renders without overflow issues", + "examples": [ + { + "id": "my_widget" + } + ] + }, + "args_loadFixture": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "path": { + "type": "string" + }, + "fixture": { + "type": "string" + }, + "statePath": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "loadFixture", + "description": "Load a JSON fixture into the test fixture map", + "examples": [ + { + "fixture": "fixtures/user.json" + } + ] + }, + "args_setStateFromFixture": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "path": { + "type": "string" + }, + "fixture": { + "type": "string" + }, + "statePath": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "setStateFromFixture", + "description": "Apply all keys from a JSON fixture to state", + "examples": [ + { + "fixture": "fixtures/user.json" + } + ] + }, + "args_expectMatchesFixture": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "path": { + "type": "string" + }, + "fixture": { + "type": "string" + }, + "statePath": { + "type": "string" + } + }, + "additionalProperties": false, + "title": "expectMatchesFixture", + "description": "Assert state or path matches a JSON fixture", + "examples": [ + { + "fixture": "fixtures/user.json" + } + ] + }, + "step": { + "oneOf": [ + { + "type": "object", + "title": "openScreen", + "description": "Navigate to another screen by name or id mid-test", + "examples": [ + { + "openScreen": { + "screen": "Home" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "openScreen": { + "$ref": "#/$defs/args_openScreen", + "description": "Navigate to another screen by name or id mid-test", + "examples": [ + { + "screen": "Home" + } + ] + } + }, + "required": [ + "openScreen" + ] + }, + { + "type": "object", + "title": "reloadScreen", + "description": "Reload the current screen (same as re-opening it)", + "examples": [ + { + "reloadScreen": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "reloadScreen": { + "$ref": "#/$defs/args_reloadScreen", + "description": "Reload the current screen (same as re-opening it)", + "examples": [ + {} + ] + } + }, + "required": [ + "reloadScreen" + ] + }, + { + "type": "object", + "title": "restartApp", + "description": "Reset runtime and reopen the test case start screen", + "examples": [ + { + "restartApp": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "restartApp": { + "$ref": "#/$defs/args_restartApp", + "description": "Reset runtime and reopen the test case start screen", + "examples": [ + {} + ] + } + }, + "required": [ + "restartApp" + ] + }, + { + "type": "object", + "title": "resetAppState", + "description": "Clear screen tracker, API call log, and public storage", + "examples": [ + { + "resetAppState": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "resetAppState": { + "$ref": "#/$defs/args_resetAppState", + "description": "Clear screen tracker, API call log, and public storage", + "examples": [ + {} + ] + } + }, + "required": [ + "resetAppState" + ] + }, + { + "type": "object", + "title": "trigger", + "description": "Fire a widget action (onLoad, onTap, onLongPress) by testId", + "examples": [ + { + "trigger": { + "action": "onTap", + "id": "submit_button" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "trigger": { + "$ref": "#/$defs/args_trigger", + "description": "Fire a widget action (onLoad, onTap, onLongPress) by testId", + "examples": [ + { + "action": "onTap", + "id": "submit_button" + } + ] + } + }, + "required": [ + "trigger" + ] + }, + { + "type": "object", + "title": "launchApp", + "description": "Alias for restartApp — bootstrap from startScreen again", + "examples": [ + { + "launchApp": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "launchApp": { + "$ref": "#/$defs/args_launchApp", + "description": "Alias for restartApp — bootstrap from startScreen again", + "examples": [ + {} + ] + } + }, + "required": [ + "launchApp" + ] + }, + { + "type": "object", + "title": "tap", + "description": "Tap a widget by testId (ValueKey)", + "examples": [ + { + "tap": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "tap": { + "$ref": "#/$defs/args_tap", + "description": "Tap a widget by testId (ValueKey)", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "tap" + ] + }, + { + "type": "object", + "title": "doubleTap", + "description": "Double-tap a widget by testId", + "examples": [ + { + "doubleTap": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "doubleTap": { + "$ref": "#/$defs/args_doubleTap", + "description": "Double-tap a widget by testId", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "doubleTap" + ] + }, + { + "type": "object", + "title": "longPress", + "description": "Long-press a widget by testId", + "examples": [ + { + "longPress": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "longPress": { + "$ref": "#/$defs/args_longPress", + "description": "Long-press a widget by testId", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "longPress" + ] + }, + { + "type": "object", + "title": "enterText", + "description": "Type text into an input field by testId", + "examples": [ + { + "enterText": { + "id": "email_field", + "value": "user@test.com" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "enterText": { + "$ref": "#/$defs/args_enterText", + "description": "Type text into an input field by testId", + "examples": [ + { + "id": "email_field", + "value": "user@test.com" + } + ] + } + }, + "required": [ + "enterText" + ] + }, + { + "type": "object", + "title": "clearText", + "description": "Clear text in an input field by testId", + "examples": [ + { + "clearText": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "clearText": { + "$ref": "#/$defs/args_clearText", + "description": "Clear text in an input field by testId", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "clearText" + ] + }, + { + "type": "object", + "title": "replaceText", + "description": "Replace the full contents of an input field", + "examples": [ + { + "replaceText": { + "id": "email_field", + "value": "user@test.com" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "replaceText": { + "$ref": "#/$defs/args_replaceText", + "description": "Replace the full contents of an input field", + "examples": [ + { + "id": "email_field", + "value": "user@test.com" + } + ] + } + }, + "required": [ + "replaceText" + ] + }, + { + "type": "object", + "title": "submitText", + "description": "Submit an input field (TextInputAction.done)", + "examples": [ + { + "submitText": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "submitText": { + "$ref": "#/$defs/args_submitText", + "description": "Submit an input field (TextInputAction.done)", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "submitText" + ] + }, + { + "type": "object", + "title": "focus", + "description": "Focus an input field by testId", + "examples": [ + { + "focus": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "focus": { + "$ref": "#/$defs/args_focus", + "description": "Focus an input field by testId", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "focus" + ] + }, + { + "type": "object", + "title": "unfocus", + "description": "Remove focus from the current field", + "examples": [ + { + "unfocus": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "unfocus": { + "$ref": "#/$defs/args_unfocus", + "description": "Remove focus from the current field", + "examples": [ + {} + ] + } + }, + "required": [ + "unfocus" + ] + }, + { + "type": "object", + "title": "select", + "description": "Open a dropdown and choose an option by visible label", + "examples": [ + { + "select": { + "id": "country_dropdown", + "value": "USA" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "select": { + "$ref": "#/$defs/args_select", + "description": "Open a dropdown and choose an option by visible label", + "examples": [ + { + "id": "country_dropdown", + "value": "USA" + } + ] + } + }, + "required": [ + "select" + ] + }, + { + "type": "object", + "title": "selectIndex", + "description": "Open a dropdown and choose the option at index", + "examples": [ + { + "selectIndex": { + "id": "country_dropdown", + "index": 0 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "selectIndex": { + "$ref": "#/$defs/args_selectIndex", + "description": "Open a dropdown and choose the option at index", + "examples": [ + { + "id": "country_dropdown", + "index": 0 + } + ] + } + }, + "required": [ + "selectIndex" + ] + }, + { + "type": "object", + "title": "check", + "description": "Check a checkbox or toggle by testId", + "examples": [ + { + "check": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "check": { + "$ref": "#/$defs/args_check", + "description": "Check a checkbox or toggle by testId", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "check" + ] + }, + { + "type": "object", + "title": "uncheck", + "description": "Uncheck a checkbox by testId if currently checked", + "examples": [ + { + "uncheck": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "uncheck": { + "$ref": "#/$defs/args_uncheck", + "description": "Uncheck a checkbox by testId if currently checked", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "uncheck" + ] + }, + { + "type": "object", + "title": "toggle", + "description": "Tap to toggle a switch or checkbox by testId", + "examples": [ + { + "toggle": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "toggle": { + "$ref": "#/$defs/args_toggle", + "description": "Tap to toggle a switch or checkbox by testId", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "toggle" + ] + }, + { + "type": "object", + "title": "setSlider", + "description": "Move a slider under testId to a normalized value (0–1)", + "examples": [ + { + "setSlider": { + "id": "volume_slider", + "value": 0.5 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setSlider": { + "$ref": "#/$defs/args_setSlider", + "description": "Move a slider under testId to a normalized value (0–1)", + "examples": [ + { + "id": "volume_slider", + "value": 0.5 + } + ] + } + }, + "required": [ + "setSlider" + ] + }, + { + "type": "object", + "title": "chooseDate", + "description": "Set a date field by testId to the given value string", + "examples": [ + { + "chooseDate": { + "id": "birth_date", + "value": "2024-01-15" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "chooseDate": { + "$ref": "#/$defs/args_chooseDate", + "description": "Set a date field by testId to the given value string", + "examples": [ + { + "id": "birth_date", + "value": "2024-01-15" + } + ] + } + }, + "required": [ + "chooseDate" + ] + }, + { + "type": "object", + "title": "chooseTime", + "description": "Set a time field by testId to the given value string", + "examples": [ + { + "chooseTime": { + "id": "birth_date", + "value": "2024-01-15" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "chooseTime": { + "$ref": "#/$defs/args_chooseTime", + "description": "Set a time field by testId to the given value string", + "examples": [ + { + "id": "birth_date", + "value": "2024-01-15" + } + ] + } + }, + "required": [ + "chooseTime" + ] + }, + { + "type": "object", + "title": "scroll", + "description": "Drag the first Scrollable by delta pixels", + "examples": [ + { + "scroll": { + "delta": 300 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "scroll": { + "$ref": "#/$defs/args_scroll", + "description": "Drag the first Scrollable by delta pixels", + "examples": [ + { + "delta": 300 + } + ] + } + }, + "required": [ + "scroll" + ] + }, + { + "type": "object", + "title": "scrollUntilVisible", + "description": "Scroll until a widget with testId is visible", + "examples": [ + { + "scrollUntilVisible": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "scrollUntilVisible": { + "$ref": "#/$defs/args_scrollUntilVisible", + "description": "Scroll until a widget with testId is visible", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "scrollUntilVisible" + ] + }, + { + "type": "object", + "title": "swipe", + "description": "Swipe on a scrollable or widget (direction: left/right/up/down)", + "examples": [ + { + "swipe": { + "direction": "left", + "id": "carousel" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "swipe": { + "$ref": "#/$defs/args_swipe", + "description": "Swipe on a scrollable or widget (direction: left/right/up/down)", + "examples": [ + { + "direction": "left", + "id": "carousel" + } + ] + } + }, + "required": [ + "swipe" + ] + }, + { + "type": "object", + "title": "drag", + "description": "Drag a widget by testId by dx/dy offset", + "examples": [ + { + "drag": { + "id": "handle", + "dx": 50, + "dy": 0 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "drag": { + "$ref": "#/$defs/args_drag", + "description": "Drag a widget by testId by dx/dy offset", + "examples": [ + { + "id": "handle", + "dx": 50, + "dy": 0 + } + ] + } + }, + "required": [ + "drag" + ] + }, + { + "type": "object", + "title": "pullToRefresh", + "description": "Pull down on a scrollable to trigger refresh", + "examples": [ + { + "pullToRefresh": { + "id": "scroll_view" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "pullToRefresh": { + "$ref": "#/$defs/args_pullToRefresh", + "description": "Pull down on a scrollable to trigger refresh", + "examples": [ + { + "id": "scroll_view" + } + ] + } + }, + "required": [ + "pullToRefresh" + ] + }, + { + "type": "object", + "title": "wait", + "description": "Alias for pump — advance frame clock by durationMs", + "examples": [ + { + "wait": { + "durationMs": 100 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "wait": { + "$ref": "#/$defs/args_wait", + "description": "Alias for pump — advance frame clock by durationMs", + "examples": [ + { + "durationMs": 100 + } + ] + } + }, + "required": [ + "wait" + ] + }, + { + "type": "object", + "title": "pump", + "description": "Advance the Flutter frame clock by durationMs", + "examples": [ + { + "pump": { + "durationMs": 100 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "pump": { + "$ref": "#/$defs/args_pump", + "description": "Advance the Flutter frame clock by durationMs", + "examples": [ + { + "durationMs": 100 + } + ] + } + }, + "required": [ + "pump" + ] + }, + { + "type": "object", + "title": "settle", + "description": "Run pumpAndSettle until idle or timeout", + "examples": [ + { + "settle": { + "timeoutMs": 5000 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "settle": { + "$ref": "#/$defs/args_settle", + "description": "Run pumpAndSettle until idle or timeout", + "examples": [ + { + "timeoutMs": 5000 + } + ] + } + }, + "required": [ + "settle" + ] + }, + { + "type": "object", + "title": "waitFor", + "description": "Poll until a widget id and/or text appears", + "examples": [ + { + "waitFor": { + "id": "loading_spinner", + "timeoutMs": 5000 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "waitFor": { + "$ref": "#/$defs/args_waitFor", + "description": "Poll until a widget id and/or text appears", + "examples": [ + { + "id": "loading_spinner", + "timeoutMs": 5000 + } + ] + } + }, + "required": [ + "waitFor" + ] + }, + { + "type": "object", + "title": "waitForText", + "description": "Poll until the given text appears on screen", + "examples": [ + { + "waitForText": { + "id": "loading_spinner", + "timeoutMs": 5000 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "waitForText": { + "$ref": "#/$defs/args_waitForText", + "description": "Poll until the given text appears on screen", + "examples": [ + { + "id": "loading_spinner", + "timeoutMs": 5000 + } + ] + } + }, + "required": [ + "waitForText" + ] + }, + { + "type": "object", + "title": "waitForGone", + "description": "Poll until a widget with testId is removed from the tree", + "examples": [ + { + "waitForGone": { + "id": "loading_spinner", + "timeoutMs": 5000 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "waitForGone": { + "$ref": "#/$defs/args_waitForGone", + "description": "Poll until a widget with testId is removed from the tree", + "examples": [ + { + "id": "loading_spinner", + "timeoutMs": 5000 + } + ] + } + }, + "required": [ + "waitForGone" + ] + }, + { + "type": "object", + "title": "waitForApi", + "description": "Poll until a mocked API is called N times", + "examples": [ + { + "waitForApi": { + "name": "login", + "times": 1 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "waitForApi": { + "$ref": "#/$defs/args_waitForApi", + "description": "Poll until a mocked API is called N times", + "examples": [ + { + "name": "login", + "times": 1 + } + ] + } + }, + "required": [ + "waitForApi" + ] + }, + { + "type": "object", + "title": "waitForNavigation", + "description": "Poll until the given screen is visible", + "examples": [ + { + "waitForNavigation": { + "screen": "Home", + "timeoutMs": 5000 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "waitForNavigation": { + "$ref": "#/$defs/args_waitForNavigation", + "description": "Poll until the given screen is visible", + "examples": [ + { + "screen": "Home", + "timeoutMs": 5000 + } + ] + } + }, + "required": [ + "waitForNavigation" + ] + }, + { + "type": "object", + "title": "waitUntil", + "description": "Poll until app state at path equals expected value", + "examples": [ + { + "waitUntil": { + "path": "user.name", + "equals": "Jane" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "waitUntil": { + "$ref": "#/$defs/args_waitUntil", + "description": "Poll until app state at path equals expected value", + "examples": [ + { + "path": "user.name", + "equals": "Jane" + } + ] + } + }, + "required": [ + "waitUntil" + ] + }, + { + "type": "object", + "title": "expectVisible", + "description": "Assert a widget with testId is visible", + "examples": [ + { + "expectVisible": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectVisible": { + "$ref": "#/$defs/args_expectVisible", + "description": "Assert a widget with testId is visible", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectVisible" + ] + }, + { + "type": "object", + "title": "expectNotVisible", + "description": "Assert a widget with testId is not visible", + "examples": [ + { + "expectNotVisible": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNotVisible": { + "$ref": "#/$defs/args_expectNotVisible", + "description": "Assert a widget with testId is not visible", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectNotVisible" + ] + }, + { + "type": "object", + "title": "expectExists", + "description": "Assert a widget with testId exists in the tree", + "examples": [ + { + "expectExists": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectExists": { + "$ref": "#/$defs/args_expectExists", + "description": "Assert a widget with testId exists in the tree", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectExists" + ] + }, + { + "type": "object", + "title": "expectNotExists", + "description": "Assert no widget with testId exists", + "examples": [ + { + "expectNotExists": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNotExists": { + "$ref": "#/$defs/args_expectNotExists", + "description": "Assert no widget with testId exists", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectNotExists" + ] + }, + { + "type": "object", + "title": "expectText", + "description": "Assert exact text is shown", + "examples": [ + { + "expectText": { + "text": "Welcome" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectText": { + "$ref": "#/$defs/args_expectText", + "description": "Assert exact text is shown", + "examples": [ + { + "text": "Welcome" + } + ] + } + }, + "required": [ + "expectText" + ] + }, + { + "type": "object", + "title": "expectNoText", + "description": "Assert text is not shown", + "examples": [ + { + "expectNoText": { + "text": "Welcome" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNoText": { + "$ref": "#/$defs/args_expectNoText", + "description": "Assert text is not shown", + "examples": [ + { + "text": "Welcome" + } + ] + } + }, + "required": [ + "expectNoText" + ] + }, + { + "type": "object", + "title": "expectTextContains", + "description": "Assert some text containing the given substring", + "examples": [ + { + "expectTextContains": { + "text": "Welcome" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectTextContains": { + "$ref": "#/$defs/args_expectTextContains", + "description": "Assert some text containing the given substring", + "examples": [ + { + "text": "Welcome" + } + ] + } + }, + "required": [ + "expectTextContains" + ] + }, + { + "type": "object", + "title": "expectEnabled", + "description": "Assert widget semantics report enabled", + "examples": [ + { + "expectEnabled": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectEnabled": { + "$ref": "#/$defs/args_expectEnabled", + "description": "Assert widget semantics report enabled", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectEnabled" + ] + }, + { + "type": "object", + "title": "expectDisabled", + "description": "Assert widget semantics report disabled", + "examples": [ + { + "expectDisabled": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectDisabled": { + "$ref": "#/$defs/args_expectDisabled", + "description": "Assert widget semantics report disabled", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectDisabled" + ] + }, + { + "type": "object", + "title": "expectValue", + "description": "Assert input value equals expected (EditableText/TextField)", + "examples": [ + { + "expectValue": { + "id": "email_field", + "equals": "user@test.com" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectValue": { + "$ref": "#/$defs/args_expectValue", + "description": "Assert input value equals expected (EditableText/TextField)", + "examples": [ + { + "id": "email_field", + "equals": "user@test.com" + } + ] + } + }, + "required": [ + "expectValue" + ] + }, + { + "type": "object", + "title": "expectChecked", + "description": "Assert checkbox checked state matches equals", + "examples": [ + { + "expectChecked": { + "id": "terms_checkbox", + "equals": true + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectChecked": { + "$ref": "#/$defs/args_expectChecked", + "description": "Assert checkbox checked state matches equals", + "examples": [ + { + "id": "terms_checkbox", + "equals": true + } + ] + } + }, + "required": [ + "expectChecked" + ] + }, + { + "type": "object", + "title": "expectProperty", + "description": "Assert a widget property (e.g. label) equals expected", + "examples": [ + { + "expectProperty": { + "id": "title", + "property": "label", + "equals": "Hello" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectProperty": { + "$ref": "#/$defs/args_expectProperty", + "description": "Assert a widget property (e.g. label) equals expected", + "examples": [ + { + "id": "title", + "property": "label", + "equals": "Hello" + } + ] + } + }, + "required": [ + "expectProperty" + ] + }, + { + "type": "object", + "title": "expectStyle", + "description": "Assert style-related property equals expected", + "examples": [ + { + "expectStyle": { + "id": "title", + "property": "label", + "equals": "Hello" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectStyle": { + "$ref": "#/$defs/args_expectStyle", + "description": "Assert style-related property equals expected", + "examples": [ + { + "id": "title", + "property": "label", + "equals": "Hello" + } + ] + } + }, + "required": [ + "expectStyle" + ] + }, + { + "type": "object", + "title": "expectSelected", + "description": "Assert selected/checked state matches equals", + "examples": [ + { + "expectSelected": { + "id": "terms_checkbox", + "equals": true + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectSelected": { + "$ref": "#/$defs/args_expectSelected", + "description": "Assert selected/checked state matches equals", + "examples": [ + { + "id": "terms_checkbox", + "equals": true + } + ] + } + }, + "required": [ + "expectSelected" + ] + }, + { + "type": "object", + "title": "expectCount", + "description": "Assert count of widgets with the same testId", + "examples": [ + { + "expectCount": { + "id": "badge", + "equals": 2 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectCount": { + "$ref": "#/$defs/args_expectCount", + "description": "Assert count of widgets with the same testId", + "examples": [ + { + "id": "badge", + "equals": 2 + } + ] + } + }, + "required": [ + "expectCount" + ] + }, + { + "type": "object", + "title": "expectListCount", + "description": "Assert number of list items under a list testId", + "examples": [ + { + "expectListCount": { + "id": "items_list", + "equals": 3 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectListCount": { + "$ref": "#/$defs/args_expectListCount", + "description": "Assert number of list items under a list testId", + "examples": [ + { + "id": "items_list", + "equals": 3 + } + ] + } + }, + "required": [ + "expectListCount" + ] + }, + { + "type": "object", + "title": "expectListContains", + "description": "Assert list contains text", + "examples": [ + { + "expectListContains": { + "id": "items_list", + "text": "Item 1" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectListContains": { + "$ref": "#/$defs/args_expectListContains", + "description": "Assert list contains text", + "examples": [ + { + "id": "items_list", + "text": "Item 1" + } + ] + } + }, + "required": [ + "expectListContains" + ] + }, + { + "type": "object", + "title": "expectListItem", + "description": "Assert a list item widget with itemId is visible", + "examples": [ + { + "expectListItem": { + "itemId": "row_0" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectListItem": { + "$ref": "#/$defs/args_expectListItem", + "description": "Assert a list item widget with itemId is visible", + "examples": [ + { + "itemId": "row_0" + } + ] + } + }, + "required": [ + "expectListItem" + ] + }, + { + "type": "object", + "title": "expectEmpty", + "description": "Assert a list has zero items", + "examples": [ + { + "expectEmpty": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectEmpty": { + "$ref": "#/$defs/args_expectEmpty", + "description": "Assert a list has zero items", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectEmpty" + ] + }, + { + "type": "object", + "title": "expectNotEmpty", + "description": "Assert a list has at least one item", + "examples": [ + { + "expectNotEmpty": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNotEmpty": { + "$ref": "#/$defs/args_expectNotEmpty", + "description": "Assert a list has at least one item", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectNotEmpty" + ] + }, + { + "type": "object", + "title": "expectScreen", + "description": "Alias for expectNavigateTo — assert current screen", + "examples": [ + { + "expectScreen": { + "screen": "Home" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectScreen": { + "$ref": "#/$defs/args_expectScreen", + "description": "Alias for expectNavigateTo — assert current screen", + "examples": [ + { + "screen": "Home" + } + ] + } + }, + "required": [ + "expectScreen" + ] + }, + { + "type": "object", + "title": "expectNavigateTo", + "description": "Assert the current visible screen name/id", + "examples": [ + { + "expectNavigateTo": { + "screen": "Home" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNavigateTo": { + "$ref": "#/$defs/args_expectNavigateTo", + "description": "Assert the current visible screen name/id", + "examples": [ + { + "screen": "Home" + } + ] + } + }, + "required": [ + "expectNavigateTo" + ] + }, + { + "type": "object", + "title": "expectVisited", + "description": "Assert a screen appears in navigation history", + "examples": [ + { + "expectVisited": { + "screen": "Login" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectVisited": { + "$ref": "#/$defs/args_expectVisited", + "description": "Assert a screen appears in navigation history", + "examples": [ + { + "screen": "Login" + } + ] + } + }, + "required": [ + "expectVisited" + ] + }, + { + "type": "object", + "title": "expectNotVisited", + "description": "Assert a screen was never visited", + "examples": [ + { + "expectNotVisited": { + "screen": "Login" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNotVisited": { + "$ref": "#/$defs/args_expectNotVisited", + "description": "Assert a screen was never visited", + "examples": [ + { + "screen": "Login" + } + ] + } + }, + "required": [ + "expectNotVisited" + ] + }, + { + "type": "object", + "title": "expectBackStack", + "description": "Assert navigation history suffix matches screens", + "examples": [ + { + "expectBackStack": { + "screens": [ + "Home", + "Details" + ] + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectBackStack": { + "$ref": "#/$defs/args_expectBackStack", + "description": "Assert navigation history suffix matches screens", + "examples": [ + { + "screens": [ + "Home", + "Details" + ] + } + ] + } + }, + "required": [ + "expectBackStack" + ] + }, + { + "type": "object", + "title": "expectCanGoBack", + "description": "Assert whether back navigation is possible", + "examples": [ + { + "expectCanGoBack": { + "equals": true + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectCanGoBack": { + "$ref": "#/$defs/args_expectCanGoBack", + "description": "Assert whether back navigation is possible", + "examples": [ + { + "equals": true + } + ] + } + }, + "required": [ + "expectCanGoBack" + ] + }, + { + "type": "object", + "title": "goBack", + "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", + "examples": [ + { + "goBack": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "goBack": { + "$ref": "#/$defs/args_goBack", + "description": "Navigate back (Ensemble navigateBack or Navigator.pop)", + "examples": [ + {} + ] + } + }, + "required": [ + "goBack" + ] + }, + { + "type": "object", + "title": "mockApi", + "description": "Register a mock HTTP API response by API name", + "examples": [ + { + "mockApi": { + "name": "login", + "response": { + "statusCode": 200, + "body": { + "token": "test-token" + } + } + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "mockApi": { + "$ref": "#/$defs/args_mockApi", + "description": "Register a mock HTTP API response by API name", + "examples": [ + { + "name": "login", + "response": { + "statusCode": 200, + "body": { + "token": "test-token" + } + } + } + ] + } + }, + "required": [ + "mockApi" + ] + }, + { + "type": "object", + "title": "mockApiError", + "description": "Mock an API to return an error status/body", + "examples": [ + { + "mockApiError": { + "name": "login", + "statusCode": 401, + "body": { + "error": "Unauthorized" + } + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "mockApiError": { + "$ref": "#/$defs/args_mockApiError", + "description": "Mock an API to return an error status/body", + "examples": [ + { + "name": "login", + "statusCode": 401, + "body": { + "error": "Unauthorized" + } + } + ] + } + }, + "required": [ + "mockApiError" + ] + }, + { + "type": "object", + "title": "mockApiFromFixture", + "description": "Load mock response body from a JSON fixture asset", + "examples": [ + { + "mockApiFromFixture": { + "name": "users", + "fixture": "fixtures/users.json" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "mockApiFromFixture": { + "$ref": "#/$defs/args_mockApiFromFixture", + "description": "Load mock response body from a JSON fixture asset", + "examples": [ + { + "name": "users", + "fixture": "fixtures/users.json" + } + ] + } + }, + "required": [ + "mockApiFromFixture" + ] + }, + { + "type": "object", + "title": "mockApiException", + "description": "Force an API call to throw an exception", + "examples": [ + { + "mockApiException": { + "name": "login", + "message": "Network error" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "mockApiException": { + "$ref": "#/$defs/args_mockApiException", + "description": "Force an API call to throw an exception", + "examples": [ + { + "name": "login", + "message": "Network error" + } + ] + } + }, + "required": [ + "mockApiException" + ] + }, + { + "type": "object", + "title": "mockTimeout", + "description": "Mock an API with a long delay (simulate timeout)", + "examples": [ + { + "mockTimeout": { + "name": "slow_api", + "delayMs": 60000 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "mockTimeout": { + "$ref": "#/$defs/args_mockTimeout", + "description": "Mock an API with a long delay (simulate timeout)", + "examples": [ + { + "name": "slow_api", + "delayMs": 60000 + } + ] + } + }, + "required": [ + "mockTimeout" + ] + }, + { + "type": "object", + "title": "mockNetworkOffline", + "description": "Simulate offline network for API calls", + "examples": [ + { + "mockNetworkOffline": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "mockNetworkOffline": { + "$ref": "#/$defs/args_mockNetworkOffline", + "description": "Simulate offline network for API calls", + "examples": [ + {} + ] + } + }, + "required": [ + "mockNetworkOffline" + ] + }, + { + "type": "object", + "title": "mockNetworkOnline", + "description": "Restore online network for API calls", + "examples": [ + { + "mockNetworkOnline": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "mockNetworkOnline": { + "$ref": "#/$defs/args_mockNetworkOnline", + "description": "Restore online network for API calls", + "examples": [ + {} + ] + } + }, + "required": [ + "mockNetworkOnline" + ] + }, + { + "type": "object", + "title": "resetApiCalls", + "description": "Clear recorded API call history", + "examples": [ + { + "resetApiCalls": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "resetApiCalls": { + "$ref": "#/$defs/args_resetApiCalls", + "description": "Clear recorded API call history", + "examples": [ + {} + ] + } + }, + "required": [ + "resetApiCalls" + ] + }, + { + "type": "object", + "title": "clearApiMocks", + "description": "Remove all registered API mocks", + "examples": [ + { + "clearApiMocks": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "clearApiMocks": { + "$ref": "#/$defs/args_clearApiMocks", + "description": "Remove all registered API mocks", + "examples": [ + {} + ] + } + }, + "required": [ + "clearApiMocks" + ] + }, + { + "type": "object", + "title": "expectApiCalled", + "description": "Assert an API was called an exact number of times", + "examples": [ + { + "expectApiCalled": { + "name": "login", + "times": 1 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectApiCalled": { + "$ref": "#/$defs/args_expectApiCalled", + "description": "Assert an API was called an exact number of times", + "examples": [ + { + "name": "login", + "times": 1 + } + ] + } + }, + "required": [ + "expectApiCalled" + ] + }, + { + "type": "object", + "title": "expectApiNotCalled", + "description": "Assert an API was never called", + "examples": [ + { + "expectApiNotCalled": { + "name": "login", + "times": 1 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectApiNotCalled": { + "$ref": "#/$defs/args_expectApiNotCalled", + "description": "Assert an API was never called", + "examples": [ + { + "name": "login", + "times": 1 + } + ] + } + }, + "required": [ + "expectApiNotCalled" + ] + }, + { + "type": "object", + "title": "expectApiRequest", + "description": "Assert last API request body/query/headers match", + "examples": [ + { + "expectApiRequest": { + "name": "login", + "body": { + "email": "user@test.com", + "password": "secret" + } + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectApiRequest": { + "$ref": "#/$defs/args_expectApiRequest", + "description": "Assert last API request body/query/headers match", + "examples": [ + { + "name": "login", + "body": { + "email": "user@test.com", + "password": "secret" + } + } + ] + } + }, + "required": [ + "expectApiRequest" + ] + }, + { + "type": "object", + "title": "expectApiRequestContains", + "description": "Assert API request contains partial body/query", + "examples": [ + { + "expectApiRequestContains": { + "name": "login", + "body": { + "email": "user@test.com", + "password": "secret" + } + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectApiRequestContains": { + "$ref": "#/$defs/args_expectApiRequestContains", + "description": "Assert API request contains partial body/query", + "examples": [ + { + "name": "login", + "body": { + "email": "user@test.com", + "password": "secret" + } + } + ] + } + }, + "required": [ + "expectApiRequestContains" + ] + }, + { + "type": "object", + "title": "expectApiHeader", + "description": "Assert an API request header equals expected", + "examples": [ + { + "expectApiHeader": { + "name": "login", + "header": "Authorization", + "equals": "Bearer test-token" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectApiHeader": { + "$ref": "#/$defs/args_expectApiHeader", + "description": "Assert an API request header equals expected", + "examples": [ + { + "name": "login", + "header": "Authorization", + "equals": "Bearer test-token" + } + ] + } + }, + "required": [ + "expectApiHeader" + ] + }, + { + "type": "object", + "title": "expectApiCallOrder", + "description": "Assert APIs were called in order", + "examples": [ + { + "expectApiCallOrder": { + "names": [ + "auth", + "profile" + ] + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectApiCallOrder": { + "$ref": "#/$defs/args_expectApiCallOrder", + "description": "Assert APIs were called in order", + "examples": [ + { + "names": [ + "auth", + "profile" + ] + } + ] + } + }, + "required": [ + "expectApiCallOrder" + ] + }, + { + "type": "object", + "title": "expectLastApiCall", + "description": "Assert the most recent API call name", + "examples": [ + { + "expectLastApiCall": { + "name": "login", + "times": 1 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectLastApiCall": { + "$ref": "#/$defs/args_expectLastApiCall", + "description": "Assert the most recent API call name", + "examples": [ + { + "name": "login", + "times": 1 + } + ] + } + }, + "required": [ + "expectLastApiCall" + ] + }, + { + "type": "object", + "title": "setState", + "description": "Set app data-context state at path to value", + "examples": [ + { + "setState": { + "path": "user.name", + "value": "Jane" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setState": { + "$ref": "#/$defs/args_setState", + "description": "Set app data-context state at path to value", + "examples": [ + { + "path": "user.name", + "value": "Jane" + } + ] + } + }, + "required": [ + "setState" + ] + }, + { + "type": "object", + "title": "expectState", + "description": "Assert app state at path equals expected", + "examples": [ + { + "expectState": { + "path": "user.name", + "equals": "Jane" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectState": { + "$ref": "#/$defs/args_expectState", + "description": "Assert app state at path equals expected", + "examples": [ + { + "path": "user.name", + "equals": "Jane" + } + ] + } + }, + "required": [ + "expectState" + ] + }, + { + "type": "object", + "title": "expectStateContains", + "description": "Assert app state at path contains subset", + "examples": [ + { + "expectStateContains": { + "path": "user.name", + "equals": "Jane" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectStateContains": { + "$ref": "#/$defs/args_expectStateContains", + "description": "Assert app state at path contains subset", + "examples": [ + { + "path": "user.name", + "equals": "Jane" + } + ] + } + }, + "required": [ + "expectStateContains" + ] + }, + { + "type": "object", + "title": "expectStateExists", + "description": "Assert state path resolves without error", + "examples": [ + { + "expectStateExists": { + "path": "user.id" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectStateExists": { + "$ref": "#/$defs/args_expectStateExists", + "description": "Assert state path resolves without error", + "examples": [ + { + "path": "user.id" + } + ] + } + }, + "required": [ + "expectStateExists" + ] + }, + { + "type": "object", + "title": "expectStateNotExists", + "description": "Assert state path is null or absent", + "examples": [ + { + "expectStateNotExists": { + "path": "user.id" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectStateNotExists": { + "$ref": "#/$defs/args_expectStateNotExists", + "description": "Assert state path is null or absent", + "examples": [ + { + "path": "user.id" + } + ] + } + }, + "required": [ + "expectStateNotExists" + ] + }, + { + "type": "object", + "title": "resetState", + "description": "Clear state at path (set to null)", + "examples": [ + { + "resetState": { + "path": "cart" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "resetState": { + "$ref": "#/$defs/args_resetState", + "description": "Clear state at path (set to null)", + "examples": [ + { + "path": "cart" + } + ] + } + }, + "required": [ + "resetState" + ] + }, + { + "type": "object", + "title": "setStorage", + "description": "Write a value to public GetStorage by key", + "examples": [ + { + "setStorage": { + "key": "onboarding_done", + "value": true + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setStorage": { + "$ref": "#/$defs/args_setStorage", + "description": "Write a value to public GetStorage by key", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + } + }, + "required": [ + "setStorage" + ] + }, + { + "type": "object", + "title": "expectStorage", + "description": "Assert public storage key equals expected", + "examples": [ + { + "expectStorage": { + "key": "onboarding_done", + "value": true + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectStorage": { + "$ref": "#/$defs/args_expectStorage", + "description": "Assert public storage key equals expected", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + } + }, + "required": [ + "expectStorage" + ] + }, + { + "type": "object", + "title": "removeStorage", + "description": "Remove a key from public storage", + "examples": [ + { + "removeStorage": { + "key": "onboarding_done", + "value": true + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "removeStorage": { + "$ref": "#/$defs/args_removeStorage", + "description": "Remove a key from public storage", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + } + }, + "required": [ + "removeStorage" + ] + }, + { + "type": "object", + "title": "clearStorage", + "description": "Clear all non-encrypted public storage keys", + "examples": [ + { + "clearStorage": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "clearStorage": { + "$ref": "#/$defs/args_clearStorage", + "description": "Clear all non-encrypted public storage keys", + "examples": [ + {} + ] + } + }, + "required": [ + "clearStorage" + ] + }, + { + "type": "object", + "title": "setEnv", + "description": "Override an environment variable for the test", + "examples": [ + { + "setEnv": { + "key": "onboarding_done", + "value": true + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setEnv": { + "$ref": "#/$defs/args_setEnv", + "description": "Override an environment variable for the test", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + } + }, + "required": [ + "setEnv" + ] + }, + { + "type": "object", + "title": "setAuth", + "description": "Simulate a signed-in user", + "examples": [ + { + "setAuth": { + "user": { + "id": "1", + "email": "user@test.com" + } + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setAuth": { + "$ref": "#/$defs/args_setAuth", + "description": "Simulate a signed-in user", + "examples": [ + { + "user": { + "id": "1", + "email": "user@test.com" + } + } + ] + } + }, + "required": [ + "setAuth" + ] + }, + { + "type": "object", + "title": "clearAuth", + "description": "Clear the signed-in user", + "examples": [ + { + "clearAuth": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "clearAuth": { + "$ref": "#/$defs/args_clearAuth", + "description": "Clear the signed-in user", + "examples": [ + {} + ] + } + }, + "required": [ + "clearAuth" + ] + }, + { + "type": "object", + "title": "setPermission", + "description": "Set a permission flag for the test runtime", + "examples": [ + { + "setPermission": { + "name": "camera", + "value": "granted" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setPermission": { + "$ref": "#/$defs/args_setPermission", + "description": "Set a permission flag for the test runtime", + "examples": [ + { + "name": "camera", + "value": "granted" + } + ] + } + }, + "required": [ + "setPermission" + ] + }, + { + "type": "object", + "title": "setDevice", + "description": "Override viewport physical size (width/height)", + "examples": [ + { + "setDevice": { + "width": 390, + "height": 844 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setDevice": { + "$ref": "#/$defs/args_setDevice", + "description": "Override viewport physical size (width/height)", + "examples": [ + { + "width": 390, + "height": 844 + } + ] + } + }, + "required": [ + "setDevice" + ] + }, + { + "type": "object", + "title": "setLocale", + "description": "Set APP_LOCALE environment override", + "examples": [ + { + "setLocale": { + "locale": "en_US" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setLocale": { + "$ref": "#/$defs/args_setLocale", + "description": "Set APP_LOCALE environment override", + "examples": [ + { + "locale": "en_US" + } + ] + } + }, + "required": [ + "setLocale" + ] + }, + { + "type": "object", + "title": "setTheme", + "description": "Set APP_THEME / theme mode override", + "examples": [ + { + "setTheme": { + "mode": "dark" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setTheme": { + "$ref": "#/$defs/args_setTheme", + "description": "Set APP_THEME / theme mode override", + "examples": [ + { + "mode": "dark" + } + ] + } + }, + "required": [ + "setTheme" + ] + }, + { + "type": "object", + "title": "runScript", + "description": "Evaluate a script expression in the data context", + "examples": [ + { + "runScript": { + "script": "1 + 1", + "equals": 2 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "runScript": { + "$ref": "#/$defs/args_runScript", + "description": "Evaluate a script expression in the data context", + "examples": [ + { + "script": "1 + 1", + "equals": 2 + } + ] + } + }, + "required": [ + "runScript" + ] + }, + { + "type": "object", + "title": "expectScriptResult", + "description": "Evaluate script and assert result equals expected", + "examples": [ + { + "expectScriptResult": { + "script": "1 + 1", + "equals": 2 + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectScriptResult": { + "$ref": "#/$defs/args_expectScriptResult", + "description": "Evaluate script and assert result equals expected", + "examples": [ + { + "script": "1 + 1", + "equals": 2 + } + ] + } + }, + "required": [ + "expectScriptResult" + ] + }, + { + "type": "object", + "title": "expectConsoleLog", + "description": "Assert a console log line contains text", + "examples": [ + { + "expectConsoleLog": { + "contains": "Screen loaded" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectConsoleLog": { + "$ref": "#/$defs/args_expectConsoleLog", + "description": "Assert a console log line contains text", + "examples": [ + { + "contains": "Screen loaded" + } + ] + } + }, + "required": [ + "expectConsoleLog" + ] + }, + { + "type": "object", + "title": "group", + "description": "Run nested steps as a named group", + "examples": [ + { + "group": { + "name": "login_flow", + "steps": [ + { + "tap": { + "id": "login_button" + } + } + ] + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "group": { + "$ref": "#/$defs/args_group", + "description": "Run nested steps as a named group", + "examples": [ + { + "name": "login_flow", + "steps": [ + { + "tap": { + "id": "login_button" + } + } + ] + } + ] + } + }, + "required": [ + "group" + ] + }, + { + "type": "object", + "title": "repeat", + "description": "Repeat nested steps N times", + "examples": [ + { + "repeat": { + "times": 3, + "steps": [ + { + "tap": { + "id": "next_button" + } + } + ] + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "repeat": { + "$ref": "#/$defs/args_repeat", + "description": "Repeat nested steps N times", + "examples": [ + { + "times": 3, + "steps": [ + { + "tap": { + "id": "next_button" + } + } + ] + } + ] + } + }, + "required": [ + "repeat" + ] + }, + { + "type": "object", + "title": "optional", + "description": "Run nested steps; swallow failures", + "examples": [ + { + "optional": { + "steps": [ + { + "tap": { + "id": "dismiss_banner" + } + } + ] + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "optional": { + "$ref": "#/$defs/args_optional", + "description": "Run nested steps; swallow failures", + "examples": [ + { + "steps": [ + { + "tap": { + "id": "dismiss_banner" + } + } + ] + } + ] + } + }, + "required": [ + "optional" + ] + }, + { + "type": "object", + "title": "ifVisible", + "description": "Run nested steps only if testId is visible", + "examples": [ + { + "ifVisible": { + "id": "promo_banner", + "steps": [ + { + "tap": { + "id": "close_banner" + } + } + ] + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "ifVisible": { + "$ref": "#/$defs/args_ifVisible", + "description": "Run nested steps only if testId is visible", + "examples": [ + { + "id": "promo_banner", + "steps": [ + { + "tap": { + "id": "close_banner" + } + } + ] + } + ] + } + }, + "required": [ + "ifVisible" + ] + }, + { + "type": "object", + "title": "logApiCalls", + "description": "Log all recorded API calls to the test log", + "examples": [ + { + "logApiCalls": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "logApiCalls": { + "$ref": "#/$defs/args_logApiCalls", + "description": "Log all recorded API calls to the test log", + "examples": [ + {} + ] + } + }, + "required": [ + "logApiCalls" + ] + }, + { + "type": "object", + "title": "screenshot", + "description": "Capture golden or dump widget tree for debugging", + "examples": [ + { + "screenshot": { + "name": "home_screen" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "screenshot": { + "$ref": "#/$defs/args_screenshot", + "description": "Capture golden or dump widget tree for debugging", + "examples": [ + { + "name": "home_screen" + } + ] + } + }, + "required": [ + "screenshot" + ] + }, + { + "type": "object", + "title": "dumpTree", + "description": "Print the widget tree to the debug console", + "examples": [ + { + "dumpTree": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "dumpTree": { + "$ref": "#/$defs/args_dumpTree", + "description": "Print the widget tree to the debug console", + "examples": [ + {} + ] + } + }, + "required": [ + "dumpTree" + ] + }, + { + "type": "object", + "title": "logState", + "description": "Log resolved state at path", + "examples": [ + { + "logState": { + "path": "user.id" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "logState": { + "$ref": "#/$defs/args_logState", + "description": "Log resolved state at path", + "examples": [ + { + "path": "user.id" + } + ] + } + }, + "required": [ + "logState" + ] + }, + { + "type": "object", + "title": "logStorage", + "description": "Log public storage value for key", + "examples": [ + { + "logStorage": { + "key": "onboarding_done", + "value": true + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "logStorage": { + "$ref": "#/$defs/args_logStorage", + "description": "Log public storage value for key", + "examples": [ + { + "key": "onboarding_done", + "value": true + } + ] + } + }, + "required": [ + "logStorage" + ] + }, + { + "type": "object", + "title": "expectNoConsoleErrors", + "description": "Assert no console errors were recorded", + "examples": [ + { + "expectNoConsoleErrors": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNoConsoleErrors": { + "$ref": "#/$defs/args_expectNoConsoleErrors", + "description": "Assert no console errors were recorded", + "examples": [ + {} + ] + } + }, + "required": [ + "expectNoConsoleErrors" + ] + }, + { + "type": "object", + "title": "expectNoRenderErrors", + "description": "Assert no Flutter render errors were recorded", + "examples": [ + { + "expectNoRenderErrors": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNoRenderErrors": { + "$ref": "#/$defs/args_expectNoRenderErrors", + "description": "Assert no Flutter render errors were recorded", + "examples": [ + {} + ] + } + }, + "required": [ + "expectNoRenderErrors" + ] + }, + { + "type": "object", + "title": "expectError", + "description": "Assert a Flutter error was recorded (optional filter)", + "examples": [ + { + "expectError": { + "contains": "overflow" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectError": { + "$ref": "#/$defs/args_expectError", + "description": "Assert a Flutter error was recorded (optional filter)", + "examples": [ + { + "contains": "overflow" + } + ] + } + }, + "required": [ + "expectError" + ] + }, + { + "type": "object", + "title": "expectNoErrors", + "description": "Alias for expectNoRenderErrors", + "examples": [ + { + "expectNoErrors": {} + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNoErrors": { + "$ref": "#/$defs/args_expectNoErrors", + "description": "Alias for expectNoRenderErrors", + "examples": [ + {} + ] + } + }, + "required": [ + "expectNoErrors" + ] + }, + { + "type": "object", + "title": "expectAccessible", + "description": "Assert widget has accessibility label or value", + "examples": [ + { + "expectAccessible": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectAccessible": { + "$ref": "#/$defs/args_expectAccessible", + "description": "Assert widget has accessibility label or value", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectAccessible" + ] + }, + { + "type": "object", + "title": "expectSemanticsLabel", + "description": "Assert semantics label equals expected", + "examples": [ + { + "expectSemanticsLabel": { + "id": "submit_button", + "label": "Submit" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectSemanticsLabel": { + "$ref": "#/$defs/args_expectSemanticsLabel", + "description": "Assert semantics label equals expected", + "examples": [ + { + "id": "submit_button", + "label": "Submit" + } + ] + } + }, + "required": [ + "expectSemanticsLabel" + ] + }, + { + "type": "object", + "title": "expectNoOverflow", + "description": "Assert widget renders without overflow issues", + "examples": [ + { + "expectNoOverflow": { + "id": "my_widget" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectNoOverflow": { + "$ref": "#/$defs/args_expectNoOverflow", + "description": "Assert widget renders without overflow issues", + "examples": [ + { + "id": "my_widget" + } + ] + } + }, + "required": [ + "expectNoOverflow" + ] + }, + { + "type": "object", + "title": "loadFixture", + "description": "Load a JSON fixture into the test fixture map", + "examples": [ + { + "loadFixture": { + "fixture": "fixtures/user.json" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "loadFixture": { + "$ref": "#/$defs/args_loadFixture", + "description": "Load a JSON fixture into the test fixture map", + "examples": [ + { + "fixture": "fixtures/user.json" + } + ] + } + }, + "required": [ + "loadFixture" + ] + }, + { + "type": "object", + "title": "setStateFromFixture", + "description": "Apply all keys from a JSON fixture to state", + "examples": [ + { + "setStateFromFixture": { + "fixture": "fixtures/user.json" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "setStateFromFixture": { + "$ref": "#/$defs/args_setStateFromFixture", + "description": "Apply all keys from a JSON fixture to state", + "examples": [ + { + "fixture": "fixtures/user.json" + } + ] + } + }, + "required": [ + "setStateFromFixture" + ] + }, + { + "type": "object", + "title": "expectMatchesFixture", + "description": "Assert state or path matches a JSON fixture", + "examples": [ + { + "expectMatchesFixture": { + "fixture": "fixtures/user.json" + } + } + ], + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "expectMatchesFixture": { + "$ref": "#/$defs/args_expectMatchesFixture", + "description": "Assert state or path matches a JSON fixture", + "examples": [ + { + "fixture": "fixtures/user.json" + } + ] + } + }, + "required": [ + "expectMatchesFixture" + ] + } + ] + } + } +} diff --git a/packages/ensemble_test_runner/bin/ensemble_test.dart b/packages/ensemble_test_runner/bin/ensemble_test.dart new file mode 100644 index 000000000..8e4902f44 --- /dev/null +++ b/packages/ensemble_test_runner/bin/ensemble_test.dart @@ -0,0 +1,4 @@ +import 'package:ensemble_test_runner/cli/ensemble_test_cli.dart'; + +/// CLI entry: `dart run ensemble_test_runner:ensemble_test` +void main(List arguments) => runEnsembleYamlTestsCli(arguments); diff --git a/packages/ensemble_test_runner/lib/actions/extended_step_handlers.dart b/packages/ensemble_test_runner/lib/actions/extended_step_handlers.dart new file mode 100644 index 000000000..957823fdd --- /dev/null +++ b/packages/ensemble_test_runner/lib/actions/extended_step_handlers.dart @@ -0,0 +1,688 @@ +import 'dart:convert'; + +import 'package:ensemble/action/navigation_action.dart'; +import 'package:ensemble/framework/bindings.dart'; +import 'package:ensemble/screen_controller.dart'; +import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/actions/state_helper.dart'; +import 'package:ensemble_test_runner/actions/test_step_executor.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// Handlers for the full declarative vocabulary beyond the original MVP set. +class ExtendedStepHandlers { + static Future tryExecute( + TestStepExecutor executor, + TestStep step, + ) async { + switch (step.type) { + case 'reloadScreen': + await _reloadScreen(executor); + return true; + case 'restartApp': + await _restartApp(executor); + return true; + case 'resetAppState': + await _resetAppState(executor); + return true; + case 'trigger': + await _trigger(executor, step); + return true; + case 'launchApp': + await _restartApp(executor); + return true; + case 'doubleTap': + await executor.tapWidget(executor.requireId(step)); + await executor.tapWidget(executor.requireId(step)); + return true; + case 'longPress': + await executor.longPressWidget(executor.requireId(step)); + return true; + case 'focus': + await executor.focusWidget(executor.requireId(step)); + return true; + case 'unfocus': + await executor.unfocus(); + return true; + case 'selectIndex': + await _selectIndex(executor, step); + return true; + case 'check': + await executor.tapWidget(executor.requireId(step)); + return true; + case 'uncheck': + await _uncheck(executor, step); + return true; + case 'setSlider': + await _setSlider(executor, step); + return true; + case 'chooseDate': + case 'chooseTime': + await executor.enterTextOn( + executor.requireId(step), + step.args['value']?.toString() ?? '', + ); + return true; + case 'scroll': + await _scroll(executor, step); + return true; + case 'swipe': + await _swipe(executor, step); + return true; + case 'drag': + await _drag(executor, step); + return true; + case 'pullToRefresh': + await _pullToRefresh(executor, step); + return true; + case 'expectExists': + executor.assertions.expectExists(executor.requireId(step)); + return true; + case 'expectNotExists': + executor.assertions.expectNotExists(executor.requireId(step)); + return true; + case 'expectTextContains': + final text = step.args['text']?.toString(); + if (text == null) { + throw EnsembleTestFailure('expectTextContains requires "text"'); + } + executor.assertions.expectTextContains(text); + return true; + case 'expectChecked': + executor.assertions.expectChecked( + executor.requireId(step), + step.args['equals'] as bool? ?? true, + ); + return true; + case 'expectSelected': + executor.assertions.expectChecked( + executor.requireId(step), + step.args['equals'] as bool? ?? true, + ); + return true; + case 'expectProperty': + executor.assertions.expectProperty( + executor.requireId(step), + step.args['property']?.toString() ?? 'label', + step.args['equals'], + ); + return true; + case 'expectStyle': + executor.assertions.expectProperty( + executor.requireId(step), + 'style', + step.args['equals'], + ); + return true; + case 'expectListCount': + executor.assertions.expectListCount( + listId: step.args['id']?.toString() ?? executor.requireId(step), + expected: step.args['equals'] as int? ?? + (throw EnsembleTestFailure('expectListCount requires "equals"')), + itemId: step.args['itemId']?.toString(), + ); + return true; + case 'expectListContains': + executor.assertions.expectListContains( + listId: step.args['id']?.toString() ?? '', + text: step.args['text']?.toString() ?? '', + ); + return true; + case 'expectListItem': + executor.assertions.expectVisible(step.args['itemId']?.toString() ?? ''); + return true; + case 'expectEmpty': + executor.assertions.expectListCount( + listId: executor.requireId(step), + expected: 0, + ); + return true; + case 'expectNotEmpty': + executor.assertions.expectListCount( + listId: executor.requireId(step), + expected: 1, + atLeast: true, + ); + return true; + case 'expectNotVisited': + final screen = step.args['screen']?.toString(); + if (screen == null) { + throw EnsembleTestFailure('expectNotVisited requires "screen"'); + } + executor.assertions.expectNotVisited(screen); + return true; + case 'expectBackStack': + executor.assertions.expectBackStack( + (step.args['screens'] as List?) + ?.map((e) => e.toString()) + .toList() ?? + [], + ); + return true; + case 'expectCanGoBack': + executor.assertions.expectCanGoBack( + step.args['equals'] as bool? ?? true, + ); + return true; + case 'goBack': + await _goBack(executor); + return true; + case 'mockApiFromFixture': + await _mockApiFromFixture(executor, step); + return true; + case 'clearApiMocks': + executor.context.mockApiProvider.clearMocks(); + return true; + case 'mockApiException': + await _mockApiException(executor, step); + return true; + case 'mockTimeout': + await _mockTimeout(executor, step); + return true; + case 'mockNetworkOffline': + _setNetworkOffline(executor, true); + return true; + case 'mockNetworkOnline': + _setNetworkOffline(executor, false); + return true; + case 'expectApiHeader': + executor.assertions.expectApiHeader( + step.args['name']?.toString() ?? '', + step.args['header']?.toString() ?? '', + step.args['equals'], + times: step.args['times'] as int?, + ); + return true; + case 'expectApiCallOrder': + executor.assertions.expectApiCallOrder( + (step.args['names'] as List?)?.map((e) => e.toString()).toList() ?? + [], + ); + return true; + case 'expectLastApiCall': + executor.assertions.expectLastApiCall(step.args['name']?.toString() ?? ''); + return true; + case 'removeStorage': + final key = step.args['key']?.toString(); + if (key == null) throw EnsembleTestFailure('removeStorage requires "key"'); + executor.context.removeStorage(key); + return true; + case 'clearStorage': + await executor.context.clearStorage(); + return true; + case 'expectStateContains': + executor.assertions.expectStateContains( + step.args['path']?.toString() ?? '', + step.args['contains'], + ); + return true; + case 'expectStateExists': + executor.assertions.expectStateExists(step.args['path']?.toString() ?? ''); + return true; + case 'expectStateNotExists': + executor.assertions.expectStateNotExists( + step.args['path']?.toString() ?? '', + ); + return true; + case 'resetState': + await _resetState(executor, step); + return true; + case 'setAuth': + _setAuth(executor, step); + return true; + case 'clearAuth': + _clearAuth(executor); + return true; + case 'setPermission': + _setPermission(executor, step); + return true; + case 'setDevice': + await _setDevice(executor, step); + return true; + case 'setLocale': + await _setLocale(executor, step); + return true; + case 'setTheme': + _setTheme(executor, step); + return true; + case 'runScript': + _runScript(executor, step, expectResult: false); + return true; + case 'expectScript': + case 'expectScriptResult': + _runScript(executor, step, expectResult: true); + return true; + case 'expectConsoleLog': + executor.assertions.expectConsoleLog(step.args['contains']?.toString() ?? ''); + return true; + case 'loadFixture': + await _loadFixture(executor, step); + return true; + case 'setStateFromFixture': + await _setStateFromFixture(executor, step); + return true; + case 'expectMatchesFixture': + await _expectMatchesFixture(executor, step); + return true; + case 'logState': + executor.context.logger.log( + 'state: ${executor.assertions.readState(step.args['path']?.toString() ?? '')}', + ); + return true; + case 'logStorage': + final key = step.args['key']?.toString(); + executor.context.logger.log( + 'storage[$key]=${StorageManager().read(key ?? '')}', + ); + return true; + case 'expectAccessible': + executor.assertions.expectAccessible(executor.requireId(step)); + return true; + case 'expectSemanticsLabel': + executor.assertions.expectSemanticsLabel( + executor.requireId(step), + step.args['label']?.toString() ?? '', + ); + return true; + case 'expectNoOverflow': + executor.assertions.expectNoOverflow(executor.requireId(step)); + return true; + case 'expectError': + executor.assertions.expectErrorRecorded( + step.args['contains']?.toString(), + ); + return true; + case 'expectNoErrors': + executor.assertions.expectNoRenderErrors(); + return true; + case 'screenshot': + await _screenshot(executor, step); + return true; + case 'dumpTree': + debugDumpApp(); + executor.context.logger.log('dumpTree: see debug console'); + return true; + case 'expectNoConsoleErrors': + executor.assertions.expectNoConsoleErrors(); + return true; + case 'expectNoRenderErrors': + executor.assertions.expectNoRenderErrors(); + return true; + default: + return false; + } + } + + static Future _reloadScreen(TestStepExecutor e) async { + final tracker = ScreenTracker(); + final current = tracker.getCurrentScreenIdentifier() ?? + e.context.testCase.startScreen; + if (current == null || current.isEmpty) { + throw EnsembleTestFailure('No current screen to reload'); + } + await e.openScreenByName(current); + } + + static Future _restartApp(TestStepExecutor e) async { + EnsembleTestHarness.resetTestRuntime(); + e.context.runtime.clear(); + e.context.mockApiProvider.resetCalls(); + final screen = e.context.testCase.startScreen ?? + ScreenTracker().getCurrentScreenIdentifier(); + if (screen == null || screen.isEmpty) { + throw EnsembleTestFailure( + 'restartApp requires startScreen or a tracked current screen', + ); + } + await e.openScreenByName(screen); + } + + static Future _resetAppState(TestStepExecutor e) async { + ScreenTracker().clearAll(); + e.context.mockApiProvider.resetCalls(); + e.context.runtime.clear(); + await StorageManager().clearPublicStorage(); + } + + static Future _trigger(TestStepExecutor e, TestStep step) async { + final action = step.args['action']?.toString() ?? 'onTap'; + switch (action) { + case 'onLoad': + await _reloadScreen(e); + case 'onTap': + case 'onLongPress': + final id = step.args['id']?.toString(); + if (id == null || id.isEmpty) { + throw EnsembleTestFailure('trigger onTap requires "id"'); + } + if (action == 'onLongPress') { + await e.longPressWidget(id); + } else { + await e.tapWidget(id); + } + default: + throw EnsembleTestFailure('Unsupported trigger action: $action'); + } + } + + static Future _selectIndex(TestStepExecutor e, TestStep step) async { + final id = e.requireId(step); + final index = step.args['index'] as int? ?? 0; + await e.tapWidget(id); + final options = find.byType(ListTile); + if (options.evaluate().length <= index) { + throw EnsembleTestFailure('selectIndex: no option at index $index'); + } + await e.tester.tap(options.at(index)); + await e.settle(); + } + + static Future _uncheck(TestStepExecutor e, TestStep step) async { + final id = e.requireId(step); + try { + e.assertions.expectChecked(id, true); + await e.tapWidget(id); + } on EnsembleTestFailure { + // already unchecked + } + } + + static Future _setSlider(TestStepExecutor e, TestStep step) async { + final id = e.requireId(step); + final value = (step.args['value'] as num?)?.toDouble() ?? 0.5; + final finder = e.assertions.finderForId(id); + final sliderFinder = find.descendant(of: finder, matching: find.byType(Slider)); + if (sliderFinder.evaluate().isEmpty) { + throw EnsembleTestFailure('setSlider: no Slider under id "$id"'); + } + final slider = e.tester.widget(sliderFinder); + final target = slider.min + (slider.max - slider.min) * value.clamp(0.0, 1.0); + await e.tester.drag(sliderFinder, Offset(target * 50, 0)); + await e.settle(); + } + + static Future _scroll(TestStepExecutor e, TestStep step) async { + final delta = step.args['delta'] as int? ?? -300; + final scrollable = find.byType(Scrollable); + if (scrollable.evaluate().isEmpty) { + throw EnsembleTestFailure('scroll: no Scrollable found'); + } + await e.tester.drag(scrollable.first, Offset(0, delta.toDouble())); + await e.settle(); + } + + static Future _swipe(TestStepExecutor e, TestStep step) async { + final direction = step.args['direction']?.toString() ?? 'left'; + Offset offset; + switch (direction) { + case 'right': + offset = const Offset(300, 0); + case 'up': + offset = const Offset(0, 300); + case 'down': + offset = const Offset(0, -300); + default: + offset = const Offset(-300, 0); + } + final target = step.args['id'] != null + ? e.assertions.finderForId(step.args['id'].toString()) + : find.byType(Scrollable).first; + await e.tester.drag(target, offset); + await e.settle(); + } + + static Future _drag(TestStepExecutor e, TestStep step) async { + final id = e.requireId(step); + final dx = (step.args['dx'] as num?)?.toDouble() ?? 0; + final dy = (step.args['dy'] as num?)?.toDouble() ?? -100; + await e.tester.drag(e.assertions.finderForId(id), Offset(dx, dy)); + await e.settle(); + } + + static Future _pullToRefresh(TestStepExecutor e, TestStep step) async { + final scrollable = step.args['id'] != null + ? e.assertions.finderForId(step.args['id'].toString()) + : find.byType(Scrollable).first; + await e.tester.drag(scrollable, const Offset(0, 300)); + await e.settle(); + } + + static Future _goBack(TestStepExecutor e) async { + final scope = e.assertions.activeScope(); + if (scope != null) { + ScreenController().executeAction( + scope.dataContext.buildContext, + NavigateBackAction.from(payload: null), + ); + await e.settle(); + return; + } + final navigator = find.byType(Navigator); + if (navigator.evaluate().isNotEmpty) { + final state = e.tester.state(navigator.first); + state.pop(); + await e.settle(); + return; + } + throw EnsembleTestFailure('goBack: no navigator or active scope'); + } + + static Future _mockApiFromFixture(TestStepExecutor e, TestStep step) async { + final name = step.args['name']?.toString(); + final fixture = step.args['fixture']?.toString(); + if (name == null || fixture == null) { + throw EnsembleTestFailure('mockApiFromFixture requires "name" and "fixture"'); + } + final body = await _readFixture(fixture); + e.context.mockApiProvider.setMock( + name, + MockAPIResponse( + statusCode: step.args['statusCode'] as int? ?? 200, + body: body, + ), + ); + } + + static Future _mockApiException(TestStepExecutor e, TestStep step) async { + final name = step.args['name']?.toString(); + if (name == null) throw EnsembleTestFailure('mockApiException requires "name"'); + e.context.mockApiProvider.setApiException( + name, + Exception(step.args['message']?.toString() ?? 'API exception (test)'), + ); + } + + static Future _mockTimeout(TestStepExecutor e, TestStep step) async { + final name = step.args['name']?.toString(); + if (name == null) throw EnsembleTestFailure('mockTimeout requires "name"'); + e.context.mockApiProvider.setMock( + name, + MockAPIResponse( + statusCode: 200, + body: {}, + delayMs: step.args['delayMs'] as int? ?? 60000, + ), + ); + } + + static void _setNetworkOffline(TestStepExecutor e, bool offline) { + e.context.mockApiProvider.simulateNetworkOffline = offline; + ConnectivityState().isOnline = !offline; + e.context.runtime.networkOffline = offline; + } + + static void _setAuth(TestStepExecutor e, TestStep step) { + final user = step.args['user']; + if (user is Map) { + e.context.runtime.authUser = Map.from(user); + StorageManager().writeToSystemStorage( + 'ensemble.auth.user', + e.context.runtime.authUser, + ); + } + } + + static void _clearAuth(TestStepExecutor e) { + e.context.runtime.authUser = null; + StorageManager().removeFromSystemStorage('ensemble.auth.user'); + } + + static void _setPermission(TestStepExecutor e, TestStep step) { + final name = step.args['name']?.toString(); + final value = step.args['value']?.toString() ?? 'granted'; + if (name != null) { + e.context.runtime.permissions[name] = value; + } + } + + static Future _setDevice(TestStepExecutor e, TestStep step) async { + final width = (step.args['width'] as num?)?.toDouble() ?? 390; + final height = (step.args['height'] as num?)?.toDouble() ?? 844; + e.tester.view.physicalSize = Size(width, height); + e.tester.view.devicePixelRatio = 1.0; + e.context.runtime.deviceSize = Size(width, height); + await e.settle(); + } + + static Future _setLocale(TestStepExecutor e, TestStep step) async { + final locale = step.args['locale']?.toString() ?? 'en'; + e.context.setEnv('APP_LOCALE', locale); + e.context.runtime.locale = Locale(locale.split('_').first); + } + + static void _setTheme(TestStepExecutor e, TestStep step) { + final theme = step.args['mode']?.toString() ?? step.args['theme']?.toString(); + if (theme != null) { + e.context.setEnv('APP_THEME', theme); + e.context.runtime.themeMode = theme; + } + } + + static void _runScript( + TestStepExecutor e, + TestStep step, { + required bool expectResult, + }) { + final scope = e.assertions.activeScope(); + if (scope == null) { + throw EnsembleTestFailure('runScript requires an active Ensemble screen'); + } + final script = step.args['script']?.toString() ?? step.args['path']?.toString(); + if (script == null) { + throw EnsembleTestFailure('runScript requires "script" or "path"'); + } + final result = scope.dataContext.eval(script); + if (expectResult) { + final expected = step.args['equals']; + if (!_deepEquals(result, expected)) { + throw EnsembleTestFailure( + 'Expected script result "$expected", got "$result".', + ); + } + } + } + + static Future _readFixture(String path) async { + final normalized = path.startsWith('ensemble/') ? path : 'ensemble/$path'; + final raw = await rootBundle.loadString(normalized); + return json.decode(raw); + } + + static Future _loadFixture(TestStepExecutor e, TestStep step) async { + final key = step.args['key']?.toString(); + final path = step.args['path']?.toString() ?? step.args['fixture']?.toString(); + if (key == null || path == null) { + throw EnsembleTestFailure('loadFixture requires "key" and "path"'); + } + e.context.runtime.fixtures[key] = await _readFixture(path); + } + + static Future _setStateFromFixture(TestStepExecutor e, TestStep step) async { + final path = step.args['fixture']?.toString() ?? step.args['path']?.toString(); + if (path == null) { + throw EnsembleTestFailure('setStateFromFixture requires "fixture" or "path"'); + } + final data = await _readFixture(path); + if (data is! Map) { + throw EnsembleTestFailure('Fixture must be a JSON object'); + } + final scope = e.assertions.activeScope(); + if (scope == null) { + throw EnsembleTestFailure('setStateFromFixture requires active screen'); + } + for (final entry in data.entries) { + setStatePath(scope, entry.key.toString(), entry.value); + } + await e.settle(); + } + + static Future _expectMatchesFixture( + TestStepExecutor e, + TestStep step, + ) async { + final path = step.args['fixture']?.toString() ?? step.args['path']?.toString(); + final statePath = step.args['statePath']?.toString(); + if (path == null) { + throw EnsembleTestFailure('expectMatchesFixture requires "fixture" or "path"'); + } + final expected = await _readFixture(path); + if (statePath != null) { + e.assertions.expectState(statePath, expected); + } else { + final actual = e.assertions.readState(step.args['path']?.toString() ?? ''); + if (!_deepEquals(actual, expected)) { + throw EnsembleTestFailure( + 'State does not match fixture $path: expected $expected, got $actual', + ); + } + } + } + + static Future _resetState(TestStepExecutor e, TestStep step) async { + final path = step.args['path']?.toString(); + final scope = e.assertions.activeScope(); + if (scope == null) { + throw EnsembleTestFailure('resetState requires active screen'); + } + if (path != null) { + setStatePath(scope, path, null); + } + await e.settle(); + } + + static Future _screenshot(TestStepExecutor e, TestStep step) async { + final name = step.args['name']?.toString(); + if (name != null && name.isNotEmpty) { + await expectLater( + find.byType(MaterialApp), + matchesGoldenFile('goldens/$name.png'), + ); + } else { + debugDumpApp(); + e.context.logger.log('screenshot: debug tree dumped (no golden name)'); + } + } + + static bool _deepEquals(dynamic a, dynamic b) { + if (a == b) return true; + if (a is Map && b is Map) { + if (a.length != b.length) return false; + for (final k in a.keys) { + if (!b.containsKey(k) || !_deepEquals(a[k], b[k])) return false; + } + return true; + } + if (a is List && b is List) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!_deepEquals(a[i], b[i])) return false; + } + return true; + } + return false; + } +} diff --git a/packages/ensemble_test_runner/lib/actions/state_helper.dart b/packages/ensemble_test_runner/lib/actions/state_helper.dart new file mode 100644 index 000000000..ed99e5a7e --- /dev/null +++ b/packages/ensemble_test_runner/lib/actions/state_helper.dart @@ -0,0 +1,34 @@ +import 'package:ensemble/framework/scope.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; + +/// Writes a dot-path value into the active [ScopeManager] data context. +void setStatePath(ScopeManager scope, String path, dynamic value) { + final parts = path.split('.').where((p) => p.isNotEmpty).toList(); + if (parts.isEmpty) { + throw EnsembleTestFailure('setState path cannot be empty'); + } + if (parts.length == 1) { + scope.dataContext.addDataContextById(parts.first, value); + return; + } + + final root = parts.first; + final existing = scope.dataContext.getContextMap()[root]; + final Map rootMap = existing is Map + ? Map.from(existing) + : {}; + + var cursor = rootMap; + for (var i = 1; i < parts.length - 1; i++) { + final key = parts[i]; + final next = cursor[key]; + if (next is Map) { + cursor[key] = Map.from(next); + } else { + cursor[key] = {}; + } + cursor = cursor[key] as Map; + } + cursor[parts.last] = value; + scope.dataContext.addDataContextById(root, rootMap); +} diff --git a/packages/ensemble_test_runner/lib/actions/test_execution_config.dart b/packages/ensemble_test_runner/lib/actions/test_execution_config.dart new file mode 100644 index 000000000..7ecff1cfc --- /dev/null +++ b/packages/ensemble_test_runner/lib/actions/test_execution_config.dart @@ -0,0 +1,14 @@ +/// Timing defaults for [TestStepExecutor] (overridable per executor instance). +class TestExecutionConfig { + final Duration settleStepDuration; + final Duration settleTimeout; + final Duration waitPollInterval; + final Duration defaultWaitTimeout; + + const TestExecutionConfig({ + this.settleStepDuration = const Duration(milliseconds: 100), + this.settleTimeout = const Duration(seconds: 10), + this.waitPollInterval = const Duration(milliseconds: 100), + this.defaultWaitTimeout = const Duration(seconds: 5), + }); +} diff --git a/packages/ensemble_test_runner/lib/actions/test_step_executor.dart b/packages/ensemble_test_runner/lib/actions/test_step_executor.dart new file mode 100644 index 000000000..0c365b4ad --- /dev/null +++ b/packages/ensemble_test_runner/lib/actions/test_step_executor.dart @@ -0,0 +1,633 @@ +import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble_test_runner/actions/extended_step_handlers.dart'; +import 'package:ensemble_test_runner/actions/state_helper.dart'; +import 'package:ensemble_test_runner/actions/test_execution_config.dart'; +import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// Translates declarative [TestStep]s into widget actions or assertions. +class TestStepExecutor { + final WidgetTester tester; + final EnsembleTestContext context; + final AssertionEngine assertions; + final EnsembleTestHarness harness; + final TestExecutionConfig config; + EnsembleConfig? _config; + + TestStepExecutor({ + required this.tester, + required this.context, + required this.assertions, + required this.harness, + EnsembleConfig? config, + TestExecutionConfig? executionConfig, + }) : _config = config, + config = executionConfig ?? const TestExecutionConfig(); + + Future execute(TestStep step) async { + if (step.type == 'group') { + for (final nested in step.nestedSteps) { + await execute(nested); + } + return; + } + if (step.type == 'repeat') { + final times = step.args['times'] as int? ?? 1; + for (var i = 0; i < times; i++) { + for (final nested in step.nestedSteps) { + await execute(nested); + } + } + return; + } + if (step.type == 'ifVisible') { + final id = step.args['id']?.toString(); + if (id == null || id.isEmpty) { + throw EnsembleTestFailure('ifVisible requires "id"'); + } + if (assertions.finderForId(id).evaluate().isNotEmpty) { + for (final nested in step.nestedSteps) { + await execute(nested); + } + } + return; + } + if (step.type == 'optional') { + try { + for (final nested in step.nestedSteps) { + await execute(nested); + } + } on EnsembleTestFailure { + // Best-effort steps (e.g. dismiss cookie banner). + } + return; + } + + switch (step.type) { + case 'wait': + await tester.pump( + Duration( + milliseconds: step.args['durationMs'] as int? ?? 500, + ), + ); + return; + case 'waitForText': + await _waitFor( + text: step.args['text']?.toString(), + timeoutMs: step.args['timeoutMs'] as int? ?? + config.defaultWaitTimeout.inMilliseconds, + ); + return; + case 'waitForGone': + await _waitForGone( + id: step.args['id']?.toString(), + timeoutMs: step.args['timeoutMs'] as int? ?? + config.defaultWaitTimeout.inMilliseconds, + ); + return; + case 'waitForApi': + await _waitForApi( + name: step.args['name']?.toString(), + times: step.args['times'] as int? ?? 1, + timeoutMs: step.args['timeoutMs'] as int? ?? + config.defaultWaitTimeout.inMilliseconds, + ); + return; + case 'waitForNavigation': + final screen = step.args['screen']?.toString(); + if (screen == null) { + throw EnsembleTestFailure('waitForNavigation requires "screen"'); + } + await _waitForNavigation( + screen: screen, + timeoutMs: step.args['timeoutMs'] as int? ?? + config.defaultWaitTimeout.inMilliseconds, + ); + return; + case 'waitUntil': + String? statePath = step.args['path']?.toString(); + dynamic expected = step.args['equals']; + final stateNode = step.args['state']; + if (stateNode is Map) { + statePath ??= stateNode['path']?.toString(); + expected ??= stateNode['equals']; + } + await _waitUntil( + path: statePath, + expected: expected, + timeoutMs: step.args['timeoutMs'] as int? ?? + config.defaultWaitTimeout.inMilliseconds, + ); + return; + case 'expectScreen': + final screen = step.args['name']?.toString() ?? + step.args['screen']?.toString(); + if (screen == null) { + throw EnsembleTestFailure('expectScreen requires "name" or "screen"'); + } + assertions.expectNavigateTo(screen); + return; + } + + final canonical = TestStepVocabulary.resolveStepType(step.type); + if (canonical != step.type) { + return execute(step.withCanonicalType(canonical)); + } + + switch (step.type) { + case 'openScreen': + await _openScreen(step); + break; + case 'tap': + await _tap(_requireId(step)); + break; + case 'enterText': + await _enterText( + _requireId(step), + step.args['value']?.toString() ?? '', + submit: step.args['submit'] == true, + ); + break; + case 'clearText': + await _clearText(_requireId(step)); + break; + case 'replaceText': + await _clearText(_requireId(step)); + await _enterText( + _requireId(step), + step.args['value']?.toString() ?? '', + submit: step.args['submit'] == true, + ); + break; + case 'submitText': + await _submitText(_requireId(step)); + break; + case 'select': + await _select(_requireId(step), step.args['value']?.toString()); + break; + case 'toggle': + await _tap(_requireId(step)); + break; + case 'waitFor': + await _waitFor( + id: step.args['id']?.toString(), + text: step.args['text']?.toString(), + timeoutMs: step.args['timeoutMs'] as int? ?? + config.defaultWaitTimeout.inMilliseconds, + ); + break; + case 'pump': + await tester.pump( + Duration( + milliseconds: step.args['durationMs'] as int? ?? + config.waitPollInterval.inMilliseconds, + ), + ); + break; + case 'settle': + await _settle( + timeout: step.args['timeoutMs'] != null + ? Duration(milliseconds: step.args['timeoutMs'] as int) + : null, + ); + break; + case 'scrollUntilVisible': + await _scrollUntilVisible(_requireId(step)); + break; + case 'expectVisible': + assertions.expectVisible(_requireId(step)); + break; + case 'expectNotVisible': + assertions.expectNotVisible(_requireId(step)); + break; + case 'expectText': + final text = step.args['text']?.toString(); + if (text == null) { + throw EnsembleTestFailure('expectText requires "text"'); + } + assertions.expectText(text); + break; + case 'expectNoText': + final text = step.args['text']?.toString(); + if (text == null) { + throw EnsembleTestFailure('expectNoText requires "text"'); + } + assertions.expectNoText(text); + break; + case 'expectEnabled': + assertions.expectEnabled(_requireId(step)); + break; + case 'expectDisabled': + assertions.expectDisabled(_requireId(step)); + break; + case 'expectValue': + assertions.expectValue(_requireId(step), step.args['equals']); + break; + case 'expectApiCalled': + final name = step.args['name']?.toString(); + if (name == null) { + throw EnsembleTestFailure('expectApiCalled requires "name"'); + } + assertions.expectApiCalled(name, step.args['times'] as int? ?? 1); + break; + case 'expectApiNotCalled': + final name = step.args['name']?.toString(); + if (name == null) { + throw EnsembleTestFailure('expectApiNotCalled requires "name"'); + } + assertions.expectApiNotCalled(name); + break; + case 'expectApiRequest': + final name = step.args['name']?.toString(); + if (name == null) { + throw EnsembleTestFailure('expectApiRequest requires "name"'); + } + assertions.expectApiRequest( + name, + body: step.args['body'], + query: step.args['query'], + headers: step.args['headers'], + times: step.args['times'] as int?, + ); + break; + case 'expectApiRequestContains': + final containsName = step.args['name']?.toString(); + if (containsName == null) { + throw EnsembleTestFailure('expectApiRequestContains requires "name"'); + } + assertions.expectApiRequestContains( + containsName, + body: step.args['body'], + query: step.args['query'], + headers: step.args['headers'], + times: step.args['times'] as int?, + ); + break; + case 'expectCount': + final expected = step.args['equals'] as int?; + if (expected == null) { + throw EnsembleTestFailure('expectCount requires "equals"'); + } + assertions.expectCount(_requireId(step), expected); + break; + case 'expectNavigateTo': + final screen = step.args['screen']?.toString(); + if (screen == null) { + throw EnsembleTestFailure('expectNavigateTo requires "screen"'); + } + assertions.expectNavigateTo(screen); + break; + case 'expectVisited': + final screen = step.args['screen']?.toString(); + if (screen == null) { + throw EnsembleTestFailure('expectVisited requires "screen"'); + } + assertions.expectVisited(screen); + break; + case 'expectStorage': + final key = step.args['key']?.toString(); + if (key == null) { + throw EnsembleTestFailure('expectStorage requires "key"'); + } + assertions.expectStorage(key, step.args['equals']); + break; + case 'expectState': + final path = step.args['path']?.toString(); + if (path == null) { + throw EnsembleTestFailure('expectState requires "path"'); + } + assertions.expectState(path, step.args['equals']); + break; + case 'setState': + final path = step.args['path']?.toString(); + if (path == null) { + throw EnsembleTestFailure('setState requires "path"'); + } + final scope = assertions.activeScope(); + if (scope == null) { + throw EnsembleTestFailure( + 'setState requires an active Ensemble screen.', + ); + } + setStatePath(scope, path, step.args['value']); + break; + case 'setStorage': + final key = step.args['key']?.toString(); + if (key == null) { + throw EnsembleTestFailure('setStorage requires "key"'); + } + context.setStorage(key, step.args['value']); + break; + case 'setEnv': + final key = step.args['key']?.toString(); + if (key == null) { + throw EnsembleTestFailure('setEnv requires "key"'); + } + context.setEnv(key, step.args['value']); + break; + case 'mockApi': + final name = step.args['name']?.toString(); + if (name == null) { + throw EnsembleTestFailure('mockApi requires "name"'); + } + context.mockApiProvider.setMock( + name, + context.mockFromStepArgs(step.args), + ); + break; + case 'mockApiError': + final name = step.args['name']?.toString(); + if (name == null) { + throw EnsembleTestFailure('mockApiError requires "name"'); + } + context.mockApiProvider.setMock( + name, + MockAPIResponse( + statusCode: step.args['statusCode'] as int? ?? 500, + body: step.args['body'], + delayMs: step.args['delayMs'] as int?, + ), + ); + break; + case 'resetApiCalls': + context.mockApiProvider.resetCalls(); + break; + case 'logApiCalls': + for (final call in context.mockApiProvider.calls) { + context.logger.log( + 'API ${call.name} body=${call.body} query=${call.query}', + ); + } + break; + default: + if (await ExtendedStepHandlers.tryExecute(this, step)) { + return; + } + throw EnsembleTestFailure( + 'Unknown test step: ${step.type}. See STEP_VOCABULARY.md for supported steps.', + ); + } + } + + String requireId(TestStep step) => _requireId(step); + + Future tapWidget(String id) => _tap(id); + + Future longPressWidget(String id) async { + final finder = assertions.finderForId(id); + expect(finder, findsOneWidget); + await tester.longPress(finder); + await _settle(); + } + + Future focusWidget(String id) async { + final finder = assertions.finderForId(id); + expect(finder, findsOneWidget); + await tester.tap(finder); + await _settle(); + } + + Future unfocus() async { + FocusManager.instance.primaryFocus?.unfocus(); + await _settle(); + } + + Future enterTextOn(String id, String value) => + _enterText(id, value, submit: false); + + Future settle({Duration? timeout}) => _settle(timeout: timeout); + + Future openScreenByName(String screen) async { + final tc = context.testCase; + _config = await harness.loadScreen( + tester: tester, + testCase: EnsembleTestCase( + id: tc.id, + startScreen: screen, + initialState: tc.initialState, + mocks: tc.mocks, + steps: const [], + ), + existingConfig: _config, + context: context, + ); + await _settle(); + } + + String _requireId(TestStep step) { + final id = step.args['id']?.toString(); + if (id == null || id.isEmpty) { + throw EnsembleTestFailure('Step "${step.type}" requires "id"'); + } + return id; + } + + Future _settle({Duration? timeout}) async { + await tester.pumpAndSettle( + config.settleStepDuration, + EnginePhase.sendSemanticsUpdate, + timeout ?? config.settleTimeout, + ); + } + + Future _tap(String id) async { + final finder = assertions.finderForId(id); + if (finder.evaluate().isEmpty) { + await _waitFor( + id: id, + timeoutMs: config.defaultWaitTimeout.inMilliseconds, + ); + } + expect(finder, findsOneWidget); + await tester.tap(finder); + await _settle(); + } + + Future _enterText(String id, String value, {bool submit = false}) async { + final finder = assertions.finderForId(id); + expect(finder, findsOneWidget); + await tester.enterText(finder, value); + if (submit) { + await tester.testTextInput.receiveAction(TextInputAction.done); + } + await _settle(); + } + + Future _submitText(String id) async { + final finder = assertions.finderForId(id); + expect(finder, findsOneWidget); + await tester.tap(finder); + await tester.testTextInput.receiveAction(TextInputAction.done); + await _settle(); + } + + Future _clearText(String id) async { + final finder = assertions.finderForId(id); + expect(finder, findsOneWidget); + await tester.enterText(finder, ''); + await _settle(); + } + + Future _select(String id, String? value) async { + if (value == null || value.isEmpty) { + throw EnsembleTestFailure('select requires "value"'); + } + await _tap(id); + final option = find.text(value); + if (option.evaluate().isEmpty) { + throw EnsembleTestFailure('select could not find option "$value"'); + } + await tester.tap(option); + await _settle(); + } + + Future _scrollUntilVisible(String id) async { + final finder = assertions.finderForId(id); + await tester.scrollUntilVisible( + finder, + 300, + scrollable: find.byType(Scrollable).first, + ); + await _settle(); + } + + Future _waitFor({ + String? id, + String? text, + required int timeoutMs, + }) async { + if (id == null && text == null) { + throw EnsembleTestFailure('waitFor requires either "id" or "text"'); + } + + final stopwatch = Stopwatch()..start(); + while (stopwatch.elapsedMilliseconds < timeoutMs) { + await tester.pump(config.waitPollInterval); + if (id != null && assertions.finderForId(id).evaluate().isNotEmpty) { + return; + } + if (text != null && find.text(text).evaluate().isNotEmpty) { + return; + } + } + + final target = id != null && text != null + ? 'id "$id" or text "$text"' + : id != null + ? 'id "$id"' + : 'text "$text"'; + throw EnsembleTestFailure( + 'Timed out after ${timeoutMs}ms waiting for $target', + ); + } + + Future _openScreen(TestStep step) async { + final screen = step.args['name']?.toString() ?? + step.args['screen']?.toString(); + if (screen == null || screen.isEmpty) { + throw EnsembleTestFailure('openScreen requires "name" or "screen"'); + } + final tc = context.testCase; + _config = await harness.loadScreen( + tester: tester, + testCase: EnsembleTestCase( + id: tc.id, + startScreen: screen, + initialState: tc.initialState, + mocks: tc.mocks, + steps: const [], + ), + existingConfig: _config, + context: context, + ); + await _settle(); + } + + Future _waitForApi({ + String? name, + required int times, + required int timeoutMs, + }) async { + if (name == null || name.isEmpty) { + throw EnsembleTestFailure('waitForApi requires "name"'); + } + + final stopwatch = Stopwatch()..start(); + while (stopwatch.elapsedMilliseconds < timeoutMs) { + await tester.pump(config.waitPollInterval); + if (context.mockApiProvider.callCount(name) >= times) { + return; + } + } + throw EnsembleTestFailure( + 'Timed out after ${timeoutMs}ms waiting for API "$name" ' + 'to be called $times time(s)', + ); + } + + Future _waitForNavigation({ + required String screen, + required int timeoutMs, + }) async { + final stopwatch = Stopwatch()..start(); + final tracker = ScreenTracker(); + while (stopwatch.elapsedMilliseconds < timeoutMs) { + await tester.pump(config.waitPollInterval); + if (tracker.isScreenVisible(screenName: screen) || + tracker.isScreenVisible(screenId: screen)) { + return; + } + } + throw EnsembleTestFailure( + 'Timed out after ${timeoutMs}ms waiting for navigation to "$screen"', + ); + } + + Future _waitUntil({ + String? path, + dynamic expected, + required int timeoutMs, + }) async { + if (path == null || path.isEmpty) { + throw EnsembleTestFailure('waitUntil requires "path"'); + } + + final stopwatch = Stopwatch()..start(); + while (stopwatch.elapsedMilliseconds < timeoutMs) { + await tester.pump(config.waitPollInterval); + if (assertions.matchesState(path, expected)) { + return; + } + } + throw EnsembleTestFailure( + 'Timed out after ${timeoutMs}ms waiting for state "$path" ' + 'to equal "$expected"', + ); + } + + Future _waitForGone({ + String? id, + required int timeoutMs, + }) async { + if (id == null || id.isEmpty) { + throw EnsembleTestFailure('waitForGone requires "id"'); + } + + final stopwatch = Stopwatch()..start(); + while (stopwatch.elapsedMilliseconds < timeoutMs) { + await tester.pump(config.waitPollInterval); + if (assertions.finderForId(id).evaluate().isEmpty) { + return; + } + } + throw EnsembleTestFailure( + 'Timed out after ${timeoutMs}ms waiting for id "$id" to disappear', + ); + } +} diff --git a/packages/ensemble_test_runner/lib/assertions/assertion_engine.dart b/packages/ensemble_test_runner/lib/assertions/assertion_engine.dart new file mode 100644 index 000000000..ecbc2552a --- /dev/null +++ b/packages/ensemble_test_runner/lib/assertions/assertion_engine.dart @@ -0,0 +1,660 @@ +import 'dart:convert'; + +import 'package:ensemble/framework/scope.dart'; +import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/framework/view/data_scope_widget.dart'; +import 'package:ensemble/framework/view/page_group.dart'; +import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; + +class AssertionEngine { + final WidgetTester tester; + final EnsembleTestContext context; + + AssertionEngine({ + required this.tester, + required this.context, + }); + + Finder finderForId(String id) => find.byKey(ValueKey(id)); + + void expectVisible(String id) { + final finder = finderForId(id); + if (finder.evaluate().isEmpty) { + throw EnsembleTestFailure('Expected widget with id "$id" to be visible.'); + } + } + + void expectNotVisible(String id) { + final finder = finderForId(id); + if (finder.evaluate().isNotEmpty) { + throw EnsembleTestFailure( + 'Expected widget with id "$id" to not be visible.', + ); + } + } + + void expectText(String text) { + final finder = find.text(text); + if (finder.evaluate().isEmpty) { + throw EnsembleTestFailure('Expected text "$text" to be visible.'); + } + } + + void expectNoText(String text) { + if (find.text(text).evaluate().isNotEmpty) { + throw EnsembleTestFailure('Expected text "$text" to not be visible.'); + } + } + + void expectEnabled(String id) { + _expectEnabledState(id, enabled: true); + } + + void expectDisabled(String id) { + _expectEnabledState(id, enabled: false); + } + + void _expectEnabledState(String id, {required bool enabled}) { + final finder = finderForId(id); + if (finder.evaluate().isEmpty) { + throw EnsembleTestFailure( + 'Expected widget with id "$id" to exist for enabled check.', + ); + } + final semantics = tester.getSemantics(finder); + final isEnabled = semantics.flagsCollection.isEnabled; + if (isEnabled != enabled) { + throw EnsembleTestFailure( + 'Expected widget "$id" to be ${enabled ? 'enabled' : 'disabled'}, ' + 'but it was ${isEnabled ? 'enabled' : 'disabled'}.', + ); + } + } + + void expectApiNotCalled(String apiName) { + final count = context.mockApiProvider.callCount(apiName); + if (count != 0) { + throw EnsembleTestFailure( + 'Expected API "$apiName" not to be called, but it was called $count times.', + ); + } + } + + void expectValue(String id, dynamic expected) { + final finder = finderForId(id); + if (finder.evaluate().isEmpty) { + throw EnsembleTestFailure( + 'Expected widget with id "$id" to be visible for expectValue.', + ); + } + + final editableFinder = find.descendant( + of: finder, + matching: find.byType(EditableText), + ); + if (editableFinder.evaluate().isNotEmpty) { + final editable = tester.widget(editableFinder); + final actual = editable.controller.text; + if (actual != expected?.toString()) { + throw EnsembleTestFailure( + 'Expected input "$id" value "$expected", but got "$actual".', + ); + } + return; + } + + final textFieldFinder = find.descendant( + of: finder, + matching: find.byType(TextField), + ); + if (textFieldFinder.evaluate().isNotEmpty) { + final field = tester.widget(textFieldFinder); + final actual = field.controller?.text; + if (actual != expected?.toString()) { + throw EnsembleTestFailure( + 'Expected input "$id" value "$expected", but got "$actual".', + ); + } + return; + } + + throw EnsembleTestFailure( + 'No EditableText or TextField found under widget id "$id".', + ); + } + + void expectApiCalled(String apiName, int times) { + final actual = context.mockApiProvider.callCount(apiName); + if (actual != times) { + throw EnsembleTestFailure( + 'Expected API "$apiName" to be called $times times, but it was called $actual times.', + ); + } + } + + void expectApiRequest( + String apiName, { + dynamic body, + dynamic query, + dynamic headers, + int? times, + }) { + final record = _selectApiCall(apiName, times: times); + + if (body != null && !_deepEquals(record.body, body)) { + throw EnsembleTestFailure( + 'Expected API "$apiName" body $body, but got ${record.body}.', + ); + } + if (query != null && !_deepEquals(record.query, query)) { + throw EnsembleTestFailure( + 'Expected API "$apiName" query $query, but got ${record.query}.', + ); + } + if (headers != null && !_deepEquals(record.headers, headers)) { + throw EnsembleTestFailure( + 'Expected API "$apiName" headers $headers, but got ${record.headers}.', + ); + } + } + + void expectApiRequestContains( + String apiName, { + dynamic body, + dynamic query, + dynamic headers, + int? times, + }) { + final record = _selectApiCall(apiName, times: times); + + if (body != null && !_deepContains(record.body, body)) { + throw EnsembleTestFailure( + 'Expected API "$apiName" body to contain $body, but got ${record.body}.', + ); + } + if (query != null && !_deepContains(record.query, query)) { + throw EnsembleTestFailure( + 'Expected API "$apiName" query to contain $query, but got ${record.query}.', + ); + } + if (headers != null && !_deepContains(record.headers, headers)) { + throw EnsembleTestFailure( + 'Expected API "$apiName" headers to contain $headers, but got ${record.headers}.', + ); + } + } + + void expectCount(String id, int expected) { + final count = finderForId(id).evaluate().length; + if (count != expected) { + throw EnsembleTestFailure( + 'Expected $expected widget(s) with id "$id", but found $count.', + ); + } + } + + void expectExists(String id) { + if (finderForId(id).evaluate().isEmpty) { + throw EnsembleTestFailure('Expected widget with id "$id" to exist.'); + } + } + + void expectNotExists(String id) { + if (finderForId(id).evaluate().isNotEmpty) { + throw EnsembleTestFailure('Expected widget with id "$id" to not exist.'); + } + } + + void expectTextContains(String text) { + if (find.textContaining(text).evaluate().isEmpty) { + throw EnsembleTestFailure('Expected text containing "$text".'); + } + } + + void expectChecked(String id, bool expected) { + final finder = finderForId(id); + if (finder.evaluate().isEmpty) { + throw EnsembleTestFailure('expectChecked: widget "$id" not found.'); + } + final semantics = tester.getSemantics(finder); + final isChecked = semantics.flagsCollection.isChecked; + if (isChecked != expected) { + throw EnsembleTestFailure( + 'Expected "$id" checked=$expected, got $isChecked.', + ); + } + } + + void expectProperty(String id, String property, dynamic expected) { + final finder = finderForId(id); + if (finder.evaluate().isEmpty) { + throw EnsembleTestFailure('expectProperty: widget "$id" not found.'); + } + if (property == 'label') { + final semantics = tester.getSemantics(finder); + final label = semantics.label; + if (label != expected?.toString()) { + throw EnsembleTestFailure( + 'Expected label "$expected", got "$label".', + ); + } + return; + } + throw EnsembleTestFailure('Unsupported property "$property".'); + } + + void expectListCount({ + required String listId, + required int expected, + String? itemId, + bool atLeast = false, + }) { + final listFinder = finderForId(listId); + if (listFinder.evaluate().isEmpty) { + throw EnsembleTestFailure('expectListCount: list "$listId" not found.'); + } + final count = itemId != null + ? find + .descendant(of: listFinder, matching: finderForId(itemId)) + .evaluate() + .length + : find.descendant(of: listFinder, matching: find.byWidgetPredicate((_) => true)).evaluate().length; + if (atLeast) { + if (count < expected) { + throw EnsembleTestFailure( + 'Expected at least $expected items in "$listId", found $count.', + ); + } + return; + } + if (count != expected) { + throw EnsembleTestFailure( + 'Expected $expected items in "$listId", found $count.', + ); + } + } + + void expectListContains({required String listId, required String text}) { + final listFinder = finderForId(listId); + final match = find.descendant(of: listFinder, matching: find.textContaining(text)); + if (match.evaluate().isEmpty) { + throw EnsembleTestFailure( + 'Expected list "$listId" to contain text "$text".', + ); + } + } + + void expectNotVisited(String screenName) { + final flow = YamlTestSession.navigationFlow.flow; + final visited = flow.contains(screenName); + if (visited) { + throw EnsembleTestFailure( + 'Expected screen "$screenName" not to be visited.', + ); + } + } + + void expectBackStack(List screens) { + final history = ScreenTracker() + .screenHistory + .map((s) => s.screenName ?? s.screenId) + .whereType() + .toList(); + if (history.length < screens.length) { + throw EnsembleTestFailure( + 'Back stack too short. Expected suffix $screens, got $history', + ); + } + final suffix = history.sublist(history.length - screens.length); + if (!_deepEquals(suffix, screens)) { + throw EnsembleTestFailure( + 'Expected back stack suffix $screens, got $suffix (full: $history)', + ); + } + } + + void expectCanGoBack(bool expected) { + final canPop = ScreenTracker().screenHistory.isNotEmpty; + if (canPop != expected) { + throw EnsembleTestFailure( + 'Expected canGoBack=$expected, but history length is ' + '${ScreenTracker().screenHistory.length}.', + ); + } + } + + void expectApiHeader( + String apiName, + String header, + dynamic expected, { + int? times, + }) { + final record = _selectApiCall(apiName, times: times); + final headers = record.headers ?? {}; + final actual = headers[header.toLowerCase()]; + if (!_deepEquals(actual, expected)) { + throw EnsembleTestFailure( + 'Expected API "$apiName" header "$header" = "$expected", got "$actual".', + ); + } + } + + void expectApiCallOrder(List names) { + final actual = context.mockApiProvider.calls.map((c) => c.name).toList(); + var index = 0; + for (final name in names) { + while (index < actual.length && actual[index] != name) { + index++; + } + if (index >= actual.length) { + throw EnsembleTestFailure( + 'Expected API call order $names, but got $actual', + ); + } + index++; + } + } + + void expectLastApiCall(String apiName) { + final calls = context.mockApiProvider.calls; + if (calls.isEmpty || calls.last.name != apiName) { + throw EnsembleTestFailure( + 'Expected last API call to be "$apiName", ' + 'but got ${calls.isEmpty ? "none" : calls.last.name}.', + ); + } + } + + void expectStateContains(String path, dynamic contains) { + final actual = readState(path); + if (!_deepContains(actual, contains)) { + throw EnsembleTestFailure( + 'Expected state "$path" to contain $contains, got $actual.', + ); + } + } + + void expectStateExists(String path) { + try { + readState(path); + } catch (_) { + throw EnsembleTestFailure('Expected state path "$path" to exist.'); + } + } + + void expectStateNotExists(String path) { + try { + final value = readState(path); + if (value != null) { + throw EnsembleTestFailure('Expected state path "$path" to be absent.'); + } + } on EnsembleTestFailure { + // expected + } + } + + void expectConsoleLog(String contains) { + final logs = context.runtime.consoleLogs; + if (!logs.any((l) => l.contains(contains))) { + throw EnsembleTestFailure( + 'Expected console log containing "$contains", got: $logs', + ); + } + } + + void expectAccessible(String id) { + final finder = finderForId(id); + if (finder.evaluate().isEmpty) { + throw EnsembleTestFailure('expectAccessible: "$id" not found.'); + } + final semantics = tester.getSemantics(finder); + if (semantics.label.isEmpty && semantics.value.isEmpty) { + throw EnsembleTestFailure( + 'Widget "$id" has no accessibility label or value.', + ); + } + } + + void expectSemanticsLabel(String id, String label) { + final finder = finderForId(id); + final semantics = tester.getSemantics(finder); + if (semantics.label != label) { + throw EnsembleTestFailure( + 'Expected semantics label "$label", got "${semantics.label}".', + ); + } + } + + void expectNoOverflow(String id) { + final finder = finderForId(id); + final renderObject = tester.renderObject(finder); + if (renderObject is RenderBox && renderObject.hasSize) { + // No direct overflow flag; presence without exception is sufficient. + return; + } + } + + void expectNoConsoleErrors() { + if (context.runtime.consoleLogs.isNotEmpty) { + throw EnsembleTestFailure( + 'Expected no console errors, got: ${context.runtime.consoleLogs}', + ); + } + } + + void expectNoRenderErrors() { + if (context.runtime.flutterErrors.isNotEmpty) { + throw EnsembleTestFailure( + 'Expected no render errors, got: ${context.runtime.flutterErrors}', + ); + } + } + + void expectErrorRecorded(String? contains) { + final errors = context.runtime.flutterErrors; + if (errors.isEmpty) { + throw EnsembleTestFailure('Expected a recorded error, but none found.'); + } + if (contains != null && + !errors.any((e) => e.contains(contains))) { + throw EnsembleTestFailure( + 'Expected error containing "$contains", got: $errors', + ); + } + } + + APICallRecord _selectApiCall(String apiName, {int? times}) { + final calls = context.mockApiProvider.callsFor(apiName); + if (calls.isEmpty) { + throw EnsembleTestFailure('Expected API "$apiName" to be called.'); + } + if (times != null) { + if (times < 1 || times > calls.length) { + throw EnsembleTestFailure( + 'Expected API "$apiName" call #$times, but it was called ${calls.length} times.', + ); + } + return calls[times - 1]; + } + return calls.last; + } + + /// Reads a data-context path without asserting. + dynamic readState(String path) { + final scope = _activeScope(); + if (scope == null) { + throw EnsembleTestFailure( + 'readState requires an active Ensemble screen (no ScopeManager found).', + ); + } + if (path.contains(r'${') || path.contains(r'$(')) { + return scope.dataContext.eval(path); + } + return scope.dataContext.eval('\${$path}'); + } + + bool matchesState(String path, dynamic expected) { + try { + expectState(path, expected); + return true; + } on EnsembleTestFailure { + return false; + } + } + + void expectStorage(String key, dynamic expected) { + final actual = StorageManager().read(key); + if (actual != expected) { + throw EnsembleTestFailure( + 'Expected storage "$key" to equal "$expected", but got "$actual".', + ); + } + } + + ScopeManager? activeScope() => _activeScope(); + + void expectState(String path, dynamic expected) { + final scope = _activeScope(); + if (scope == null) { + throw EnsembleTestFailure( + 'expectState requires an active Ensemble screen (no ScopeManager found).', + ); + } + + dynamic actual; + if (path.contains(r'${') || path.contains(r'$(')) { + actual = scope.dataContext.eval(path); + } else { + actual = scope.dataContext.eval('\${$path}'); + } + + if (!_deepEquals(actual, expected)) { + throw EnsembleTestFailure( + 'Expected state "$path" to equal "$expected", but got "$actual".', + ); + } + } + + void expectNavigateTo(String screenName) { + final tracker = ScreenTracker(); + if (!tracker.isScreenVisible(screenName: screenName) && + !tracker.isScreenVisible(screenId: screenName)) { + final current = tracker.getCurrentScreenIdentifier(); + final history = tracker.screenHistory + .map((s) => s.screenName ?? s.screenId) + .whereType() + .toList(); + throw EnsembleTestFailure( + 'Expected navigation to "$screenName", but current screen is "$current". ' + 'History: $history', + ); + } + } + + void expectVisited(String screenName) { + final flow = YamlTestSession.navigationFlow.flow; + final visited = flow.contains(screenName); + if (!visited) { + throw EnsembleTestFailure( + 'Expected screen "$screenName" in navigation history, but visited ' + '$flow', + ); + } + } + + ScopeManager? _activeScope() { + for (final element in find.byType(DataScopeWidget).evaluate()) { + final scope = DataScopeWidget.getScope(element); + if (scope != null) return scope; + } + for (final element in find.byType(PageGroupWidget).evaluate()) { + final scope = PageGroupWidget.getScope(element); + if (scope != null) return scope; + } + return null; + } + + /// True when [subset] is contained in [value] (maps/lists recurse). + bool _deepContains(dynamic value, dynamic subset) { + if (subset == null) return true; + if (value == null) return false; + + final normalizedValue = _normalizeForCompare(value); + final normalizedSubset = _normalizeForCompare(subset); + + if (normalizedSubset is Map) { + if (normalizedValue is! Map) return false; + for (final entry in normalizedSubset.entries) { + if (!normalizedValue.containsKey(entry.key)) return false; + if (!_deepContains(normalizedValue[entry.key], entry.value)) { + return false; + } + } + return true; + } + + if (normalizedSubset is List) { + if (normalizedValue is! List) return false; + for (final item in normalizedSubset) { + final found = normalizedValue.any((v) => _deepContains(v, item)); + if (!found) return false; + } + return true; + } + + return normalizedValue == normalizedSubset; + } + + bool _deepEquals(dynamic a, dynamic b) { + if (a == b) return true; + + final normalizedA = _normalizeForCompare(a); + final normalizedB = _normalizeForCompare(b); + + if (normalizedA is Map && normalizedB is Map) { + if (normalizedA.length != normalizedB.length) return false; + for (final key in normalizedA.keys) { + if (!normalizedB.containsKey(key)) return false; + if (!_deepEquals(normalizedA[key], normalizedB[key])) return false; + } + return true; + } + + if (normalizedA is List && normalizedB is List) { + if (normalizedA.length != normalizedB.length) return false; + for (var i = 0; i < normalizedA.length; i++) { + if (!_deepEquals(normalizedA[i], normalizedB[i])) return false; + } + return true; + } + + return false; + } + + dynamic _normalizeForCompare(dynamic value) { + if (value == null) return null; + if (value is Map) { + return value.map( + (key, val) => MapEntry(key.toString(), _normalizeForCompare(val)), + ); + } + if (value is List) { + return value.map(_normalizeForCompare).toList(); + } + if (value is String) { + final trimmed = value.trim(); + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + try { + return _normalizeForCompare(json.decode(trimmed)); + } catch (_) {} + } + } + return value; + } +} diff --git a/packages/ensemble_test_runner/lib/cli/ensemble_test_cli.dart b/packages/ensemble_test_runner/lib/cli/ensemble_test_cli.dart new file mode 100644 index 000000000..87013b663 --- /dev/null +++ b/packages/ensemble_test_runner/lib/cli/ensemble_test_cli.dart @@ -0,0 +1,116 @@ +import 'dart:io'; + +import 'package:ensemble_test_runner/cli/ensemble_test_cli_output.dart'; +import 'package:ensemble_test_runner/cli/yaml_test_app_patcher.dart'; + +/// Runs declarative YAML tests in an Ensemble app. +/// +/// The host app must list `ensemble_test_runner` in `dev_dependencies`. +/// +/// Options: +/// --app-dir= App directory (default: current directory) +/// --verbose Full `flutter pub get` / `flutter test` output +Future runEnsembleYamlTestsCli(List arguments) async { + final verbose = isVerboseCli(arguments); + final appDir = _resolveAppDir(arguments); + final patcher = YamlTestAppPatcher(appDir); + + if (!Directory(appDir).existsSync()) { + stderr.writeln('App directory not found: $appDir'); + exit(1); + } + + if (!File('${appDir}/pubspec.yaml').existsSync()) { + stderr.writeln('No pubspec.yaml in $appDir'); + exit(1); + } + + var exitCode = 0; + try { + patcher.enable(); + + if (patcher.pubspecChanged) { + final pubGet = await _runProcess( + 'flutter', + ['pub', 'get', '--suppress-analytics'], + workingDirectory: appDir, + ); + if (pubGet.exitCode != 0 || verbose) { + _writeProcessStreams(pubGet); + } + if (pubGet.exitCode != 0) { + exitCode = pubGet.exitCode; + return; + } + } + + final testArgs = [ + 'test', + YamlTestAppPatcher.testEntryRelativePath, + '--no-pub', + '--dart-define=testmode=true', + '--reporter', + verbose ? 'expanded' : 'silent', + ...flutterTestArguments(arguments), + ]; + + final testRun = await _runProcess( + 'flutter', + testArgs, + workingDirectory: appDir, + ); + + if (verbose || testRun.exitCode != 0) { + _writeProcessStreams(testRun); + } else { + final out = testRun.stdout?.toString() ?? ''; + final report = extractSuiteReport(out); + if (report.isNotEmpty) { + stdout.write(report); + } else { + _writeProcessStreams(testRun); + } + final err = testRun.stderr?.toString() ?? ''; + if (err.isNotEmpty && !isBenignFlutterTestStderr(err)) { + stderr.write(err); + } + } + + exitCode = testRun.exitCode; + } on StateError catch (error) { + stderr.writeln(error.message); + exitCode = 1; + } finally { + patcher.restore(); + } + exit(exitCode); +} + +Future _runProcess( + String executable, + List arguments, { + required String workingDirectory, +}) { + return Process.run( + executable, + arguments, + workingDirectory: workingDirectory, + runInShell: false, + ); +} + +String _resolveAppDir(List arguments) { + for (final arg in arguments) { + if (arg.startsWith('--app-dir=')) { + return arg.substring('--app-dir='.length); + } + } + return Directory.current.path; +} + +void _writeProcessStreams(ProcessResult result) { + final out = result.stdout?.toString() ?? ''; + final err = result.stderr?.toString() ?? ''; + if (out.isNotEmpty) stdout.write(out); + if (err.isNotEmpty) stderr.write(err); +} diff --git a/packages/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart b/packages/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart new file mode 100644 index 000000000..1f88258d3 --- /dev/null +++ b/packages/ensemble_test_runner/lib/cli/ensemble_test_cli_output.dart @@ -0,0 +1,52 @@ +/// CLI output filtering for [runEnsembleYamlTestsCli]. +library; + +const suiteReportStart = '┌─ Ensemble YAML tests'; +const screenTrackerPrefix = 'SCREEN TRACKER:'; + +/// Strips Flutter test framework noise; keeps navigation logs and the suite report. +String extractSuiteReport(String output) { + final lines = output.split('\n'); + final kept = []; + var inReport = false; + + for (final line in lines) { + if (line.startsWith(screenTrackerPrefix)) { + kept.add(line); + continue; + } + if (line.startsWith(suiteReportStart)) { + inReport = true; + } + if (inReport) { + kept.add(line); + if (line.startsWith('└─')) { + inReport = false; + } + } + } + + if (kept.isEmpty) return ''; + return '${kept.join('\n')}\n'; +} + +/// Whether the user asked for full subprocess output (`--verbose`). +bool isVerboseCli(List arguments) => + arguments.contains('--verbose'); + +/// Arguments forwarded to `flutter test` (CLI-only flags removed). +List flutterTestArguments(List arguments) { + return arguments + .where( + (a) => + !a.startsWith('--app-dir') && a != '--verbose' && a != '--quiet', + ) + .toList(); +} + +/// Flutter sometimes prints asset cleanup warnings after a successful run. +bool isBenignFlutterTestStderr(String stderr) { + if (stderr.trim().isEmpty) return true; + return stderr.contains('Failed to clean up asset directory') && + stderr.contains('flutter clean'); +} diff --git a/packages/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart b/packages/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart new file mode 100644 index 000000000..d466194aa --- /dev/null +++ b/packages/ensemble_test_runner/lib/cli/yaml_test_app_patcher.dart @@ -0,0 +1,146 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; + +/// Temporarily wires [appDir] to run declarative YAML tests, then restores files. +class YamlTestAppPatcher { + YamlTestAppPatcher(String appDir) : appDir = p.normalize(p.absolute(appDir)); + + final String appDir; + + static const testEntryRelativePath = 'test/ensemble_tests.dart'; + static const testsAssetLine = ' - ensemble/tests/'; + static const testsDirRelative = 'ensemble/tests'; + + static const testEntryContents = ''' +// Generated by ensemble_test_runner — run `dart run ensemble_test_runner:ensemble_test`. +import 'package:ensemble_test_runner/entry/ensemble_test_entry.dart'; + +void main() => runEnsembleYamlTests(); +'''; + + final Map _backups = {}; + bool _enabled = false; + bool _removeTestEntryOnRestore = false; + bool _pubspecChanged = false; + + bool get pubspecChanged => _pubspecChanged; + + String get _pubspecPath => p.join(appDir, 'pubspec.yaml'); + String get _testEntryPath => p.join(appDir, testEntryRelativePath); + String get _testDirPath => p.join(appDir, 'test'); + + void enable() { + if (_enabled) return; + + _backup(_pubspecPath); + _backup(_testEntryPath, optional: true); + + final priorTestEntry = _backups[_testEntryPath] ?? ''; + _removeTestEntryOnRestore = + priorTestEntry.isEmpty || priorTestEntry == testEntryContents; + + final pubspec = File(_pubspecPath).readAsStringSync(); + final activated = _activatePubspec(pubspec); + _pubspecChanged = activated != pubspec; + if (_pubspecChanged) { + File(_pubspecPath).writeAsStringSync(activated); + } + + Directory(_testDirPath).createSync(recursive: true); + final testEntry = File(_testEntryPath); + if (!testEntry.existsSync() || + testEntry.readAsStringSync() != testEntryContents) { + testEntry.writeAsStringSync(testEntryContents); + } + + _enabled = true; + } + + void restore() { + if (!_enabled) return; + + _restore(_pubspecPath); + if (_removeTestEntryOnRestore) { + _deleteTestEntry(); + } else { + _restore(_testEntryPath); + } + + _backups.clear(); + _enabled = false; + _removeTestEntryOnRestore = false; + _pubspecChanged = false; + } + + void _backup(String path, {bool optional = false}) { + final file = File(path); + if (!file.existsSync()) { + if (!optional) { + throw StateError('Expected file not found: $path'); + } + _backups[path] = ''; + return; + } + _backups[path] = file.readAsStringSync(); + } + + void _restore(String path) { + final backup = _backups[path]; + if (backup == null || backup.isEmpty) return; + File(path).writeAsStringSync(backup); + } + + void _deleteTestEntry() { + final file = File(_testEntryPath); + if (file.existsSync()) { + file.deleteSync(); + } + final testDir = Directory(_testDirPath); + if (testDir.existsSync() && testDir.listSync().isEmpty) { + testDir.deleteSync(); + } + } + + String _activatePubspec(String content) { + if (_hasTestYamlOnDisk) { + return _activateTestAssets(content); + } + return content; + } + + bool get _hasTestYamlOnDisk { + final testsDir = Directory(p.join(appDir, testsDirRelative)); + if (!testsDir.existsSync()) return false; + return testsDir + .listSync() + .any((entity) => entity.path.endsWith('.test.yaml')); + } + + static String _activateTestAssets(String content) { + if (content.contains(testsAssetLine) || + content.contains('- ensemble/tests/')) { + return content; + } + + const ensembleDirLine = ' - ensemble/\n'; + if (content.contains(ensembleDirLine)) { + return content.replaceFirst( + ensembleDirLine, + '$ensembleDirLine$testsAssetLine\n', + ); + } + + const assetsMarker = '\n assets:\n'; + if (content.contains(assetsMarker)) { + return content.replaceFirst( + assetsMarker, + '$assetsMarker$testsAssetLine\n', + ); + } + + throw StateError( + 'Could not patch pubspec.yaml assets: add a flutter.assets section.', + ); + } +} diff --git a/packages/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart b/packages/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart new file mode 100644 index 000000000..8f3abab80 --- /dev/null +++ b/packages/ensemble_test_runner/lib/discovery/ensemble_test_discovery.dart @@ -0,0 +1,78 @@ +import 'package:ensemble/framework/ensemble_config_service.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:flutter/services.dart'; + +/// App target resolved from `ensemble/ensemble-config.yaml` (`definitions.local`). +class EnsembleTestAppTarget { + final String appPath; + final String appHome; + final String? i18nPath; + + const EnsembleTestAppTarget({ + required this.appPath, + required this.appHome, + this.i18nPath, + }); +} + +/// Discovers declarative tests and app settings from the host Flutter app bundle. +class EnsembleTestDiscovery { + static const String testsAssetPrefix = 'ensemble/tests/'; + + /// All `*.test.yaml` files bundled under [testsAssetPrefix]. + static Future> findTestYamlAssets() async { + final manifest = await AssetManifest.loadFromAssetBundle(rootBundle); + final files = manifest + .listAssets() + .where( + (path) => + path.startsWith(testsAssetPrefix) && path.endsWith('.test.yaml'), + ) + .toList() + ..sort(); + return files; + } + + /// Reads `definitions.local` from `ensemble/ensemble-config.yaml`. + static Future loadAppTarget() async { + if (!EnsembleConfigService.isInitialized) { + await EnsembleConfigService.initialize(); + } + + final definitions = EnsembleConfigService.config['definitions']; + if (definitions is! Map) { + throw EnsembleTestFailure( + 'ensemble/ensemble-config.yaml must define "definitions"', + ); + } + + final local = definitions['local']; + if (local is! Map) { + throw EnsembleTestFailure( + 'Declarative tests require definitions.local in ensemble-config.yaml ' + '(path, appHome, i18n.path)', + ); + } + + final path = local['path']?.toString(); + final appHome = local['appHome']?.toString(); + if (path == null || path.isEmpty || appHome == null || appHome.isEmpty) { + throw EnsembleTestFailure( + 'definitions.local.path and definitions.local.appHome are required', + ); + } + + String? i18nPath; + final i18n = local['i18n']; + if (i18n is Map && i18n['path'] != null) { + i18nPath = i18n['path'].toString(); + } + + return EnsembleTestAppTarget( + appPath: EnsembleTestHarness.normalizeAppPath(path), + appHome: appHome, + i18nPath: i18nPath, + ); + } +} diff --git a/packages/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart b/packages/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart new file mode 100644 index 000000000..bf8b457ca --- /dev/null +++ b/packages/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart @@ -0,0 +1,169 @@ +import 'package:flutter/foundation.dart'; + +import 'package:ensemble_test_runner/discovery/ensemble_test_discovery.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; + +/// A parsed `*.test.yaml` file with its asset path. +class EnsembleTestDefinition { + final String assetPath; + final EnsembleTestCase testCase; + + const EnsembleTestDefinition({ + required this.assetPath, + required this.testCase, + }); +} + +/// Topologically sorted test run order (each test id appears once). +class EnsembleTestExecutionPlan { + final List ordered; + + const EnsembleTestExecutionPlan({required this.ordered}); +} + +/// Builds a dependency-ordered execution plan for all declarative tests. +class EnsembleTestExecutionPlanner { + /// Discovers assets, parses every file, validates graph, returns run order. + static Future build() async { + final paths = await EnsembleTestDiscovery.findTestYamlAssets(); + if (paths.isEmpty) { + throw EnsembleTestFailure( + 'No declarative tests found. Add *.test.yaml files under ensemble/tests/ ', + ); + } + + final byId = {}; + for (final path in paths) { + final testCase = await EnsembleTestParser.parseFile(path); + final existing = byId[testCase.id]; + if (existing != null) { + throw EnsembleTestFailure( + 'Duplicate test id "${testCase.id}" in ${existing.assetPath} and $path', + ); + } + byId[testCase.id] = EnsembleTestDefinition( + assetPath: path, + testCase: testCase, + ); + } + + for (final def in byId.values) { + final prereq = def.testCase.prerequisite; + if (prereq != null && !byId.containsKey(prereq)) { + throw EnsembleTestFailure( + 'Test "${def.testCase.id}" in ${def.assetPath} references unknown ' + 'prerequisite "$prereq"', + ); + } + } + + final ordered = _topologicalSort(byId); + return EnsembleTestExecutionPlan(ordered: ordered); + } + + /// IDs that participate in a shared session (`prerequisite` chain). + static Set _sessionConnectedIds( + Map byId, + ) { + final connected = {}; + for (final def in byId.values) { + final prereq = def.testCase.prerequisite; + if (prereq == null) continue; + connected.add(def.testCase.id); + connected.add(prereq); + } + var expanded = true; + while (expanded) { + expanded = false; + for (final def in byId.values) { + final prereq = def.testCase.prerequisite; + if (prereq != null && + connected.contains(prereq) && + connected.add(def.testCase.id)) { + expanded = true; + } + } + } + return connected; + } + + /// Kahn's algorithm: edge from test → its prerequisite (prereq runs first). + /// + /// Tests with [EnsembleTestCase.hasStartScreen] that are not in a prerequisite + /// chain run **after** the chain so [EnsembleTestHarness.loadScreen] does not + /// reset the session for continuation tests. + static List _topologicalSort( + Map byId, + ) { + final sessionConnected = _sessionConnectedIds(byId); + final inDegree = {}; + final dependents = >{}; + + for (final id in byId.keys) { + inDegree[id] = 0; + dependents[id] = []; + } + + for (final entry in byId.entries) { + final prereq = entry.value.testCase.prerequisite; + if (prereq == null) continue; + inDegree[entry.key] = (inDegree[entry.key] ?? 0) + 1; + dependents[prereq]!.add(entry.key); + } + + bool sessionChainComplete(List ordered) { + if (sessionConnected.isEmpty) return true; + return sessionConnected.every(ordered.contains); + } + + bool canSchedule(String id, List ordered) { + if (inDegree[id] != 0) return false; + if (sessionConnected.contains(id)) return true; + return sessionChainComplete(ordered); + } + + final ready = []; + for (final id in byId.keys) { + if (canSchedule(id, const [])) ready.add(id); + } + ready.sort((a, b) => byId[a]!.assetPath.compareTo(byId[b]!.assetPath)); + + final orderedIds = []; + while (ready.isNotEmpty) { + ready.sort((a, b) => byId[a]!.assetPath.compareTo(byId[b]!.assetPath)); + final id = ready.removeAt(0); + orderedIds.add(id); + for (final dependent in dependents[id]!) { + inDegree[dependent] = inDegree[dependent]! - 1; + if (inDegree[dependent] == 0 && canSchedule(dependent, orderedIds)) { + ready.add(dependent); + } + } + for (final candidate in byId.keys) { + if (!orderedIds.contains(candidate) && + !ready.contains(candidate) && + canSchedule(candidate, orderedIds)) { + ready.add(candidate); + } + } + } + + if (orderedIds.length != byId.length) { + throw EnsembleTestFailure( + 'Circular prerequisite dependency among tests: ' + '${byId.keys.where((id) => !orderedIds.contains(id)).join(", ")}', + ); + } + + return orderedIds.map((id) => byId[id]!).toList(); + } + + /// Exposed for unit tests only. + @visibleForTesting + static List orderIdsForTest( + Map byId, + ) { + return _topologicalSort(byId).map((d) => d.testCase.id).toList(); + } +} diff --git a/packages/ensemble_test_runner/lib/ensemble_test_runner.dart b/packages/ensemble_test_runner/lib/ensemble_test_runner.dart new file mode 100644 index 000000000..744bd8549 --- /dev/null +++ b/packages/ensemble_test_runner/lib/ensemble_test_runner.dart @@ -0,0 +1,17 @@ +library ensemble_test_runner; + +export 'discovery/ensemble_test_discovery.dart'; +export 'entry/ensemble_test_entry.dart'; +export 'vocabulary/test_step_vocabulary.dart'; +export 'actions/test_execution_config.dart'; +export 'actions/test_step_executor.dart'; +export 'assertions/assertion_engine.dart'; +export 'models/ensemble_test_models.dart'; +export 'mocks/mock_api_provider.dart'; +export 'mocks/test_logger.dart'; +export 'parser/ensemble_test_parser.dart'; +export 'schema/ensemble_test_schema_builder.dart'; +export 'reporters/test_reporter.dart'; +export 'runner/ensemble_test_context.dart'; +export 'runner/ensemble_test_harness.dart'; +export 'runner/ensemble_test_runner.dart'; diff --git a/packages/ensemble_test_runner/lib/entry/ensemble_test_entry.dart b/packages/ensemble_test_runner/lib/entry/ensemble_test_entry.dart new file mode 100644 index 000000000..4e19f5636 --- /dev/null +++ b/packages/ensemble_test_runner/lib/entry/ensemble_test_entry.dart @@ -0,0 +1,70 @@ +import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; +import 'package:ensemble_test_runner/ensemble_test_runner.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// Flutter test entry: discovers `ensemble/tests/*.test.yaml` and runs them. +void runEnsembleYamlTests() { + EnsembleTestHarness.ensureTestPlugins(); + tearDown(() { + TestErrorTracker.reset(); + EnsembleTestHarness.resetTestRuntime(); + YamlTestSession.dispose(); + }); + + testWidgets( + 'ensemble/tests/*.test.yaml', + (tester) async { + final plan = await EnsembleTestExecutionPlanner.build(); + final target = await EnsembleTestDiscovery.loadAppTarget(); + final harness = EnsembleTestHarness( + appPath: target.appPath, + appHome: target.appHome, + i18nPath: target.i18nPath, + ); + + final runner = EnsembleTestRunner(harness: harness); + final resultsById = await runner.runPlan(plan, tester); + + final failures = []; + final orderedResults = []; + + for (final def in plan.ordered) { + final result = resultsById[def.testCase.id]!; + orderedResults.add( + EnsembleSingleTestResult( + testId: '${result.testId} (${def.assetPath})', + status: result.status, + durationMs: result.durationMs, + failedStepIndex: result.failedStepIndex, + failedStep: result.failedStep, + message: result.message, + stackTrace: result.stackTrace, + logs: result.logs, + report: result.report, + ), + ); + + if (result.status == TestStatus.failed) { + failures.add(def.assetPath); + } + } + + final suiteSummary = TestReporter().formatSummary( + EnsembleTestRunResult(results: orderedResults), + testFile: 'ensemble/tests/*.test.yaml', + ); + print(suiteSummary); + + if (failures.isNotEmpty) { + fail( + 'Failed YAML tests:\n' + '${failures.map((p) => '- $p').join('\n')}\n\n' + '$suiteSummary', + ); + } + }, + timeout: const Timeout(Duration(minutes: 10)), + ); +} diff --git a/packages/ensemble_test_runner/lib/mocks/mock_api_provider.dart b/packages/ensemble_test_runner/lib/mocks/mock_api_provider.dart new file mode 100644 index 000000000..5e5407851 --- /dev/null +++ b/packages/ensemble_test_runner/lib/mocks/mock_api_provider.dart @@ -0,0 +1,186 @@ +import 'dart:convert'; + +import 'package:ensemble/framework/apiproviders/api_provider.dart'; +import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; +import 'package:ensemble/framework/data_context.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:flutter/widgets.dart'; +import 'package:yaml/yaml.dart'; + +class APICallRecord { + final String name; + final YamlMap apiDefinition; + final DateTime timestamp; + final dynamic body; + final Map? query; + final Map? headers; + + APICallRecord({ + required this.name, + required this.apiDefinition, + required this.timestamp, + this.body, + this.query, + this.headers, + }); +} + +/// Records API calls and returns YAML-configured mock responses by API name. +class MockAPIProvider extends HTTPAPIProvider { + MockAPIProvider({ + required Map mocks, + HTTPAPIProvider? delegate, + }) : _mocks = mocks, + _delegate = delegate ?? HTTPAPIProvider(); + + final Map _mocks; + final HTTPAPIProvider _delegate; + final List calls = []; + + int callCount(String apiName) => + calls.where((c) => c.name == apiName).length; + + List callsFor(String apiName) => + calls.where((c) => c.name == apiName).toList(); + + void setMock(String apiName, MockAPIResponse response) { + _mocks[apiName] = response; + } + + void resetCalls() => calls.clear(); + + void clearMocks() => _mocks.clear(); + + final Map _forcedExceptions = {}; + + void setApiException(String apiName, Exception error) { + _forcedExceptions[apiName] = error; + } + + void clearApiExceptions() => _forcedExceptions.clear(); + + @override + Future init(String appId, Map config) => + _delegate.init(appId, config); + + /// When true, [invokeApi] returns an offline error without calling the delegate. + bool simulateNetworkOffline = false; + + @override + Future invokeApi( + BuildContext context, + YamlMap api, + DataContext eContext, + String apiName, + ) async { + if (simulateNetworkOffline) { + calls.add(APICallRecord( + name: apiName, + apiDefinition: api, + timestamp: DateTime.now(), + )); + return HttpResponse.fromBody( + {'message': 'Network offline (test)'}, + null, + 503, + null, + APIState.error, + ); + } + + final forced = _forcedExceptions[apiName]; + if (forced != null) { + calls.add(APICallRecord( + name: apiName, + apiDefinition: api, + timestamp: DateTime.now(), + )); + throw forced; + } + + final captured = _captureRequest(api, eContext); + calls.add(APICallRecord( + name: apiName, + apiDefinition: api, + timestamp: DateTime.now(), + body: captured.body, + query: captured.query, + headers: captured.headers, + )); + + final mock = _mocks[apiName]; + if (mock == null) { + return _delegate.invokeApi(context, api, eContext, apiName); + } + + if (mock.delayMs != null && mock.delayMs! > 0) { + await Future.delayed(Duration(milliseconds: mock.delayMs!)); + } + + return HttpResponse.fromBody( + mock.body, + mock.headers?.map((k, v) => MapEntry(k, v.toString())), + mock.statusCode, + null, + APIState.success, + ); + } + + @override + Future invokeMockAPI(DataContext eContext, dynamic mock) => + _delegate.invokeMockAPI(eContext, mock); + + /// Same instance as config — keeps call recording aligned with [EnsembleTestContext]. + @override + MockAPIProvider clone() => this; + + @override + void dispose() => _delegate.dispose(); +} + +class _CapturedRequest { + final dynamic body; + final Map? query; + final Map? headers; + + const _CapturedRequest({this.body, this.query, this.headers}); +} + +_CapturedRequest _captureRequest(YamlMap api, DataContext eContext) { + Map? headers; + if (api['headers'] is YamlMap) { + headers = {}; + (api['headers'] as YamlMap).forEach((key, value) { + if (value != null) { + headers![key.toString().toLowerCase()] = + eContext.eval(value)?.toString() ?? ''; + } + }); + } + + dynamic body; + if (api['body'] != null) { + final evaluated = eContext.eval(api['body']); + if (evaluated is Map || evaluated is List) { + body = evaluated; + } else if (evaluated is String) { + try { + body = json.decode(evaluated); + } catch (_) { + body = evaluated; + } + } else { + body = evaluated; + } + } + + Map? query; + if (api['parameters'] is YamlMap) { + query = {}; + (api['parameters'] as YamlMap).forEach((key, value) { + query![key.toString()] = eContext.eval(value)?.toString() ?? ''; + }); + } + + return _CapturedRequest(body: body, query: query, headers: headers); +} diff --git a/packages/ensemble_test_runner/lib/mocks/test_logger.dart b/packages/ensemble_test_runner/lib/mocks/test_logger.dart new file mode 100644 index 000000000..ee3ee0b1c --- /dev/null +++ b/packages/ensemble_test_runner/lib/mocks/test_logger.dart @@ -0,0 +1,10 @@ +/// Simple in-memory logger for test runs. +class TestLogger { + final List logs = []; + + void log(String message) { + logs.add(message); + } + + void clear() => logs.clear(); +} diff --git a/packages/ensemble_test_runner/lib/models/ensemble_test_models.dart b/packages/ensemble_test_runner/lib/models/ensemble_test_models.dart new file mode 100644 index 000000000..54ddfc50d --- /dev/null +++ b/packages/ensemble_test_runner/lib/models/ensemble_test_models.dart @@ -0,0 +1,223 @@ +/// Declarative test document and run results. +library; + +class EnsembleTestRunRequest { + final String? appPath; + final String? appHome; + final String? i18nPath; + final List tests; + final EnsembleTestEnvironment environment; + + const EnsembleTestRunRequest({ + this.appPath, + this.appHome, + this.i18nPath, + required this.tests, + this.environment = const EnsembleTestEnvironment(), + }); +} + +class EnsembleTestEnvironment { + final Map env; + + const EnsembleTestEnvironment({this.env = const {}}); +} + +class EnsembleTestCase { + final String id; + final String? type; + /// Cold-start screen. Omit when [prerequisite] is set. + final String? startScreen; + /// Test [id] that must run before this one (same app session). + final String? prerequisite; + final Map initialState; + final TestMocks mocks; + final List steps; + + const EnsembleTestCase({ + required this.id, + this.type, + this.startScreen, + this.prerequisite, + this.initialState = const {}, + this.mocks = const TestMocks(), + required this.steps, + }); + + bool get hasStartScreen => + startScreen != null && startScreen!.isNotEmpty; + + bool get hasPrerequisite => + prerequisite != null && prerequisite!.isNotEmpty; +} + +class TestMocks { + final Map apis; + + const TestMocks({this.apis = const {}}); +} + +class MockAPIResponse { + final int statusCode; + final dynamic body; + final Map? headers; + final int? delayMs; + + const MockAPIResponse({ + this.statusCode = 200, + this.body, + this.headers, + this.delayMs, + }); +} + +class TestStep { + /// YAML step key (e.g. `expectVisible`, `group`). + final String type; + final Map args; + + /// Nested steps for [group], [repeat], [optional], [ifVisible]. + final List nestedSteps; + + const TestStep({ + required this.type, + required this.args, + this.nestedSteps = const [], + }); + + /// Canonical handler name after alias resolution. + String get canonicalType => type; + + TestStep withCanonicalType(String canonical) => TestStep( + type: canonical, + args: args, + nestedSteps: nestedSteps, + ); + + Map toJson() => { + type: args, + if (nestedSteps.isNotEmpty) + 'steps': nestedSteps.map((s) => s.toJson()).toList(), + }; +} + +class EnsembleTestRunResult { + final List results; + + const EnsembleTestRunResult({required this.results}); + + int get passedCount => results.where((r) => r.status == TestStatus.passed).length; + int get failedCount => results.where((r) => r.status == TestStatus.failed).length; + + String get summary => + '$passedCount passed, $failedCount failed (${results.length} total)'; + + Map toJson() => { + 'status': failedCount > 0 ? 'failed' : 'passed', + 'total': results.length, + 'passed': passedCount, + 'failed': failedCount, + 'results': results.map((r) => r.toJson()).toList(), + }; +} + +enum TestStatus { passed, failed } + +class EnsembleSingleTestResult { + final String testId; + final TestStatus status; + final int durationMs; + final int? failedStepIndex; + final TestStep? failedStep; + final String? message; + final String? stackTrace; + final List logs; + final EnsembleTestReportDetails? report; + + const EnsembleSingleTestResult({ + required this.testId, + required this.status, + required this.durationMs, + this.failedStepIndex, + this.failedStep, + this.message, + this.stackTrace, + this.logs = const [], + this.report, + }); + + factory EnsembleSingleTestResult.passed({ + required String testId, + required int durationMs, + List logs = const [], + EnsembleTestReportDetails? report, + }) => + EnsembleSingleTestResult( + testId: testId, + status: TestStatus.passed, + durationMs: durationMs, + logs: logs, + report: report, + ); + + factory EnsembleSingleTestResult.failed({ + required String testId, + required int durationMs, + int? failedStepIndex, + TestStep? failedStep, + String? error, + String? stackTrace, + List logs = const [], + EnsembleTestReportDetails? report, + }) => + EnsembleSingleTestResult( + testId: testId, + status: TestStatus.failed, + durationMs: durationMs, + failedStepIndex: failedStepIndex, + failedStep: failedStep, + message: error, + stackTrace: stackTrace, + logs: logs, + report: report, + ); + + Map toJson() => { + 'testId': testId, + 'status': status.name, + 'durationMs': durationMs, + if (failedStepIndex != null) 'failedStepIndex': failedStepIndex, + if (failedStep != null) 'failedStep': failedStep!.toJson(), + if (message != null) 'message': message, + if (stackTrace != null) 'stackTrace': stackTrace, + 'logs': logs, + }; +} + +/// Human-readable run metadata for console reports (see [TestReporter]). +class EnsembleTestReportDetails { + /// Display start screen (explicit or inherited from runtime). + final String startScreen; + final String? endScreen; + final String? prerequisite; + final List screensVisited; + final List stepsOutline; + + const EnsembleTestReportDetails({ + required this.startScreen, + this.endScreen, + this.prerequisite, + this.screensVisited = const [], + this.stepsOutline = const [], + }); +} + +/// Thrown when a test step or assertion fails. +class EnsembleTestFailure implements Exception { + final String message; + + EnsembleTestFailure(this.message); + + @override + String toString() => message; +} diff --git a/packages/ensemble_test_runner/lib/parser/ensemble_test_parser.dart b/packages/ensemble_test_runner/lib/parser/ensemble_test_parser.dart new file mode 100644 index 000000000..ecd6f5014 --- /dev/null +++ b/packages/ensemble_test_runner/lib/parser/ensemble_test_parser.dart @@ -0,0 +1,226 @@ +import 'dart:io'; + +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; +import 'package:yaml/yaml.dart'; + +class EnsembleTestParser { + /// Loads a test file from [rootBundle] (widget/integration tests) or [File] (CLI). + static Future parseFile(String path) async { + Object? bundleError; + try { + final content = await rootBundle.loadString(path); + return parseString(content, sourcePath: path); + } catch (error) { + bundleError = error; + } + + // dart:io can hang under widget test bindings' fake async zone. + if (_isWidgetTestBinding) { + throw EnsembleTestFailure( + 'Test file not found in asset bundle: $path ($bundleError)', + ); + } + + final file = File(path); + if (!await file.exists()) { + throw EnsembleTestFailure('Test file not found: $path'); + } + return parseString(await file.readAsString(), sourcePath: path); + } + + static bool get _isWidgetTestBinding { + final type = WidgetsBinding.instance.runtimeType.toString(); + return type.contains('TestWidgetsFlutterBinding') || + type.contains('LiveTestWidgetsFlutterBinding'); + } + + static EnsembleTestCase parseString(String content, {String? sourcePath}) { + final dynamic doc = loadYaml(content); + if (doc is! YamlMap) { + throw EnsembleTestFailure( + 'Invalid test file${sourcePath != null ? ' ($sourcePath)' : ''}: root must be a map'); + } + + if (doc.containsKey('tests')) { + throw EnsembleTestFailure( + 'Each *.test.yaml file defines one test at the root — remove the "tests" ' + 'wrapper and put id, startScreen, and steps at the top level.', + ); + } + + return _parseTestCase(doc); + } + + static EnsembleTestCase _parseTestCase(YamlMap map) { + final id = map['id']?.toString(); + if (id == null || id.isEmpty) { + throw EnsembleTestFailure('Each test must have an "id"'); + } + + final startScreen = map['startScreen']?.toString(); + final hasStartScreen = + startScreen != null && startScreen.isNotEmpty; + final prerequisite = map['prerequisite']?.toString(); + final hasPrerequisite = + prerequisite != null && prerequisite.isNotEmpty; + + if (hasStartScreen && hasPrerequisite) { + throw EnsembleTestFailure( + 'Test "$id" must have either "startScreen" or "prerequisite", not both', + ); + } + if (!hasStartScreen && !hasPrerequisite) { + throw EnsembleTestFailure( + 'Test "$id" must have either "startScreen" or "prerequisite"', + ); + } + + final stepsNode = map['steps']; + if (stepsNode is! YamlList || stepsNode.isEmpty) { + throw EnsembleTestFailure('Test "$id" must have a non-empty "steps" list'); + } + + return EnsembleTestCase( + id: id, + type: map['type']?.toString(), + startScreen: hasStartScreen ? startScreen : null, + prerequisite: hasPrerequisite ? prerequisite : null, + initialState: _toStringDynamicMap(map['initialState']), + mocks: _parseMocks(map['mocks']), + steps: _parseSteps(stepsNode, testId: id), + ); + } + + static TestMocks _parseMocks(dynamic node) { + if (node == null) return const TestMocks(); + if (node is! YamlMap) { + throw EnsembleTestFailure('"mocks" must be a map'); + } + + final apisNode = node['apis']; + if (apisNode == null) return const TestMocks(); + if (apisNode is! YamlMap) { + throw EnsembleTestFailure('"mocks.apis" must be a map'); + } + + final apis = {}; + apisNode.forEach((key, value) { + if (value is! YamlMap) { + throw EnsembleTestFailure('Mock for API "$key" must be a map'); + } + apis[key.toString()] = _parseMockApiResponse(value); + }); + + return TestMocks(apis: apis); + } + + static MockAPIResponse _parseMockApiResponse(YamlMap map) { + final response = map['response']; + if (response is! YamlMap) { + throw EnsembleTestFailure('API mock must include a "response" map'); + } + + return MockAPIResponse( + statusCode: response['statusCode'] as int? ?? 200, + body: _unwrapYaml(response['body']), + headers: response['headers'] is YamlMap + ? _toStringDynamicMap(response['headers']) + : null, + delayMs: map['delayMs'] as int?, + ); + } + + static List _parseSteps(YamlList steps, {required String testId}) => + _parseStepsList(steps, testId: testId); + + static List _parseStepsList(dynamic steps, {required String testId}) { + if (steps is! List || steps.isEmpty) { + throw EnsembleTestFailure('Test "$testId" requires a non-empty "steps" list'); + } + final result = []; + for (var i = 0; i < steps.length; i++) { + result.add(_parseStep(steps[i], testId: testId, index: i)); + } + return result; + } + + static TestStep _parseStep(dynamic step, {required String testId, int? index}) { + final String type; + final dynamic argsNode; + + if (step is YamlMap && step.length == 1) { + type = step.keys.first.toString(); + argsNode = step[type]; + } else if (step is Map && step.length == 1) { + type = step.keys.first.toString(); + argsNode = step[type]; + } else { + throw EnsembleTestFailure( + 'Test "$testId" step ${index ?? ''} must be a single-key map ' + '(e.g. expectVisible: {...})', + ); + } + + final args = _argsFromNode(argsNode); + + List nested = const []; + if (type == 'group' || type == 'repeat') { + final stepsNode = args['steps']; + if (stepsNode is List && stepsNode.isNotEmpty) { + nested = _parseStepsList(stepsNode, testId: testId); + } else { + throw EnsembleTestFailure('"$type" requires a non-empty "steps" list'); + } + } else if (type == 'optional' || type == 'ifVisible') { + final single = args['step']; + if (single is YamlMap && single.length == 1) { + nested = [_parseStep(single, testId: testId)]; + } else if (single is Map && single.length == 1) { + nested = [_parseStep(single, testId: testId)]; + } else if (args['steps'] is List && (args['steps'] as List).isNotEmpty) { + nested = _parseStepsList(args['steps'], testId: testId); + } + } + + return TestStep(type: type, args: args, nestedSteps: nested); + } + + static Map _argsFromNode(dynamic node) { + if (node is YamlMap) { + return _toStringDynamicMap(node); + } + if (node is Map) { + return node.map( + (key, value) => MapEntry(key.toString(), _unwrapYaml(value)), + ); + } + if (node != null) { + return {'value': _unwrapYaml(node)}; + } + return {}; + } + + static Map _toStringDynamicMap(dynamic node) { + if (node == null) return {}; + if (node is! YamlMap) { + throw EnsembleTestFailure('Expected a map'); + } + final out = {}; + node.forEach((key, value) { + out[key.toString()] = _unwrapYaml(value); + }); + return out; + } + + static dynamic _unwrapYaml(dynamic value) { + if (value is YamlMap) { + return _toStringDynamicMap(value); + } + if (value is YamlList) { + return value.map(_unwrapYaml).toList(); + } + return value; + } +} diff --git a/packages/ensemble_test_runner/lib/reporters/test_reporter.dart b/packages/ensemble_test_runner/lib/reporters/test_reporter.dart new file mode 100644 index 000000000..86431f1a4 --- /dev/null +++ b/packages/ensemble_test_runner/lib/reporters/test_reporter.dart @@ -0,0 +1,148 @@ +import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; + +EnsembleTestReportDetails buildTestReportDetails(EnsembleTestCase testCase) { + final effectiveStart = testCase.startScreen ?? + ScreenTracker().getCurrentScreenIdentifier() ?? + '(unknown)'; + return EnsembleTestReportDetails( + startScreen: effectiveStart, + endScreen: ScreenTracker().getCurrentScreenIdentifier(), + prerequisite: testCase.prerequisite, + screensVisited: collectScreensVisited(effectiveStart), + stepsOutline: outlineSteps(testCase.steps), + ); +} + +List collectScreensVisited(String effectiveStart) { + final flow = List.from(YamlTestSession.navigationFlow.flow); + if (flow.isEmpty) { + return [effectiveStart]; + } + if (flow.first != effectiveStart) { + return [effectiveStart, ...flow]; + } + return flow; +} + +List outlineSteps(List steps) { + final lines = []; + for (final step in steps) { + lines.addAll(_outlineStep(step)); + } + return lines; +} + +List _outlineStep(TestStep step) { + if (step.nestedSteps.isNotEmpty) { + final nested = outlineSteps(step.nestedSteps); + return [ + '${step.type} (${nested.length} steps)', + ...nested.map((line) => ' $line'), + ]; + } + return [formatStepBrief(step)]; +} + +/// Short label for a step, e.g. `expectVisible(greeting_text)`. +String formatStepBrief(TestStep step) { + final type = step.type; + final args = step.args; + + String? detail; + final id = args['id']; + if (id != null) { + final action = args['action']; + detail = action != null ? '$action $id' : id.toString(); + } else if (args['screen'] != null) { + detail = args['screen'].toString(); + } else if (args['name'] != null) { + detail = args['name'].toString(); + } else if (args['text'] != null) { + var text = args['text'].toString(); + if (text.length > 40) { + text = '${text.substring(0, 37)}...'; + } + detail = '"$text"'; + } else if (args['value'] != null) { + detail = args['value'].toString(); + } + + if (detail == null && args.isNotEmpty) { + final entry = args.entries.first; + detail = '${entry.key}=${entry.value}'; + } + + return detail != null ? '$type($detail)' : type; +} + +class TestReporter { + /// Formats a multi-test run report for console output. + String formatSummary( + EnsembleTestRunResult result, { + String? testFile, + }) { + final buffer = StringBuffer(); + final totalMs = + result.results.fold(0, (sum, r) => sum + r.durationMs); + + buffer.writeln('┌─ Ensemble YAML tests ─────────────────────────────'); + if (testFile != null) { + buffer.writeln('│ $testFile'); + buffer.writeln('│'); + } + + for (var i = 0; i < result.results.length; i++) { + final r = result.results[i]; + if (i > 0) buffer.writeln('│'); + _writeTestCase(buffer, r); + } + + buffer.writeln('│'); + buffer.writeln( + '└─ ${result.summary} · ${totalMs}ms total', + ); + return buffer.toString(); + } + + void _writeTestCase(StringBuffer buffer, EnsembleSingleTestResult r) { + final icon = r.status == TestStatus.passed ? '✓' : '✗'; + buffer.writeln('│ $icon ${r.testId} (${r.durationMs}ms)'); + + final report = r.report; + if (report != null) { + if (report.prerequisite != null) { + buffer.writeln('│ after: ${report.prerequisite}'); + } + buffer.writeln('│ start: ${report.startScreen}'); + if (report.endScreen != null && report.endScreen != report.startScreen) { + buffer.writeln('│ end: ${report.endScreen}'); + } + if (report.screensVisited.length > 1) { + buffer.writeln('│ flow: ${report.screensVisited.join(' → ')}'); + } + if (report.stepsOutline.isNotEmpty) { + buffer.writeln('│ steps (${report.stepsOutline.length}):'); + for (var i = 0; i < report.stepsOutline.length; i++) { + final prefix = r.status == TestStatus.failed && + r.failedStepIndex == i + ? '>>' + : ' '; + buffer.writeln('│ $prefix ${i + 1}. ${report.stepsOutline[i]}'); + } + } + } + + if (r.status == TestStatus.failed) { + if (r.message != null) { + buffer.writeln('│ error: ${r.message}'); + } + if (r.failedStep != null) { + buffer.writeln('│ failed: ${formatStepBrief(r.failedStep!)}'); + } else if (r.failedStepIndex != null) { + buffer.writeln('│ at step: ${r.failedStepIndex! + 1}'); + } + } + } +} diff --git a/packages/ensemble_test_runner/lib/runner/ensemble_test_context.dart b/packages/ensemble_test_runner/lib/runner/ensemble_test_context.dart new file mode 100644 index 000000000..046bdcbbb --- /dev/null +++ b/packages/ensemble_test_runner/lib/runner/ensemble_test_context.dart @@ -0,0 +1,93 @@ +import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_test_runner/mocks/test_logger.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; + +class EnsembleTestContext { + final EnsembleTestCase testCase; + final MockAPIProvider mockApiProvider; + final TestLogger logger; + final EnsembleTestSetup setup; + + /// Runtime env overrides applied via [setEnv] steps. + final Map envOverrides = {}; + + final TestRuntimeState runtime = TestRuntimeState(); + + EnsembleTestContext({ + required this.testCase, + required this.mockApiProvider, + required this.logger, + required this.setup, + }); + + factory EnsembleTestContext.fromTestCase(EnsembleTestCase testCase) { + final logger = TestLogger(); + final mockApi = MockAPIProvider( + mocks: Map.from(testCase.mocks.apis), + ); + + final storage = testCase.initialState['storage']; + final env = testCase.initialState['env']; + + final envMap = env is Map + ? Map.from(env) + : {}; + + final setup = EnsembleTestSetup( + envOverrides: envMap.isEmpty ? null : envMap, + initialPublicStorage: storage is Map + ? Map.from(storage) + : null, + ); + + final ctx = EnsembleTestContext( + testCase: testCase, + mockApiProvider: mockApi, + logger: logger, + setup: setup, + ); + ctx.envOverrides.addAll(envMap); + return ctx; + } + + void applyRuntimeEnv() { + if (envOverrides.isEmpty) return; + Ensemble().getConfig()?.updateEnvOverrides(envOverrides); + } + + void setEnv(String key, dynamic value) { + envOverrides[key] = value; + applyRuntimeEnv(); + } + + void setStorage(String key, dynamic value) { + StorageManager().write(key, value); + } + + void removeStorage(String key) { + StorageManager().remove(key); + } + + Future clearStorage() async { + await StorageManager().clearPublicStorage(); + } + + MockAPIResponse mockFromStepArgs(Map args) { + final response = args['response']; + if (response is! Map) { + throw EnsembleTestFailure('mockApi requires a "response" map'); + } + return MockAPIResponse( + statusCode: response['statusCode'] as int? ?? 200, + body: response['body'], + headers: response['headers'] is Map + ? Map.from(response['headers'] as Map) + : null, + delayMs: args['delayMs'] as int?, + ); + } +} diff --git a/packages/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/packages/ensemble_test_runner/lib/runner/ensemble_test_harness.dart new file mode 100644 index 000000000..35a4e4dae --- /dev/null +++ b/packages/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -0,0 +1,224 @@ +import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/ensemble_app.dart'; +import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; +import 'package:ensemble/framework/definition_providers/local_provider.dart'; +import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/page_model.dart'; +import 'package:ensemble_test_runner/mocks/mock_api_provider.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// Per-test bootstrap data applied before the widget tree mounts. +class EnsembleTestSetup { + final Map? envOverrides; + final Map? initialPublicStorage; + + const EnsembleTestSetup({ + this.envOverrides, + this.initialPublicStorage, + }); +} + +void applyYamlTestBootstrap(EnsembleConfig config, EnsembleTestSetup setup) { + if (setup.envOverrides != null && setup.envOverrides!.isNotEmpty) { + config.updateEnvOverrides(setup.envOverrides!); + } + setup.initialPublicStorage?.forEach((key, value) { + StorageManager().write(key, value); + }); +} + +/// Boots the real Ensemble runtime for widget tests. +class EnsembleTestHarness { + static void ensureTestPlugins() { + TestWidgetsFlutterBinding.ensureInitialized(); + const pathProviderChannel = + MethodChannel('plugins.flutter.io/path_provider'); + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(pathProviderChannel, (call) async { + switch (call.method) { + case 'getApplicationDocumentsDirectory': + case 'getTemporaryDirectory': + case 'getApplicationSupportDirectory': + return '.'; + default: + return null; + } + }); + + const packageInfoChannel = + MethodChannel('dev.fluttercommunity.plus/package_info'); + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(packageInfoChannel, (call) async { + if (call.method == 'getAll') { + return { + 'appName': 'EnsembleTest', + 'packageName': 'com.ensemble.test', + 'version': '1.0.0', + 'buildNumber': '1', + }; + } + return null; + }); + + YamlTestSession.navigationFlow.startListening(); + } + + final String appPath; + final String appHome; + final String? i18nPath; + + EnsembleTestHarness({ + required this.appPath, + required this.appHome, + this.i18nPath, + }); + + static String normalizeAppPath(String path) { + if (path.endsWith('/')) return path; + return '$path/'; + } + + Future buildConfig({Locale? forcedLocale}) async { + final normalized = normalizeAppPath(appPath); + final i18n = I18nProps(i18nPath ?? '${normalized}translations'); + + final provider = await LocalDefinitionProvider( + normalized, + appHome, + i18nProps: i18n, + initialForcedLocale: forcedLocale, + ).init(); + + final config = EnsembleConfig(definitionProvider: provider); + return config.updateAppBundle(); + } + + static void installHttpMockProvider( + EnsembleConfig config, + MockAPIProvider mock, + ) { + config.apiProviders = { + ...?config.apiProviders, + 'http': mock, + }; + } + + Future bootstrapRuntime( + EnsembleConfig config, + EnsembleTestSetup setup, { + MockAPIProvider? httpMock, + }) async { + ensureTestPlugins(); + + applyYamlTestBootstrap(config, setup); + Ensemble().setEnsembleConfig(config); + + if (httpMock != null) { + installHttpMockProvider(config, httpMock); + } + + await Ensemble().initManagers(); + + config.apiProviders ??= {'http': HTTPAPIProvider()}; + config.apiProviders!.putIfAbsent('http', () => HTTPAPIProvider()); + + YamlTestSession.markRuntimeBootstrapped(); + return config; + } + + Future loadScreen({ + required WidgetTester tester, + required EnsembleTestCase testCase, + EnsembleConfig? existingConfig, + EnsembleTestContext? context, + }) async { + resetTestRuntime(); + ScreenTracker().clearAll(); + YamlTestSession.navigationFlow.clear(); + + final ctx = context ?? EnsembleTestContext.fromTestCase(testCase); + var config = existingConfig ?? await buildConfig(); + final bootstrapped = await tester.runAsync(() async { + return bootstrapRuntime( + config, + ctx.setup, + httpMock: ctx.mockApiProvider, + ); + }); + config = bootstrapped!; + + final startScreen = testCase.startScreen; + if (startScreen == null || startScreen.isEmpty) { + throw EnsembleTestFailure( + 'loadScreen requires startScreen on test "${testCase.id}"', + ); + } + + await tester.pumpWidget( + EnsembleApp( + ensembleConfig: config, + screenPayload: ScreenPayload(screenId: startScreen), + ), + ); + + await waitForInitialWidgets(tester, testCase: testCase); + return config; + } + + static Future waitForInitialWidgets( + WidgetTester tester, { + EnsembleTestCase? testCase, + }) async { + final keysToWait = []; + if (testCase != null) { + for (final step in testCase.steps) { + if (step.type != 'expectVisible' && step.type != 'tap') { + break; + } + final id = step.args['id']?.toString(); + if (id != null && id.isNotEmpty) keysToWait.add(id); + } + } + + await tester.pump(); + for (var i = 0; i < 80; i++) { + if (keysToWait.isEmpty || + keysToWait.every( + (id) => find.byKey(ValueKey(id)).evaluate().isNotEmpty, + )) { + return; + } + await tester.pump(const Duration(milliseconds: 100)); + } + + if (keysToWait.isNotEmpty) { + throw EnsembleTestFailure( + 'Timed out waiting for widgets: ${keysToWait.join(", ")}', + ); + } + } + + static void applyInPlaceSetup(EnsembleTestContext ctx) { + final config = Ensemble().getConfig(); + if (config != null) { + applyYamlTestBootstrap(config, ctx.setup); + } + ctx.applyRuntimeEnv(); + for (final entry in ctx.testCase.mocks.apis.entries) { + ctx.mockApiProvider.setMock(entry.key, entry.value); + } + if (config != null) { + installHttpMockProvider(config, ctx.mockApiProvider); + } + } + + static void resetTestRuntime() { + YamlTestSession.reset(); + } +} diff --git a/packages/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/packages/ensemble_test_runner/lib/runner/ensemble_test_runner.dart new file mode 100644 index 000000000..06934dc53 --- /dev/null +++ b/packages/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -0,0 +1,162 @@ +import 'package:ensemble/ensemble.dart'; +import 'package:ensemble_test_runner/actions/test_step_executor.dart'; +import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; +import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/reporters/test_reporter.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/runner/test_runtime_state.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter_test/flutter_test.dart'; + +typedef EnsembleTestRunOutput = ({ + EnsembleSingleTestResult result, + EnsembleConfig config, +}); + +class EnsembleTestRunner { + final EnsembleTestHarness harness; + + EnsembleTestRunner({required this.harness}); + + Future> runPlan( + EnsembleTestExecutionPlan plan, + WidgetTester tester, + ) async { + final resultsById = {}; + var config = await harness.buildConfig(); + + for (final def in plan.ordered) { + final test = def.testCase; + final prereq = test.prerequisite; + if (prereq != null) { + final prereqResult = resultsById[prereq]; + if (prereqResult == null) { + throw EnsembleTestFailure( + 'Internal error: prerequisite "$prereq" for "${test.id}" was not scheduled', + ); + } + if (prereqResult.status == TestStatus.failed) { + resultsById[test.id] = EnsembleSingleTestResult.failed( + testId: test.id, + error: 'Prerequisite "$prereq" failed', + durationMs: 0, + report: buildTestReportDetails(test), + ); + continue; + } + } + + final out = await runOne( + test, + tester, + existingConfig: config, + continuation: test.hasPrerequisite, + ); + resultsById[test.id] = out.result; + config = out.config; + } + + return resultsById; + } + + Future runOne( + EnsembleTestCase test, + WidgetTester tester, { + EnsembleConfig? existingConfig, + bool continuation = false, + }) async { + final stopwatch = Stopwatch()..start(); + + try { + final ctx = EnsembleTestContext.fromTestCase(test); + TestErrorTracker.install(ctx.runtime); + + late final EnsembleConfig config; + if (continuation) { + if (!YamlTestSession.runtimeBootstrapped) { + throw EnsembleTestFailure( + 'Test "${test.id}" has prerequisite "${test.prerequisite}" but the ' + 'runtime is not bootstrapped — ensure the prerequisite test runs first', + ); + } + EnsembleTestHarness.applyInPlaceSetup(ctx); + config = existingConfig ?? Ensemble().getConfig()!; + await EnsembleTestHarness.waitForInitialWidgets(tester, testCase: test); + } else { + config = await harness.loadScreen( + tester: tester, + testCase: test, + existingConfig: existingConfig, + context: ctx, + ); + } + + final result = await _executeSteps( + test: test, + tester: tester, + ctx: ctx, + config: config, + stopwatch: stopwatch, + ); + return (result: result, config: config); + } catch (error, stackTrace) { + final config = existingConfig ?? Ensemble().getConfig(); + return ( + result: EnsembleSingleTestResult.failed( + testId: test.id, + error: error.toString(), + stackTrace: stackTrace.toString(), + durationMs: stopwatch.elapsedMilliseconds, + report: buildTestReportDetails(test), + ), + config: config ?? await harness.buildConfig(), + ); + } finally { + TestErrorTracker.reset(); + } + } + + Future _executeSteps({ + required EnsembleTestCase test, + required WidgetTester tester, + required EnsembleTestContext ctx, + required EnsembleConfig config, + required Stopwatch stopwatch, + }) async { + final assertions = AssertionEngine(tester: tester, context: ctx); + final executor = TestStepExecutor( + tester: tester, + context: ctx, + assertions: assertions, + harness: harness, + config: config, + ); + + for (var i = 0; i < test.steps.length; i++) { + final step = test.steps[i]; + try { + await executor.execute(step); + } catch (error, stackTrace) { + return EnsembleSingleTestResult.failed( + testId: test.id, + failedStepIndex: i, + failedStep: step, + error: error.toString(), + stackTrace: stackTrace.toString(), + durationMs: stopwatch.elapsedMilliseconds, + logs: ctx.logger.logs, + report: buildTestReportDetails(test), + ); + } + } + + return EnsembleSingleTestResult.passed( + testId: test.id, + durationMs: stopwatch.elapsedMilliseconds, + logs: ctx.logger.logs, + report: buildTestReportDetails(test), + ); + } +} diff --git a/packages/ensemble_test_runner/lib/runner/test_runtime_state.dart b/packages/ensemble_test_runner/lib/runner/test_runtime_state.dart new file mode 100644 index 000000000..a87b17712 --- /dev/null +++ b/packages/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -0,0 +1,47 @@ +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; + +/// Mutable runtime flags and logs for declarative test steps. +class TestRuntimeState { + bool networkOffline = false; + final List consoleLogs = []; + final List flutterErrors = []; + Map? authUser; + final Map permissions = {}; + Size? deviceSize; + Locale? locale; + String? themeMode; + final Map fixtures = {}; + + void clear() { + networkOffline = false; + consoleLogs.clear(); + flutterErrors.clear(); + authUser = null; + permissions.clear(); + deviceSize = null; + locale = null; + themeMode = null; + fixtures.clear(); + } +} + +/// Captures Flutter framework errors for quality assertion steps. +class TestErrorTracker { + static FlutterExceptionHandler? _previousHandler; + + static void install(TestRuntimeState runtime) { + _previousHandler = FlutterError.onError; + FlutterError.onError = (FlutterErrorDetails details) { + runtime.flutterErrors.add(details.exceptionAsString()); + _previousHandler?.call(details); + }; + } + + static void reset() { + if (_previousHandler != null) { + FlutterError.onError = _previousHandler; + _previousHandler = null; + } + } +} diff --git a/packages/ensemble_test_runner/lib/runner/yaml_test_session.dart b/packages/ensemble_test_runner/lib/runner/yaml_test_session.dart new file mode 100644 index 000000000..bf90c5d19 --- /dev/null +++ b/packages/ensemble_test_runner/lib/runner/yaml_test_session.dart @@ -0,0 +1,69 @@ +import 'dart:async'; + +import 'package:ensemble/ensemble.dart'; +import 'package:ensemble/framework/screen_tracker.dart'; + +/// Records screen visits for YAML tests (including back-navigation revisits). +class NavigationFlowRecorder { + final List _flow = []; + StreamSubscription? _subscription; + + List get flow => List.unmodifiable(_flow); + + void startListening() { + _subscription?.cancel(); + _subscription = ScreenTracker().onScreenChange.listen(_onScreenChange); + } + + void clear() => _flow.clear(); + + /// For unit tests only. + void seed(Iterable names) { + _flow + ..clear() + ..addAll(names); + } + + void dispose() { + _subscription?.cancel(); + _subscription = null; + _flow.clear(); + } + + /// Visible for unit tests; production uses [startListening]. + void recordScreenChange(VisibleScreen? screen) => _onScreenChange(screen); + + void _onScreenChange(VisibleScreen? screen) { + if (screen == null) return; + final name = screen.screenName ?? screen.screenId; + if (name == null) return; + if (_flow.isEmpty || _flow.last != name) { + _flow.add(name); + } + } +} + +/// Per–widget-test session state for declarative YAML tests. +class YamlTestSession { + YamlTestSession._(); + + static final NavigationFlowRecorder navigationFlow = NavigationFlowRecorder(); + + static bool runtimeBootstrapped = false; + + static void markRuntimeBootstrapped() { + runtimeBootstrapped = true; + } + + /// Between cold starts within a suite (keeps the screen-change listener). + static void reset() { + runtimeBootstrapped = false; + navigationFlow.clear(); + Ensemble.resetInitManagersForTest(); + } + + /// End of the widget test (cancels listeners). + static void dispose() { + navigationFlow.dispose(); + } +} diff --git a/packages/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/packages/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart new file mode 100644 index 000000000..c59763be7 --- /dev/null +++ b/packages/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -0,0 +1,161 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; + +/// Builds JSON Schema for `ensemble/tests/*.test.yaml` files. +class EnsembleTestSchemaBuilder { + static const schemaId = 'https://ensembleui.com/ensemble_test.schema.json'; + static const schemaVersion = 'https://json-schema.org/draft/2020-12/schema'; + + static Map build() { + final defs = { + 'mockResponse': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'statusCode': {'type': 'integer'}, + 'body': true, + 'headers': { + 'type': 'object', + 'additionalProperties': true, + }, + }, + }, + 'mockApiEntry': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'response': {'\$ref': '#/\$defs/mockResponse'}, + 'delayMs': {'type': 'integer'}, + }, + 'required': ['response'], + }, + 'initialState': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'storage': {'type': 'object', 'additionalProperties': true}, + 'env': {'type': 'object', 'additionalProperties': true}, + }, + }, + 'mocks': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'apis': { + 'type': 'object', + 'additionalProperties': {'\$ref': '#/\$defs/mockApiEntry'}, + }, + }, + }, + 'testCase': { + 'type': 'object', + 'additionalProperties': false, + 'properties': { + 'id': { + 'type': 'string', + 'minLength': 1, + 'description': 'Unique test identifier', + }, + 'type': {'type': 'string'}, + 'startScreen': { + 'type': 'string', + 'minLength': 1, + 'description': 'Ensemble screen name or id to load first', + }, + 'prerequisite': { + 'type': 'string', + 'minLength': 1, + 'description': + 'ID of another test that must run before this one in the same app session', + }, + 'initialState': {'\$ref': '#/\$defs/initialState'}, + 'mocks': {'\$ref': '#/\$defs/mocks'}, + 'steps': { + 'type': 'array', + 'minItems': 1, + 'items': {'\$ref': '#/\$defs/step'}, + }, + }, + 'required': ['id', 'steps'], + 'oneOf': [ + { + 'required': ['startScreen'], + 'not': { + 'required': ['prerequisite'], + }, + }, + { + 'required': ['prerequisite'], + 'not': { + 'required': ['startScreen'], + }, + }, + ], + }, + }; + + // Register per-step arg defs and step wrappers. + final stepOneOf = >[]; + final registeredArgDefs = {}; + + for (final yamlKey in TestStepVocabulary.yamlStepKeys) { + final entry = TestStepRegistry.entries[yamlKey]!; + final description = entry.description; + final argsSchema = TestStepVocabulary.argJsonSchemaForYamlKey(yamlKey); + final argsDefName = 'args_$yamlKey'; + + final exampleArgs = Map.from(entry.example); + final exampleStep = {yamlKey: exampleArgs}; + + if (registeredArgDefs.add(argsDefName)) { + defs[argsDefName] = { + ...argsSchema, + 'title': yamlKey, + 'description': description, + 'examples': [exampleArgs], + }; + } + + stepOneOf.add({ + 'type': 'object', + 'title': yamlKey, + 'description': description, + 'examples': [exampleStep], + 'additionalProperties': false, + 'minProperties': 1, + 'maxProperties': 1, + 'properties': { + yamlKey: { + '\$ref': '#/\$defs/$argsDefName', + 'description': description, + 'examples': [exampleArgs], + }, + }, + 'required': [yamlKey], + }); + } + + defs['step'] = {'oneOf': stepOneOf}; + + final testCase = defs.remove('testCase') as Map; + + return { + '\$schema': schemaVersion, + '\$id': schemaId, + 'title': 'Ensemble declarative test file', + 'description': + 'Schema for ensemble/tests/*.test.yaml — one test per file; ' + 'see packages/ensemble_test_runner/STEP_VOCABULARY.md', + ...testCase, + '\$defs': defs, + }; + } + + static String buildJson({bool pretty = true}) { + final encoder = pretty + ? const JsonEncoder.withIndent(' ') + : const JsonEncoder(); + return encoder.convert(build()); + } +} diff --git a/packages/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart b/packages/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart new file mode 100644 index 000000000..d9250cfa9 --- /dev/null +++ b/packages/ensemble_test_runner/lib/vocabulary/test_step_arg_kind.dart @@ -0,0 +1,420 @@ +/// Argument shape for a declarative test step (JSON Schema for editor validation). +enum TestStepArgKind { + empty, + openScreen, + trigger, + idRequired, + idOptional, + enterText, + select, + selectIndex, + setSlider, + chooseValue, + scroll, + swipe, + drag, + pump, + timeoutOptional, + waitFor, + waitForGone, + waitForNavigation, + waitUntil, + textRequired, + expectEquals, + expectChecked, + expectProperty, + expectCount, + expectListCount, + screenRequired, + expectVisited, + mockApi, + mockApiError, + mockApiFromFixture, + mockApiException, + mockTimeout, + apiName, + apiRequest, + expectApiHeader, + setState, + expectState, + storageKey, + group, + repeat, + optional, + ifVisible, + screenshot, + expectStatePath, + resetStatePath, + setAuth, + setPermission, + setDevice, + setLocale, + setTheme, + runScript, + expectConsoleLog, + expectErrorContains, + fixturePath, + expectApiCallOrder, + expectListContains, + expectListItem, + expectBackStack, + expectCanGoBack, + expectSemanticsLabel, +} + +extension TestStepArgKindSchema on TestStepArgKind { + static const _string = {'type': 'string'}; + static const _integer = {'type': 'integer'}; + static const _boolean = {'type': 'boolean'}; + static const _any = true; + + static Map _object({ + Map? properties, + List? required, + bool additionalProperties = false, + }) => + { + 'type': 'object', + if (properties != null) 'properties': properties, + if (required != null && required.isNotEmpty) 'required': required, + 'additionalProperties': additionalProperties, + }; + + static Map _ref(String name) => {'\$ref': '#/\$defs/$name'}; + + /// JSON Schema for this step's YAML argument object. + Map get jsonSchema { + switch (this) { + case TestStepArgKind.empty: + return _object(); + case TestStepArgKind.openScreen: + return _object(properties: {'screen': _string, 'name': _string}); + case TestStepArgKind.trigger: + return _object( + properties: { + 'action': { + 'type': 'string', + 'enum': ['onLoad', 'onTap', 'onLongPress'], + }, + 'id': _string, + }, + required: ['action'], + ); + case TestStepArgKind.idRequired: + return _object(properties: {'id': _string}, required: ['id']); + case TestStepArgKind.idOptional: + return _object(properties: {'id': _string}); + case TestStepArgKind.enterText: + return _object( + properties: {'id': _string, 'value': _any, 'submit': _boolean}, + required: ['id'], + ); + case TestStepArgKind.select: + return _object( + properties: {'id': _string, 'value': _string}, + required: ['id', 'value'], + ); + case TestStepArgKind.selectIndex: + return _object( + properties: {'id': _string, 'index': _integer}, + required: ['id'], + ); + case TestStepArgKind.setSlider: + return _object( + properties: {'id': _string, 'value': {'type': 'number'}}, + required: ['id'], + ); + case TestStepArgKind.chooseValue: + return _object( + properties: {'id': _string, 'value': _string}, + required: ['id', 'value'], + ); + case TestStepArgKind.scroll: + return _object(properties: {'delta': _integer}); + case TestStepArgKind.swipe: + return _object( + properties: { + 'direction': { + 'type': 'string', + 'enum': ['left', 'right', 'up', 'down'], + }, + 'id': _string, + }, + ); + case TestStepArgKind.drag: + return _object( + properties: { + 'id': _string, + 'dx': {'type': 'number'}, + 'dy': {'type': 'number'}, + }, + required: ['id'], + ); + case TestStepArgKind.pump: + return _object(properties: {'durationMs': _integer}); + case TestStepArgKind.timeoutOptional: + return _object(properties: {'timeoutMs': _integer}); + case TestStepArgKind.waitFor: + return _object( + properties: {'id': _string, 'text': _string, 'timeoutMs': _integer}, + ); + case TestStepArgKind.waitForGone: + return _object( + properties: {'id': _string, 'timeoutMs': _integer}, + required: ['id'], + ); + case TestStepArgKind.waitForNavigation: + return _object( + properties: {'screen': _string, 'timeoutMs': _integer}, + required: ['screen'], + ); + case TestStepArgKind.waitUntil: + return _object( + properties: { + 'path': _string, + 'equals': _any, + 'state': _object( + properties: {'path': _string, 'equals': _any}, + ), + 'timeoutMs': _integer, + }, + ); + case TestStepArgKind.textRequired: + return _object(properties: {'text': _string}, required: ['text']); + case TestStepArgKind.expectEquals: + return _object( + properties: {'id': _string, 'equals': _any}, + required: ['id', 'equals'], + ); + case TestStepArgKind.expectChecked: + return _object( + properties: {'id': _string, 'equals': _boolean}, + required: ['id'], + ); + case TestStepArgKind.expectProperty: + return _object( + properties: {'id': _string, 'property': _string, 'equals': _any}, + required: ['id', 'equals'], + ); + case TestStepArgKind.expectCount: + return _object( + properties: {'id': _string, 'equals': _integer}, + required: ['id', 'equals'], + ); + case TestStepArgKind.expectListCount: + return _object( + properties: {'id': _string, 'itemId': _string, 'equals': _integer}, + required: ['equals'], + ); + case TestStepArgKind.screenRequired: + return _object( + properties: {'screen': _string, 'name': _string}, + required: ['screen'], + ); + case TestStepArgKind.expectVisited: + return _object( + properties: {'screen': _string}, + required: ['screen'], + ); + case TestStepArgKind.mockApi: + return _object( + properties: { + 'name': _string, + 'response': _ref('mockResponse'), + 'delayMs': _integer, + }, + required: ['name', 'response'], + ); + case TestStepArgKind.mockApiError: + return _object( + properties: { + 'name': _string, + 'statusCode': _integer, + 'body': _any, + 'delayMs': _integer, + }, + required: ['name'], + ); + case TestStepArgKind.mockApiFromFixture: + return _object( + properties: { + 'name': _string, + 'fixture': _string, + 'statusCode': _integer, + }, + required: ['name', 'fixture'], + ); + case TestStepArgKind.mockApiException: + return _object( + properties: {'name': _string, 'message': _string}, + required: ['name'], + ); + case TestStepArgKind.mockTimeout: + return _object( + properties: {'name': _string, 'delayMs': _integer}, + required: ['name'], + ); + case TestStepArgKind.apiName: + return _object( + properties: {'name': _string, 'times': _integer}, + required: ['name'], + ); + case TestStepArgKind.apiRequest: + return _object( + properties: { + 'name': _string, + 'body': _any, + 'query': _any, + 'headers': _any, + 'times': _integer, + }, + required: ['name'], + ); + case TestStepArgKind.expectApiHeader: + return _object( + properties: { + 'name': _string, + 'header': _string, + 'equals': _any, + 'times': _integer, + }, + required: ['name', 'header', 'equals'], + ); + case TestStepArgKind.setState: + return _object( + properties: {'path': _string, 'value': _any}, + required: ['path'], + ); + case TestStepArgKind.expectState: + return _object( + properties: {'path': _string, 'equals': _any, 'contains': _any}, + required: ['path'], + ); + case TestStepArgKind.storageKey: + return _object( + properties: {'key': _string, 'equals': _any, 'value': _any}, + required: ['key'], + ); + case TestStepArgKind.group: + return _object( + properties: { + 'name': _string, + 'steps': {'type': 'array', 'items': _ref('step'), 'minItems': 1}, + }, + required: ['steps'], + ); + case TestStepArgKind.repeat: + return _object( + properties: { + 'times': _integer, + 'steps': {'type': 'array', 'items': _ref('step'), 'minItems': 1}, + }, + required: ['times', 'steps'], + ); + case TestStepArgKind.optional: + return _object( + properties: { + 'step': _ref('step'), + 'steps': {'type': 'array', 'items': _ref('step'), 'minItems': 1}, + }, + ); + case TestStepArgKind.ifVisible: + return _object( + properties: { + 'id': _string, + 'step': _ref('step'), + 'steps': {'type': 'array', 'items': _ref('step'), 'minItems': 1}, + }, + required: ['id'], + ); + case TestStepArgKind.screenshot: + return _object(properties: {'name': _string}); + case TestStepArgKind.expectStatePath: + return _object(properties: {'path': _string}, required: ['path']); + case TestStepArgKind.resetStatePath: + return _object(properties: {'path': _string}); + case TestStepArgKind.setAuth: + return _object( + properties: { + 'user': {'type': 'object', 'additionalProperties': true}, + }, + required: ['user'], + ); + case TestStepArgKind.setPermission: + return _object( + properties: {'name': _string, 'value': _string}, + required: ['name'], + ); + case TestStepArgKind.setDevice: + return _object( + properties: { + 'width': {'type': 'number'}, + 'height': {'type': 'number'}, + }, + ); + case TestStepArgKind.setLocale: + return _object(properties: {'locale': _string}); + case TestStepArgKind.setTheme: + return _object(properties: {'mode': _string, 'theme': _string}); + case TestStepArgKind.runScript: + return _object( + properties: {'script': _string, 'path': _string, 'equals': _any}, + ); + case TestStepArgKind.expectConsoleLog: + return _object( + properties: {'contains': _string}, + required: ['contains'], + ); + case TestStepArgKind.expectErrorContains: + return _object(properties: {'contains': _string}); + case TestStepArgKind.fixturePath: + return _object( + properties: { + 'key': _string, + 'path': _string, + 'fixture': _string, + 'statePath': _string, + }, + ); + case TestStepArgKind.expectApiCallOrder: + return _object( + properties: { + 'names': { + 'type': 'array', + 'items': _string, + 'minItems': 1, + }, + }, + required: ['names'], + ); + case TestStepArgKind.expectListContains: + return _object( + properties: {'id': _string, 'text': _string}, + required: ['id', 'text'], + ); + case TestStepArgKind.expectListItem: + return _object( + properties: {'itemId': _string}, + required: ['itemId'], + ); + case TestStepArgKind.expectBackStack: + return _object( + properties: { + 'screens': { + 'type': 'array', + 'items': _string, + 'minItems': 1, + }, + }, + required: ['screens'], + ); + case TestStepArgKind.expectCanGoBack: + return _object(properties: {'equals': _boolean}); + case TestStepArgKind.expectSemanticsLabel: + return _object( + properties: {'id': _string, 'label': _string}, + required: ['id', 'label'], + ); + } + } +} diff --git a/packages/ensemble_test_runner/lib/vocabulary/test_step_registry.dart b/packages/ensemble_test_runner/lib/vocabulary/test_step_registry.dart new file mode 100644 index 000000000..75d8e9c53 --- /dev/null +++ b/packages/ensemble_test_runner/lib/vocabulary/test_step_registry.dart @@ -0,0 +1,874 @@ +// GENERATED by tool/generate_step_registry.dart — do not edit by hand. +// Re-run: dart run tool/generate_step_registry.dart + +import 'package:ensemble_test_runner/vocabulary/test_step_arg_kind.dart'; +import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; + +/// Single source of truth for declarative test steps (metadata + JSON Schema args). +class TestStepRegistryEntry { + const TestStepRegistryEntry({ + required this.category, + required this.tier, + required this.argKind, + required this.description, + required this.example, + this.executorCanonical, + }); + + final TestStepCategory category; + final TestStepTier tier; + final TestStepArgKind argKind; + final String description; + + /// Example YAML args object for this step (also used in JSON Schema). + final Map example; + + /// When set, [TestStepVocabulary.resolveStepType] maps this YAML key here + /// (e.g. `wait` → `pump`). Schema/definition use this entry's [argKind]. + final String? executorCanonical; +} + +abstract final class TestStepRegistry { + TestStepRegistry._(); + + static const Map entries = { + 'openScreen': TestStepRegistryEntry( + category: TestStepCategory.lifecycle, + tier: TestStepTier.core, + argKind: TestStepArgKind.openScreen, + description: 'Navigate to another screen by name or id mid-test', + example: const {'screen': 'Home'}, + ), + 'reloadScreen': TestStepRegistryEntry( + category: TestStepCategory.lifecycle, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Reload the current screen (same as re-opening it)', + example: const {}, + ), + 'restartApp': TestStepRegistryEntry( + category: TestStepCategory.lifecycle, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Reset runtime and reopen the test case start screen', + example: const {}, + ), + 'resetAppState': TestStepRegistryEntry( + category: TestStepCategory.lifecycle, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Clear screen tracker, API call log, and public storage', + example: const {}, + ), + 'trigger': TestStepRegistryEntry( + category: TestStepCategory.lifecycle, + tier: TestStepTier.core, + argKind: TestStepArgKind.trigger, + description: 'Fire a widget action (onLoad, onTap, onLongPress) by testId', + example: const {'action': 'onTap', 'id': 'submit_button'}, + ), + 'launchApp': TestStepRegistryEntry( + category: TestStepCategory.lifecycle, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Alias for restartApp — bootstrap from startScreen again', + example: const {}, + executorCanonical: 'restartApp', + ), + 'tap': TestStepRegistryEntry( + category: TestStepCategory.interaction, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Tap a widget by testId (ValueKey)', + example: const {'id': 'my_widget'}, + ), + 'doubleTap': TestStepRegistryEntry( + category: TestStepCategory.interaction, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Double-tap a widget by testId', + example: const {'id': 'my_widget'}, + ), + 'longPress': TestStepRegistryEntry( + category: TestStepCategory.interaction, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Long-press a widget by testId', + example: const {'id': 'my_widget'}, + ), + 'enterText': TestStepRegistryEntry( + category: TestStepCategory.interaction, + tier: TestStepTier.core, + argKind: TestStepArgKind.enterText, + description: 'Type text into an input field by testId', + example: const {'id': 'email_field', 'value': 'user@test.com'}, + ), + 'clearText': TestStepRegistryEntry( + category: TestStepCategory.interaction, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Clear text in an input field by testId', + example: const {'id': 'my_widget'}, + ), + 'replaceText': TestStepRegistryEntry( + category: TestStepCategory.interaction, + tier: TestStepTier.core, + argKind: TestStepArgKind.enterText, + description: 'Replace the full contents of an input field', + example: const {'id': 'email_field', 'value': 'user@test.com'}, + ), + 'submitText': TestStepRegistryEntry( + category: TestStepCategory.interaction, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Submit an input field (TextInputAction.done)', + example: const {'id': 'my_widget'}, + ), + 'focus': TestStepRegistryEntry( + category: TestStepCategory.interaction, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Focus an input field by testId', + example: const {'id': 'my_widget'}, + ), + 'unfocus': TestStepRegistryEntry( + category: TestStepCategory.interaction, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Remove focus from the current field', + example: const {}, + ), + 'select': TestStepRegistryEntry( + category: TestStepCategory.formControl, + tier: TestStepTier.core, + argKind: TestStepArgKind.select, + description: 'Open a dropdown and choose an option by visible label', + example: const {'id': 'country_dropdown', 'value': 'USA'}, + ), + 'selectIndex': TestStepRegistryEntry( + category: TestStepCategory.formControl, + tier: TestStepTier.core, + argKind: TestStepArgKind.selectIndex, + description: 'Open a dropdown and choose the option at index', + example: const {'id': 'country_dropdown', 'index': 0}, + ), + 'check': TestStepRegistryEntry( + category: TestStepCategory.formControl, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Check a checkbox or toggle by testId', + example: const {'id': 'my_widget'}, + ), + 'uncheck': TestStepRegistryEntry( + category: TestStepCategory.formControl, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Uncheck a checkbox by testId if currently checked', + example: const {'id': 'my_widget'}, + ), + 'toggle': TestStepRegistryEntry( + category: TestStepCategory.formControl, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Tap to toggle a switch or checkbox by testId', + example: const {'id': 'my_widget'}, + ), + 'setSlider': TestStepRegistryEntry( + category: TestStepCategory.formControl, + tier: TestStepTier.core, + argKind: TestStepArgKind.setSlider, + description: 'Move a slider under testId to a normalized value (0–1)', + example: const {'id': 'volume_slider', 'value': 0.5}, + ), + 'chooseDate': TestStepRegistryEntry( + category: TestStepCategory.formControl, + tier: TestStepTier.core, + argKind: TestStepArgKind.chooseValue, + description: 'Set a date field by testId to the given value string', + example: const {'id': 'birth_date', 'value': '2024-01-15'}, + ), + 'chooseTime': TestStepRegistryEntry( + category: TestStepCategory.formControl, + tier: TestStepTier.core, + argKind: TestStepArgKind.chooseValue, + description: 'Set a time field by testId to the given value string', + example: const {'id': 'birth_date', 'value': '2024-01-15'}, + ), + 'scroll': TestStepRegistryEntry( + category: TestStepCategory.gesture, + tier: TestStepTier.core, + argKind: TestStepArgKind.scroll, + description: 'Drag the first Scrollable by delta pixels', + example: const {'delta': 300}, + ), + 'scrollUntilVisible': TestStepRegistryEntry( + category: TestStepCategory.gesture, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Scroll until a widget with testId is visible', + example: const {'id': 'my_widget'}, + ), + 'swipe': TestStepRegistryEntry( + category: TestStepCategory.gesture, + tier: TestStepTier.core, + argKind: TestStepArgKind.swipe, + description: 'Swipe on a scrollable or widget (direction: left/right/up/down)', + example: const {'direction': 'left', 'id': 'carousel'}, + ), + 'drag': TestStepRegistryEntry( + category: TestStepCategory.gesture, + tier: TestStepTier.core, + argKind: TestStepArgKind.drag, + description: 'Drag a widget by testId by dx/dy offset', + example: const {'id': 'handle', 'dx': 50, 'dy': 0}, + ), + 'pullToRefresh': TestStepRegistryEntry( + category: TestStepCategory.gesture, + tier: TestStepTier.core, + argKind: TestStepArgKind.idOptional, + description: 'Pull down on a scrollable to trigger refresh', + example: const {'id': 'scroll_view'}, + ), + 'wait': TestStepRegistryEntry( + category: TestStepCategory.wait, + tier: TestStepTier.extended, + argKind: TestStepArgKind.pump, + description: 'Alias for pump — advance frame clock by durationMs', + example: const {'durationMs': 100}, + executorCanonical: 'pump', + ), + 'pump': TestStepRegistryEntry( + category: TestStepCategory.wait, + tier: TestStepTier.core, + argKind: TestStepArgKind.pump, + description: 'Advance the Flutter frame clock by durationMs', + example: const {'durationMs': 100}, + ), + 'settle': TestStepRegistryEntry( + category: TestStepCategory.wait, + tier: TestStepTier.core, + argKind: TestStepArgKind.timeoutOptional, + description: 'Run pumpAndSettle until idle or timeout', + example: const {'timeoutMs': 5000}, + ), + 'waitFor': TestStepRegistryEntry( + category: TestStepCategory.wait, + tier: TestStepTier.core, + argKind: TestStepArgKind.waitFor, + description: 'Poll until a widget id and/or text appears', + example: const {'id': 'loading_spinner', 'timeoutMs': 5000}, + ), + 'waitForText': TestStepRegistryEntry( + category: TestStepCategory.wait, + tier: TestStepTier.core, + argKind: TestStepArgKind.waitFor, + description: 'Poll until the given text appears on screen', + example: const {'id': 'loading_spinner', 'timeoutMs': 5000}, + executorCanonical: 'waitFor', + ), + 'waitForGone': TestStepRegistryEntry( + category: TestStepCategory.wait, + tier: TestStepTier.core, + argKind: TestStepArgKind.waitForGone, + description: 'Poll until a widget with testId is removed from the tree', + example: const {'id': 'loading_spinner', 'timeoutMs': 5000}, + ), + 'waitForApi': TestStepRegistryEntry( + category: TestStepCategory.wait, + tier: TestStepTier.core, + argKind: TestStepArgKind.apiName, + description: 'Poll until a mocked API is called N times', + example: const {'name': 'login', 'times': 1}, + ), + 'waitForNavigation': TestStepRegistryEntry( + category: TestStepCategory.wait, + tier: TestStepTier.core, + argKind: TestStepArgKind.waitForNavigation, + description: 'Poll until the given screen is visible', + example: const {'screen': 'Home', 'timeoutMs': 5000}, + ), + 'waitUntil': TestStepRegistryEntry( + category: TestStepCategory.wait, + tier: TestStepTier.core, + argKind: TestStepArgKind.waitUntil, + description: 'Poll until app state at path equals expected value', + example: const {'path': 'user.name', 'equals': 'Jane'}, + ), + 'expectVisible': TestStepRegistryEntry( + category: TestStepCategory.uiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert a widget with testId is visible', + example: const {'id': 'my_widget'}, + ), + 'expectNotVisible': TestStepRegistryEntry( + category: TestStepCategory.uiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert a widget with testId is not visible', + example: const {'id': 'my_widget'}, + ), + 'expectExists': TestStepRegistryEntry( + category: TestStepCategory.uiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert a widget with testId exists in the tree', + example: const {'id': 'my_widget'}, + ), + 'expectNotExists': TestStepRegistryEntry( + category: TestStepCategory.uiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert no widget with testId exists', + example: const {'id': 'my_widget'}, + ), + 'expectText': TestStepRegistryEntry( + category: TestStepCategory.uiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.textRequired, + description: 'Assert exact text is shown', + example: const {'text': 'Welcome'}, + ), + 'expectNoText': TestStepRegistryEntry( + category: TestStepCategory.uiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.textRequired, + description: 'Assert text is not shown', + example: const {'text': 'Welcome'}, + ), + 'expectTextContains': TestStepRegistryEntry( + category: TestStepCategory.uiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.textRequired, + description: 'Assert some text containing the given substring', + example: const {'text': 'Welcome'}, + ), + 'expectEnabled': TestStepRegistryEntry( + category: TestStepCategory.uiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert widget semantics report enabled', + example: const {'id': 'my_widget'}, + ), + 'expectDisabled': TestStepRegistryEntry( + category: TestStepCategory.uiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert widget semantics report disabled', + example: const {'id': 'my_widget'}, + ), + 'expectValue': TestStepRegistryEntry( + category: TestStepCategory.valueAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectEquals, + description: 'Assert input value equals expected (EditableText/TextField)', + example: const {'id': 'email_field', 'equals': 'user@test.com'}, + ), + 'expectChecked': TestStepRegistryEntry( + category: TestStepCategory.valueAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectChecked, + description: 'Assert checkbox checked state matches equals', + example: const {'id': 'terms_checkbox', 'equals': true}, + ), + 'expectProperty': TestStepRegistryEntry( + category: TestStepCategory.valueAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectProperty, + description: 'Assert a widget property (e.g. label) equals expected', + example: const {'id': 'title', 'property': 'label', 'equals': 'Hello'}, + ), + 'expectStyle': TestStepRegistryEntry( + category: TestStepCategory.valueAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectProperty, + description: 'Assert style-related property equals expected', + example: const {'id': 'title', 'property': 'label', 'equals': 'Hello'}, + ), + 'expectSelected': TestStepRegistryEntry( + category: TestStepCategory.valueAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectChecked, + description: 'Assert selected/checked state matches equals', + example: const {'id': 'terms_checkbox', 'equals': true}, + ), + 'expectCount': TestStepRegistryEntry( + category: TestStepCategory.listAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectCount, + description: 'Assert count of widgets with the same testId', + example: const {'id': 'badge', 'equals': 2}, + ), + 'expectListCount': TestStepRegistryEntry( + category: TestStepCategory.listAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectListCount, + description: 'Assert number of list items under a list testId', + example: const {'id': 'items_list', 'equals': 3}, + ), + 'expectListContains': TestStepRegistryEntry( + category: TestStepCategory.listAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectListContains, + description: 'Assert list contains text', + example: const {'id': 'items_list', 'text': 'Item 1'}, + ), + 'expectListItem': TestStepRegistryEntry( + category: TestStepCategory.listAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectListItem, + description: 'Assert a list item widget with itemId is visible', + example: const {'itemId': 'row_0'}, + ), + 'expectEmpty': TestStepRegistryEntry( + category: TestStepCategory.listAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert a list has zero items', + example: const {'id': 'my_widget'}, + ), + 'expectNotEmpty': TestStepRegistryEntry( + category: TestStepCategory.listAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert a list has at least one item', + example: const {'id': 'my_widget'}, + ), + 'expectScreen': TestStepRegistryEntry( + category: TestStepCategory.navigation, + tier: TestStepTier.core, + argKind: TestStepArgKind.screenRequired, + description: 'Alias for expectNavigateTo — assert current screen', + example: const {'screen': 'Home'}, + executorCanonical: 'expectNavigateTo', + ), + 'expectNavigateTo': TestStepRegistryEntry( + category: TestStepCategory.navigation, + tier: TestStepTier.core, + argKind: TestStepArgKind.screenRequired, + description: 'Assert the current visible screen name/id', + example: const {'screen': 'Home'}, + ), + 'expectVisited': TestStepRegistryEntry( + category: TestStepCategory.navigation, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectVisited, + description: 'Assert a screen appears in navigation history', + example: const {'screen': 'Login'}, + ), + 'expectNotVisited': TestStepRegistryEntry( + category: TestStepCategory.navigation, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectVisited, + description: 'Assert a screen was never visited', + example: const {'screen': 'Login'}, + ), + 'expectBackStack': TestStepRegistryEntry( + category: TestStepCategory.navigation, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectBackStack, + description: 'Assert navigation history suffix matches screens', + example: const {'screens': const ['Home', 'Details']}, + ), + 'expectCanGoBack': TestStepRegistryEntry( + category: TestStepCategory.navigation, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectCanGoBack, + description: 'Assert whether back navigation is possible', + example: const {'equals': true}, + ), + 'goBack': TestStepRegistryEntry( + category: TestStepCategory.navigation, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Navigate back (Ensemble navigateBack or Navigator.pop)', + example: const {}, + ), + 'mockApi': TestStepRegistryEntry( + category: TestStepCategory.apiMock, + tier: TestStepTier.core, + argKind: TestStepArgKind.mockApi, + description: 'Register a mock HTTP API response by API name', + example: const {'name': 'login', 'response': const {'statusCode': 200, 'body': const {'token': 'test-token'}}}, + ), + 'mockApiError': TestStepRegistryEntry( + category: TestStepCategory.apiMock, + tier: TestStepTier.core, + argKind: TestStepArgKind.mockApiError, + description: 'Mock an API to return an error status/body', + example: const {'name': 'login', 'statusCode': 401, 'body': const {'error': 'Unauthorized'}}, + ), + 'mockApiFromFixture': TestStepRegistryEntry( + category: TestStepCategory.fixture, + tier: TestStepTier.core, + argKind: TestStepArgKind.mockApiFromFixture, + description: 'Load mock response body from a JSON fixture asset', + example: const {'name': 'users', 'fixture': 'fixtures/users.json'}, + ), + 'mockApiException': TestStepRegistryEntry( + category: TestStepCategory.apiMock, + tier: TestStepTier.core, + argKind: TestStepArgKind.mockApiException, + description: 'Force an API call to throw an exception', + example: const {'name': 'login', 'message': 'Network error'}, + ), + 'mockTimeout': TestStepRegistryEntry( + category: TestStepCategory.network, + tier: TestStepTier.core, + argKind: TestStepArgKind.mockTimeout, + description: 'Mock an API with a long delay (simulate timeout)', + example: const {'name': 'slow_api', 'delayMs': 60000}, + ), + 'mockNetworkOffline': TestStepRegistryEntry( + category: TestStepCategory.network, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Simulate offline network for API calls', + example: const {}, + ), + 'mockNetworkOnline': TestStepRegistryEntry( + category: TestStepCategory.network, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Restore online network for API calls', + example: const {}, + ), + 'resetApiCalls': TestStepRegistryEntry( + category: TestStepCategory.apiMock, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Clear recorded API call history', + example: const {}, + ), + 'clearApiMocks': TestStepRegistryEntry( + category: TestStepCategory.apiMock, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Remove all registered API mocks', + example: const {}, + ), + 'expectApiCalled': TestStepRegistryEntry( + category: TestStepCategory.apiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.apiName, + description: 'Assert an API was called an exact number of times', + example: const {'name': 'login', 'times': 1}, + ), + 'expectApiNotCalled': TestStepRegistryEntry( + category: TestStepCategory.apiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.apiName, + description: 'Assert an API was never called', + example: const {'name': 'login', 'times': 1}, + ), + 'expectApiRequest': TestStepRegistryEntry( + category: TestStepCategory.apiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.apiRequest, + description: 'Assert last API request body/query/headers match', + example: const {'name': 'login', 'body': const {'email': 'user@test.com', 'password': 'secret'}}, + ), + 'expectApiRequestContains': TestStepRegistryEntry( + category: TestStepCategory.apiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.apiRequest, + description: 'Assert API request contains partial body/query', + example: const {'name': 'login', 'body': const {'email': 'user@test.com', 'password': 'secret'}}, + ), + 'expectApiHeader': TestStepRegistryEntry( + category: TestStepCategory.apiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectApiHeader, + description: 'Assert an API request header equals expected', + example: const {'name': 'login', 'header': 'Authorization', 'equals': 'Bearer test-token'}, + ), + 'expectApiCallOrder': TestStepRegistryEntry( + category: TestStepCategory.apiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectApiCallOrder, + description: 'Assert APIs were called in order', + example: const {'names': const ['auth', 'profile']}, + ), + 'expectLastApiCall': TestStepRegistryEntry( + category: TestStepCategory.apiAssertion, + tier: TestStepTier.core, + argKind: TestStepArgKind.apiName, + description: 'Assert the most recent API call name', + example: const {'name': 'login', 'times': 1}, + ), + 'setState': TestStepRegistryEntry( + category: TestStepCategory.state, + tier: TestStepTier.core, + argKind: TestStepArgKind.setState, + description: 'Set app data-context state at path to value', + example: const {'path': 'user.name', 'value': 'Jane'}, + ), + 'expectState': TestStepRegistryEntry( + category: TestStepCategory.state, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectState, + description: 'Assert app state at path equals expected', + example: const {'path': 'user.name', 'equals': 'Jane'}, + ), + 'expectStateContains': TestStepRegistryEntry( + category: TestStepCategory.state, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectState, + description: 'Assert app state at path contains subset', + example: const {'path': 'user.name', 'equals': 'Jane'}, + ), + 'expectStateExists': TestStepRegistryEntry( + category: TestStepCategory.state, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectStatePath, + description: 'Assert state path resolves without error', + example: const {'path': 'user.id'}, + ), + 'expectStateNotExists': TestStepRegistryEntry( + category: TestStepCategory.state, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectStatePath, + description: 'Assert state path is null or absent', + example: const {'path': 'user.id'}, + ), + 'resetState': TestStepRegistryEntry( + category: TestStepCategory.state, + tier: TestStepTier.core, + argKind: TestStepArgKind.resetStatePath, + description: 'Clear state at path (set to null)', + example: const {'path': 'cart'}, + ), + 'setStorage': TestStepRegistryEntry( + category: TestStepCategory.storage, + tier: TestStepTier.core, + argKind: TestStepArgKind.storageKey, + description: 'Write a value to public GetStorage by key', + example: const {'key': 'onboarding_done', 'value': true}, + ), + 'expectStorage': TestStepRegistryEntry( + category: TestStepCategory.storage, + tier: TestStepTier.core, + argKind: TestStepArgKind.storageKey, + description: 'Assert public storage key equals expected', + example: const {'key': 'onboarding_done', 'value': true}, + ), + 'removeStorage': TestStepRegistryEntry( + category: TestStepCategory.storage, + tier: TestStepTier.core, + argKind: TestStepArgKind.storageKey, + description: 'Remove a key from public storage', + example: const {'key': 'onboarding_done', 'value': true}, + ), + 'clearStorage': TestStepRegistryEntry( + category: TestStepCategory.storage, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Clear all non-encrypted public storage keys', + example: const {}, + ), + 'setEnv': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.storageKey, + description: 'Override an environment variable for the test', + example: const {'key': 'onboarding_done', 'value': true}, + ), + 'setAuth': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.setAuth, + description: 'Simulate a signed-in user', + example: const {'user': const {'id': '1', 'email': 'user@test.com'}}, + ), + 'clearAuth': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Clear the signed-in user', + example: const {}, + ), + 'setPermission': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.setPermission, + description: 'Set a permission flag for the test runtime', + example: const {'name': 'camera', 'value': 'granted'}, + ), + 'setDevice': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.setDevice, + description: 'Override viewport physical size (width/height)', + example: const {'width': 390, 'height': 844}, + ), + 'setLocale': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.setLocale, + description: 'Set APP_LOCALE environment override', + example: const {'locale': 'en_US'}, + ), + 'setTheme': TestStepRegistryEntry( + category: TestStepCategory.runtime, + tier: TestStepTier.core, + argKind: TestStepArgKind.setTheme, + description: 'Set APP_THEME / theme mode override', + example: const {'mode': 'dark'}, + ), + 'runScript': TestStepRegistryEntry( + category: TestStepCategory.script, + tier: TestStepTier.core, + argKind: TestStepArgKind.runScript, + description: 'Evaluate a script expression in the data context', + example: const {'script': '1 + 1', 'equals': 2}, + ), + 'expectScriptResult': TestStepRegistryEntry( + category: TestStepCategory.script, + tier: TestStepTier.core, + argKind: TestStepArgKind.runScript, + description: 'Evaluate script and assert result equals expected', + example: const {'script': '1 + 1', 'equals': 2}, + ), + 'expectConsoleLog': TestStepRegistryEntry( + category: TestStepCategory.script, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectConsoleLog, + description: 'Assert a console log line contains text', + example: const {'contains': 'Screen loaded'}, + ), + 'group': TestStepRegistryEntry( + category: TestStepCategory.control, + tier: TestStepTier.core, + argKind: TestStepArgKind.group, + description: 'Run nested steps as a named group', + example: const {'name': 'login_flow', 'steps': const [const {'tap': const {'id': 'login_button'}}]}, + ), + 'repeat': TestStepRegistryEntry( + category: TestStepCategory.control, + tier: TestStepTier.core, + argKind: TestStepArgKind.repeat, + description: 'Repeat nested steps N times', + example: const {'times': 3, 'steps': const [const {'tap': const {'id': 'next_button'}}]}, + ), + 'optional': TestStepRegistryEntry( + category: TestStepCategory.control, + tier: TestStepTier.core, + argKind: TestStepArgKind.optional, + description: 'Run nested steps; swallow failures', + example: const {'steps': const [const {'tap': const {'id': 'dismiss_banner'}}]}, + ), + 'ifVisible': TestStepRegistryEntry( + category: TestStepCategory.control, + tier: TestStepTier.core, + argKind: TestStepArgKind.ifVisible, + description: 'Run nested steps only if testId is visible', + example: const {'id': 'promo_banner', 'steps': const [const {'tap': const {'id': 'close_banner'}}]}, + ), + 'logApiCalls': TestStepRegistryEntry( + category: TestStepCategory.debug, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Log all recorded API calls to the test log', + example: const {}, + ), + 'screenshot': TestStepRegistryEntry( + category: TestStepCategory.debug, + tier: TestStepTier.core, + argKind: TestStepArgKind.screenshot, + description: 'Capture golden or dump widget tree for debugging', + example: const {'name': 'home_screen'}, + ), + 'dumpTree': TestStepRegistryEntry( + category: TestStepCategory.debug, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Print the widget tree to the debug console', + example: const {}, + ), + 'logState': TestStepRegistryEntry( + category: TestStepCategory.debug, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectStatePath, + description: 'Log resolved state at path', + example: const {'path': 'user.id'}, + ), + 'logStorage': TestStepRegistryEntry( + category: TestStepCategory.debug, + tier: TestStepTier.core, + argKind: TestStepArgKind.storageKey, + description: 'Log public storage value for key', + example: const {'key': 'onboarding_done', 'value': true}, + ), + 'expectNoConsoleErrors': TestStepRegistryEntry( + category: TestStepCategory.quality, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Assert no console errors were recorded', + example: const {}, + ), + 'expectNoRenderErrors': TestStepRegistryEntry( + category: TestStepCategory.quality, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Assert no Flutter render errors were recorded', + example: const {}, + ), + 'expectError': TestStepRegistryEntry( + category: TestStepCategory.quality, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectErrorContains, + description: 'Assert a Flutter error was recorded (optional filter)', + example: const {'contains': 'overflow'}, + ), + 'expectNoErrors': TestStepRegistryEntry( + category: TestStepCategory.quality, + tier: TestStepTier.core, + argKind: TestStepArgKind.empty, + description: 'Alias for expectNoRenderErrors', + example: const {}, + ), + 'expectAccessible': TestStepRegistryEntry( + category: TestStepCategory.quality, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert widget has accessibility label or value', + example: const {'id': 'my_widget'}, + ), + 'expectSemanticsLabel': TestStepRegistryEntry( + category: TestStepCategory.quality, + tier: TestStepTier.core, + argKind: TestStepArgKind.expectSemanticsLabel, + description: 'Assert semantics label equals expected', + example: const {'id': 'submit_button', 'label': 'Submit'}, + ), + 'expectNoOverflow': TestStepRegistryEntry( + category: TestStepCategory.quality, + tier: TestStepTier.core, + argKind: TestStepArgKind.idRequired, + description: 'Assert widget renders without overflow issues', + example: const {'id': 'my_widget'}, + ), + 'loadFixture': TestStepRegistryEntry( + category: TestStepCategory.fixture, + tier: TestStepTier.core, + argKind: TestStepArgKind.fixturePath, + description: 'Load a JSON fixture into the test fixture map', + example: const {'fixture': 'fixtures/user.json'}, + ), + 'setStateFromFixture': TestStepRegistryEntry( + category: TestStepCategory.fixture, + tier: TestStepTier.core, + argKind: TestStepArgKind.fixturePath, + description: 'Apply all keys from a JSON fixture to state', + example: const {'fixture': 'fixtures/user.json'}, + ), + 'expectMatchesFixture': TestStepRegistryEntry( + category: TestStepCategory.fixture, + tier: TestStepTier.core, + argKind: TestStepArgKind.fixturePath, + description: 'Assert state or path matches a JSON fixture', + example: const {'fixture': 'fixtures/user.json'}, + ), + }; +} + diff --git a/packages/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart b/packages/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart new file mode 100644 index 000000000..cffcc153f --- /dev/null +++ b/packages/ensemble_test_runner/lib/vocabulary/test_step_vocabulary.dart @@ -0,0 +1,121 @@ +import 'package:ensemble_test_runner/vocabulary/test_step_arg_kind.dart'; +import 'package:ensemble_test_runner/vocabulary/test_step_registry.dart'; + +/// Official Ensemble declarative test step vocabulary. +/// +/// Single source of truth for step metadata and JSON Schema args: +/// [TestStepRegistry.entries]. Executor aliases are declared via +/// [TestStepRegistryEntry.executorCanonical]. +export 'test_step_arg_kind.dart'; +export 'test_step_registry.dart'; + +enum TestStepCategory { + lifecycle, + interaction, + formControl, + gesture, + wait, + uiAssertion, + valueAssertion, + listAssertion, + navigation, + apiMock, + apiAssertion, + state, + storage, + runtime, + script, + network, + fixture, + control, + debug, + quality, +} + +enum TestStepTier { + core, + extended, +} + +class TestStepDefinition { + final String name; + final TestStepCategory category; + final TestStepTier tier; + final TestStepArgKind argKind; + final String description; + final Map example; + final List aliases; + + const TestStepDefinition({ + required this.name, + required this.category, + required this.tier, + required this.argKind, + required this.description, + required this.example, + this.aliases = const [], + }); +} + +/// Canonical step names, aliases, and JSON Schema args. +class TestStepVocabulary { + TestStepVocabulary._(); + + static final Map definitions = { + for (final e in TestStepRegistry.entries.entries) + e.key: TestStepDefinition( + name: e.key, + category: e.value.category, + tier: e.value.tier, + argKind: e.value.argKind, + description: e.value.description, + example: e.value.example, + ), + }; + + static final Map _aliasToCanonical = () { + final map = {}; + for (final e in TestStepRegistry.entries.entries) { + map[e.key] = e.value.executorCanonical ?? e.key; + } + return map; + }(); + + /// Maps YAML step keys (including aliases) to canonical handler names. + static String resolveStepType(String yamlKey) => + _aliasToCanonical[yamlKey] ?? yamlKey; + + static TestStepDefinition? lookup(String yamlKey) { + final entry = TestStepRegistry.entries[yamlKey]; + if (entry != null) { + return definitions[yamlKey]; + } + final canonical = resolveStepType(yamlKey); + return definitions[canonical]; + } + + static Iterable byTier(TestStepTier tier) => + definitions.values.where((d) => d.tier == tier); + + static Iterable get coreSteps => + byTier(TestStepTier.core); + + /// Every YAML key that may appear as a single-key step map. + static Iterable get yamlStepKeys => TestStepRegistry.entries.keys; + + /// JSON Schema object for a step's YAML argument map (by YAML key). + static Map argJsonSchemaForYamlKey(String yamlKey) { + final entry = TestStepRegistry.entries[yamlKey]; + if (entry != null) { + return entry.argKind.jsonSchema; + } + final canonical = resolveStepType(yamlKey); + return definitions[canonical]?.argKind.jsonSchema ?? + TestStepArgKind.empty.jsonSchema; + } + + /// JSON Schema for a canonical executor step name. + static Map argJsonSchemaFor(String canonicalStep) => + definitions[canonicalStep]?.argKind.jsonSchema ?? + TestStepArgKind.empty.jsonSchema; +} diff --git a/packages/ensemble_test_runner/pubspec.yaml b/packages/ensemble_test_runner/pubspec.yaml new file mode 100644 index 000000000..cbf86aca9 --- /dev/null +++ b/packages/ensemble_test_runner/pubspec.yaml @@ -0,0 +1,30 @@ +name: ensemble_test_runner +description: Declarative YAML test runner for Ensemble apps — wraps the real runtime with mocks and Flutter widget assertions. +version: 0.1.0 +publish_to: none + +executables: + ensemble_test: ensemble_test + +environment: + sdk: ">=3.5.0" + flutter: ">=3.24.0" + +dependencies: + ensemble: + git: + url: https://github.com/EnsembleUI/ensemble.git + ref: ensemble-v1.2.43 + path: modules/ensemble + flutter: + sdk: flutter + yaml: ^3.1.2 + path: ^1.9.0 + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^2.0.3 + +flutter: + uses-material-design: true diff --git a/packages/ensemble_test_runner/test/ensemble_test_cli_output_test.dart b/packages/ensemble_test_runner/test/ensemble_test_cli_output_test.dart new file mode 100644 index 000000000..fa4096209 --- /dev/null +++ b/packages/ensemble_test_runner/test/ensemble_test_cli_output_test.dart @@ -0,0 +1,39 @@ +import 'package:ensemble_test_runner/cli/ensemble_test_cli_output.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('extractSuiteReport keeps screen tracker and boxed summary', () { + const noisy = ''' +00:00 +0: loading test/ensemble_tests.dart +SCREEN TRACKER: Hello Home +name is ={first: John} +┌─ Ensemble YAML tests ───────────────────────────── +│ ✓ hello_home_renders +└─ 5 passed, 0 failed (5 total) · 653ms total + +00:00 +1: All tests passed! +'''; + + expect( + extractSuiteReport(noisy), + '''SCREEN TRACKER: Hello Home +┌─ Ensemble YAML tests ───────────────────────────── +│ ✓ hello_home_renders +└─ 5 passed, 0 failed (5 total) · 653ms total +''', + ); + }); + + test('flutterTestArguments strips CLI-only flags', () { + expect( + flutterTestArguments([ + '--app-dir=foo', + '--verbose', + '--quiet', + '--name', + 'x', + ]), + ['--name', 'x'], + ); + }); +} diff --git a/packages/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart b/packages/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart new file mode 100644 index 000000000..ed2555df7 --- /dev/null +++ b/packages/ensemble_test_runner/test/ensemble_test_execution_planner_test.dart @@ -0,0 +1,69 @@ +import 'package:ensemble_test_runner/discovery/ensemble_test_execution_planner.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/parser/ensemble_test_parser.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('EnsembleTestExecutionPlanner', () { + test('orders prerequisite chain before independent startScreen tests', () { + final byId = { + 'chain_root': _def('a/chain_root.test.yaml', ''' +id: chain_root +startScreen: Home +steps: + - expectVisible: + id: x +'''), + 'chain_child': _def('b/chain_child.test.yaml', ''' +id: chain_child +prerequisite: chain_root +steps: + - expectVisible: + id: y +'''), + 'standalone': _def('z/standalone.test.yaml', ''' +id: standalone +startScreen: Other +steps: + - expectVisible: + id: z +'''), + }; + + final order = EnsembleTestExecutionPlanner.orderIdsForTest(byId); + expect(order.indexOf('chain_root'), lessThan(order.indexOf('chain_child'))); + expect(order.indexOf('chain_child'), lessThan(order.indexOf('standalone'))); + }); + + test('detects circular prerequisites', () { + final byId = { + 'a': _def('a.test.yaml', ''' +id: a +prerequisite: b +steps: + - expectVisible: + id: x +'''), + 'b': _def('b.test.yaml', ''' +id: b +prerequisite: a +steps: + - expectVisible: + id: y +'''), + }; + + expect( + () => EnsembleTestExecutionPlanner.orderIdsForTest(byId), + throwsA(isA()), + ); + }); + }); +} + +EnsembleTestDefinition _def(String assetPath, String yaml) { + return EnsembleTestDefinition( + assetPath: assetPath, + testCase: EnsembleTestParser.parseString(yaml), + ); +} diff --git a/packages/ensemble_test_runner/test/ensemble_test_parser_nested_test.dart b/packages/ensemble_test_runner/test/ensemble_test_parser_nested_test.dart new file mode 100644 index 000000000..68265003d --- /dev/null +++ b/packages/ensemble_test_runner/test/ensemble_test_parser_nested_test.dart @@ -0,0 +1,41 @@ +import 'package:ensemble_test_runner/ensemble_test_runner.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('parses group with nested steps', () { + const yaml = ''' +id: grouped +startScreen: Home +steps: + - group: + name: validation + steps: + - expectVisible: + id: submit +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.steps.length, 1); + expect(test.steps.first.type, 'group'); + expect(test.steps.first.nestedSteps.length, 1); + expect(test.steps.first.nestedSteps.first.type, 'expectVisible'); + }); + + test('parses ifVisible with nested steps', () { + const yaml = ''' +id: conditional +startScreen: Home +steps: + - ifVisible: + id: banner + steps: + - tap: + id: dismiss +'''; + + final step = EnsembleTestParser.parseString(yaml).steps.first; + expect(step.type, 'ifVisible'); + expect(step.args['id'], 'banner'); + expect(step.nestedSteps.single.type, 'tap'); + }); +} diff --git a/packages/ensemble_test_runner/test/ensemble_test_parser_test.dart b/packages/ensemble_test_runner/test/ensemble_test_parser_test.dart new file mode 100644 index 000000000..f09bf0ed4 --- /dev/null +++ b/packages/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -0,0 +1,148 @@ +import 'package:ensemble_test_runner/ensemble_test_runner.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('EnsembleTestParser', () { + test('parses minimal test file', () { + const yaml = ''' +id: login_screen_renders +startScreen: LoginScreen +steps: + - expectVisible: + id: emailInput +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.id, 'login_screen_renders'); + expect(test.startScreen, 'LoginScreen'); + expect(test.steps.length, 1); + expect(test.steps.first.type, 'expectVisible'); + expect(test.steps.first.args['id'], 'emailInput'); + }); + + test('parses API mocks', () { + const yaml = ''' +id: login_success +startScreen: Login +mocks: + apis: + loginApi: + delayMs: 100 + response: + statusCode: 200 + body: + token: test-token +steps: + - expectApiCalled: + name: loginApi + times: 1 +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.mocks.apis['loginApi']?.statusCode, 200); + expect(test.mocks.apis['loginApi']?.delayMs, 100); + expect( + (test.mocks.apis['loginApi']?.body as Map)['token'], + 'test-token', + ); + }); + + test('rejects legacy tests wrapper', () { + expect( + () => EnsembleTestParser.parseString(''' +tests: + - id: old_format + startScreen: Home + steps: + - expectVisible: + id: x +'''), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('tests'), + ), + ), + ); + }); + }); + + group('prerequisite and startScreen XOR', () { + test('parses prerequisite-only test', () { + const yaml = ''' +id: continuation_flow +prerequisite: hello_home_renders +steps: + - expectVisible: + id: goodbye_title +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.id, 'continuation_flow'); + expect(test.startScreen, isNull); + expect(test.prerequisite, 'hello_home_renders'); + expect(test.steps.single.type, 'expectVisible'); + }); + + test('rejects when both startScreen and prerequisite are set', () { + const yaml = ''' +id: invalid_both +startScreen: Home +prerequisite: other +steps: + - expectVisible: + id: x +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('startScreen'), + ), + ), + ); + }); + + test('rejects when neither startScreen nor prerequisite is set', () { + const yaml = ''' +id: invalid_neither +steps: + - expectVisible: + id: x +'''; + + expect( + () => EnsembleTestParser.parseString(yaml), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('startScreen'), + ), + ), + ); + }); + }); + + group('EnsembleTestRunResult', () { + test('summary counts pass and fail', () { + final result = EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.passed(testId: 'a', durationMs: 1), + EnsembleSingleTestResult.failed( + testId: 'b', + durationMs: 2, + error: 'oops', + ), + ], + ); + expect(result.passedCount, 1); + expect(result.failedCount, 1); + expect(result.summary, '1 passed, 1 failed (2 total)'); + }); + }); +} diff --git a/packages/ensemble_test_runner/test/ensemble_test_schema_test.dart b/packages/ensemble_test_runner/test/ensemble_test_schema_test.dart new file mode 100644 index 000000000..6b5098f12 --- /dev/null +++ b/packages/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -0,0 +1,53 @@ +import 'dart:convert'; + +import 'package:ensemble_test_runner/schema/ensemble_test_schema_builder.dart'; +import 'package:ensemble_test_runner/vocabulary/test_step_registry.dart'; +import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('schema includes all vocabulary steps', () { + final schema = EnsembleTestSchemaBuilder.build(); + final stepDef = schema['\$defs']['step'] as Map; + final oneOf = stepDef['oneOf'] as List; + + final yamlKeys = oneOf + .map((e) { + final props = (e as Map)['properties'] as Map; + return props.keys.first as String; + }) + .toSet(); + + for (final name in TestStepRegistry.entries.keys) { + expect(yamlKeys, contains(name), reason: 'missing step $name in schema'); + } + + for (final variant in oneOf) { + final map = variant as Map; + expect(map['description'], isNotEmpty); + expect(map['examples'], isNotEmpty); + final yamlKey = (map['properties'] as Map).keys.first as String; + final entry = TestStepRegistry.entries[yamlKey]!; + final argsDefName = 'args_$yamlKey'; + final argsDef = schema['\$defs'][argsDefName] as Map; + expect(argsDef['description'], map['description']); + expect(argsDef['examples'], [entry.example]); + expect(map['examples'], [ + {yamlKey: entry.example}, + ]); + } + }); + + test('generated JSON is valid and requires id, steps with XOR start/prerequisite', () { + final json = EnsembleTestSchemaBuilder.buildJson(); + final decoded = jsonDecode(json) as Map; + expect(decoded['\$schema'], EnsembleTestSchemaBuilder.schemaVersion); + expect(decoded['required'], containsAll(['id', 'steps'])); + expect((decoded['required'] as List), isNot(contains('startScreen'))); + expect(decoded['properties'], contains('id')); + expect(decoded['properties'], isNot(contains('tests'))); + expect(decoded['properties'], contains('startScreen')); + expect(decoded['properties'], contains('prerequisite')); + expect(decoded['oneOf'], isA()); + }); +} diff --git a/packages/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart b/packages/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart new file mode 100644 index 000000000..73835454f --- /dev/null +++ b/packages/ensemble_test_runner/test/ensemble_test_schema_up_to_date_test.dart @@ -0,0 +1,19 @@ +import 'dart:io'; + +import 'package:ensemble_test_runner/schema/ensemble_test_schema_builder.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('committed schema matches generator output', () { + final path = + 'assets/schema/ensemble_test.schema.json'; + final committed = File(path).readAsStringSync().trim(); + final generated = EnsembleTestSchemaBuilder.buildJson().trim(); + expect( + committed, + generated, + reason: + 'Run: cd packages/ensemble_test_runner && dart run tool/generate_schema.dart', + ); + }); +} diff --git a/packages/ensemble_test_runner/test/navigation_flow_recorder_test.dart b/packages/ensemble_test_runner/test/navigation_flow_recorder_test.dart new file mode 100644 index 000000000..f337763a0 --- /dev/null +++ b/packages/ensemble_test_runner/test/navigation_flow_recorder_test.dart @@ -0,0 +1,27 @@ +import 'package:ensemble/framework/screen_tracker.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('dedupes consecutive screen names', () { + final recorder = NavigationFlowRecorder(); + + recorder.recordScreenChange( + VisibleScreen(screenName: 'Hello Home', visibleSince: DateTime.now()), + ); + recorder.recordScreenChange( + VisibleScreen(screenName: 'Goodbye', visibleSince: DateTime.now()), + ); + recorder.recordScreenChange( + VisibleScreen(screenName: 'Hello Home', visibleSince: DateTime.now()), + ); + recorder.recordScreenChange( + VisibleScreen(screenName: 'Hello Home', visibleSince: DateTime.now()), + ); + + expect( + recorder.flow, + ['Hello Home', 'Goodbye', 'Hello Home'], + ); + }); +} diff --git a/packages/ensemble_test_runner/test/test_reporter_test.dart b/packages/ensemble_test_runner/test/test_reporter_test.dart new file mode 100644 index 000000000..b7cb93205 --- /dev/null +++ b/packages/ensemble_test_runner/test/test_reporter_test.dart @@ -0,0 +1,91 @@ +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/yaml_test_session.dart'; +import 'package:ensemble_test_runner/reporters/test_reporter.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('formatStepBrief', () { + test('formats id-based steps', () { + expect( + formatStepBrief(const TestStep(type: 'tap', args: {'id': 'btn'})), + 'tap(btn)', + ); + expect( + formatStepBrief(const TestStep( + type: 'trigger', + args: {'action': 'onTap', 'id': 'nav'}, + )), + 'trigger(onTap nav)', + ); + }); + }); + + group('collectScreensVisited', () { + test('reads navigation flow including back navigation', () { + YamlTestSession.navigationFlow + .seed(['Hello Home', 'Goodbye', 'Hello Home']); + + expect( + collectScreensVisited('Hello Home'), + ['Hello Home', 'Goodbye', 'Hello Home'], + ); + }); + }); + + group('TestReporter', () { + test('includes file, flow, and steps on pass', () { + final report = EnsembleTestReportDetails( + startScreen: 'Hello Home', + endScreen: 'Hello Home', + screensVisited: ['Hello Home', 'Goodbye', 'Hello Home'], + stepsOutline: [ + 'expectVisible(greeting_text)', + 'trigger(onTap navigate_button)', + ], + ); + final output = TestReporter().formatSummary( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.passed( + testId: 'hello_home_renders', + durationMs: 610, + report: report, + ), + ], + ), + testFile: 'ensemble/tests/hello_home.test.yaml', + ); + + expect(output, contains('hello_home.test.yaml')); + expect(output, contains('hello_home_renders')); + expect(output, contains('Hello Home → Goodbye → Hello Home')); + expect(output, contains('expectVisible(greeting_text)')); + expect(output, contains('1 passed, 0 failed')); + expect(output, contains('610ms total')); + }); + + test('marks failed step in outline', () { + final report = EnsembleTestReportDetails( + startScreen: 'Login', + stepsOutline: ['tap(email)', 'tap(submit)'], + ); + final output = TestReporter().formatSummary( + EnsembleTestRunResult( + results: [ + EnsembleSingleTestResult.failed( + testId: 'login', + durationMs: 100, + failedStepIndex: 1, + failedStep: const TestStep(type: 'tap', args: {'id': 'submit'}), + error: 'not found', + report: report, + ), + ], + ), + ); + + expect(output, contains('>> 2. tap(submit)')); + expect(output, contains('error: not found')); + }); + }); +} diff --git a/packages/ensemble_test_runner/test/test_step_executor_test.dart b/packages/ensemble_test_runner/test/test_step_executor_test.dart new file mode 100644 index 000000000..8e128a273 --- /dev/null +++ b/packages/ensemble_test_runner/test/test_step_executor_test.dart @@ -0,0 +1,36 @@ +import 'package:ensemble_test_runner/actions/test_step_executor.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; +import 'package:ensemble_test_runner/assertions/assertion_engine.dart'; +import 'package:ensemble_test_runner/models/ensemble_test_models.dart'; +import 'package:ensemble_test_runner/runner/ensemble_test_context.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + testWidgets('waitFor requires id or text', (tester) async { + final context = EnsembleTestContext.fromTestCase( + const EnsembleTestCase( + id: 't', + startScreen: 'Home', + steps: [], + ), + ); + final harness = EnsembleTestHarness(appPath: 'ensemble/apps/', appHome: 'x'); + final executor = TestStepExecutor( + tester: tester, + context: context, + assertions: AssertionEngine(tester: tester, context: context), + harness: harness, + ); + + await expectLater( + executor.execute(const TestStep(type: 'waitFor', args: {})), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('either "id" or "text"'), + ), + ), + ); + }); +} diff --git a/packages/ensemble_test_runner/test/vocabulary_arg_schema_sync_test.dart b/packages/ensemble_test_runner/test/vocabulary_arg_schema_sync_test.dart new file mode 100644 index 000000000..240d1c542 --- /dev/null +++ b/packages/ensemble_test_runner/test/vocabulary_arg_schema_sync_test.dart @@ -0,0 +1,50 @@ +import 'package:ensemble_test_runner/vocabulary/test_step_registry.dart'; +import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('registry and vocabulary definitions stay in sync', () { + expect( + TestStepRegistry.entries.length, + TestStepVocabulary.definitions.length, + ); + for (final name in TestStepRegistry.entries.keys) { + expect(TestStepVocabulary.definitions, contains(name)); + expect( + TestStepVocabulary.definitions[name]!.argKind, + TestStepRegistry.entries[name]!.argKind, + ); + } + }); + + test('every registry entry has a JSON Schema', () { + for (final entry in TestStepRegistry.entries.values) { + expect(entry.argKind.jsonSchema, isA>()); + expect(entry.argKind.jsonSchema['type'], 'object'); + } + }); + + test('every registry entry has a non-empty description', () { + for (final e in TestStepRegistry.entries.entries) { + expect( + e.value.description.trim(), + isNotEmpty, + reason: 'missing description for ${e.key}', + ); + } + }); + + test('every registry entry has an example args map', () { + for (final e in TestStepRegistry.entries.entries) { + expect( + e.value.example, + isA>(), + reason: 'missing example for ${e.key}', + ); + expect( + TestStepVocabulary.definitions[e.key]!.example, + e.value.example, + ); + } + }); +} diff --git a/packages/ensemble_test_runner/test/yaml_test_app_patcher_test.dart b/packages/ensemble_test_runner/test/yaml_test_app_patcher_test.dart new file mode 100644 index 000000000..4a2795d0d --- /dev/null +++ b/packages/ensemble_test_runner/test/yaml_test_app_patcher_test.dart @@ -0,0 +1,114 @@ +import 'dart:io'; + +import 'package:ensemble_test_runner/cli/yaml_test_app_patcher.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('enable injects test assets and restores', () { + final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); + addTearDown(() => dir.deleteSync(recursive: true)); + + const pubspec = ''' +name: sample_app +dev_dependencies: + flutter_test: + sdk: flutter + +flutter: + assets: + - ensemble/ +'''; + File('${dir.path}/pubspec.yaml').writeAsStringSync(pubspec); + Directory('${dir.path}/ensemble/tests').createSync(recursive: true); + File('${dir.path}/ensemble/tests/sample.test.yaml').writeAsStringSync(''' +id: sample +startScreen: Home +steps: [] +'''); + + final patcher = YamlTestAppPatcher(dir.path); + patcher.enable(); + + expect(patcher.pubspecChanged, isTrue); + final enabled = File('${dir.path}/pubspec.yaml').readAsStringSync(); + expect(enabled, contains(YamlTestAppPatcher.testsAssetLine)); + expect( + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') + .readAsStringSync(), + contains('runEnsembleYamlTests'), + ); + + patcher.restore(); + + expect(File('${dir.path}/pubspec.yaml').readAsStringSync(), pubspec); + expect( + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}').existsSync(), + isFalse, + ); + }); + + test('restore deletes leftover generated test entry from a prior run', () { + final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); + addTearDown(() => dir.deleteSync(recursive: true)); + + File('${dir.path}/pubspec.yaml').writeAsStringSync(''' +name: sample_app +dev_dependencies: + flutter_test: + sdk: flutter +flutter: + assets: + - ensemble/ +'''); + Directory('${dir.path}/ensemble/tests').createSync(recursive: true); + File('${dir.path}/ensemble/tests/sample.test.yaml').writeAsStringSync(''' +id: sample +startScreen: Home +steps: [] +'''); + Directory('${dir.path}/test').createSync(recursive: true); + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}') + .writeAsStringSync(YamlTestAppPatcher.testEntryContents); + + final patcher = YamlTestAppPatcher(dir.path); + patcher.enable(); + patcher.restore(); + + expect( + File('${dir.path}/${YamlTestAppPatcher.testEntryRelativePath}').existsSync(), + isFalse, + ); + }); + + test('enable skips pubspec write when test assets already present', () { + final dir = Directory.systemTemp.createTempSync('yaml_test_patcher_'); + addTearDown(() => dir.deleteSync(recursive: true)); + + const pubspec = ''' +name: sample_app +dev_dependencies: + flutter_test: + sdk: flutter + +flutter: + assets: + - ensemble/ + - ensemble/tests/ +'''; + File('${dir.path}/pubspec.yaml').writeAsStringSync(pubspec); + Directory('${dir.path}/ensemble/tests').createSync(recursive: true); + File('${dir.path}/ensemble/tests/sample.test.yaml').writeAsStringSync(''' +id: sample +startScreen: Home +steps: [] +'''); + + final patcher = YamlTestAppPatcher(dir.path); + patcher.enable(); + + expect(patcher.pubspecChanged, isFalse); + expect(File('${dir.path}/pubspec.yaml').readAsStringSync(), pubspec); + + patcher.restore(); + }); +} diff --git a/packages/ensemble_test_runner/tool/generate_schema.dart b/packages/ensemble_test_runner/tool/generate_schema.dart new file mode 100644 index 000000000..14c135e3e --- /dev/null +++ b/packages/ensemble_test_runner/tool/generate_schema.dart @@ -0,0 +1,20 @@ +import 'dart:io'; + +import 'package:ensemble_test_runner/schema/ensemble_test_schema_builder.dart'; + +/// Regenerates [assets/schema/ensemble_test.schema.json] from the step registry. +/// +/// Run from this package: +/// dart run tool/generate_schema.dart +void main() { + final packageRoot = Directory.current; + if (!File('${packageRoot.path}/pubspec.yaml').existsSync()) { + stderr.writeln('Run from packages/ensemble_test_runner'); + exit(1); + } + + final outFile = File('assets/schema/ensemble_test.schema.json'); + outFile.parent.createSync(recursive: true); + outFile.writeAsStringSync('${EnsembleTestSchemaBuilder.buildJson()}\n'); + stdout.writeln('Wrote ${outFile.path}'); +} diff --git a/packages/ensemble_test_runner/tool/generate_step_registry.dart b/packages/ensemble_test_runner/tool/generate_step_registry.dart new file mode 100644 index 000000000..2d61fe5e1 --- /dev/null +++ b/packages/ensemble_test_runner/tool/generate_step_registry.dart @@ -0,0 +1,994 @@ +// ignore_for_file: avoid_print + +import 'dart:io'; + +/// One-time / maintenance generator for [test_step_registry.dart]. +/// Run: dart run tool/generate_step_registry.dart +void main() { + Map step( + String cat, + String tier, + String arg, + String desc, { + Map? example, + }) => + { + 'cat': cat, + 'tier': tier, + 'arg': arg, + 'desc': desc, + 'example': example ?? defaultExampleForArg(arg), + }; + + final entries = >{ + 'openScreen': step( + 'lifecycle', + 'core', + 'openScreen', + 'Navigate to another screen by name or id mid-test', + ), + 'reloadScreen': step( + 'lifecycle', + 'core', + 'empty', + 'Reload the current screen (same as re-opening it)', + ), + 'restartApp': step( + 'lifecycle', + 'core', + 'empty', + 'Reset runtime and reopen the test case start screen', + ), + 'resetAppState': step( + 'lifecycle', + 'core', + 'empty', + 'Clear screen tracker, API call log, and public storage', + ), + 'trigger': step( + 'lifecycle', + 'core', + 'trigger', + 'Fire a widget action (onLoad, onTap, onLongPress) by testId', + ), + 'launchApp': step( + 'lifecycle', + 'core', + 'empty', + 'Alias for restartApp — bootstrap from startScreen again', + ), + 'tap': step( + 'interaction', + 'core', + 'idRequired', + 'Tap a widget by testId (ValueKey)', + ), + 'doubleTap': step( + 'interaction', + 'core', + 'idRequired', + 'Double-tap a widget by testId', + ), + 'longPress': step( + 'interaction', + 'core', + 'idRequired', + 'Long-press a widget by testId', + ), + 'enterText': step( + 'interaction', + 'core', + 'enterText', + 'Type text into an input field by testId', + ), + 'clearText': step( + 'interaction', + 'core', + 'idRequired', + 'Clear text in an input field by testId', + ), + 'replaceText': step( + 'interaction', + 'core', + 'enterText', + 'Replace the full contents of an input field', + ), + 'submitText': step( + 'interaction', + 'core', + 'idRequired', + 'Submit an input field (TextInputAction.done)', + ), + 'focus': step( + 'interaction', + 'core', + 'idRequired', + 'Focus an input field by testId', + ), + 'unfocus': step( + 'interaction', + 'core', + 'empty', + 'Remove focus from the current field', + ), + 'select': step( + 'formControl', + 'core', + 'select', + 'Open a dropdown and choose an option by visible label', + ), + 'selectIndex': step( + 'formControl', + 'core', + 'selectIndex', + 'Open a dropdown and choose the option at index', + ), + 'check': step( + 'formControl', + 'core', + 'idRequired', + 'Check a checkbox or toggle by testId', + ), + 'uncheck': step( + 'formControl', + 'core', + 'idRequired', + 'Uncheck a checkbox by testId if currently checked', + ), + 'toggle': step( + 'formControl', + 'core', + 'idRequired', + 'Tap to toggle a switch or checkbox by testId', + ), + 'setSlider': step( + 'formControl', + 'core', + 'setSlider', + 'Move a slider under testId to a normalized value (0–1)', + ), + 'chooseDate': step( + 'formControl', + 'core', + 'chooseValue', + 'Set a date field by testId to the given value string', + ), + 'chooseTime': step( + 'formControl', + 'core', + 'chooseValue', + 'Set a time field by testId to the given value string', + ), + 'scroll': step( + 'gesture', + 'core', + 'scroll', + 'Drag the first Scrollable by delta pixels', + ), + 'scrollUntilVisible': step( + 'gesture', + 'core', + 'idRequired', + 'Scroll until a widget with testId is visible', + ), + 'swipe': step( + 'gesture', + 'core', + 'swipe', + 'Swipe on a scrollable or widget (direction: left/right/up/down)', + ), + 'drag': step( + 'gesture', + 'core', + 'drag', + 'Drag a widget by testId by dx/dy offset', + ), + 'pullToRefresh': step( + 'gesture', + 'core', + 'idOptional', + 'Pull down on a scrollable to trigger refresh', + ), + 'wait': step( + 'wait', + 'extended', + 'pump', + 'Alias for pump — advance frame clock by durationMs', + ), + 'pump': step( + 'wait', + 'core', + 'pump', + 'Advance the Flutter frame clock by durationMs', + ), + 'settle': step( + 'wait', + 'core', + 'timeoutOptional', + 'Run pumpAndSettle until idle or timeout', + ), + 'waitFor': step( + 'wait', + 'core', + 'waitFor', + 'Poll until a widget id and/or text appears', + ), + 'waitForText': step( + 'wait', + 'core', + 'waitFor', + 'Poll until the given text appears on screen', + ), + 'waitForGone': step( + 'wait', + 'core', + 'waitForGone', + 'Poll until a widget with testId is removed from the tree', + ), + 'waitForApi': step( + 'wait', + 'core', + 'apiName', + 'Poll until a mocked API is called N times', + ), + 'waitForNavigation': step( + 'wait', + 'core', + 'waitForNavigation', + 'Poll until the given screen is visible', + ), + 'waitUntil': step( + 'wait', + 'core', + 'waitUntil', + 'Poll until app state at path equals expected value', + ), + 'expectVisible': step( + 'uiAssertion', + 'core', + 'idRequired', + 'Assert a widget with testId is visible', + ), + 'expectNotVisible': step( + 'uiAssertion', + 'core', + 'idRequired', + 'Assert a widget with testId is not visible', + ), + 'expectExists': step( + 'uiAssertion', + 'core', + 'idRequired', + 'Assert a widget with testId exists in the tree', + ), + 'expectNotExists': step( + 'uiAssertion', + 'core', + 'idRequired', + 'Assert no widget with testId exists', + ), + 'expectText': step( + 'uiAssertion', + 'core', + 'textRequired', + 'Assert exact text is shown', + ), + 'expectNoText': step( + 'uiAssertion', + 'core', + 'textRequired', + 'Assert text is not shown', + ), + 'expectTextContains': step( + 'uiAssertion', + 'core', + 'textRequired', + 'Assert some text containing the given substring', + ), + 'expectEnabled': step( + 'uiAssertion', + 'core', + 'idRequired', + 'Assert widget semantics report enabled', + ), + 'expectDisabled': step( + 'uiAssertion', + 'core', + 'idRequired', + 'Assert widget semantics report disabled', + ), + 'expectValue': step( + 'valueAssertion', + 'core', + 'expectEquals', + 'Assert input value equals expected (EditableText/TextField)', + ), + 'expectChecked': step( + 'valueAssertion', + 'core', + 'expectChecked', + 'Assert checkbox checked state matches equals', + ), + 'expectProperty': step( + 'valueAssertion', + 'core', + 'expectProperty', + 'Assert a widget property (e.g. label) equals expected', + ), + 'expectStyle': step( + 'valueAssertion', + 'core', + 'expectProperty', + 'Assert style-related property equals expected', + ), + 'expectSelected': step( + 'valueAssertion', + 'core', + 'expectChecked', + 'Assert selected/checked state matches equals', + ), + 'expectCount': step( + 'listAssertion', + 'core', + 'expectCount', + 'Assert count of widgets with the same testId', + ), + 'expectListCount': step( + 'listAssertion', + 'core', + 'expectListCount', + 'Assert number of list items under a list testId', + ), + 'expectListContains': step( + 'listAssertion', + 'core', + 'expectListContains', + 'Assert list contains text', + ), + 'expectListItem': step( + 'listAssertion', + 'core', + 'expectListItem', + 'Assert a list item widget with itemId is visible', + ), + 'expectEmpty': step( + 'listAssertion', + 'core', + 'idRequired', + 'Assert a list has zero items', + ), + 'expectNotEmpty': step( + 'listAssertion', + 'core', + 'idRequired', + 'Assert a list has at least one item', + ), + 'expectScreen': step( + 'navigation', + 'core', + 'screenRequired', + 'Alias for expectNavigateTo — assert current screen', + ), + 'expectNavigateTo': step( + 'navigation', + 'core', + 'screenRequired', + 'Assert the current visible screen name/id', + ), + 'expectVisited': step( + 'navigation', + 'core', + 'expectVisited', + 'Assert a screen appears in navigation history', + ), + 'expectNotVisited': step( + 'navigation', + 'core', + 'expectVisited', + 'Assert a screen was never visited', + ), + 'expectBackStack': step( + 'navigation', + 'core', + 'expectBackStack', + 'Assert navigation history suffix matches screens', + ), + 'expectCanGoBack': step( + 'navigation', + 'core', + 'expectCanGoBack', + 'Assert whether back navigation is possible', + ), + 'goBack': step( + 'navigation', + 'core', + 'empty', + 'Navigate back (Ensemble navigateBack or Navigator.pop)', + ), + 'mockApi': step( + 'apiMock', + 'core', + 'mockApi', + 'Register a mock HTTP API response by API name', + ), + 'mockApiError': step( + 'apiMock', + 'core', + 'mockApiError', + 'Mock an API to return an error status/body', + ), + 'mockApiFromFixture': step( + 'fixture', + 'core', + 'mockApiFromFixture', + 'Load mock response body from a JSON fixture asset', + ), + 'mockApiException': step( + 'apiMock', + 'core', + 'mockApiException', + 'Force an API call to throw an exception', + ), + 'mockTimeout': step( + 'network', + 'core', + 'mockTimeout', + 'Mock an API with a long delay (simulate timeout)', + ), + 'mockNetworkOffline': step( + 'network', + 'core', + 'empty', + 'Simulate offline network for API calls', + ), + 'mockNetworkOnline': step( + 'network', + 'core', + 'empty', + 'Restore online network for API calls', + ), + 'resetApiCalls': step( + 'apiMock', + 'core', + 'empty', + 'Clear recorded API call history', + ), + 'clearApiMocks': step( + 'apiMock', + 'core', + 'empty', + 'Remove all registered API mocks', + ), + 'expectApiCalled': step( + 'apiAssertion', + 'core', + 'apiName', + 'Assert an API was called an exact number of times', + ), + 'expectApiNotCalled': step( + 'apiAssertion', + 'core', + 'apiName', + 'Assert an API was never called', + ), + 'expectApiRequest': step( + 'apiAssertion', + 'core', + 'apiRequest', + 'Assert last API request body/query/headers match', + ), + 'expectApiRequestContains': step( + 'apiAssertion', + 'core', + 'apiRequest', + 'Assert API request contains partial body/query', + ), + 'expectApiHeader': step( + 'apiAssertion', + 'core', + 'expectApiHeader', + 'Assert an API request header equals expected', + ), + 'expectApiCallOrder': step( + 'apiAssertion', + 'core', + 'expectApiCallOrder', + 'Assert APIs were called in order', + ), + 'expectLastApiCall': step( + 'apiAssertion', + 'core', + 'apiName', + 'Assert the most recent API call name', + ), + 'setState': step( + 'state', + 'core', + 'setState', + 'Set app data-context state at path to value', + ), + 'expectState': step( + 'state', + 'core', + 'expectState', + 'Assert app state at path equals expected', + ), + 'expectStateContains': step( + 'state', + 'core', + 'expectState', + 'Assert app state at path contains subset', + ), + 'expectStateExists': step( + 'state', + 'core', + 'expectStatePath', + 'Assert state path resolves without error', + ), + 'expectStateNotExists': step( + 'state', + 'core', + 'expectStatePath', + 'Assert state path is null or absent', + ), + 'resetState': step( + 'state', + 'core', + 'resetStatePath', + 'Clear state at path (set to null)', + ), + 'setStorage': step( + 'storage', + 'core', + 'storageKey', + 'Write a value to public GetStorage by key', + ), + 'expectStorage': step( + 'storage', + 'core', + 'storageKey', + 'Assert public storage key equals expected', + ), + 'removeStorage': step( + 'storage', + 'core', + 'storageKey', + 'Remove a key from public storage', + ), + 'clearStorage': step( + 'storage', + 'core', + 'empty', + 'Clear all non-encrypted public storage keys', + ), + 'setEnv': step( + 'runtime', + 'core', + 'storageKey', + 'Override an environment variable for the test', + ), + 'setAuth': step( + 'runtime', + 'core', + 'setAuth', + 'Simulate a signed-in user', + ), + 'clearAuth': step( + 'runtime', + 'core', + 'empty', + 'Clear the signed-in user', + ), + 'setPermission': step( + 'runtime', + 'core', + 'setPermission', + 'Set a permission flag for the test runtime', + ), + 'setDevice': step( + 'runtime', + 'core', + 'setDevice', + 'Override viewport physical size (width/height)', + ), + 'setLocale': step( + 'runtime', + 'core', + 'setLocale', + 'Set APP_LOCALE environment override', + ), + 'setTheme': step( + 'runtime', + 'core', + 'setTheme', + 'Set APP_THEME / theme mode override', + ), + 'runScript': step( + 'script', + 'core', + 'runScript', + 'Evaluate a script expression in the data context', + ), + 'expectScriptResult': step( + 'script', + 'core', + 'runScript', + 'Evaluate script and assert result equals expected', + ), + 'expectConsoleLog': step( + 'script', + 'core', + 'expectConsoleLog', + 'Assert a console log line contains text', + ), + 'group': step( + 'control', + 'core', + 'group', + 'Run nested steps as a named group', + ), + 'repeat': step( + 'control', + 'core', + 'repeat', + 'Repeat nested steps N times', + ), + 'optional': step( + 'control', + 'core', + 'optional', + 'Run nested steps; swallow failures', + ), + 'ifVisible': step( + 'control', + 'core', + 'ifVisible', + 'Run nested steps only if testId is visible', + ), + 'logApiCalls': step( + 'debug', + 'core', + 'empty', + 'Log all recorded API calls to the test log', + ), + 'screenshot': step( + 'debug', + 'core', + 'screenshot', + 'Capture golden or dump widget tree for debugging', + ), + 'dumpTree': step( + 'debug', + 'core', + 'empty', + 'Print the widget tree to the debug console', + ), + 'logState': step( + 'debug', + 'core', + 'expectStatePath', + 'Log resolved state at path', + ), + 'logStorage': step( + 'debug', + 'core', + 'storageKey', + 'Log public storage value for key', + ), + 'expectNoConsoleErrors': step( + 'quality', + 'core', + 'empty', + 'Assert no console errors were recorded', + ), + 'expectNoRenderErrors': step( + 'quality', + 'core', + 'empty', + 'Assert no Flutter render errors were recorded', + ), + 'expectError': step( + 'quality', + 'core', + 'expectErrorContains', + 'Assert a Flutter error was recorded (optional filter)', + ), + 'expectNoErrors': step( + 'quality', + 'core', + 'empty', + 'Alias for expectNoRenderErrors', + ), + 'expectAccessible': step( + 'quality', + 'core', + 'idRequired', + 'Assert widget has accessibility label or value', + ), + 'expectSemanticsLabel': step( + 'quality', + 'core', + 'expectSemanticsLabel', + 'Assert semantics label equals expected', + ), + 'expectNoOverflow': step( + 'quality', + 'core', + 'idRequired', + 'Assert widget renders without overflow issues', + ), + 'loadFixture': step( + 'fixture', + 'core', + 'fixturePath', + 'Load a JSON fixture into the test fixture map', + ), + 'setStateFromFixture': step( + 'fixture', + 'core', + 'fixturePath', + 'Apply all keys from a JSON fixture to state', + ), + 'expectMatchesFixture': step( + 'fixture', + 'core', + 'fixturePath', + 'Assert state or path matches a JSON fixture', + ), + }; + + const executorAliases = { + 'waitForText': 'waitFor', + 'expectScreen': 'expectNavigateTo', + 'wait': 'pump', + 'launchApp': 'restartApp', + 'expectScript': 'expectScriptResult', + }; + + final buffer = StringBuffer(''' +// GENERATED by tool/generate_step_registry.dart — do not edit by hand. +// Re-run: dart run tool/generate_step_registry.dart + +import 'package:ensemble_test_runner/vocabulary/test_step_arg_kind.dart'; +import 'package:ensemble_test_runner/vocabulary/test_step_vocabulary.dart'; + +/// Single source of truth for declarative test steps (metadata + JSON Schema args). +class TestStepRegistryEntry { + const TestStepRegistryEntry({ + required this.category, + required this.tier, + required this.argKind, + required this.description, + required this.example, + this.executorCanonical, + }); + + final TestStepCategory category; + final TestStepTier tier; + final TestStepArgKind argKind; + final String description; + + /// Example YAML args object for this step (also used in JSON Schema). + final Map example; + + /// When set, [TestStepVocabulary.resolveStepType] maps this YAML key here + /// (e.g. `wait` → `pump`). Schema/definition use this entry's [argKind]. + final String? executorCanonical; +} + +abstract final class TestStepRegistry { + TestStepRegistry._(); + + static const Map entries = { +'''); + + for (final e in entries.entries) { + final name = e.key; + final v = e.value; + final cat = v['cat'] as String; + final tier = v['tier'] as String; + final arg = v['arg'] as String; + final desc = v['desc'] as String; + final example = v['example'] as Map; + final execAlias = executorAliases[name]; + buffer.writeln(" '$name': TestStepRegistryEntry("); + buffer.writeln(' category: TestStepCategory.$cat,'); + buffer.writeln(' tier: TestStepTier.$tier,'); + buffer.writeln(' argKind: TestStepArgKind.$arg,'); + buffer.writeln(" description: '${_escapeDartString(desc)}',"); + buffer.writeln(' example: ${_dartLiteral(example)},'); + if (execAlias != null && execAlias != name) { + buffer.writeln(" executorCanonical: '$execAlias',"); + } + buffer.writeln(' ),'); + } + + buffer.writeln(' };'); + buffer.writeln('}'); + buffer.writeln(); + + File('lib/vocabulary/test_step_registry.dart') + .writeAsStringSync(buffer.toString()); + print('Wrote lib/vocabulary/test_step_registry.dart'); +} + +String _escapeDartString(String s) => s.replaceAll(r'\', r'\\').replaceAll("'", r"\'"); + +/// Default argument map per [TestStepArgKind] name (override per step when needed). +Map defaultExampleForArg(String arg) { + switch (arg) { + case 'empty': + return {}; + case 'openScreen': + return {'screen': 'Home'}; + case 'trigger': + return {'action': 'onTap', 'id': 'submit_button'}; + case 'idRequired': + return {'id': 'my_widget'}; + case 'idOptional': + return {'id': 'scroll_view'}; + case 'enterText': + return {'id': 'email_field', 'value': 'user@test.com'}; + case 'select': + return {'id': 'country_dropdown', 'value': 'USA'}; + case 'selectIndex': + return {'id': 'country_dropdown', 'index': 0}; + case 'setSlider': + return {'id': 'volume_slider', 'value': 0.5}; + case 'chooseValue': + return {'id': 'birth_date', 'value': '2024-01-15'}; + case 'scroll': + return {'delta': 300}; + case 'swipe': + return {'direction': 'left', 'id': 'carousel'}; + case 'drag': + return {'id': 'handle', 'dx': 50, 'dy': 0}; + case 'pump': + return {'durationMs': 100}; + case 'timeoutOptional': + return {'timeoutMs': 5000}; + case 'waitFor': + return {'id': 'loading_spinner', 'timeoutMs': 5000}; + case 'waitForGone': + return {'id': 'loading_spinner', 'timeoutMs': 5000}; + case 'waitForNavigation': + return {'screen': 'Home', 'timeoutMs': 5000}; + case 'waitUntil': + return {'path': 'user.name', 'equals': 'Jane'}; + case 'textRequired': + return {'text': 'Welcome'}; + case 'expectEquals': + return {'id': 'email_field', 'equals': 'user@test.com'}; + case 'expectChecked': + return {'id': 'terms_checkbox', 'equals': true}; + case 'expectProperty': + return {'id': 'title', 'property': 'label', 'equals': 'Hello'}; + case 'expectCount': + return {'id': 'badge', 'equals': 2}; + case 'expectListCount': + return {'id': 'items_list', 'equals': 3}; + case 'screenRequired': + return {'screen': 'Home'}; + case 'expectVisited': + return {'screen': 'Login'}; + case 'mockApi': + return { + 'name': 'login', + 'response': { + 'statusCode': 200, + 'body': {'token': 'test-token'}, + }, + }; + case 'mockApiError': + return {'name': 'login', 'statusCode': 401, 'body': {'error': 'Unauthorized'}}; + case 'mockApiFromFixture': + return {'name': 'users', 'fixture': 'fixtures/users.json'}; + case 'mockApiException': + return {'name': 'login', 'message': 'Network error'}; + case 'mockTimeout': + return {'name': 'slow_api', 'delayMs': 60000}; + case 'apiName': + return {'name': 'login', 'times': 1}; + case 'apiRequest': + return { + 'name': 'login', + 'body': {'email': 'user@test.com', 'password': 'secret'}, + }; + case 'expectApiHeader': + return { + 'name': 'login', + 'header': 'Authorization', + 'equals': 'Bearer test-token', + }; + case 'setState': + return {'path': 'user.name', 'value': 'Jane'}; + case 'expectState': + return {'path': 'user.name', 'equals': 'Jane'}; + case 'storageKey': + return {'key': 'onboarding_done', 'value': true}; + case 'group': + return { + 'name': 'login_flow', + 'steps': [ + {'tap': {'id': 'login_button'}}, + ], + }; + case 'repeat': + return { + 'times': 3, + 'steps': [ + {'tap': {'id': 'next_button'}}, + ], + }; + case 'optional': + return { + 'steps': [ + {'tap': {'id': 'dismiss_banner'}}, + ], + }; + case 'ifVisible': + return { + 'id': 'promo_banner', + 'steps': [ + {'tap': {'id': 'close_banner'}}, + ], + }; + case 'screenshot': + return {'name': 'home_screen'}; + case 'expectStatePath': + return {'path': 'user.id'}; + case 'resetStatePath': + return {'path': 'cart'}; + case 'setAuth': + return { + 'user': {'id': '1', 'email': 'user@test.com'}, + }; + case 'setPermission': + return {'name': 'camera', 'value': 'granted'}; + case 'setDevice': + return {'width': 390, 'height': 844}; + case 'setLocale': + return {'locale': 'en_US'}; + case 'setTheme': + return {'mode': 'dark'}; + case 'runScript': + return {'script': '1 + 1', 'equals': 2}; + case 'expectConsoleLog': + return {'contains': 'Screen loaded'}; + case 'expectErrorContains': + return {'contains': 'overflow'}; + case 'fixturePath': + return {'fixture': 'fixtures/user.json'}; + case 'expectApiCallOrder': + return {'names': ['auth', 'profile']}; + case 'expectListContains': + return {'id': 'items_list', 'text': 'Item 1'}; + case 'expectListItem': + return {'itemId': 'row_0'}; + case 'expectBackStack': + return {'screens': ['Home', 'Details']}; + case 'expectCanGoBack': + return {'equals': true}; + case 'expectSemanticsLabel': + return {'id': 'submit_button', 'label': 'Submit'}; + default: + throw ArgumentError('No default example for arg kind: $arg'); + } +} + +String _dartLiteral(dynamic value) { + if (value is Map) { + if (value.isEmpty) return 'const {}'; + final parts = value.entries + .map((e) => "'${e.key}': ${_dartLiteral(e.value)}") + .join(', '); + return 'const {$parts}'; + } + if (value is List) { + if (value.isEmpty) return 'const []'; + return 'const [${value.map(_dartLiteral).join(', ')}]'; + } + if (value is String) return "'${_escapeDartString(value)}'"; + if (value is bool || value is int || value is double) return value.toString(); + throw ArgumentError('Unsupported literal type: ${value.runtimeType}'); +} diff --git a/starter/ensemble/apps/helloApp/screens/Hello Home.yaml b/starter/ensemble/apps/helloApp/screens/Hello Home.yaml index 78e1c6798..9fee5801f 100644 --- a/starter/ensemble/apps/helloApp/screens/Hello Home.yaml +++ b/starter/ensemble/apps/helloApp/screens/Hello Home.yaml @@ -56,6 +56,7 @@ View: message: Hello ${ensemble.storage.helloApp.name.first} - Button: + testId: call_api_button label: Call API (page reusable action) onTap: executeAction: diff --git a/starter/ensemble/tests/hello_goodbye_back.test.yaml b/starter/ensemble/tests/hello_goodbye_back.test.yaml new file mode 100644 index 000000000..7f03624b2 --- /dev/null +++ b/starter/ensemble/tests/hello_goodbye_back.test.yaml @@ -0,0 +1,11 @@ +# yaml-language-server: $schema=../../../packages/ensemble_test_runner/assets/schema/ensemble_test.schema.json +id: hello_goodbye_back +prerequisite: hello_goodbye_message +steps: + - trigger: + action: onTap + id: back_button + - expectVisible: + id: greeting_text + - expectVisible: + id: navigate_button diff --git a/starter/ensemble/tests/hello_goodbye_continuation.test.yaml b/starter/ensemble/tests/hello_goodbye_continuation.test.yaml new file mode 100644 index 000000000..6639cef99 --- /dev/null +++ b/starter/ensemble/tests/hello_goodbye_continuation.test.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=../../../packages/ensemble_test_runner/assets/schema/ensemble_test.schema.json +id: hello_goodbye_continuation +prerequisite: hello_home_renders +steps: + - expectVisible: + id: goodbye_title diff --git a/starter/ensemble/tests/hello_goodbye_message.test.yaml b/starter/ensemble/tests/hello_goodbye_message.test.yaml new file mode 100644 index 000000000..d5c5a8a97 --- /dev/null +++ b/starter/ensemble/tests/hello_goodbye_message.test.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=../../../packages/ensemble_test_runner/assets/schema/ensemble_test.schema.json +id: hello_goodbye_message +prerequisite: hello_goodbye_continuation +steps: + - expectVisible: + id: goodbye_message diff --git a/starter/ensemble/tests/hello_home.test.yaml b/starter/ensemble/tests/hello_home.test.yaml new file mode 100644 index 000000000..f90e27627 --- /dev/null +++ b/starter/ensemble/tests/hello_home.test.yaml @@ -0,0 +1,19 @@ +# yaml-language-server: $schema=../../../packages/ensemble_test_runner/assets/schema/ensemble_test.schema.json +id: hello_home_renders +startScreen: Hello Home +steps: + - expectVisible: + id: greeting_text + + - expectVisible: + id: description_text + + - expectVisible: + id: navigate_button + - trigger: + action: onTap + id: call_api_button + - tap: + id: navigate_button + - expectVisible: + id: goodbye_title diff --git a/starter/ensemble/tests/hello_navigation_flow.test.yaml b/starter/ensemble/tests/hello_navigation_flow.test.yaml new file mode 100644 index 000000000..7ba66a956 --- /dev/null +++ b/starter/ensemble/tests/hello_navigation_flow.test.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=../../../packages/ensemble_test_runner/assets/schema/ensemble_test.schema.json +id: hello_navigation_flow +prerequisite: hello_goodbye_back +steps: + - expectVisited: + screen: Hello Home + - expectVisited: + screen: Goodbye diff --git a/starter/macos/Flutter/ephemeral/Flutter-Generated.xcconfig b/starter/macos/Flutter/ephemeral/Flutter-Generated.xcconfig index f0890f40b..5d11cc95e 100644 --- a/starter/macos/Flutter/ephemeral/Flutter-Generated.xcconfig +++ b/starter/macos/Flutter/ephemeral/Flutter-Generated.xcconfig @@ -1,6 +1,6 @@ // This is a generated file; do not edit or check into version control. -FLUTTER_ROOT=/Users/hemishpancholi/flutter -FLUTTER_APPLICATION_PATH=/Users/hemishpancholi/Documents/Flutter Projects/ensemble/ensemble/examples/starter +FLUTTER_ROOT=/Users/sharjeelyunus/fvm/versions/3.35.4 +FLUTTER_APPLICATION_PATH=/Users/sharjeelyunus/Desktop/Ensemble/ensemble/starter COCOAPODS_PARALLEL_CODE_SIGN=true FLUTTER_BUILD_DIR=build FLUTTER_BUILD_NAME=1.0.0 diff --git a/starter/macos/Flutter/ephemeral/flutter_export_environment.sh b/starter/macos/Flutter/ephemeral/flutter_export_environment.sh index a6b99d2bb..b1431b028 100755 --- a/starter/macos/Flutter/ephemeral/flutter_export_environment.sh +++ b/starter/macos/Flutter/ephemeral/flutter_export_environment.sh @@ -1,7 +1,7 @@ #!/bin/sh # This is a generated file; do not edit or check into version control. -export "FLUTTER_ROOT=/Users/hemishpancholi/flutter" -export "FLUTTER_APPLICATION_PATH=/Users/hemishpancholi/Documents/Flutter Projects/ensemble/ensemble/examples/starter" +export "FLUTTER_ROOT=/Users/sharjeelyunus/fvm/versions/3.35.4" +export "FLUTTER_APPLICATION_PATH=/Users/sharjeelyunus/Desktop/Ensemble/ensemble/starter" export "COCOAPODS_PARALLEL_CODE_SIGN=true" export "FLUTTER_BUILD_DIR=build" export "FLUTTER_BUILD_NAME=1.0.0"