From b441c4d4888e52effb6355ec3e42d42f5b7569c8 Mon Sep 17 00:00:00 2001 From: Daniel Jette Date: Wed, 9 Sep 2026 18:34:37 -0400 Subject: [PATCH 1/3] 245: Add support for com.android.test modules --- .gitignore | 1 + CHANGELOG.md | 7 ++ .../kotlin/dev/testify/TestifyExtension.kt | 8 +- .../main/kotlin/dev/testify/TestifyPlugin.kt | 11 +- .../internal/GradleProjectExtensions.kt | 5 + Samples/Flix/FlixTest/.gitignore | 1 + Samples/Flix/FlixTest/build.gradle | 94 ++++++++++++++++++ .../FlixTest/src/main/AndroidManifest.xml | 3 + ...trumentedTestModuleExampleTest_default.png | Bin 0 -> 2897 bytes .../test/InstrumentedTestModuleExampleTest.kt | 49 +++++++++ build.gradle.kts | 1 + gradle/libs.versions.toml | 1 + settings.gradle | 2 + 13 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 Samples/Flix/FlixTest/.gitignore create mode 100644 Samples/Flix/FlixTest/build.gradle create mode 100644 Samples/Flix/FlixTest/src/main/AndroidManifest.xml create mode 100644 Samples/Flix/FlixTest/src/main/assets/screenshots/37-1080x2220@440dp-en_US/InstrumentedTestModuleExampleTest_default.png create mode 100644 Samples/Flix/FlixTest/src/main/java/dev/testify/samples/flix/test/InstrumentedTestModuleExampleTest.kt diff --git a/.gitignore b/.gitignore index 4dcc83c84..67c843906 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,6 @@ Plugins/Gradle/gradlew Plugins/Gradle/gradlew.bat .intellijPlatform/ .antigravitycli/ +Plugins/IntelliJ/.kotlin/ Samples/Paparazzi/.idea/ .kotlin/ diff --git a/CHANGELOG.md b/CHANGELOG.md index ce5cb028b..279cf2321 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,13 @@ - Fix `RootViewWithoutFocusException` being intermittently thrown by `Espresso.closeSoftKeyboard()` - **Breaking:** `dev.testify.internal.helpers.closeSoftKeyboard()` now requires an `Activity` parameter - Fix `ClassCastException` when passing `-PshardCount` and `-PshardIndex` to `screenshotTest` or `screenshotRecord` +- Add support for test-only modules. The plugin can now be applied to a `com.android.test` module; + previously it failed with `Gradle project must contain an 'android' closure` + - `baselineSourceDir` defaults to `src/main/assets` for a test module, since a `com.android.test` + module has no `androidTest` source set — its tests are its `main` sources + - The Testify library is added as an `implementation` dependency rather than + `androidTestImplementation` for the same reason + - See the new `Samples/Flix/FlixTest` module for a worked example ## 6.0.0 diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyExtension.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyExtension.kt index 2d15863bf..3696f98dc 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyExtension.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyExtension.kt @@ -26,6 +26,7 @@ package dev.testify import dev.testify.internal.android +import dev.testify.internal.isTestModule import dev.testify.internal.applicationTargetPackageId import dev.testify.internal.inferredAndroidTestInstallTask import dev.testify.internal.inferredDefaultTestVariantId @@ -113,9 +114,12 @@ internal data class TestifySettings( val android = project.android val extension = project.getTestifyExtension() + // A `com.android.test` module has no `androidTest` source set; its tests are its + // `main` sources, so that is where its baselines belong. + val testSourceSet = if (project.isTestModule) "main" else "androidTest" val baselineSourceDir = extension.baselineSourceDir - ?: project.android.sourceSets.findByName("androidTest")?.assets?.directories?.firstOrNull() - ?: "src/androidTest/assets" + ?: project.android.sourceSets.findByName(testSourceSet)?.assets?.directories?.firstOrNull() + ?: "src/$testSourceSet/assets" val testRunner = extension.testRunner ?: android.defaultConfig.testInstrumentationRunner ?: "unknown" val pullWaitTime = extension.pullWaitTime ?: 0L val testPackageId = extension.testPackageId ?: project.inferredDefaultTestVariantId diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyPlugin.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyPlugin.kt index 67ac95b13..dcd7b971f 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyPlugin.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyPlugin.kt @@ -28,6 +28,7 @@ package dev.testify import dev.testify.TestifyPlugin.Companion.EVALUATED_SETTINGS import dev.testify.internal.Style.Description import dev.testify.internal.android +import dev.testify.internal.isTestModule import dev.testify.internal.isVerbose import dev.testify.internal.println import dev.testify.tasks.internal.TaskNameProvider @@ -78,8 +79,14 @@ class TestifyPlugin : Plugin { if (settings.autoImplementLibrary) { val version = javaClass.getPackage().implementationVersion.orEmpty() val dependency = "dev.testify:testify:$version" - if (project.isVerbose) println(Description, "Adding androidTestImplementation($dependency)") - project.dependencies.add("androidTestImplementation", dependency) + + if (project.isTestModule) { + if (project.isVerbose) println(Description, "Adding implementation($dependency)") + project.dependencies.add("implementation", dependency) + } else { + if (project.isVerbose) println(Description, "Adding androidTestImplementation($dependency)") + project.dependencies.add("androidTestImplementation", dependency) + } } } diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt index fe43c5d89..66d5ae205 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt @@ -28,14 +28,19 @@ package dev.testify.internal import com.android.build.api.dsl.ApplicationExtension import com.android.build.api.dsl.CommonExtension import com.android.build.api.dsl.LibraryExtension +import com.android.build.api.dsl.TestExtension import org.gradle.api.GradleException import org.gradle.api.Project val Project.android: CommonExtension<*, *, *, *, *, *> get() = this.extensions.findByType(ApplicationExtension::class.java) ?: this.extensions.findByType(LibraryExtension::class.java) + ?: this.extensions.findByType(TestExtension::class.java) ?: throw GradleException("Gradle project must contain an `android` closure") +val Project.isTestModule: Boolean + get() = this.extensions.findByType(TestExtension::class.java) != null + val Project.isVerbose: Boolean get() = (this.properties["verbose"] as? String)?.toBoolean() ?: false diff --git a/Samples/Flix/FlixTest/.gitignore b/Samples/Flix/FlixTest/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/Samples/Flix/FlixTest/.gitignore @@ -0,0 +1 @@ +/build diff --git a/Samples/Flix/FlixTest/build.gradle b/Samples/Flix/FlixTest/build.gradle new file mode 100644 index 000000000..e1ad6f68e --- /dev/null +++ b/Samples/Flix/FlixTest/build.gradle @@ -0,0 +1,94 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +buildscript { + ext { + versions = [ + 'compose': [ + 'compilerExt': '1.5.14', + 'core' : '1.5.14', + 'material' : '1.6.7', + 'ui' : '1.6.7', + ] + ] + } +} + +plugins { + id 'com.android.test' + id 'dev.testify' + alias(libs.plugins.compose.compiler) +} + +android { + namespace 'dev.testify.samples.flix.test' + compileSdk 36 + + // Target the Flix application module + targetProjectPath ':FlixSample' + + defaultConfig { + minSdkVersion 26 + targetSdk 36 + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + + compileOptions { + sourceCompatibility JavaVersion.VERSION_25 + targetCompatibility JavaVersion.VERSION_25 + } + kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_25) + } + } + buildFeatures { + compose true + } + composeOptions { + kotlinCompilerExtensionVersion "${versions.compose.compilerExt}" + } +} + +dependencies { + + implementation project(":Library") + implementation project(":ComposeExtensions") + + // AndroidX Test + implementation 'androidx.test.ext:junit:1.3.0' + implementation 'androidx.test.espresso:espresso-core:3.7.0' + implementation "androidx.test:rules:1.7.0" + implementation "androidx.test:runner:1.7.0" + + // Align lifecycle version with the target app to prevent classpath conflicts + // between the test APK and app APK. + implementation "androidx.lifecycle:lifecycle-common:2.9.3" + implementation "androidx.lifecycle:lifecycle-runtime:2.9.3" + + + def composeBom = platform('androidx.compose:compose-bom:2025.08.01') + implementation composeBom + //noinspection UseTomlInstead + implementation "androidx.appcompat:appcompat:1.7.1" + implementation "androidx.hilt:hilt-navigation-compose:1.2.0" + implementation "androidx.lifecycle:lifecycle-runtime-compose:2.9.3" + implementation "androidx.test.espresso.idling:idling-concurrent:3.7.0" + implementation "com.google.dagger:hilt-android:2.60.1" + implementation 'androidx.compose.material3:material3:1.3.2' + implementation 'androidx.core:core-ktx:1.17.0' + implementation 'io.ktor:ktor-client-android:3.2.3' + implementation 'org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0' + implementation("io.coil-kt:coil-compose:2.7.0") +} + +testify { + // The app under test, named by `targetProjectPath` above. Screenshots are written into this + // application's data directory, so `screenshotPull` looks for them here. + applicationPackageId "dev.testify.samples.flix" + + // This module's own APK, which carries the instrumentation. + testPackageId "dev.testify.samples.flix.test" + + autoImplementLibrary false +} diff --git a/Samples/Flix/FlixTest/src/main/AndroidManifest.xml b/Samples/Flix/FlixTest/src/main/AndroidManifest.xml new file mode 100644 index 000000000..9a40236b9 --- /dev/null +++ b/Samples/Flix/FlixTest/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + + diff --git a/Samples/Flix/FlixTest/src/main/assets/screenshots/37-1080x2220@440dp-en_US/InstrumentedTestModuleExampleTest_default.png b/Samples/Flix/FlixTest/src/main/assets/screenshots/37-1080x2220@440dp-en_US/InstrumentedTestModuleExampleTest_default.png new file mode 100644 index 0000000000000000000000000000000000000000..fa9446aacd1bdbee1802e4042f9acd86b1570af5 GIT binary patch literal 2897 zcmY*bXH-*57fn!Uf)q8<5*{5vAc9H@K{|>eA_9*lf=CHH5NV-G2O&V{)kq1%P)sPH zh*EsC2r5C5F1?FL2fypj@2V|s`Byi~Ii%Ih4ie~kC*wB_FG2#TXY!i7MfhIcNuA*X$4=T+_>`PcJK%9~aW~!EWL8&Kz32P+U-ve|bav{t1pZ+GAIXYDB4@KCFR)u*JTB4rOU-g3k=T1}(3sg0ii?b7dfmqXLs+8z z^!MKpQpa!{AMP52gtRagzCh$Hzw>LCaR2B|Mx~?AFf)t7;Wc;ZXFvt&&m@=1-u3#Q zQsH3tmy^9j5@L>a2$!j^3kUn3f*}IJWi7~4>*LJ<#|PJgwim$>H^Y04CGGiO6_^Vcg=-c2nwB^o{so?4zNUF^lMp337CzIt+P~wHonW@>Gb<(x#e8=W2W@1tn&RcZW6Rc<%GqV+=~#1A_A5& ztMI7JW=(Z>?YqHngY!q2MA3B=BQ-=*Q&Z+%^>}!e!lU=&<4`H%{Nfj9Pc!+gjpEfZ z)6yc_+ixBn?Wglb+y9!SW`#k4IS|-SCXuT6A}{MS3i22adg9Lf`e$rUhH#%*_ddBD=S-U zJs5Ewwyx>!>cS;oU@#bsK0lZ<<*Z{ST7yryTv(o(3g6!L>+I}I^|fyB)C&w$MbR7_ z94c+0&HZP(Y=2M)Mk z2)gYBl|zpLl6zhH-j3xl?6^FbOQ}?9l8LlEk^dX5~=Nmvci<58P(kVoTpEF$ohs#@P#Ex&@_(gGX?kfN| zpgnmiKBAFPQPw2F!$c;4PIO{8hOE4I*T;wF!;TN|btRb)~3>g!=&{i^P(Gt zwBp$gfZ(#V&%;uQTsqq`3O`tn=D!m2^{eIb*E*-Zsl~;L`+-r9JUQ9ehG+{pe5b(t94<2LQ5Z#ye8X35D1}5@-`-R8(xtPs7k)Za2O37f)oZ=DoOyH7yR57%8;Mn`Rj_NRs3otARHsv@)ax;v zf@NrIj4>9gp{=d`pJ`QM;@PycwEwihAYFa^v@CEQ!J^uZoAop>7C)DB;mflE&AUFF zytZ!e-k8ux?j8Q}WeNzSd(ZoPRkWd+rlv5!)N+}`ii&oem_ZzroBO7Qsv7&@?#73K zJmvUS)QEg`PL7C*3ZIz4i_@D8>cuLYM9v^Kb75iOWt<={Z)8FOJJrbzg__>j_*-;! zBHAPU`Ksr2V$Vrno;-Oqiay@|#R8NYkdoQ#%2GyUrA$U<=1U4CWAx8AJL$`pOP+#W zzI-_~JA1m=$J_~kytsn8x;0@vTxOe>q!GNMd;2zfZ4n|c^4l9{LQRcHY1sYyJO~6L zH9b8lUFx1F0+D~wc7A^TOJOu1d3$?%6*iQ;oi#yi9UY&&&Bwl1kVv~Z$)AA4sHmv4 zzk3J%H9z2}*W6Z7U2Wj)4G%m5xZdpt52&%Z`N1DFHxCcE){<1$R#&I^{P}bHrB9`e z*uxw6@&;lR&;|eoa0lvFZ zX=zNzqUza8!on#=S@K$cl$2EO!rJ!+@q5)??0(B%*oKCN0(MsUff6T%l0@lrx=QGg z3fZoeJ>uq|l~f?J8a;a=NePZaql zS^}e%m)!vEJ#TJqUPGliVIRbm(ZqS#fZO&$kgIEJe~^V%?IR;uh*5ymE!Vm!EHW}O zCcrDf!NGOYPDSTGG|1w~@geC?)4##dhm)Yp9y(nzVtHofXtsQn26|%9Ej?phvbKHX{{R`fiY5R6 literal 0 HcmV?d00001 diff --git a/Samples/Flix/FlixTest/src/main/java/dev/testify/samples/flix/test/InstrumentedTestModuleExampleTest.kt b/Samples/Flix/FlixTest/src/main/java/dev/testify/samples/flix/test/InstrumentedTestModuleExampleTest.kt new file mode 100644 index 000000000..d0c56c0c5 --- /dev/null +++ b/Samples/Flix/FlixTest/src/main/java/dev/testify/samples/flix/test/InstrumentedTestModuleExampleTest.kt @@ -0,0 +1,49 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026 ndtp + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package dev.testify.samples.flix.test + +import androidx.compose.material3.Text +import dev.testify.ComposableScreenshotRule +import dev.testify.annotation.ScreenshotInstrumentation +import org.junit.Rule +import org.junit.Test + +/** + * Demonstrates how to use the testify-compose extension library to test + * Jetpack Compose @Composable functions. + */ +class InstrumentedTestModuleExampleTest { + @get:Rule + val rule = ComposableScreenshotRule() + + @ScreenshotInstrumentation + @Test + fun default() { + rule + .setCompose { + Text(text = "Hello, Flix!") + } + .assertSame() + } +} diff --git a/build.gradle.kts b/build.gradle.kts index 52e554fde..236537a8e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,7 @@ plugins { alias(libs.plugins.android.application) apply false alias(libs.plugins.android.library) apply false + alias(libs.plugins.android.test) apply false alias(libs.plugins.google.android.libraries.mapsplatform.secrets) apply false alias(libs.plugins.google.dagger.hilt.android) apply false alias(libs.plugins.google.devtools.ksp) apply false diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index faec06420..d44417a19 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -73,6 +73,7 @@ truth = { module = "com.google.truth:truth", version.ref = "truth" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } android-library = { id = "com.android.library", version.ref = "agp" } +android-test = { id = "com.android.test", version.ref = "agp" } compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } google-android-libraries-mapsplatform-secrets = { id = "com.google.android.libraries.mapsplatform.secrets-gradle-plugin", version.ref = "mapsplatformSecrets" } google-dagger-hilt-android = { id = "com.google.dagger.hilt.android", version.ref = "hilt" } diff --git a/settings.gradle b/settings.gradle index 4e3ba1aea..b3d089078 100644 --- a/settings.gradle +++ b/settings.gradle @@ -21,6 +21,7 @@ include ':Accessibility' include ':ComposeExtensions' include ':FlixLibrary' include ':FlixSample' +include ':FlixTest' include ':FullscreenCaptureMethod' include ':GmdSample' include ':Ktx' @@ -34,6 +35,7 @@ project(':Accessibility').projectDir = new File("./Ext/Accessibility") project(':ComposeExtensions').projectDir = new File("./Ext/Compose") project(':FlixLibrary').projectDir = new File("./Samples/Flix/FlixLibrary") project(':FlixSample').projectDir = new File("./Samples/Flix") +project(':FlixTest').projectDir = new File("./Samples/Flix/FlixTest") project(':FullscreenCaptureMethod').projectDir = new File("./Ext/Fullscreen") project(':GmdSample').projectDir = new File("./Samples/Gmd") project(':Ktx').projectDir = new File("./Ext/Ktx") From 169d96d2944ee156bd44121390bf9d54a81b0cb0 Mon Sep 17 00:00:00 2001 From: Daniel Jette Date: Fri, 2 Oct 2026 17:22:20 -0400 Subject: [PATCH 2/3] 245: Install the application under test and infer its package id --- .gitignore | 1 - CHANGELOG.md | 8 +++ .../kotlin/dev/testify/TestifyExtension.kt | 24 ++++++++- .../main/kotlin/dev/testify/internal/Adb.kt | 4 +- .../dev/testify/internal/ClientUtilities.kt | 19 ++++++- .../internal/GradleProjectExtensions.kt | 23 +++++++++ .../testify/tasks/main/ScreenshotTestTask.kt | 38 +++++++++++++- Samples/Flix/FlixTest/build.gradle | 50 ++++--------------- bitrise.yml | 16 ++++++ docs/docs/recipes/21-library-projects.md | 25 ++++++++++ 10 files changed, 159 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index 67c843906..4dcc83c84 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,5 @@ Plugins/Gradle/gradlew Plugins/Gradle/gradlew.bat .intellijPlatform/ .antigravitycli/ -Plugins/IntelliJ/.kotlin/ Samples/Paparazzi/.idea/ .kotlin/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 279cf2321..ea8ff3907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,11 +22,19 @@ - Fix `ClassCastException` when passing `-PshardCount` and `-PshardIndex` to `screenshotTest` or `screenshotRecord` - Add support for test-only modules. The plugin can now be applied to a `com.android.test` module; previously it failed with `Gradle project must contain an 'android' closure` + - `applicationPackageId` and `testPackageId` are both inferred: the application under test comes + from `targetProjectPath`, and the test package is the module's own namespace + - `screenshotTest` and `screenshotRecord` also depend on the target project's install task, so + the application under test is installed and not merely assumed to be present - `baselineSourceDir` defaults to `src/main/assets` for a test module, since a `com.android.test` module has no `androidTest` source set — its tests are its `main` sources - The Testify library is added as an `implementation` dependency rather than `androidTestImplementation` for the same reason - See the new `Samples/Flix/FlixTest` module for a worked example +- `screenshotTest` and `screenshotRecord` now fail when `am instrument` could not run the tests at + all — an uninstalled application under test, for example. Previously the task reported success + having run nothing, because the error is reported on standard error and the plugin only read + standard output ## 6.0.0 diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyExtension.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyExtension.kt index 3696f98dc..841f20bb1 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyExtension.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/TestifyExtension.kt @@ -27,6 +27,8 @@ package dev.testify import dev.testify.internal.android import dev.testify.internal.isTestModule +import dev.testify.internal.targetProjectPath +import dev.testify.internal.testModulePackageId import dev.testify.internal.applicationTargetPackageId import dev.testify.internal.inferredAndroidTestInstallTask import dev.testify.internal.inferredDefaultTestVariantId @@ -122,7 +124,9 @@ internal data class TestifySettings( ?: "src/$testSourceSet/assets" val testRunner = extension.testRunner ?: android.defaultConfig.testInstrumentationRunner ?: "unknown" val pullWaitTime = extension.pullWaitTime ?: 0L - val testPackageId = extension.testPackageId ?: project.inferredDefaultTestVariantId + val testPackageId = extension.testPackageId + ?: project.testModulePackageId + ?: project.inferredDefaultTestVariantId val targetPackageId = extension.applicationPackageId ?: project.inferredTargetPackageId val version = TestifySettings::class.java.getPackage().implementationVersion val isSnapshot = version?.contains("SNAPSHOT", ignoreCase = true) ?: false @@ -192,6 +196,13 @@ private val Project.inferredTargetPackageId: String get() { var targetPackageId: String? = this.applicationTargetPackageId + // A `com.android.test` module has no applicationId of its own: the application under test is + // the project named by `targetProjectPath`. Screenshots are written into that application's + // data directory, so this has to be its id or `screenshotPull` looks in the wrong package. + if (targetPackageId.isNullOrEmpty()) { + targetPackageId = this.targetProjectApplicationId + } + // If we still do not have a targetPackageId, it is likely a library project // Infer the package from the test configuration if (targetPackageId.isNullOrEmpty()) { @@ -201,6 +212,17 @@ private val Project.inferredTargetPackageId: String return targetPackageId } +/** + * The applicationId of the project a `com.android.test` module targets, or `null` for any other + * module type or when the target cannot be resolved. + */ +private val Project.targetProjectApplicationId: String? + get() { + val targetPath = this.targetProjectPath ?: return null + val targetProject = this.rootProject.findProject(targetPath) ?: return null + return targetProject.applicationTargetPackageId + } + open class TestifyExtension { var baselineSourceDir: String? = null diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/Adb.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/Adb.kt index 6d1f1776e..8fd560030 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/Adb.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/Adb.kt @@ -106,7 +106,7 @@ class Adb( return this } - fun execute(targetsDevice: Boolean = true): String { + fun execute(targetsDevice: Boolean = true, redirectErrorStream: Boolean = false): String { if (targetsDevice) { val deviceTarget = Device.targets(adbService)[deviceTargetIndex] if (deviceTarget != null) { @@ -121,7 +121,7 @@ class Adb( println(Description, command) } - return runProcess(command, streamData ?: BufferedStream()) + return runProcess(command, streamData ?: BufferedStream(), redirectErrorStream) } fun testOptions(testOptionsBuilder: TestOptionsBuilder): Adb { diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/ClientUtilities.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/ClientUtilities.kt index 27f5e7234..a5d8bda63 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/ClientUtilities.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/ClientUtilities.kt @@ -88,8 +88,23 @@ sealed class StreamData { internal val Project.destinationImageDirectory: String get() = "${project.testifySettings.baselineSourceDir}${File.separatorChar}" -fun runProcess(command: String, streamData: StreamData = BufferedStream()): String { - val process = Runtime.getRuntime().exec(command) +/** + * Run [command] and return what it wrote. + * + * @param redirectErrorStream merge the process's standard error into the returned output. `adb` + * writes most of what it has to say to standard output, but `am instrument` reports a failure to + * start the tests at all — an uninstalled target package, for instance — on standard error. A + * caller that inspects the output for failures has to see it, or it reports success having run + * nothing. + */ +fun runProcess( + command: String, + streamData: StreamData = BufferedStream(), + redirectErrorStream: Boolean = false +): String { + val process = ProcessBuilder(*command.split(" ").toTypedArray()) + .redirectErrorStream(redirectErrorStream) + .start() val result = streamData.handleInputStream(process.inputStream) process.waitFor() return result diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt index 66d5ae205..0ef01528f 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt @@ -41,6 +41,29 @@ val Project.android: CommonExtension<*, *, *, *, *, *> val Project.isTestModule: Boolean get() = this.extensions.findByType(TestExtension::class.java) != null +/** + * The Gradle path of the application under test, for a `com.android.test` module. + * + * A test-only module declares its target with `targetProjectPath`. The instrumentation runs against + * that application, and its screenshots are written into that application's data directory, so the + * target's APK has to be installed for the tests to run at all. + * + * `null` for any other module type. + */ +val Project.targetProjectPath: String? + get() = this.extensions.findByType(TestExtension::class.java)?.targetProjectPath + +/** + * The applicationId of a `com.android.test` module's own APK. + * + * A test module's APK carries the instrumentation, so this is the package `am instrument` is invoked + * against. It has no `applicationId` in `defaultConfig`, so AGP uses its namespace. + * + * `null` for any other module type. + */ +val Project.testModulePackageId: String? + get() = this.extensions.findByType(TestExtension::class.java)?.namespace + val Project.isVerbose: Boolean get() = (this.properties["verbose"] as? String)?.toBoolean() ?: false diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/ScreenshotTestTask.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/ScreenshotTestTask.kt index 1a5bda8d6..a82c0ddfe 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/ScreenshotTestTask.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/ScreenshotTestTask.kt @@ -30,7 +30,9 @@ import dev.testify.internal.StreamData.ConsoleStream import dev.testify.internal.Style.Failure import dev.testify.internal.TestOptionsBuilder import dev.testify.internal.fromEnv +import dev.testify.internal.inferredInstallTask import dev.testify.internal.println +import dev.testify.internal.targetProjectPath import dev.testify.tasks.internal.TaskDependencyProvider import dev.testify.tasks.internal.TaskNameProvider import dev.testify.tasks.internal.TestifyDefaultTask @@ -168,14 +170,19 @@ open class ScreenshotTestTask : TestifyDefaultTask() { .argument("-w") .argument("$testPackageId/$testRunner") .stream(ConsoleStream) - .execute() + // `am instrument` reports a failure to start the tests on standard error, and + // finalizeTaskAction has to see it. + .execute(redirectErrorStream = true) finalizeTaskAction(log) } protected open fun finalizeTaskAction(log: String) { if (log.contains("FAILURES!!!") || log.contains("INSTRUMENTATION_CODE: 0") || - log.contains("Process crashed while executing") + log.contains("Process crashed while executing") || + // `am instrument` could not run the tests at all, for example because the application + // under test is not installed. Without this the task reports success having run nothing. + log.contains("INSTRUMENTATION_STATUS: Error=") ) { println(Failure, "SCREENSHOT TESTS HAVE FAILED!!!") throw RuntimeException("Screenshot tests have failed") @@ -200,6 +207,33 @@ open class ScreenshotTestTask : TestifyDefaultTask() { getInstallDebugTask(project)?.let { installDebugTask -> task.dependsOn(installDebugTask) } + addTargetProjectInstallDependency(task, project) + } + + /** + * For a `com.android.test` module, also install the application it targets. + * + * A test module's own `installDebug` installs the test APK; nothing installs the application + * named by `targetProjectPath`. Without it `am instrument` cannot find its target package, + * so no tests run — and the screenshots would have nowhere to go, because they are written + * into the target application's data directory. + */ + private fun addTargetProjectInstallDependency(task: Task, project: Project) { + val targetPath = project.targetProjectPath ?: return + val targetProject = project.rootProject.findProject(targetPath) ?: return + + fun dependOnInstallTaskOf(evaluated: Project) { + evaluated.inferredInstallTask?.let { taskName -> + task.dependsOn("${evaluated.path}:$taskName") + } + } + + // The target may be configured before or after this module, depending on project order. + if (targetProject.state.executed) { + dependOnInstallTaskOf(targetProject) + } else { + targetProject.afterEvaluate(::dependOnInstallTaskOf) + } } } } diff --git a/Samples/Flix/FlixTest/build.gradle b/Samples/Flix/FlixTest/build.gradle index e1ad6f68e..2b46dacb6 100644 --- a/Samples/Flix/FlixTest/build.gradle +++ b/Samples/Flix/FlixTest/build.gradle @@ -1,18 +1,5 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget -buildscript { - ext { - versions = [ - 'compose': [ - 'compilerExt': '1.5.14', - 'core' : '1.5.14', - 'material' : '1.6.7', - 'ui' : '1.6.7', - ] - ] - } -} - plugins { id 'com.android.test' id 'dev.testify' @@ -45,9 +32,6 @@ android { buildFeatures { compose true } - composeOptions { - kotlinCompilerExtensionVersion "${versions.compose.compilerExt}" - } } dependencies { @@ -55,40 +39,24 @@ dependencies { implementation project(":Library") implementation project(":ComposeExtensions") + def composeBom = platform('androidx.compose:compose-bom:2025.08.01') + implementation composeBom + implementation 'androidx.compose.material3:material3:1.3.2' + // AndroidX Test implementation 'androidx.test.ext:junit:1.3.0' - implementation 'androidx.test.espresso:espresso-core:3.7.0' + // ComposableScreenshotRule extends ActivityTestRule, which lives here. implementation "androidx.test:rules:1.7.0" implementation "androidx.test:runner:1.7.0" - // Align lifecycle version with the target app to prevent classpath conflicts - // between the test APK and app APK. + // Align lifecycle with the application under test, to keep the test APK and app APK from + // disagreeing on the version. implementation "androidx.lifecycle:lifecycle-common:2.9.3" implementation "androidx.lifecycle:lifecycle-runtime:2.9.3" - - - def composeBom = platform('androidx.compose:compose-bom:2025.08.01') - implementation composeBom - //noinspection UseTomlInstead - implementation "androidx.appcompat:appcompat:1.7.1" - implementation "androidx.hilt:hilt-navigation-compose:1.2.0" - implementation "androidx.lifecycle:lifecycle-runtime-compose:2.9.3" - implementation "androidx.test.espresso.idling:idling-concurrent:3.7.0" - implementation "com.google.dagger:hilt-android:2.60.1" - implementation 'androidx.compose.material3:material3:1.3.2' - implementation 'androidx.core:core-ktx:1.17.0' - implementation 'io.ktor:ktor-client-android:3.2.3' - implementation 'org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0' - implementation("io.coil-kt:coil-compose:2.7.0") } testify { - // The app under test, named by `targetProjectPath` above. Screenshots are written into this - // application's data directory, so `screenshotPull` looks for them here. - applicationPackageId "dev.testify.samples.flix" - - // This module's own APK, which carries the instrumentation. - testPackageId "dev.testify.samples.flix.test" - + // applicationPackageId and testPackageId are both inferred for a test module: + // the application under test comes from `targetProjectPath`, and the test APK is this module. autoImplementLibrary false } diff --git a/bitrise.yml b/bitrise.yml index 491cfde31..27bd4aa88 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -431,6 +431,22 @@ workflows: - test_package: dev.testify.samples.flix.library.test - test_runner: androidx.test.runner.AndroidJUnitRunner - verbose: true + - android-build@1: + inputs: + - variant: Debug + - module: FlixTest + - git::https://github.com/ndtp/bitrise-step-android-testify.git@main: + timeout: 1800 + title: Run Screenshot Tests + inputs: + - adb_command: "./gradlew FlixTest:screenshotTest" + - module: ":FlixTest" + - app_apk: "./Samples/Flix/build/outputs/apk/debug/FlixSample-debug.apk" + - app_package: dev.testify.samples.flix + - test_apk: "./Samples/Flix/FlixTest/build/outputs/apk/debug/FlixTest-debug.apk" + - test_package: dev.testify.samples.flix.test + - test_runner: androidx.test.runner.AndroidJUnitRunner + - verbose: true - custom-test-results-export@1.0: is_always_run: true timeout: 120 diff --git a/docs/docs/recipes/21-library-projects.md b/docs/docs/recipes/21-library-projects.md index 05645c973..5771163d2 100644 --- a/docs/docs/recipes/21-library-projects.md +++ b/docs/docs/recipes/21-library-projects.md @@ -50,3 +50,28 @@ Please see the [Flix Library sample](https://github.com/ndtp/android-testify/tre - [Instrumentation tests ](https://source.android.com/docs/core/tests/development/instrumentation) --- + +## Test-only modules + +A [test-only module ](https://developer.android.com/studio/test/advanced-test-setup#use-separate-test-modules-for-instrumented-tests) applies `com.android.test` and names the application it tests with `targetProjectPath`. Apply `dev.testify` to it as usual and no extra configuration is needed: + +```groovy +plugins { + id 'com.android.test' + id 'dev.testify' +} + +android { + namespace 'com.example.screenshots' + targetProjectPath ':app' +} +``` + +Two things differ from an application or library module, and the plugin handles both: + +- **`applicationPackageId` is the application under test**, taken from `targetProjectPath` — not this module. Screenshots are written into the application's data directory, so `screenshotPull` looks there. `testPackageId` is this module's own namespace, because its APK carries the instrumentation. +- **Baselines live in `src/main/assets`**, because a test module has no `androidTest` source set. Its tests *are* its `main` sources. + +`screenshotTest` and `screenshotRecord` install both APKs, so a clean device or CI agent needs no extra setup. + +See [FlixTest ](https://github.com/ndtp/android-testify/tree/main/Samples/Flix/FlixTest) in the samples for a working module. From a4deb79bda6a3d946030b8eaf7bee8c8d02a47f3 Mon Sep 17 00:00:00 2001 From: Daniel Jette Date: Fri, 2 Oct 2026 21:37:48 -0400 Subject: [PATCH 3/3] 245: Fail when the instrumentation does not run, and give FlixTest its own namespace --- CHANGELOG.md | 13 ++++--- .../internal/GradleProjectExtensions.kt | 13 +++++-- .../testify/tasks/main/ScreenshotTestTask.kt | 36 ++++++++++++++++--- .../InternalScreenshotTestRecordTask.kt | 5 ++- Samples/Flix/FlixTest/build.gradle | 5 ++- bitrise.yml | 2 +- docs/docs/recipes/21-library-projects.md | 2 +- 7 files changed, 62 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea8ff3907..a9a604112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,10 +31,15 @@ - The Testify library is added as an `implementation` dependency rather than `androidTestImplementation` for the same reason - See the new `Samples/Flix/FlixTest` module for a worked example -- `screenshotTest` and `screenshotRecord` now fail when `am instrument` could not run the tests at - all — an uninstalled application under test, for example. Previously the task reported success - having run nothing, because the error is reported on standard error and the plugin only read - standard output +- `screenshotTest` and `screenshotRecord` now fail when `am instrument` did not run the tests at all + — an uninstalled application under test, for example. Previously the task reported success having + run nothing + - The check is that the output carries a JUnit summary, `OK (n tests)` or `FAILURES!!!`. A run + that executed always produces one, so this does not depend on recognising the ways + instrumentation can fail to start + - A recording run still never fails because a screenshot differs, but it does now fail if + nothing ran + - The plugin also reads the instrumentation's standard error, which it previously discarded ## 6.0.0 diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt index 0ef01528f..3ed54bc7d 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/internal/GradleProjectExtensions.kt @@ -57,12 +57,21 @@ val Project.targetProjectPath: String? * The applicationId of a `com.android.test` module's own APK. * * A test module's APK carries the instrumentation, so this is the package `am instrument` is invoked - * against. It has no `applicationId` in `defaultConfig`, so AGP uses its namespace. + * against. + * + * It is the namespace, deliberately and not as a fallback. AGP 9 does not let a `com.android.test` + * module choose its own applicationId: `TestDefaultConfig` does not expose `applicationId` in the + * typed DSL, and setting it from the Groovy DSL is accepted silently and then ignored — the built + * APK, and the installed package, use the namespace regardless. Reading `defaultConfig.applicationId` + * would therefore report an id that is not installed. * * `null` for any other module type. */ val Project.testModulePackageId: String? - get() = this.extensions.findByType(TestExtension::class.java)?.namespace + get() { + val testExtension = this.extensions.findByType(TestExtension::class.java) ?: return null + return testExtension.namespace + } val Project.isVerbose: Boolean get() = (this.properties["verbose"] as? String)?.toBoolean() ?: false diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/ScreenshotTestTask.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/ScreenshotTestTask.kt index a82c0ddfe..e57957462 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/ScreenshotTestTask.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/ScreenshotTestTask.kt @@ -177,19 +177,47 @@ open class ScreenshotTestTask : TestifyDefaultTask() { } protected open fun finalizeTaskAction(log: String) { + verifyInstrumentationRan(log) + if (log.contains("FAILURES!!!") || log.contains("INSTRUMENTATION_CODE: 0") || - log.contains("Process crashed while executing") || - // `am instrument` could not run the tests at all, for example because the application - // under test is not installed. Without this the task reports success having run nothing. - log.contains("INSTRUMENTATION_STATUS: Error=") + log.contains("Process crashed while executing") ) { println(Failure, "SCREENSHOT TESTS HAVE FAILED!!!") throw RuntimeException("Screenshot tests have failed") } } + /** + * Fail if `am instrument` did not get as far as running the tests. + * + * A run that executed always ends with JUnit's summary — `OK (n tests)` when everything passed, + * `FAILURES!!!` otherwise — so a log with neither means nothing ran. Testing for the summary + * rather than for particular error strings is deliberate: the ways instrumentation can fail to + * start are open-ended, and matching them individually kept missing variants. An uninstalled + * application under test reports `Error=Unable to find instrumentation target package` on some + * runs and only `INSTRUMENTATION_FAILED` on others, and either way the task used to report + * success having run no tests at all. + */ + protected fun verifyInstrumentationRan(log: String) { + if (log.contains(JUNIT_PASS_SUMMARY) || log.contains(JUNIT_FAIL_SUMMARY)) return + + println(Failure, "THE SCREENSHOT TESTS DID NOT RUN!!!") + throw RuntimeException( + "`am instrument` did not run any tests. The output has no test summary, which means " + + "the instrumentation could not start. Check that the application under test is " + + "installed and that testPackageId and testRunner are correct; run with " + + "`-Pverbose=true` to see the command and its full output." + ) + } + companion object : TaskNameProvider, TaskDependencyProvider { + /** JUnit's summary for a run where every test passed, including a run of no tests. */ + private const val JUNIT_PASS_SUMMARY = "OK (" + + /** JUnit's summary for a run with at least one failure. */ + private const val JUNIT_FAIL_SUMMARY = "FAILURES!!!" + override fun taskName() = "screenshotTest" override fun setDependencies(taskNameProvider: TaskNameProvider, project: Project) { diff --git a/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/internal/InternalScreenshotTestRecordTask.kt b/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/internal/InternalScreenshotTestRecordTask.kt index 649a2d1a5..e3f7882bd 100644 --- a/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/internal/InternalScreenshotTestRecordTask.kt +++ b/Plugins/Gradle/src/main/kotlin/dev/testify/tasks/main/internal/InternalScreenshotTestRecordTask.kt @@ -40,7 +40,10 @@ open class InternalScreenshotTestRecordTask : ScreenshotTestTask() { } override fun finalizeTaskAction(log: String) { - // Do nothing + // A recording run never fails because a screenshot differs - that is the point of it. It + // must still fail if the instrumentation never ran, or it reports success having recorded + // nothing. + verifyInstrumentationRan(log) } companion object : TaskNameProvider { diff --git a/Samples/Flix/FlixTest/build.gradle b/Samples/Flix/FlixTest/build.gradle index 2b46dacb6..3e89e361d 100644 --- a/Samples/Flix/FlixTest/build.gradle +++ b/Samples/Flix/FlixTest/build.gradle @@ -7,7 +7,10 @@ plugins { } android { - namespace 'dev.testify.samples.flix.test' + // Deliberately not `…flix.test`: that is FlixSample's own androidTest APK id (its + // applicationId plus AGP's default `.test` suffix), and the two APKs would install over each + // other. + namespace 'dev.testify.samples.flix.screenshots' compileSdk 36 // Target the Flix application module diff --git a/bitrise.yml b/bitrise.yml index 27bd4aa88..3098ff14e 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -444,7 +444,7 @@ workflows: - app_apk: "./Samples/Flix/build/outputs/apk/debug/FlixSample-debug.apk" - app_package: dev.testify.samples.flix - test_apk: "./Samples/Flix/FlixTest/build/outputs/apk/debug/FlixTest-debug.apk" - - test_package: dev.testify.samples.flix.test + - test_package: dev.testify.samples.flix.screenshots - test_runner: androidx.test.runner.AndroidJUnitRunner - verbose: true - custom-test-results-export@1.0: diff --git a/docs/docs/recipes/21-library-projects.md b/docs/docs/recipes/21-library-projects.md index 5771163d2..27156b03d 100644 --- a/docs/docs/recipes/21-library-projects.md +++ b/docs/docs/recipes/21-library-projects.md @@ -69,7 +69,7 @@ android { Two things differ from an application or library module, and the plugin handles both: -- **`applicationPackageId` is the application under test**, taken from `targetProjectPath` — not this module. Screenshots are written into the application's data directory, so `screenshotPull` looks there. `testPackageId` is this module's own namespace, because its APK carries the instrumentation. +- **`applicationPackageId` is the application under test**, taken from `targetProjectPath` — not this module. Screenshots are written into the application's data directory, so `screenshotPull` looks there. `testPackageId` is this module's own namespace, because its APK carries the instrumentation — AGP does not let a test module choose a separate `applicationId`, so the namespace is the installed package. Give it a namespace of its own: reusing the application's `applicationId` plus `.test` collides with the application module's own androidTest APK, and the two install over each other. - **Baselines live in `src/main/assets`**, because a test module has no `androidTest` source set. Its tests *are* its `main` sources. `screenshotTest` and `screenshotRecord` install both APKs, so a clean device or CI agent needs no extra setup.