diff --git a/CHANGELOG.md b/CHANGELOG.md index ce5cb028b..93bb170d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ - Fix `RootViewWithoutFocusException` being intermittently thrown by `Espresso.closeSoftKeyboard()` - **Breaking:** `dev.testify.internal.helpers.closeSoftKeyboard()` now requires an `Activity` parameter - Fix `ClassCastException` when passing `-PshardCount` and `-PshardIndex` to `screenshotTest` or `screenshotRecord` +- Fix `baseline_image` in `report.yml` naming a file that does not exist. The path was built from the + test method name alone, while baselines are stored as `_.png` — the same entry's + `test_image` already used the correct name +- Fix duplicate `baseline_image` / `test_image` keys in `report.yml` when a test calls `assertSame()` + more than once, which produced invalid YAML ## 6.0.0 diff --git a/Library/src/main/java/dev/testify/report/Reporter.kt b/Library/src/main/java/dev/testify/report/Reporter.kt index a7e0728ae..b38542bff 100644 --- a/Library/src/main/java/dev/testify/report/Reporter.kt +++ b/Library/src/main/java/dev/testify/report/Reporter.kt @@ -70,6 +70,16 @@ internal open class Reporter protected constructor( */ private lateinit var testDescription: TestDescription + /** + * Whether [captureOutput] has already recorded the image paths for the current test. + * + * [startTest] is called once per test method, but [captureOutput] is called from every + * `assertSame()`. A test that asserts more than once would otherwise append a second + * `baseline_image:` / `test_image:` pair inside the same `- test:` entry, producing duplicate + * keys and invalid YAML. + */ + private var hasCapturedOutput: Boolean = false + /** * Creates a unique session ID for the given test run */ @@ -83,6 +93,7 @@ internal open class Reporter protected constructor( */ fun startTest(description: TestDescription) { testDescription = description + hasCapturedOutput = false session.addTest() builder.appendLine("- test:", indent = 4) @@ -97,6 +108,9 @@ internal open class Reporter protected constructor( * modifications have been applied */ fun captureOutput() { + if (hasCapturedOutput) return + hasCapturedOutput = true + builder.appendLine("baseline_image: assets/${getBaselinePath()}", indent = 8) builder.appendLine("test_image: ${getOutputPath()}", indent = 8) } @@ -165,7 +179,7 @@ internal open class Reporter protected constructor( internal open fun getBaselinePath(): String = getFileRelativeToRoot( subpath = getDeviceDescription(context), - fileName = testDescription.methodName, + fileName = testDescription.name, extension = PNG_EXTENSION ) diff --git a/Library/src/test/java/dev/testify/report/ReporterTest.kt b/Library/src/test/java/dev/testify/report/ReporterTest.kt index aad1931f4..55abb8612 100644 --- a/Library/src/test/java/dev/testify/report/ReporterTest.kt +++ b/Library/src/test/java/dev/testify/report/ReporterTest.kt @@ -227,7 +227,7 @@ internal open class ReporterTest { " name: startTest\n" + " class: ReporterTest\n" + " package: dev.testify.report\n" + - " baseline_image: assets/screenshots/device/startTest.png\n" + + " baseline_image: assets/screenshots/device/ReporterTest_startTest.png\n" + " test_image: path\n" + " status: PASS\n", yaml @@ -270,7 +270,7 @@ internal open class ReporterTest { assertEquals(" name: skipTest", lines[9]) assertEquals(" class: ReporterTest", lines[10]) assertEquals(" package: dev.testify.report", lines[11]) - assertEquals(" baseline_image: assets/screenshots/device/skipTest.png", lines[12]) + assertEquals(" baseline_image: assets/screenshots/device/ReporterTest_skipTest.png", lines[12]) assertEquals(" test_image: path", lines[13]) assertEquals(" status: SKIP", lines[14]) assertEquals(" - test:", lines[15]) @@ -291,6 +291,50 @@ internal open class ReporterTest { assertEquals(" status: PASS", lines[30]) } + @Test + fun `getBaselinePath() matches the file name the baseline is loaded from`() { + reporter.startTest(mockDescription) + + // loadBaselineBitmapForComparison() resolves the baseline with TestDescription.name, + // which is "_". The report has to name the same file. + assertEquals("screenshots/device/${mockDescription.name}.png", reporter.getBaselinePath()) + assertEquals("screenshots/device/ReporterTest_startTest.png", reporter.getBaselinePath()) + } + + @Test + fun `captureOutput() called twice in one test records the image paths once`() { + every { reporter.getBaselinePath() } returns "path" + + reporter.startTest(mockDescription) + reporter.captureOutput() + reporter.captureOutput() + reporter.pass() + + assertEquals( + " - test:\n" + + " name: startTest\n" + + " class: ReporterTest\n" + + " package: dev.testify.report\n" + + " baseline_image: assets/path\n" + + " test_image: path\n" + + " status: PASS\n", + reporter.yaml + ) + } + + @Test + fun `startTest() allows the next test to record its own image paths`() { + every { reporter.getBaselinePath() } returns "path" + + reporter.startTest(mockDescription) + reporter.captureOutput() + reporter.captureOutput() + reporter.startTest(mockDescription.copy(methodName = "secondTest")) + reporter.captureOutput() + + assertEquals(2, Regex("baseline_image:").findAll(reporter.yaml).count()) + } + private val Reporter.yaml: String get() { return this.builder.toString() diff --git a/Plugins/Gradle/README.md b/Plugins/Gradle/README.md index 8e15f32d6..79960e9d1 100644 --- a/Plugins/Gradle/README.md +++ b/Plugins/Gradle/README.md @@ -460,13 +460,14 @@ Print the test result report to the console - date: 2021-03-19@20:37:32 - failed: 0 - passed: 1 +- skipped: 0 - total: 1 - tests: - test: name: withFocusOnBackground class: ClientListActivityScreenshotTest package: dev.testify.sample.clients.index - baseline_image: assets/screenshots/29-1080x2220@440dp-en_US/withFocusOnBackground.png + baseline_image: assets/screenshots/29-1080x2220@440dp-en_US/ClientListActivityScreenshotTest_withFocusOnBackground.png test_image: /data/user/0/dev.testify.sample/app_images/screenshots/29-1080x2220@440dp-en_US/ClientListActivityScreenshotTest_withFocusOnBackground.png status: PASS ``` diff --git a/docs/docs/get-started/8-use-gradle-plugin.md b/docs/docs/get-started/8-use-gradle-plugin.md index 9586a6355..6948ace40 100644 --- a/docs/docs/get-started/8-use-gradle-plugin.md +++ b/docs/docs/get-started/8-use-gradle-plugin.md @@ -523,13 +523,14 @@ Print the test result report to the console - date: 2021-03-19@20:37:32 - failed: 0 - passed: 1 +- skipped: 0 - total: 1 - tests: - test: name: withFocusOnBackground class: ClientListActivityScreenshotTest package: dev.testify.sample.clients.index - baseline_image: assets/screenshots/29-1080x2220@440dp-en_US/withFocusOnBackground.png + baseline_image: assets/screenshots/29-1080x2220@440dp-en_US/ClientListActivityScreenshotTest_withFocusOnBackground.png test_image: /data/user/0/dev.testify.sample/app_images/screenshots/29-1080x2220@440dp-en_US/ClientListActivityScreenshotTest_withFocusOnBackground.png status: PASS ```