diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index bad828f0c5..0f37490b48 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -1129,6 +1129,15 @@ class EMConfig { */ fun usesDriver() = !blackBox || bbExperiments || problemType == ProblemType.ASYNCAPI + /** + * Whether what the driver measures of the SUT, i.e., its units and boot-time targets, is + * read to report the code coverage of this run. + * + * The same as [usesDriver], except for a black-box AsyncAPI run, where the driver is only + * the way onto the broker and no coverage is read from it. + */ + fun readsSutCoverage() = !blackBox || bbExperiments + /** * Whether the driver should be asked to stop SUT code still running once the coverage * of a test has been collected. Read this, not [killSwitch], when talking to the driver. diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 8cf1eae007..14721d415a 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -366,7 +366,7 @@ class Main { solution: Solution<*>, idMapper: IdMapper ) { - if (config.usesDriver()) { + if (config.readsSutCoverage()) { val rc = injector.getInstance(RemoteController::class.java) val unitsInfo = rc.getSutInfo()?.unitsInfoDto val bootTimeInfo = rc.getSutInfo()?.bootTimeInfoDto diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index 4a98e0fb2e..761397cb16 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -151,14 +151,16 @@ class AsyncApiFitness : ApiWsFitness() { /* There is no separate black-box fitness here, unlike REST and GraphQL: AsyncAPI publishes - through the driver in both modes, which is what EMConfig.usesDriver() states. What - changes is only what this call adds on top of the outcome and reply targets above: an - instrumented SUT reports the lines and branches the messages reached, an uninstrumented - one reports none. + through the driver in both modes (see EMConfig.usesDriver()). Only in white-box mode is + the coverage of the SUT read on top of the outcome and reply targets. In black-box mode, + as for REST, it is not: it would add whatever the driver instruments to the targets, + and, with the kill switch on, stop the service's threads after every test. */ - val dto = updateFitnessAfterEvaluation(targets, allTargets, fullyCovered, descriptiveIds, individual, fv) - ?: return null - handleExtra(dto, fv) + if (!config.blackBox) { + val dto = updateFitnessAfterEvaluation(targets, allTargets, fullyCovered, descriptiveIds, individual, fv) + ?: return null + handleExtra(dto, fv) + } return EvaluatedIndividual( fv, diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt index b47035b828..7619fd5376 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt @@ -596,7 +596,7 @@ class Statistics : SearchListener { fun getData(solution: Solution<*>): List { - val sutInfo : SutInfoDto? = if(config.usesDriver()) { + val sutInfo : SutInfoDto? = if(config.readsSutCoverage()) { remoteController?.getSutInfo() } else { null @@ -987,7 +987,7 @@ class Statistics : SearchListener { // append boot-time targets - if(config.usesDriver()) { + if(config.readsSutCoverage()) { remoteController?.getSutInfo()?.bootTimeInfoDto?.targets?.map { it.descriptiveId }?.sorted()?.apply { if (isNotEmpty()){ content.add(System.lineSeparator()) diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index b5425f114e..33423903d2 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -794,6 +794,34 @@ internal class EMConfigTest{ assertTrue(whiteBox.usesDriver()) } + @Test + fun testABlackBoxAsyncApiRunReadsNoCoverageOfTheService(){ + + fun configFor(vararg args: String) = + EMConfig().apply { updateProperties(EMConfig.getOptionParser().parse(*args)) } + + //the driver takes part, but only to reach the broker + val asyncApi = configFor("--$blackBox", "true", "--problemType", "ASYNCAPI", "--createTests", "false") + assertTrue(asyncApi.usesDriver()) + assertFalse(asyncApi.readsSutCoverage()) + + //for every other problem type, the two say the same + val rest = configFor("--$blackBox", "true", "--problemType", "REST") + assertFalse(rest.usesDriver()) + assertFalse(rest.readsSutCoverage()) + + val restExperiments = configFor("--$blackBox", "true", "--problemType", "REST", "--bbExperiments", "true") + assertTrue(restExperiments.usesDriver()) + assertTrue(restExperiments.readsSutCoverage()) + + val asyncApiExperiments = configFor("--$blackBox", "true", "--problemType", "ASYNCAPI", + "--createTests", "false", "--bbExperiments", "true") + assertTrue(asyncApiExperiments.readsSutCoverage()) + + val whiteBox = configFor("--$blackBox", "false", "--problemType", "ASYNCAPI", "--createTests", "false") + assertTrue(whiteBox.readsSutCoverage()) + } + @Test fun testAsyncApiWritesTestsOnlyInTheLanguagesItCanPublishFrom(){ diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt index 2c8c86c2b4..7905611936 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt @@ -5,8 +5,11 @@ import com.google.inject.Injector import com.google.inject.Key import com.google.inject.TypeLiteral import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.TargetInfoDto +import org.evomaster.client.java.controller.api.dto.TestResultsDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming import org.evomaster.core.problem.asyncapi.data.AsyncApiAction import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual @@ -41,6 +44,18 @@ class AsyncApiFitnessTest { private const val ERROR = """{"error": {"code": 400, "message": "n must be >= 3"}}""" + private const val BLACK_BOX = "--blackBox" + + /** + * A line of the SUT, as an instrumented driver would report it once a message reached it. + */ + private const val SUT_LINE = "${ObjectiveNaming.LINE}_at_com.foo.NcsService_00042" + + /** + * The numeric id the driver gives [SUT_LINE]. + */ + private const val SUT_LINE_ID = 1 + /** * One operation that expects nothing back, and one whose message carries a header * besides the one the correlation id is stamped into. @@ -138,7 +153,9 @@ class AsyncApiFitnessTest { ) { driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(schemaText), answer) - injector = AsyncApiTestInjector.create(driver, "--blackBox=false", *options) + //white-box unless a test asks otherwise: EMConfig refuses an option given twice + val mode = if (options.any { it.startsWith("$BLACK_BOX=") }) arrayOf() else arrayOf("$BLACK_BOX=false") + injector = AsyncApiTestInjector.create(driver, *mode, *options) sampler = injector.getInstance(AsyncApiSampler::class.java) fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) @@ -179,6 +196,52 @@ class AsyncApiFitnessTest { private fun results(evaluated: EvaluatedIndividual): List = evaluated.evaluatedMainActions().map { it.result as AsyncApiCallResult } + /** + * What an instrumented driver reports once a message reached [SUT_LINE]. + */ + private fun lineReached() = TestResultsDto().apply { + targets.add(TargetInfoDto().apply { + id = SUT_LINE_ID + descriptiveId = SUT_LINE + value = 1.0 + actionIndex = 0 + }) + } + + @Test + fun testAWhiteBoxRunReadsTheCoverageOfTheService() { + + startNcs { replied(DOUBLE_RESULT) } + driver.testResults = { lineReached() } + + val evaluated = evaluate("bessj") + val covered = coveredIds(evaluated) + + assertTrue(driver.testResultsRequests > 0, "the coverage of the service was never asked for") + assertTrue(covered.contains(SUT_LINE), "$covered") + assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:bessj"), "$covered") + } + + @Test + fun testABlackBoxRunReadsNoCoverage() { + + /* + The driver is there only to reach the broker. Even an instrumented one is never + asked for lines: the targets come from the replies. + */ + startNcs("$BLACK_BOX=true") { replied(DOUBLE_RESULT) } + driver.testResults = { lineReached() } + + val evaluated = evaluate("bessj", "expint") + val covered = coveredIds(evaluated) + + assertEquals(0, driver.testResultsRequests, "a black-box run asked the driver for coverage") + assertFalse(covered.contains(SUT_LINE), "$covered") + assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:bessj"), "$covered") + assertTrue(covered.contains("ASYNCAPI_REPLY:doubleResult:expint"), "$covered") + assertEquals(2, driver.published.size) + } + @Test fun testTheDriverIsToldEverythingItNeeds() { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt index 0d9ff66d70..8a9ae728c0 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt @@ -11,7 +11,8 @@ import org.evomaster.core.remote.service.RemoteController /** * A driver that publishes nothing: it records what it was asked to publish, and answers each - * message with whatever [answer] decides. No instrumentation, so it reports no code targets. + * message with whatever [answer] decides. Unless told otherwise through [testResults], it is + * not instrumented, so it reports no code targets. */ class FakeAsyncApiDriver( private val info: SutInfoDto, @@ -23,6 +24,18 @@ class FakeAsyncApiDriver( */ val published: MutableList = mutableListOf() + /** + * What this driver answers when asked for the coverage of a test, as an instrumented one + * would: code targets, and what the SUT did with its inputs. + */ + var testResults: () -> TestResultsDto = { TestResultsDto() } + + /** + * How many times this driver was asked for the coverage of a test. + */ + var testResultsRequests = 0 + private set + companion object { fun replied(payload: String, correlationMatched: Boolean = true) = AsyncApiReplyDto().apply { @@ -72,5 +85,8 @@ class FakeAsyncApiDriver( ignoreKillSwitch: Boolean, fullyCovered: Boolean, descriptiveIds: Boolean - ) = TestResultsDto() + ): TestResultsDto { + testResultsRequests++ + return testResults() + } } diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt index c8a4c4ae0c..04e1f7c1fd 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt @@ -1,10 +1,86 @@ package org.evomaster.core.search.service +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.BootTimeInfoDto +import org.evomaster.client.java.controller.api.dto.TargetInfoDto +import org.evomaster.client.java.controller.api.dto.UnitsInfoDto +import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.AsyncApiTestInjector +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions.assertEquals class StatisticsTest { + companion object { + + /** + * How many lines an instrumented AsyncAPI driver says the SUT has. + */ + private const val SUT_LINES = 120 + + /** + * A line the same driver says was covered while the SUT booted. + */ + private const val BOOT_TIME_LINE = "${ObjectiveNaming.LINE}_at_com.foo.NcsApplication_00010" + } + + /** + * The statistics an AsyncAPI run would write, with a driver that is instrumented and says so. + */ + private fun asyncApiStatistics(vararg options: String): Map { + + RestActionBuilderV3.cleanCache() + + val info = AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS)).apply { + unitsInfoDto = UnitsInfoDto().apply { + numberOfLines = SUT_LINES + unitNames.add("com.foo.NcsApplication") + } + bootTimeInfoDto = BootTimeInfoDto().apply { + targets = listOf(TargetInfoDto().apply { + id = 1 + descriptiveId = BOOT_TIME_LINE + value = 1.0 + actionIndex = -1 + }) + } + } + val injector = AsyncApiTestInjector.create(FakeAsyncApiDriver(info) { null }, *options) + + val archive = injector.getInstance(Key.get(object : TypeLiteral>() {})) + val statistics = injector.getInstance(Statistics::class.java) + + return statistics.getData(archive.extractSolution()).associate { it.header to it.element } + } + + @Test + fun testAWhiteBoxAsyncApiRunReportsTheCoverageOfTheService() { + + val data = asyncApiStatistics("--blackBox=false") + + assertEquals("$SUT_LINES", data[Statistics.TOTAL_LINES]) + assertEquals("1", data[Statistics.COVERED_LINES], "the line covered at boot time is not counted") + } + + @Test + fun testABlackBoxAsyncApiRunReportsNoCoverageOfTheService() { + + /* + The driver takes part only to reach the broker, and the search never asks it for + coverage. Reporting the lines it covered at boot time out of the total it knows of + would present a black-box run as having covered code. + */ + val data = asyncApiStatistics("--blackBox=true") + + assertEquals("0", data[Statistics.TOTAL_LINES]) + assertEquals("0", data[Statistics.COVERED_LINES]) + } + @Test fun testMongoHeuristicsAverage() { val statistics = Statistics() diff --git a/docs/write_driver.md b/docs/write_driver.md index b450df0221..2492030e02 100644 --- a/docs/write_driver.md +++ b/docs/write_driver.md @@ -374,6 +374,8 @@ REST API, but no one client for every broker: the transport may be Kafka, AMQP, share nothing. So publishing a message is the driver's job, by overriding `executeAsyncApiAction`. What is asked of it is only to put the message on the wire and report what came back, never to judge it: deciding what an outcome means is _EvoMaster_'s job, so that it means the same thing whatever the transport. +In black-box mode the driver is only the way onto the broker: _EvoMaster_ does not ask it for the code +coverage of the service, even if the service is instrumented, and reports none. Two things are worth knowing when writing one: