AsyncAPI 3.x: a black-box run reads no coverage

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -1129,6 +1129,15 @@ class EMConfig {
      */
     fun usesDriver() = !blackBox || bbExperiments || problemType == ProblemType.ASYNCAPI
 
+    /**
+     * Whether what the driver measures of the SUT, i.e., its units and boot-time targets, is
+     * read to report the code coverage of this run.
+     *
+     * The same as [usesDriver], except for a black-box AsyncAPI run, where the driver is only
+     * the way onto the broker and no coverage is read from it.
+     */
+    fun readsSutCoverage() = !blackBox || bbExperiments
+
     /**
      * Whether the driver should be asked to stop SUT code still running once the coverage
      * of a test has been collected. Read this, not [killSwitch], when talking to the driver.

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -366,7 +366,7 @@ class Main {
             solution: Solution<*>,
             idMapper: IdMapper
         ) {
-            if (config.usesDriver()) {
+            if (config.readsSutCoverage()) {
                 val rc = injector.getInstance(RemoteController::class.java)
                 val unitsInfo = rc.getSutInfo()?.unitsInfoDto
                 val bootTimeInfo = rc.getSutInfo()?.bootTimeInfoDto

core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt

@@ -151,14 +151,16 @@ class AsyncApiFitness : ApiWsFitness<AsyncApiIndividual>() {
 
         /*
             There is no separate black-box fitness here, unlike REST and GraphQL: AsyncAPI publishes
-            through the driver in both modes, which is what EMConfig.usesDriver() states. What
-            changes is only what this call adds on top of the outcome and reply targets above: an
-            instrumented SUT reports the lines and branches the messages reached, an uninstrumented
-            one reports none.
+            through the driver in both modes (see EMConfig.usesDriver()). Only in white-box mode is
+            the coverage of the SUT read on top of the outcome and reply targets. In black-box mode,
+            as for REST, it is not: it would add whatever the driver instruments to the targets,
+            and, with the kill switch on, stop the service's threads after every test.
          */
-        val dto = updateFitnessAfterEvaluation(targets, allTargets, fullyCovered, descriptiveIds, individual, fv)
-            ?: return null
-        handleExtra(dto, fv)
+        if (!config.blackBox) {
+            val dto = updateFitnessAfterEvaluation(targets, allTargets, fullyCovered, descriptiveIds, individual, fv)
+                ?: return null
+            handleExtra(dto, fv)
+        }
 
         return EvaluatedIndividual(
             fv,

core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt

@@ -596,7 +596,7 @@ class Statistics : SearchListener {
 
     fun getData(solution: Solution<*>): List<Pair> {
 
-        val sutInfo : SutInfoDto? = if(config.usesDriver()) {
+        val sutInfo : SutInfoDto? = if(config.readsSutCoverage()) {
             remoteController?.getSutInfo()
         } else {
             null
@@ -987,7 +987,7 @@ class Statistics : SearchListener {
 
 
         // append boot-time targets
-        if(config.usesDriver()) {
+        if(config.readsSutCoverage()) {
             remoteController?.getSutInfo()?.bootTimeInfoDto?.targets?.map { it.descriptiveId }?.sorted()?.apply {
                 if (isNotEmpty()){
                     content.add(System.lineSeparator())

core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt

@@ -794,6 +794,34 @@ internal class EMConfigTest{
         assertTrue(whiteBox.usesDriver())
     }
 
+    @Test
+    fun testABlackBoxAsyncApiRunReadsNoCoverageOfTheService(){
+
+        fun configFor(vararg args: String) =
+            EMConfig().apply { updateProperties(EMConfig.getOptionParser().parse(*args)) }
+
+        //the driver takes part, but only to reach the broker
+        val asyncApi = configFor("--$blackBox", "true", "--problemType", "ASYNCAPI", "--createTests", "false")
+        assertTrue(asyncApi.usesDriver())
+        assertFalse(asyncApi.readsSutCoverage())
+
+        //for every other problem type, the two say the same
+        val rest = configFor("--$blackBox", "true", "--problemType", "REST")
+        assertFalse(rest.usesDriver())
+        assertFalse(rest.readsSutCoverage())
+
+        val restExperiments = configFor("--$blackBox", "true", "--problemType", "REST", "--bbExperiments", "true")
+        assertTrue(restExperiments.usesDriver())
+        assertTrue(restExperiments.readsSutCoverage())
+
+        val asyncApiExperiments = configFor("--$blackBox", "true", "--problemType", "ASYNCAPI",
+            "--createTests", "false", "--bbExperiments", "true")
+        assertTrue(asyncApiExperiments.readsSutCoverage())
+
+        val whiteBox = configFor("--$blackBox", "false", "--problemType", "ASYNCAPI", "--createTests", "false")
+        assertTrue(whiteBox.readsSutCoverage())
+    }
+
     @Test
     fun testAsyncApiWritesTestsOnlyInTheLanguagesItCanPublishFrom(){
 

core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt

@@ -5,8 +5,11 @@ import com.google.inject.Injector
 import com.google.inject.Key
 import com.google.inject.TypeLiteral
 import com.webfuzzing.asyncapi.access.AsyncApiAccess
+import org.evomaster.client.java.controller.api.dto.TargetInfoDto
+import org.evomaster.client.java.controller.api.dto.TestResultsDto
 import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto
 import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto
+import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming
 import org.evomaster.core.problem.asyncapi.data.AsyncApiAction
 import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult
 import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual
@@ -41,6 +44,18 @@ class AsyncApiFitnessTest {
 
         private const val ERROR = """{"error": {"code": 400, "message": "n must be >= 3"}}"""
 
+        private const val BLACK_BOX = "--blackBox"
+
+        /**
+         * A line of the SUT, as an instrumented driver would report it once a message reached it.
+         */
+        private const val SUT_LINE = "${ObjectiveNaming.LINE}_at_com.foo.NcsService_00042"
+
+        /**
+         * The numeric id the driver gives [SUT_LINE].
+         */
+        private const val SUT_LINE_ID = 1
+
         /**
          * One operation that expects nothing back, and one whose message carries a header
          * besides the one the correlation id is stamped into.
@@ -138,7 +153,9 @@ class AsyncApiFitnessTest {
     ) {
 
         driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(schemaText), answer)
-        injector = AsyncApiTestInjector.create(driver, "--blackBox=false", *options)
+        //white-box unless a test asks otherwise: EMConfig refuses an option given twice
+        val mode = if (options.any { it.startsWith("$BLACK_BOX=") }) arrayOf() else arrayOf("$BLACK_BOX=false")
+        injector = AsyncApiTestInjector.create(driver, *mode, *options)
 
         sampler = injector.getInstance(AsyncApiSampler::class.java)
         fitness = injector.getInstance(Key.get(object : TypeLiteral<FitnessFunction<AsyncApiIndividual>>() {}))
@@ -179,6 +196,52 @@ class AsyncApiFitnessTest {
     private fun results(evaluated: EvaluatedIndividual<AsyncApiIndividual>): List<AsyncApiCallResult> =
         evaluated.evaluatedMainActions().map { it.result as AsyncApiCallResult }
 
+    /**
+     * What an instrumented driver reports once a message reached [SUT_LINE].
+     */
+    private fun lineReached() = TestResultsDto().apply {
+        targets.add(TargetInfoDto().apply {
+            id = SUT_LINE_ID
+            descriptiveId = SUT_LINE
+            value = 1.0
+            actionIndex = 0
+        })
+    }
+
+    @Test
+    fun testAWhiteBoxRunReadsTheCoverageOfTheService() {
+
+        startNcs { replied(DOUBLE_RESULT) }
+        driver.testResults = { lineReached() }
+
+        val evaluated = evaluate("bessj")
+        val covered = coveredIds(evaluated)
+
+        assertTrue(driver.testResultsRequests > 0, "the coverage of the service was never asked for")
+        assertTrue(covered.contains(SUT_LINE), "$covered")
+        assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:bessj"), "$covered")
+    }
+
+    @Test
+    fun testABlackBoxRunReadsNoCoverage() {
+
+        /*
+            The driver is there only to reach the broker. Even an instrumented one is never
+            asked for lines: the targets come from the replies.
+         */
+        startNcs("$BLACK_BOX=true") { replied(DOUBLE_RESULT) }
+        driver.testResults = { lineReached() }
+
+        val evaluated = evaluate("bessj", "expint")
+        val covered = coveredIds(evaluated)
+
+        assertEquals(0, driver.testResultsRequests, "a black-box run asked the driver for coverage")
+        assertFalse(covered.contains(SUT_LINE), "$covered")
+        assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:bessj"), "$covered")
+        assertTrue(covered.contains("ASYNCAPI_REPLY:doubleResult:expint"), "$covered")
+        assertEquals(2, driver.published.size)
+    }
+
     @Test
     fun testTheDriverIsToldEverythingItNeeds() {
 

core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt

@@ -11,7 +11,8 @@ import org.evomaster.core.remote.service.RemoteController
 
 /**
  * A driver that publishes nothing: it records what it was asked to publish, and answers each
- * message with whatever [answer] decides. No instrumentation, so it reports no code targets.
+ * message with whatever [answer] decides. Unless told otherwise through [testResults], it is
+ * not instrumented, so it reports no code targets.
  */
 class FakeAsyncApiDriver(
     private val info: SutInfoDto,
@@ -23,6 +24,18 @@ class FakeAsyncApiDriver(
      */
     val published: MutableList<AsyncApiActionDto> = mutableListOf()
 
+    /**
+     * What this driver answers when asked for the coverage of a test, as an instrumented one
+     * would: code targets, and what the SUT did with its inputs.
+     */
+    var testResults: () -> TestResultsDto = { TestResultsDto() }
+
+    /**
+     * How many times this driver was asked for the coverage of a test.
+     */
+    var testResultsRequests = 0
+        private set
+
     companion object {
 
         fun replied(payload: String, correlationMatched: Boolean = true) = AsyncApiReplyDto().apply {
@@ -72,5 +85,8 @@ class FakeAsyncApiDriver(
         ignoreKillSwitch: Boolean,
         fullyCovered: Boolean,
         descriptiveIds: Boolean
-    ) = TestResultsDto()
+    ): TestResultsDto {
+        testResultsRequests++
+        return testResults()
+    }
 }

core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt

@@ -1,10 +1,86 @@
 package org.evomaster.core.search.service
 
+import com.google.inject.Key
+import com.google.inject.TypeLiteral
+import com.webfuzzing.asyncapi.access.AsyncApiAccess
+import org.evomaster.client.java.controller.api.dto.BootTimeInfoDto
+import org.evomaster.client.java.controller.api.dto.TargetInfoDto
+import org.evomaster.client.java.controller.api.dto.UnitsInfoDto
+import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming
+import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual
+import org.evomaster.core.problem.asyncapi.service.AsyncApiTestInjector
+import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver
+import org.evomaster.core.problem.rest.builder.RestActionBuilderV3
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.Assertions.assertEquals
 
 class StatisticsTest {
 
+    companion object {
+
+        /**
+         * How many lines an instrumented AsyncAPI driver says the SUT has.
+         */
+        private const val SUT_LINES = 120
+
+        /**
+         * A line the same driver says was covered while the SUT booted.
+         */
+        private const val BOOT_TIME_LINE = "${ObjectiveNaming.LINE}_at_com.foo.NcsApplication_00010"
+    }
+
+    /**
+     * The statistics an AsyncAPI run would write, with a driver that is instrumented and says so.
+     */
+    private fun asyncApiStatistics(vararg options: String): Map<String, String> {
+
+        RestActionBuilderV3.cleanCache()
+
+        val info = AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS)).apply {
+            unitsInfoDto = UnitsInfoDto().apply {
+                numberOfLines = SUT_LINES
+                unitNames.add("com.foo.NcsApplication")
+            }
+            bootTimeInfoDto = BootTimeInfoDto().apply {
+                targets = listOf(TargetInfoDto().apply {
+                    id = 1
+                    descriptiveId = BOOT_TIME_LINE
+                    value = 1.0
+                    actionIndex = -1
+                })
+            }
+        }
+        val injector = AsyncApiTestInjector.create(FakeAsyncApiDriver(info) { null }, *options)
+
+        val archive = injector.getInstance(Key.get(object : TypeLiteral<Archive<AsyncApiIndividual>>() {}))
+        val statistics = injector.getInstance(Statistics::class.java)
+
+        return statistics.getData(archive.extractSolution()).associate { it.header to it.element }
+    }
+
+    @Test
+    fun testAWhiteBoxAsyncApiRunReportsTheCoverageOfTheService() {
+
+        val data = asyncApiStatistics("--blackBox=false")
+
+        assertEquals("$SUT_LINES", data[Statistics.TOTAL_LINES])
+        assertEquals("1", data[Statistics.COVERED_LINES], "the line covered at boot time is not counted")
+    }
+
+    @Test
+    fun testABlackBoxAsyncApiRunReportsNoCoverageOfTheService() {
+
+        /*
+            The driver takes part only to reach the broker, and the search never asks it for
+            coverage. Reporting the lines it covered at boot time out of the total it knows of
+            would present a black-box run as having covered code.
+         */
+        val data = asyncApiStatistics("--blackBox=true")
+
+        assertEquals("0", data[Statistics.TOTAL_LINES])
+        assertEquals("0", data[Statistics.COVERED_LINES])
+    }
+
     @Test
     fun testMongoHeuristicsAverage() {
         val statistics = Statistics()

docs/write_driver.md

@@ -374,6 +374,8 @@ REST API, but no one client for every broker: the transport may be Kafka, AMQP,
 share nothing. So publishing a message is the driver's job, by overriding `executeAsyncApiAction`. What is
 asked of it is only to put the message on the wire and report what came back, never to judge it: deciding
 what an outcome means is _EvoMaster_'s job, so that it means the same thing whatever the transport.
+In black-box mode the driver is only the way onto the broker: _EvoMaster_ does not ask it for the code
+coverage of the service, even if the service is instrumented, and reports none.
 
 Two things are worth knowing when writing one: