From 3956f6196feae7b438de315cbbdbbd436ecb3481 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Sat, 10 Oct 2026 21:19:00 -0300 Subject: [PATCH 1/2] AsyncAPI 3.x: a black-box run reads no coverage AsyncAPI needs the driver in black-box mode too, as the only way onto the broker, and AsyncApiFitness has one code path for both modes. That path asked the driver for the coverage of every test regardless of the mode, and Main and Statistics, which gate on EMConfig.usesDriver(), read the units and boot-time targets of the SUT in black-box runs as well. A black-box AsyncAPI run therefore reported line coverage whenever the driver happened to be instrumented, and added those lines to the search targets. REST and GraphQL never read coverage in black-box mode, not even with --bbExperiments. AsyncAPI does the same: - AsyncApiFitness calls updateFitnessAfterEvaluation and handleExtra only in white-box mode. resetSUT is still called in both. - EMConfig.readsSutCoverage(), i.e. !blackBox || bbExperiments, is what Main.logCodeCoverage, Statistics.getData and Statistics.writeCoveredTargets check. For every problem type other than AsyncAPI it is exactly what usesDriver() was, so their output is unchanged. A black-box AsyncAPI run reports its AsyncAPI targets and zero lines and branches, and logs no coverage, as a black-box REST run does. With --bbExperiments it reads the units and boot-time targets, as REST does. FakeAsyncApiDriver answers coverage requests with given test results, and counts them. AsyncApiFitnessTest checks that a white-box run asks for coverage and gets the SUT lines into the fitness, and that a black-box run never asks. StatisticsTest checks the reported totals in both modes with an instrumented driver, and EMConfigTest the new predicate against usesDriver() for REST and AsyncAPI, with and without --bbExperiments. docs/write_driver.md says the driver is not asked for coverage in black-box mode. No change to white-box runs of any problem type, nor to black-box runs of REST, GraphQL or RPC. --- .../kotlin/org/evomaster/core/EMConfig.kt | 9 +++ .../main/kotlin/org/evomaster/core/Main.kt | 2 +- .../asyncapi/service/AsyncApiFitness.kt | 16 ++-- .../core/search/service/Statistics.kt | 4 +- .../kotlin/org/evomaster/core/EMConfigTest.kt | 28 +++++++ .../asyncapi/service/AsyncApiFitnessTest.kt | 65 +++++++++++++++- .../asyncapi/service/FakeAsyncApiDriver.kt | 20 ++++- .../core/search/service/StatisticsTest.kt | 76 +++++++++++++++++++ docs/write_driver.md | 2 + 9 files changed, 209 insertions(+), 13 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index f86bd2d732..e0e7047947 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -1125,6 +1125,15 @@ class EMConfig { */ fun usesDriver() = !blackBox || bbExperiments || problemType == ProblemType.ASYNCAPI + /** + * Whether what the driver measures of the SUT, i.e., its units and boot-time targets, is + * read to report the code coverage of this run. + * + * The same as [usesDriver], except for a black-box AsyncAPI run, where the driver is only + * the way onto the broker and no coverage is read from it. + */ + fun readsSutCoverage() = !blackBox || bbExperiments + /** * Whether the driver should be asked to stop SUT code still running once the coverage * of a test has been collected. Read this, not [killSwitch], when talking to the driver. diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 8cf1eae007..14721d415a 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -366,7 +366,7 @@ class Main { solution: Solution<*>, idMapper: IdMapper ) { - if (config.usesDriver()) { + if (config.readsSutCoverage()) { val rc = injector.getInstance(RemoteController::class.java) val unitsInfo = rc.getSutInfo()?.unitsInfoDto val bootTimeInfo = rc.getSutInfo()?.bootTimeInfoDto diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index 3b8001d0dc..b3431eb46a 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -141,14 +141,16 @@ class AsyncApiFitness : ApiWsFitness() { /* There is no separate black-box fitness here, unlike REST and GraphQL: AsyncAPI publishes - through the driver in both modes, which is what EMConfig.usesDriver() states. What - changes is only what this call adds on top of the outcome and reply targets above: an - instrumented SUT reports the lines and branches the messages reached, an uninstrumented - one reports none. + through the driver in both modes (see EMConfig.usesDriver()). Only in white-box mode is + the coverage of the SUT read on top of the outcome and reply targets. In black-box mode, + as for REST, it is not: it would add whatever the driver instruments to the targets, + and, with the kill switch on, stop the service's threads after every test. */ - val dto = updateFitnessAfterEvaluation(targets, allTargets, fullyCovered, descriptiveIds, individual, fv) - ?: return null - handleExtra(dto, fv) + if (!config.blackBox) { + val dto = updateFitnessAfterEvaluation(targets, allTargets, fullyCovered, descriptiveIds, individual, fv) + ?: return null + handleExtra(dto, fv) + } return EvaluatedIndividual( fv, diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt index 9a0add74e6..ba77216a1d 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt @@ -611,7 +611,7 @@ class Statistics : SearchListener { fun getData(solution: Solution<*>): List { - val sutInfo : SutInfoDto? = if(config.usesDriver()) { + val sutInfo : SutInfoDto? = if(config.readsSutCoverage()) { remoteController?.getSutInfo() } else { null @@ -1005,7 +1005,7 @@ class Statistics : SearchListener { // append boot-time targets - if(config.usesDriver()) { + if(config.readsSutCoverage()) { remoteController?.getSutInfo()?.bootTimeInfoDto?.targets?.map { it.descriptiveId }?.sorted()?.apply { if (isNotEmpty()){ content.add(System.lineSeparator()) diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index da283e80fc..f4e24498d9 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -794,6 +794,34 @@ internal class EMConfigTest{ assertTrue(whiteBox.usesDriver()) } + @Test + fun testABlackBoxAsyncApiRunReadsNoCoverageOfTheService(){ + + fun configFor(vararg args: String) = + EMConfig().apply { updateProperties(EMConfig.getOptionParser().parse(*args)) } + + //the driver takes part, but only to reach the broker + val asyncApi = configFor("--$blackBox", "true", "--problemType", "ASYNCAPI", "--createTests", "false") + assertTrue(asyncApi.usesDriver()) + assertFalse(asyncApi.readsSutCoverage()) + + //for every other problem type, the two say the same + val rest = configFor("--$blackBox", "true", "--problemType", "REST") + assertFalse(rest.usesDriver()) + assertFalse(rest.readsSutCoverage()) + + val restExperiments = configFor("--$blackBox", "true", "--problemType", "REST", "--bbExperiments", "true") + assertTrue(restExperiments.usesDriver()) + assertTrue(restExperiments.readsSutCoverage()) + + val asyncApiExperiments = configFor("--$blackBox", "true", "--problemType", "ASYNCAPI", + "--createTests", "false", "--bbExperiments", "true") + assertTrue(asyncApiExperiments.readsSutCoverage()) + + val whiteBox = configFor("--$blackBox", "false", "--problemType", "ASYNCAPI", "--createTests", "false") + assertTrue(whiteBox.readsSutCoverage()) + } + @Test fun testAsyncApiWritesTestsOnlyInJavaOrKotlin(){ diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt index aec91a42d1..d2e51dc1dc 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt @@ -3,8 +3,11 @@ package org.evomaster.core.problem.asyncapi.service import com.fasterxml.jackson.databind.ObjectMapper import com.google.inject.Injector import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.TargetInfoDto +import org.evomaster.client.java.controller.api.dto.TestResultsDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming import org.evomaster.core.problem.asyncapi.data.AsyncApiAction import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual @@ -32,6 +35,18 @@ class AsyncApiFitnessTest { private const val ERROR = """{"error": {"code": 400, "message": "n must be >= 3"}}""" + private const val BLACK_BOX = "--blackBox" + + /** + * A line of the SUT, as an instrumented driver would report it once a message reached it. + */ + private const val SUT_LINE = "${ObjectiveNaming.LINE}_at_com.foo.NcsService_00042" + + /** + * The numeric id the driver gives [SUT_LINE]. + */ + private const val SUT_LINE_ID = 1 + /** * One operation that expects nothing back, and one whose message carries a header * besides the one the correlation id is stamped into. @@ -128,7 +143,9 @@ class AsyncApiFitnessTest { ) { driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(schemaText), answer) - injector = AsyncApiTestInjector.create(driver, "--blackBox=false", *options) + //white-box unless a test asks otherwise: EMConfig refuses an option given twice + val mode = if (options.any { it.startsWith("$BLACK_BOX=") }) arrayOf() else arrayOf("$BLACK_BOX=false") + injector = AsyncApiTestInjector.create(driver, *mode, *options) sampler = injector.getInstance(AsyncApiSampler::class.java) idMapper = injector.getInstance(IdMapper::class.java) @@ -152,6 +169,52 @@ class AsyncApiFitnessTest { private fun results(evaluated: EvaluatedIndividual): List = evaluated.evaluatedMainActions().map { it.result as AsyncApiCallResult } + /** + * What an instrumented driver reports once a message reached [SUT_LINE]. + */ + private fun lineReached() = TestResultsDto().apply { + targets.add(TargetInfoDto().apply { + id = SUT_LINE_ID + descriptiveId = SUT_LINE + value = 1.0 + actionIndex = 0 + }) + } + + @Test + fun testAWhiteBoxRunReadsTheCoverageOfTheService() { + + startNcs { replied(DOUBLE_RESULT) } + driver.testResults = { lineReached() } + + val evaluated = evaluate("bessj") + val covered = coveredIds(evaluated) + + assertTrue(driver.testResultsRequests > 0, "the coverage of the service was never asked for") + assertTrue(covered.contains(SUT_LINE), "$covered") + assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:bessj"), "$covered") + } + + @Test + fun testABlackBoxRunReadsNoCoverage() { + + /* + The driver is there only to reach the broker. Even an instrumented one is never + asked for lines: the targets come from the replies. + */ + startNcs("$BLACK_BOX=true") { replied(DOUBLE_RESULT) } + driver.testResults = { lineReached() } + + val evaluated = evaluate("bessj", "expint") + val covered = coveredIds(evaluated) + + assertEquals(0, driver.testResultsRequests, "a black-box run asked the driver for coverage") + assertFalse(covered.contains(SUT_LINE), "$covered") + assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:bessj"), "$covered") + assertTrue(covered.contains("ASYNCAPI_REPLY:doubleResult:expint"), "$covered") + assertEquals(2, driver.published.size) + } + @Test fun testTheDriverIsToldEverythingItNeeds() { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt index 0d9ff66d70..8a9ae728c0 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt @@ -11,7 +11,8 @@ import org.evomaster.core.remote.service.RemoteController /** * A driver that publishes nothing: it records what it was asked to publish, and answers each - * message with whatever [answer] decides. No instrumentation, so it reports no code targets. + * message with whatever [answer] decides. Unless told otherwise through [testResults], it is + * not instrumented, so it reports no code targets. */ class FakeAsyncApiDriver( private val info: SutInfoDto, @@ -23,6 +24,18 @@ class FakeAsyncApiDriver( */ val published: MutableList = mutableListOf() + /** + * What this driver answers when asked for the coverage of a test, as an instrumented one + * would: code targets, and what the SUT did with its inputs. + */ + var testResults: () -> TestResultsDto = { TestResultsDto() } + + /** + * How many times this driver was asked for the coverage of a test. + */ + var testResultsRequests = 0 + private set + companion object { fun replied(payload: String, correlationMatched: Boolean = true) = AsyncApiReplyDto().apply { @@ -72,5 +85,8 @@ class FakeAsyncApiDriver( ignoreKillSwitch: Boolean, fullyCovered: Boolean, descriptiveIds: Boolean - ) = TestResultsDto() + ): TestResultsDto { + testResultsRequests++ + return testResults() + } } diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt index fed8f9279f..2ba6a6d4cc 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt @@ -1,10 +1,86 @@ package org.evomaster.core.search.service +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.BootTimeInfoDto +import org.evomaster.client.java.controller.api.dto.TargetInfoDto +import org.evomaster.client.java.controller.api.dto.UnitsInfoDto +import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.AsyncApiTestInjector +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions.assertEquals class StatisticsTest { + companion object { + + /** + * How many lines an instrumented AsyncAPI driver says the SUT has. + */ + private const val SUT_LINES = 120 + + /** + * A line the same driver says was covered while the SUT booted. + */ + private const val BOOT_TIME_LINE = "${ObjectiveNaming.LINE}_at_com.foo.NcsApplication_00010" + } + + /** + * The statistics an AsyncAPI run would write, with a driver that is instrumented and says so. + */ + private fun asyncApiStatistics(vararg options: String): Map { + + RestActionBuilderV3.cleanCache() + + val info = AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS)).apply { + unitsInfoDto = UnitsInfoDto().apply { + numberOfLines = SUT_LINES + unitNames.add("com.foo.NcsApplication") + } + bootTimeInfoDto = BootTimeInfoDto().apply { + targets = listOf(TargetInfoDto().apply { + id = 1 + descriptiveId = BOOT_TIME_LINE + value = 1.0 + actionIndex = -1 + }) + } + } + val injector = AsyncApiTestInjector.create(FakeAsyncApiDriver(info) { null }, *options) + + val archive = injector.getInstance(Key.get(object : TypeLiteral>() {})) + val statistics = injector.getInstance(Statistics::class.java) + + return statistics.getData(archive.extractSolution()).associate { it.header to it.element } + } + + @Test + fun testAWhiteBoxAsyncApiRunReportsTheCoverageOfTheService() { + + val data = asyncApiStatistics("--blackBox=false") + + assertEquals("$SUT_LINES", data[Statistics.TOTAL_LINES]) + assertEquals("1", data[Statistics.COVERED_LINES], "the line covered at boot time is not counted") + } + + @Test + fun testABlackBoxAsyncApiRunReportsNoCoverageOfTheService() { + + /* + The driver takes part only to reach the broker, and the search never asks it for + coverage. Reporting the lines it covered at boot time out of the total it knows of + would present a black-box run as having covered code. + */ + val data = asyncApiStatistics("--blackBox=true") + + assertEquals("0", data[Statistics.TOTAL_LINES]) + assertEquals("0", data[Statistics.COVERED_LINES]) + } + @Test fun testMongoHeuristicsAverage() { val statistics = Statistics() diff --git a/docs/write_driver.md b/docs/write_driver.md index 6cb785a51d..47bffd04ec 100644 --- a/docs/write_driver.md +++ b/docs/write_driver.md @@ -376,6 +376,8 @@ REST API, but no one client for every broker: the transport may be Kafka, AMQP, share nothing. So publishing a message is the driver's job, by overriding `executeAsyncApiAction`. What is asked of it is only to put the message on the wire and report what came back, never to judge it: deciding what an outcome means is _EvoMaster_'s job, so that it means the same thing whatever the transport. +In black-box mode the driver is only the way onto the broker: _EvoMaster_ does not ask it for the code +coverage of the service, even if the service is instrumented, and reports none. Two things are worth knowing when writing one: From 6e0ee7589dedd9e673dbf86f1a15eadb5c86feae Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Tue, 27 Oct 2026 16:34:10 -0300 Subject: [PATCH 2/2] Keep AsyncAPI black-box coverage reporting consistent in experiments --- core/src/main/kotlin/org/evomaster/core/EMConfig.kt | 4 ++-- core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt | 2 +- .../core/problem/asyncapi/service/AsyncApiFitnessTest.kt | 9 +++++++++ .../org/evomaster/core/search/service/StatisticsTest.kt | 7 +++++++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index e0e7047947..56ebb24d00 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -1130,9 +1130,9 @@ class EMConfig { * read to report the code coverage of this run. * * The same as [usesDriver], except for a black-box AsyncAPI run, where the driver is only - * the way onto the broker and no coverage is read from it. + * the way onto the broker and no coverage is read from it, including with [bbExperiments]. */ - fun readsSutCoverage() = !blackBox || bbExperiments + fun readsSutCoverage() = !blackBox || (bbExperiments && problemType != ProblemType.ASYNCAPI) /** * Whether the driver should be asked to stop SUT code still running once the coverage diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index f4e24498d9..1533506b58 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -816,7 +816,7 @@ internal class EMConfigTest{ val asyncApiExperiments = configFor("--$blackBox", "true", "--problemType", "ASYNCAPI", "--createTests", "false", "--bbExperiments", "true") - assertTrue(asyncApiExperiments.readsSutCoverage()) + assertFalse(asyncApiExperiments.readsSutCoverage()) val whiteBox = configFor("--$blackBox", "false", "--problemType", "ASYNCAPI", "--createTests", "false") assertTrue(whiteBox.readsSutCoverage()) diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt index d2e51dc1dc..8edb58743b 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt @@ -215,6 +215,15 @@ class AsyncApiFitnessTest { assertEquals(2, driver.published.size) } + @Test + fun testBlackBoxExperimentsStillReadNoCoverage() { + startNcs("$BLACK_BOX=true", "--bbExperiments=true") { replied(DOUBLE_RESULT) } + driver.testResults = { fail("AsyncAPI black-box experiments must not request coverage") } + val covered = coveredIds(evaluate("bessj")) + assertEquals(0, driver.testResultsRequests) + assertFalse(covered.contains(SUT_LINE)) + } + @Test fun testTheDriverIsToldEverythingItNeeds() { diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt index 2ba6a6d4cc..9da4dc67bd 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt @@ -81,6 +81,13 @@ class StatisticsTest { assertEquals("0", data[Statistics.COVERED_LINES]) } + @Test + fun testBlackBoxExperimentsDoNotReportAsyncApiCoverage() { + val data = asyncApiStatistics("--blackBox=true", "--bbExperiments=true") + assertEquals("0", data[Statistics.TOTAL_LINES]) + assertEquals("0", data[Statistics.COVERED_LINES]) + } + @Test fun testMongoHeuristicsAverage() { val statistics = Statistics()