From 3c5546e4530efdd760696a1bbc49b9c3963415bc Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Sat, 12 Sep 2026 19:39:13 -0300 Subject: [PATCH 1/9] AsyncAPI 3.x: the driver call, the reply timeout, and the two faults RemoteController.executeNewAsyncApiActionAndGetReply PUTs the action to the driver and reads back an AsyncApiReplyDto, as its RPC counterpart does. It has a default that throws, so the controllers that never publish need not say so one by one. EMConfig gains asyncApiReplyTimeoutMs, experimental, and a constraint: there is no test writer for AsyncAPI yet, so a run must say --createTests false rather than search for an hour and fail at the end. The two tests that already parsed --problemType ASYNCAPI say so now. Two experimental fault categories: a promised reply that never arrives, and a reply matching none of the messages the contract declares. --- .../kotlin/org/evomaster/core/EMConfig.kt | 12 +++++++++++ .../enterprise/ExperimentalFaultCategory.kt | 7 +++++++ .../core/remote/service/RemoteController.kt | 12 +++++++++++ .../service/RemoteControllerImplementation.kt | 20 +++++++++++++++++++ .../kotlin/org/evomaster/core/EMConfigTest.kt | 17 +++++++++++++++- .../asyncapi/service/AsyncApiSamplerTest.kt | 2 +- docs/options.md | 1 + 7 files changed, 69 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 9015d09803..95b9a0ab7d 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -806,6 +806,11 @@ class EMConfig { throw ConfigProblemException("When using the seedTestCases option, you must specify the file path of the test cases with the seedTestCasesPath option") } + if (problemType == ProblemType.ASYNCAPI && createTests) { + throw ConfigProblemException("Test generation for AsyncAPI services is not available yet." + + " For the time being, run with '--createTests false' to only search for faults.") + } + if (problemType == ProblemType.RPC && createTests && (enablePureRPCTestGeneration || enableRPCAssertionWithInstance) @@ -2836,6 +2841,13 @@ class EMConfig { @Cfg("Whether to enable extra targets for responses, e.g., regarding nullable response, having extra targets for whether it is null") var enableRPCExtraResponseTargets = true + @Experimental + @Cfg("When testing an AsyncAPI service, how long to wait for the reply to a published message before" + + " treating it as unanswered, in milliseconds. A slow service and a stuck one look the same from" + + " outside, so this is a tuning parameter with no equivalent in a synchronous protocol.") + @Min(1.0) + var asyncApiReplyTimeoutMs = 5000 + @Cfg("Whether to enable customized responses indicating business logic") var enableRPCCustomizedResponseTargets = true diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt index ad4b452e90..4cf81db2e3 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt @@ -11,6 +11,13 @@ enum class ExperimentalFaultCategory( //9xx for experimental, work-in-progress oracles + //AsyncAPI + // the contract promised a reply, and none arrived within the time waited + ASYNCAPI_NO_REPLY(950, "No Reply", "getsNoReply", + "TODO"), + // a reply arrived, but it matches none of the messages the contract says a reply can be + ASYNCAPI_UNDECLARED_REPLY(951, "Undeclared Reply", "repliesWithUndeclaredMessage", + "TODO"), //Implemented HTTP_TIMEOUT(960, "Request Timeout", "requestTimeout", "TODO"), diff --git a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteController.kt b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteController.kt index fb08a4b1c2..600a535931 100644 --- a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteController.kt +++ b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteController.kt @@ -1,6 +1,7 @@ package org.evomaster.core.remote.service import org.evomaster.client.java.controller.api.dto.* +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto import org.evomaster.client.java.controller.api.dto.problem.param.DeriveParamResponseDto import org.evomaster.client.java.controller.api.dto.problem.param.DerivedParamChangeReqDto import org.evomaster.core.scheduletask.ScheduleTaskExecutor @@ -51,6 +52,17 @@ interface RemoteController : DatabaseExecutor, ScheduleTaskExecutor { fun executeNewRPCActionAndGetResponse(actionDto: ActionDto) : ActionResponseDto? + /** + * Have the driver publish the message in [ActionDto.asyncApiCall] and, when one is + * expected, wait for its reply. Null when the driver could not be reached, or refused. + * + * Only a driver for an AsyncAPI service can do this, which is why the default throws + * rather than every other controller having to say it cannot. + */ + fun executeNewAsyncApiActionAndGetReply(actionDto: ActionDto) : AsyncApiReplyDto? { + throw IllegalStateException("This controller does not publish AsyncAPI messages") + } + fun postSearchAction(postSearchActionDto: PostSearchActionDto) : Boolean fun registerNewAction(actionDto: ActionDto) : Boolean diff --git a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt index 06e025ad1c..e850ecaebc 100644 --- a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt +++ b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt @@ -5,6 +5,7 @@ import com.google.inject.Inject import org.evomaster.client.java.controller.api.ControllerConstants import org.evomaster.client.java.controller.api.dto.* import org.evomaster.client.java.controller.api.dto.database.operations.* +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto import org.evomaster.client.java.controller.api.dto.problem.param.DeriveParamResponseDto import org.evomaster.client.java.controller.api.dto.problem.param.DerivedParamChangeReqDto import org.evomaster.client.java.controller.api.dto.problem.rpc.ScheduleTaskInvocationsDto @@ -411,6 +412,25 @@ class RemoteControllerImplementation() : RemoteController{ return dto?.data } + override fun executeNewAsyncApiActionAndGetReply(actionDto: ActionDto): AsyncApiReplyDto? { + + val response = makeHttpCall { + getWebTarget() + .path(ControllerConstants.NEW_ACTION) + .queryParam("queryFromDatabase", !config.useInsertionForSqlHeuristics) + .request() + .put(Entity.entity(actionDto, MediaType.APPLICATION_JSON_TYPE)) + } + + val dto = getDtoFromResponse(response, object : GenericType>() {}) + + if (!checkResponse(response, dto, "Failed to publish an AsyncAPI message")) { + return null + } + + return dto?.data + } + /** * process post actions after search based on [postSearchActionDto] */ diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 4a67d5de98..0696df06da 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -777,11 +777,26 @@ internal class EMConfigTest{ connection to the broker even when the service itself is a black box. */ val asyncApi = EMConfig() - asyncApi.updateProperties(parser.parse("--$blackBox", "true", "--problemType", "ASYNCAPI")) + asyncApi.updateProperties(parser.parse("--$blackBox", "true", "--problemType", "ASYNCAPI", "--createTests", "false")) assertTrue(asyncApi.usesDriver()) val whiteBox = EMConfig() whiteBox.updateProperties(parser.parse("--$blackBox", "false")) assertTrue(whiteBox.usesDriver()) } + + @Test + fun testAsyncApiCannotWriteTestsYet(){ + + val parser = EMConfig.getOptionParser() + + //createTests is on by default, and there is no test writer for AsyncAPI yet + val e = assertThrows { + EMConfig().updateProperties(parser.parse("--problemType", "ASYNCAPI")) + } + assertTrue(e.message!!.contains("createTests"), e.message) + + //so a run has to say it only wants the search + EMConfig().updateProperties(parser.parse("--problemType", "ASYNCAPI", "--createTests", "false")) + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt index 6c225e8105..5c75267638 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt @@ -49,7 +49,7 @@ class AsyncApiSamplerTest { private fun injector(info: SutInfoDto, starts: Boolean = true, vararg options: String): Injector { - val args = arrayOf("--seed=42", "--problemType=ASYNCAPI") + options + val args = arrayOf("--seed=42", "--problemType=ASYNCAPI", "--createTests=false") + options val modules = listOf(BaseModule(args), object : AbstractModule() { override fun configure() { diff --git a/docs/options.md b/docs/options.md index ba38a4c4e5..7ab8e627d3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -278,6 +278,7 @@ There are 3 types of options: |`aiResponseClassifierWarmup`| __Int__. Number of training iterations required to update classifier parameters. For example, in the Gaussian model this affects mean and variance updates. For neural network (NN) models, the warm-up should typically be larger than 1000. *Default value*: `100`.| |`appendToTargetHeuristicsFile`| __Boolean__. Whether should add to an existing target heuristics file, instead of replacing it. It is only used when processFormat is TARGET_HEURISTIC. *Default value*: `false`.| |`arazzoLocation`| __String__. arazzo location on disk. *Default value*: `""`.| +|`asyncApiReplyTimeoutMs`| __Int__. When testing an AsyncAPI service, how long to wait for the reply to a published message before treating it as unanswered, in milliseconds. A slow service and a stuck one look the same from outside, so this is a tuning parameter with no equivalent in a synchronous protocol. *Constraints*: `min=1.0`. *Default value*: `5000`.| |`breederParentsMin`| __Int__. Breeder GA: minimum number of individuals in parents pool after truncation. *Constraints*: `min=2.0`. *Default value*: `2`.| |`breederTruncationFraction`| __Double__. Breeder GA: fraction of top individuals to keep in parents pool (truncation). *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`callbackURLHostname`| __String__. HTTP callback verifier hostname. Default is set to 'localhost'. If the SUT is running inside a container (i.e., Docker), 'localhost' will refer to the container. This can be used to change the hostname. *Default value*: `localhost`.| From 997e363f862a62c55f311a7452dd309ceccd1e07 Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Sat, 12 Sep 2026 19:39:14 -0300 Subject: [PATCH 2/9] AsyncAPI 3.x: the fitness, and --problemType ASYNCAPI runs AsyncApiBlackBoxFitness publishes each message of a test through the driver and turns what comes back into targets: for every operation, what publishing to it was seen to do, and, when a reply came back, which of the messages the contract declares for the reply it was. A contract listing a result and an error thus gives the search two things to reach, which is the AsyncAPI analogue of REST's (status x endpoint). AsyncApiReplyClassifier is what recognises a reply: a structural match against each declared payload schema, reading the parts of JSON Schema that tell one message from another and giving the benefit of the doubt on the rest. The most specific match wins. AsyncApiModule binds it all, with the driver bound unconditionally, and Main uses it in place of the message it showed until now. One test runs a whole MIO search against a stand-in for the NCS service and sees both declared replies of an operation covered. --- .../main/kotlin/org/evomaster/core/Main.kt | 5 +- .../asyncapi/data/AsyncApiCallResult.kt | 58 ++++ .../problem/asyncapi/data/AsyncApiOutcome.kt | 30 ++ .../service/AsyncApiBlackBoxFitness.kt | 308 ++++++++++++++++++ .../asyncapi/service/AsyncApiModule.kt | 105 ++++++ .../service/AsyncApiReplyClassifier.kt | 182 +++++++++++ .../asyncapi/service/AsyncApiSampler.kt | 13 +- .../service/AsyncApiStructureMutator.kt | 76 +++++ .../service/AsyncApiBlackBoxFitnessTest.kt | 304 +++++++++++++++++ .../service/AsyncApiReplyClassifierTest.kt | 83 +++++ .../asyncapi/service/AsyncApiSearchTest.kt | 147 +++++++++ .../asyncapi/service/FakeAsyncApiDriver.kt | 76 +++++ 12 files changed, 1382 insertions(+), 5 deletions(-) create mode 100644 core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt create mode 100644 core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt create mode 100644 core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt create mode 100644 core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt create mode 100644 core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt create mode 100644 core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt create mode 100644 core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt create mode 100644 core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt create mode 100644 core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSearchTest.kt create mode 100644 core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 435b6fc8ad..8cf1eae007 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -20,6 +20,7 @@ import org.evomaster.core.output.TestSuiteSplitter import org.evomaster.core.output.clustering.SplitResult import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.AsyncApiModule import org.evomaster.core.problem.enterprise.service.WFCReportWriter import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHttpWsResponseHandler import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler @@ -627,8 +628,8 @@ class Main { } EMConfig.ProblemType.ASYNCAPI -> { - //the sampler, fitness and module for it are being added one at a time - throw IllegalStateException("AsyncAPI is not wired into the search yet") + //one module for both modes: the driver is needed either way, see EMConfig.usesDriver + AsyncApiModule() } //this should never happen, unless we add new type and forget to add it here diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt new file mode 100644 index 0000000000..1fdbbbd671 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt @@ -0,0 +1,58 @@ +package org.evomaster.core.problem.asyncapi.data + +import org.evomaster.core.problem.enterprise.EnterpriseActionResult + +/** + * What happened when one message was published: the outcome, and the reply when there was one. + */ +class AsyncApiCallResult : EnterpriseActionResult { + + companion object { + const val OUTCOME = "OUTCOME" + const val REPLY_PAYLOAD = "REPLY_PAYLOAD" + const val REPLY_MESSAGE = "REPLY_MESSAGE" + const val CORRELATION_MATCHED = "CORRELATION_MATCHED" + const val WAITED_MS = "WAITED_MS" + } + + constructor(sourceLocalId: String, stopping: Boolean = false) : super(sourceLocalId, stopping) + + private constructor(other: AsyncApiCallResult) : super(other) + + override fun copy(): AsyncApiCallResult { + return AsyncApiCallResult(this) + } + + fun setOutcome(outcome: AsyncApiOutcome) { + addResultValue(OUTCOME, outcome.name) + } + + fun getOutcome(): AsyncApiOutcome? = getResultValue(OUTCOME)?.let { AsyncApiOutcome.valueOf(it) } + + fun setReplyPayload(payload: String) { + addResultValue(REPLY_PAYLOAD, payload) + } + + fun getReplyPayload(): String? = getResultValue(REPLY_PAYLOAD) + + /** + * Which of the messages the contract declares for the reply this one was recognised as. + */ + fun setReplyMessage(messageId: String) { + addResultValue(REPLY_MESSAGE, messageId) + } + + fun getReplyMessage(): String? = getResultValue(REPLY_MESSAGE) + + fun setCorrelationMatched(matched: Boolean) { + addResultValue(CORRELATION_MATCHED, matched.toString()) + } + + fun getCorrelationMatched(): Boolean? = getResultValue(CORRELATION_MATCHED)?.toBoolean() + + fun setWaitedMs(ms: Long) { + addResultValue(WAITED_MS, ms.toString()) + } + + fun getWaitedMs(): Long? = getResultValue(WAITED_MS)?.toLong() +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt new file mode 100644 index 0000000000..7a05c00af5 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt @@ -0,0 +1,30 @@ +package org.evomaster.core.problem.asyncapi.data + +/** + * What came of publishing one message, as the driver reported it. + * + * Only [NO_REPLY] is a fault in itself. [PUBLISH_FAILED] is a broken setup rather than a finding + * about the service, and the other two are the service doing what its contract says. + */ +enum class AsyncApiOutcome { + + /** + * Published, and no reply was expected. + */ + PUBLISHED, + + /** + * Published, and a reply arrived that answers it. + */ + REPLIED, + + /** + * Published, a reply was expected, and none arrived within the time waited. + */ + NO_REPLY, + + /** + * The driver could not put the message on the wire at all. + */ + PUBLISH_FAILED +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt new file mode 100644 index 0000000000..a3231b4482 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt @@ -0,0 +1,308 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.inject.Inject +import com.webfuzzing.asyncapi.models.AsyncApiCorrelationId +import com.webfuzzing.asyncapi.models.AsyncApiReply +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.database.sql.SqlAction +import org.evomaster.core.logging.LoggingUtil +import org.evomaster.core.problem.api.service.ApiWsFitness +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome +import org.evomaster.core.problem.asyncapi.param.AsyncApiParam +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.FitnessValue +import org.evomaster.core.search.action.ActionResult +import org.evomaster.core.search.gene.utils.GeneUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * Publishes the messages of a test through the driver and turns what comes back into targets. + * + * The targets are the AsyncAPI analogue of REST's `(status x endpoint)`: for every operation, + * what publishing to it was seen to do ([AsyncApiOutcome]), and, when a reply came back, which + * of the messages the contract declares for the reply it was recognised as. A contract that + * enumerates a result and an error thus gives the search two things to reach. + * + * Two outcomes are faults: a promised reply that never arrives, and a reply matching none of + * the declared messages. A message the driver could not publish is neither; it is a broken + * setup, and the test stops there. + */ +class AsyncApiBlackBoxFitness : ApiWsFitness() { + + companion object { + private val log: Logger = LoggerFactory.getLogger(AsyncApiBlackBoxFitness::class.java) + + /** + * Prefix of the `(outcome x operation)` targets, written as PREFIX:OUTCOME:action. + */ + const val OUTCOME_TARGET_PREFIX = "ASYNCAPI_OUTCOME" + + /** + * Prefix of the `(declared reply x operation)` targets, written as PREFIX:messageId:action. + */ + const val REPLY_TARGET_PREFIX = "ASYNCAPI_REPLY" + + private const val DEFAULT_CONTENT_TYPE = "application/json" + + private val mapper = ObjectMapper() + } + + @Inject + private lateinit var asyncApiSampler: AsyncApiSampler + + /** + * Tells this run's correlation ids from those of an earlier run against the same broker. + * Drawn from [randomness], so that a seeded run is reproducible. + */ + private val runId: String by lazy { Integer.toHexString(randomness.nextInt()) } + + /** + * How many messages this run has published, which is what makes each correlation id unique. + */ + private var published = 0L + + override fun doCalculateCoverage( + individual: AsyncApiIndividual, + targets: Set, + allTargets: Boolean, + fullyCovered: Boolean, + descriptiveIds: Boolean, + ): EvaluatedIndividual? { + + rc.resetSUT() + + val actionResults: MutableList = mutableListOf() + + doDbCalls(individual.seeInitializingActions().filterIsInstance(), actionResults = actionResults) + + val fv = FitnessValue(individual.size().toDouble()) + + val actions = individual.seeMainExecutableActions().filterIsInstance() + + for ((index, action) in actions.withIndex()) { + val ok = publish(action, index, actionResults, fv) + if (!ok) { + break + } + } + + val dto = updateFitnessAfterEvaluation(targets, allTargets, fullyCovered, descriptiveIds, individual, fv) + ?: return null + handleExtra(dto, fv) + + return EvaluatedIndividual( + fv, + individual.copy() as AsyncApiIndividual, + actionResults, + trackOperator = individual.trackOperator, + index = time.evaluatedIndividuals, + config = config + ) + } + + /** + * @return whether the message reached the wire, so that the test can go on + */ + private fun publish( + action: AsyncApiAction, + index: Int, + actionResults: MutableList, + fv: FitnessValue + ): Boolean { + + searchTimeController.waitForRateLimiter() + + val result = AsyncApiCallResult(action.getLocalId()) + actionResults.add(result) + + val dto = getActionDto(action, index) + dto.asyncApiCall = toDto(action) + + val reply = rc.executeNewAsyncApiActionAndGetReply(dto) + + if (reply == null || !reply.published) { + /* + Not a finding about the service: the driver could not put the message on the + wire, or could not be reached at all. Nothing published after this point would + mean anything, so the test stops here, and no target is registered for it. + */ + result.setOutcome(AsyncApiOutcome.PUBLISH_FAILED) + result.setErrorMessage(reply?.errorMessage ?: "No response from the driver") + result.stopping = true + return false + } + + record(reply, result) + handleTargets(fv, action, result, index) + + return true + } + + private fun record(reply: AsyncApiReplyDto, result: AsyncApiCallResult) { + + val outcome = when { + !reply.replyExpected -> AsyncApiOutcome.PUBLISHED + reply.replyReceived -> AsyncApiOutcome.REPLIED + else -> AsyncApiOutcome.NO_REPLY + } + + result.setOutcome(outcome) + reply.waitedMs?.let { result.setWaitedMs(it) } + + if (outcome == AsyncApiOutcome.REPLIED) { + reply.replyPayload?.let { result.setReplyPayload(it) } + result.setCorrelationMatched(reply.correlationMatched) + } + } + + private fun handleTargets(fv: FitnessValue, action: AsyncApiAction, result: AsyncApiCallResult, index: Int) { + + val name = action.getName() + val outcome = result.getOutcome()!! + + fv.updateTarget(idMapper.handleLocalTarget("$OUTCOME_TARGET_PREFIX:${outcome.name}:$name"), 1.0, index) + + when (outcome) { + + AsyncApiOutcome.REPLIED -> handleReplyTargets(fv, action, result, index) + + AsyncApiOutcome.NO_REPLY -> { + val fault = idMapper.getFaultDescriptiveId(ExperimentalFaultCategory.ASYNCAPI_NO_REPLY, name) + fv.updateTarget(idMapper.handleLocalTarget(fault), 1.0, index) + } + + AsyncApiOutcome.PUBLISHED, AsyncApiOutcome.PUBLISH_FAILED -> Unit + } + } + + private fun handleReplyTargets(fv: FitnessValue, action: AsyncApiAction, result: AsyncApiCallResult, index: Int) { + + val name = action.getName() + val document = asyncApiSampler.document + val operation = document.operations[action.operationId] ?: return + val declared = document.replyMessagesOf(operation) + + if (declared.isEmpty()) { + //the contract says a reply comes, but not what it is: nothing to recognise it as + return + } + + val recognised = AsyncApiReplyClassifier.classify(result.getReplyPayload(), declared, document.componentSchemas) + + if (recognised == null) { + val fault = idMapper.getFaultDescriptiveId(ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY, name) + fv.updateTarget(idMapper.handleLocalTarget(fault), 1.0, index) + return + } + + result.setReplyMessage(recognised.id) + fv.updateTarget(idMapper.handleLocalTarget("$REPLY_TARGET_PREFIX:${recognised.id}:$name"), 1.0, index) + } + + /** + * Everything the driver needs to publish the message and wait for its reply, resolved + * against the document so that the driver never has to read it. + */ + private fun toDto(action: AsyncApiAction): AsyncApiActionDto { + + val document = asyncApiSampler.document + val message = document.messages[action.messageId] + val channel = document.channels[action.channelName] + + val dto = AsyncApiActionDto() + dto.operationId = action.operationId + dto.channelName = action.channelName + dto.messageId = action.messageId + + /* + A channel may declare no address, meaning it is decided at run time. The driver is + then given the channel's name and left to map it, being the one that knows the broker. + */ + dto.address = channel?.address ?: action.channelName + + dto.payload = action.parameters.firstOrNull { it.name == AsyncApiParam.PAYLOAD } + ?.gene?.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = null) + dto.contentType = message?.contentType ?: document.defaultContentType ?: DEFAULT_CONTENT_TYPE + dto.headers = headersOf(action) + + dto.correlationId = "$runId-${published++}" + message?.correlationId?.let { + dto.correlationLocation = if (it.source == AsyncApiCorrelationId.Source.HEADER) { + AsyncApiActionDto.CORRELATION_IN_HEADER + } else { + AsyncApiActionDto.CORRELATION_IN_PAYLOAD + } + dto.correlationPointer = it.pointer + } + + action.replyTemplate?.let { reply -> + dto.replyAddress = replyAddressOf(reply, action) + if (dto.replyAddress != null) { + dto.replyTimeoutMs = config.asyncApiReplyTimeoutMs.toLong() + } + } + + return dto + } + + /** + * The headers gene as a map, by way of its own JSON printing, which is what knows which + * optional headers are on. + */ + private fun headersOf(action: AsyncApiAction): MutableMap { + + val headers = LinkedHashMap() + + val gene = action.parameters.firstOrNull { it.name == AsyncApiParam.HEADERS }?.gene + ?: return headers + + val json = gene.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = null) + + val node = try { + mapper.readTree(json) + } catch (e: JsonProcessingException) { + log.warn("The headers of '{}' did not print as JSON: {}", action.getName(), e.message) + return headers + } + + if (node.isObject) { + node.fields().forEach { (name, value) -> + headers[name] = if (value.isValueNode) value.asText() else value.toString() + } + } + + return headers + } + + /** + * Where the driver should wait for the reply, or null when there is nowhere to wait yet. + */ + private fun replyAddressOf(reply: AsyncApiReply, action: AsyncApiAction): String? { + + val channelName = reply.channelName + + if (channelName == null) { + /* + The reply address is announced inside the request (reply.address.location) rather + than fixed by the contract. Supporting that means minting an address and stamping + it into the message, which is not done yet; until then such an operation is + published without waiting. + */ + LoggingUtil.uniqueUserWarn( + "Operation '${action.operationId}' announces its reply address at run time, which is" + + " not supported yet: its replies will not be waited for" + ) + return null + } + + return asyncApiSampler.document.channels[channelName]?.address ?: channelName + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt new file mode 100644 index 0000000000..cc709ee442 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt @@ -0,0 +1,105 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.TypeLiteral +import org.evomaster.core.output.service.NoTestCaseWriter +import org.evomaster.core.output.service.TestCaseWriter +import org.evomaster.core.output.service.TestSuiteWriter +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.enterprise.service.EnterpriseModule +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler +import org.evomaster.core.remote.service.RemoteController +import org.evomaster.core.remote.service.RemoteControllerImplementation +import org.evomaster.core.search.service.Archive +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector +import org.evomaster.core.search.service.Minimizer +import org.evomaster.core.search.service.Sampler +import org.evomaster.core.search.service.mutator.Mutator +import org.evomaster.core.search.service.mutator.StandardMutator +import org.evomaster.core.search.service.mutator.StructureMutator + +/** + * The services a search over an AsyncAPI service is made of. + * + * One module serves both white-box and black-box mode, and the driver is bound unconditionally: + * it is what holds the connection to the broker, so it takes part either way (see + * [org.evomaster.core.EMConfig.usesDriver]). + * + * No test cases are written yet, which [org.evomaster.core.EMConfig] enforces by requiring + * `--createTests false`. + */ +class AsyncApiModule : EnterpriseModule() { + + override fun configure() { + + /* + No super.configure(): what EnterpriseModule binds there is REST-only, and the RPC, + GraphQL and Web modules leave it out the same way. + */ + + bind(object : TypeLiteral>() {}) + .to(AsyncApiSampler::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(AsyncApiSampler::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(AsyncApiSampler::class.java) + .asEagerSingleton() + + bind(AsyncApiSampler::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(AsyncApiBlackBoxFitness::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(AsyncApiBlackBoxFitness::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(object : TypeLiteral>() {}) + + bind(Archive::class.java) + .to(object : TypeLiteral>() {}) + + bind(RemoteController::class.java) + .to(RemoteControllerImplementation::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(StructureMutator::class.java) + .to(AsyncApiStructureMutator::class.java) + .asEagerSingleton() + + bind(TestCaseWriter::class.java) + .to(NoTestCaseWriter::class.java) + .asEagerSingleton() + + bind(TestSuiteWriter::class.java) + .asEagerSingleton() + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt new file mode 100644 index 0000000000..8170bfccab --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt @@ -0,0 +1,182 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.webfuzzing.asyncapi.models.AsyncApiMessage +import com.webfuzzing.asyncapi.resolver.AsyncApiRefResolver + +/** + * Recognises which of the messages a contract declares for a reply an observed reply is. + * + * A reply channel often carries several messages -- a result and an error, say -- and telling + * them apart is what gives a black-box search distinct outcomes to cover. There are no status + * codes to read, so the payload is matched against each declared schema instead. + * + * The matching is structural and deliberately lenient: it checks the parts of JSON Schema that + * tell one message from another (type, required fields, const and enum discriminators, the + * combinators), and gives the benefit of the doubt on anything it does not understand. It is a + * classifier, not a validator: its job is to tell which declared message a reply is, not to + * find every way in which it deviates from its schema. + */ +object AsyncApiReplyClassifier { + + private const val REF = "\$ref" + private const val TYPE = "type" + private const val PROPERTIES = "properties" + private const val REQUIRED = "required" + private const val ITEMS = "items" + private const val CONST = "const" + private const val ENUM = "enum" + private const val ONE_OF = "oneOf" + private const val ANY_OF = "anyOf" + private const val ALL_OF = "allOf" + + private const val TYPE_OBJECT = "object" + private const val TYPE_ARRAY = "array" + private const val TYPE_STRING = "string" + private const val TYPE_INTEGER = "integer" + private const val TYPE_NUMBER = "number" + private const val TYPE_BOOLEAN = "boolean" + private const val TYPE_NULL = "null" + + /** + * How far to follow references and nesting before giving up. A schema that refers to + * itself is legitimate, and the payload it describes is finite, so this is only reached by + * a cycle in the schema that the data never enters. + */ + private const val MAX_DEPTH = 32 + + private val mapper = ObjectMapper() + + /** + * The declared message [payload] is an instance of, or null when it is none of them. + * + * When more than one matches, the most specific wins: the one pinning down the most fields, + * through `required` and through `const`/`enum` properties. Ties go to the first declared. + * + * @param candidates the messages the contract declares for the reply, in declaration order + * @param componentSchemas where a `$ref` inside a payload schema is resolved + */ + fun classify( + payload: String?, + candidates: List, + componentSchemas: Map + ): AsyncApiMessage? { + + if (payload == null) { + return null + } + + val node = try { + mapper.readTree(payload) + } catch (e: JsonProcessingException) { + //not JSON, so it is none of the JSON-described messages + return null + } ?: return null + + return candidates + .filter { it.payload != null && matches(node, it.payload, componentSchemas, 0) } + .maxByOrNull { specificity(it.payload, componentSchemas) } + } + + private fun matches(node: JsonNode, schema: JsonNode, schemas: Map, depth: Int): Boolean { + + if (depth > MAX_DEPTH) { + return true + } + + //a reference that cannot be followed is something this cannot judge, so it does not reject + val s = resolve(schema, schemas) ?: return true + + if (!s.isObject) { + return true + } + + s.get(CONST)?.let { if (node != it) return false } + + s.get(ENUM)?.let { allowed -> if (allowed.isArray && allowed.none { it == node }) return false } + + s.get(TYPE)?.let { if (!isOfType(node, it)) return false } + + s.get(ALL_OF)?.let { all -> if (all.any { !matches(node, it, schemas, depth + 1) }) return false } + + s.get(ANY_OF)?.let { any -> if (any.none { matches(node, it, schemas, depth + 1) }) return false } + + //oneOf is read as "at least one": exclusivity is a validator's concern, not a classifier's + s.get(ONE_OF)?.let { one -> if (one.none { matches(node, it, schemas, depth + 1) }) return false } + + if (node.isObject) { + s.get(REQUIRED)?.let { required -> if (required.any { !node.has(it.asText()) }) return false } + + s.get(PROPERTIES)?.fields()?.forEach { (name, property) -> + val value = node.get(name) + if (value != null && !matches(value, property, schemas, depth + 1)) { + return false + } + } + } + + if (node.isArray) { + s.get(ITEMS)?.let { items -> + if (items.isObject && node.any { !matches(it, items, schemas, depth + 1) }) return false + } + } + + return true + } + + /** + * Whether [node] is of one of the types [type] names. JSON Schema writes it as one name or a + * list of them, and counts a number with no fractional part as an integer. + */ + private fun isOfType(node: JsonNode, type: JsonNode): Boolean { + + val names = if (type.isArray) type.map { it.asText() } else listOf(type.asText()) + + return names.any { name -> + when (name) { + TYPE_OBJECT -> node.isObject + TYPE_ARRAY -> node.isArray + TYPE_STRING -> node.isTextual + TYPE_INTEGER -> node.isIntegralNumber + || (node.isNumber && node.decimalValue().stripTrailingZeros().scale() <= 0) + TYPE_NUMBER -> node.isNumber + TYPE_BOOLEAN -> node.isBoolean + TYPE_NULL -> node.isNull + else -> true + } + } + } + + /** + * The schema itself, once any chain of `$ref` to a component schema is followed. Null when + * a reference points at something other than a whole component schema. + */ + private fun resolve(schema: JsonNode, schemas: Map): JsonNode? { + + var current = schema + + repeat(MAX_DEPTH) { + val ref = AsyncApiRefResolver.refOf(current) ?: return current + val key = AsyncApiRefResolver.refKey(ref, AsyncApiRefResolver.SCHEMA_PREFIX) ?: return null + current = schemas[key] ?: return null + } + + return null + } + + /** + * How many fields the schema pins down at its top level, which is what tells a specific + * message from a permissive one when both match. + */ + private fun specificity(schema: JsonNode, schemas: Map): Int { + + val s = resolve(schema, schemas) ?: return 0 + + val required = s.get(REQUIRED)?.size() ?: 0 + val pinned = s.get(PROPERTIES)?.count { it.has(CONST) || it.has(ENUM) } ?: 0 + + return required + pinned + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt index bbfea75981..6f494ccd26 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt @@ -41,6 +41,13 @@ class AsyncApiSampler : ApiWsSampler() { */ private val adHocInitialIndividuals: MutableList = mutableListOf() + /** + * The document the actions were built from. The fitness needs it again at execution time, + * to resolve addresses and to recognise which declared reply came back. + */ + lateinit var document: AsyncApiDocument + private set + /** * Start the service through the driver, read its document, and build one action per * publishable message. Anything the parser or the builder had to skip is reported. @@ -63,14 +70,14 @@ class AsyncApiSampler : ApiWsSampler() { val problem = infoDto.asyncApiProblem ?: throw SutProblemException("Missing problem definition object") - val schema = readSchema(problem) + document = readSchema(problem) val messages = AsyncApiActionBuilder.addActionsFromSchema( - schema, + document, actionCluster, AsyncApiGeneBuilder.options(config) ) - handleMessages(schema.warnings + messages) + handleMessages(document.warnings + messages) initSqlInfo(infoDto) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt new file mode 100644 index 0000000000..0bb530c748 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt @@ -0,0 +1,76 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.Inject +import org.evomaster.core.database.sql.SqlInsertBuilder +import org.evomaster.core.problem.api.service.ApiWsStructureMutator +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.Individual +import org.evomaster.core.search.service.mutator.MutatedGeneSpecification + +/** + * Changes how many messages a test publishes: one more, drawn from the action templates, or one + * fewer. Which messages, and what they say, is left to the gene mutations. + */ +class AsyncApiStructureMutator : ApiWsStructureMutator() { + + @Inject + private lateinit var sampler: AsyncApiSampler + + override fun mutateStructure( + individual: Individual, + evaluatedIndividual: EvaluatedIndividual<*>, + mutatedGenes: MutatedGeneSpecification?, + targets: Set + ) { + if (individual !is AsyncApiIndividual) { + throw IllegalArgumentException( + "Invalid: individual type to be mutated with AsyncApiStructureMutator should be" + + " AsyncApiIndividual but ${individual::class.java.simpleName}" + ) + } + + if (!individual.canMutateStructure()) return + if (config.maxTestSize == 1) return + + addOrRemoveAMessage(individual, mutatedGenes) + + if (config.trackingEnabled()) tag(individual, time.evaluatedIndividuals) + } + + private fun addOrRemoveAMessage(individual: AsyncApiIndividual, mutatedGenes: MutatedGeneSpecification?) { + + val size = individual.seeMainExecutableActions().size + + if ((size + 1 < config.maxTestSize) && (size <= 1 || randomness.nextBoolean())) { + val added = sampler.sampleRandomAction() + individual.addAction(action = added) + mutatedGenes?.addRemovedOrAddedByAction( + added, + individual.seeFixedMainActions().indexOf(added), + null, + false, + size + ) + } else { + val chosen = randomness.choose(individual.seeMainActionComponents().indices) + val removed = individual.seeMainExecutableActions()[chosen] + mutatedGenes?.addRemovedOrAddedByAction( + removed, + individual.seeFixedMainActions().indexOf(removed), + null, + true, + size + ) + individual.removeAction(chosen) + } + } + + override fun addInitializingActions(individual: EvaluatedIndividual<*>, mutatedGenes: MutatedGeneSpecification?) { + addInitializingActions(individual, mutatedGenes, sampler) + } + + override fun getSqlInsertBuilder(): SqlInsertBuilder? { + return sampler.sqlInsertBuilder + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt new file mode 100644 index 0000000000..2c0c5faba4 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt @@ -0,0 +1,304 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.inject.AbstractModule +import com.google.inject.Injector +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.google.inject.util.Modules +import com.netflix.governator.guice.LifecycleInjector +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.BaseModule +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.fireAndForget +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.silence +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.core.problem.enterprise.SampleType +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 +import org.evomaster.core.remote.service.RemoteController +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.IdMapper +import org.evomaster.core.search.service.Randomness +import org.evomaster.core.search.service.SearchGlobalState +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class AsyncApiBlackBoxFitnessTest { + + companion object { + private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" + + private const val ERROR = """{"error": {"code": 400, "message": "n must be >= 3"}}""" + + /** + * One operation that expects nothing back, and one whose message carries a header + * besides the one the correlation id is stamped into. + */ + private val EVENTS = """ + asyncapi: 3.0.0 + info: + title: Events + version: 1.0.0 + channels: + events: + address: app.events + messages: + event: + headers: + type: object + required: [tenant] + properties: + tenant: + type: string + correlationId: + type: string + correlationId: + location: '${'$'}message.header#/correlationId' + payload: + type: object + required: [id] + properties: + id: + type: string + operations: + publishEvent: + action: receive + channel: + ${'$'}ref: '#/channels/events' + """.trimIndent() + } + + private lateinit var injector: Injector + private lateinit var driver: FakeAsyncApiDriver + private lateinit var sampler: AsyncApiSampler + private lateinit var fitness: FitnessFunction + private lateinit var idMapper: IdMapper + + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + + private fun start(schemaText: String, answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) { + + val info = SutInfoDto().apply { + asyncApiProblem = AsyncApiProblemDto().apply { this.schemaText = schemaText } + defaultOutputFormat = SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 + } + driver = FakeAsyncApiDriver(info, answer) + + val args = arrayOf("--seed=42", "--problemType=ASYNCAPI", "--blackBox=false", "--createTests=false") + + val fake = object : AbstractModule() { + override fun configure() { + bind(RemoteController::class.java).toInstance(driver) + } + } + + injector = LifecycleInjector.builder() + .withModules(listOf(BaseModule(args), Modules.override(AsyncApiModule()).with(fake))) + .build().createInjector() + + sampler = injector.getInstance(AsyncApiSampler::class.java) + fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + idMapper = injector.getInstance(IdMapper::class.java) + } + + private fun startNcs(answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) = + start(AsyncApiAccess.readFromResource(NCS), answer) + + private fun individualOf(vararg names: String): AsyncApiIndividual { + + val randomness = injector.getInstance(Randomness::class.java) + + val actions = names.map { name -> + val template = sampler.seeAvailableActions().first { it.getName() == name } + (template.copy() as AsyncApiAction).apply { doInitialize(randomness) } + } + + return AsyncApiIndividual(SampleType.RANDOM, actions.toMutableList()).apply { + doGlobalInitialize(injector.getInstance(SearchGlobalState::class.java)) + } + } + + private fun evaluate(vararg names: String): EvaluatedIndividual = + fitness.calculateCoverage(individualOf(*names), modifiedSpec = null) + ?: fail("the fitness gave up on the individual") + + private fun coveredIds(evaluated: EvaluatedIndividual): Set = + evaluated.fitness.coveredTargets().map { idMapper.getDescriptiveId(it) }.toSet() + + private fun results(evaluated: EvaluatedIndividual): List = + evaluated.evaluatedMainActions().map { it.result as AsyncApiCallResult } + + @Test + fun testTheDriverIsToldEverythingItNeeds() { + + startNcs { replied(DOUBLE_RESULT) } + + evaluate("bessj") + + val dto = driver.published.single() + + assertEquals("bessj", dto.operationId) + assertEquals("bessjRequest", dto.channelName) + assertEquals("bessjRequest", dto.messageId) + //resolved from the document, so the driver never has to read it + assertEquals("ncs.bessj.request", dto.address) + assertEquals("application/json", dto.contentType) + assertEquals("ncs.bessj.reply", dto.replyAddress) + assertEquals(5000L, dto.replyTimeoutMs) + + val payload = ObjectMapper().readTree(dto.payload) + assertTrue(payload.isObject, "payload is not a JSON object: ${dto.payload}") + assertEquals(setOf("n", "x"), payload.fieldNames().asSequence().toSet()) + + assertEquals(AsyncApiActionDto.CORRELATION_IN_HEADER, dto.correlationLocation) + assertEquals("/correlationId", dto.correlationPointer) + assertFalse(dto.correlationId.isNullOrBlank()) + assertTrue(dto.headers.isEmpty()) + } + + @Test + fun testEachPublishedMessageGetsItsOwnCorrelationId() { + + startNcs { replied(DOUBLE_RESULT) } + + evaluate("bessj", "expint", "gammq") + evaluate("bessj", "expint", "gammq") + + val ids = driver.published.map { it.correlationId } + assertEquals(6, ids.size) + assertEquals(6, ids.toSet().size, "correlation ids repeat: $ids") + } + + @Test + fun testARecognisedReplyCoversTheOperationAndTheDeclaredMessage() { + + startNcs { replied(DOUBLE_RESULT) } + + val evaluated = evaluate("bessj") + val covered = coveredIds(evaluated) + + assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:bessj"), "$covered") + assertTrue(covered.contains("ASYNCAPI_REPLY:doubleResult:bessj"), "$covered") + assertTrue(covered.none { IdMapper.isFault(it) }, "$covered") + + val result = results(evaluated).single() + assertEquals(AsyncApiOutcome.REPLIED, result.getOutcome()) + assertEquals("doubleResult", result.getReplyMessage()) + assertEquals(DOUBLE_RESULT, result.getReplyPayload()) + assertEquals(true, result.getCorrelationMatched()) + assertFalse(result.stopping) + } + + @Test + fun testTheErrorReplyIsADifferentTargetFromTheResult() { + + startNcs { replied(ERROR) } + + val covered = coveredIds(evaluate("bessj")) + + assertTrue(covered.contains("ASYNCAPI_REPLY:error:bessj"), "$covered") + assertFalse(covered.contains("ASYNCAPI_REPLY:doubleResult:bessj")) + //an error reply the contract declares is the service behaving, not a fault + assertTrue(covered.none { IdMapper.isFault(it) }, "$covered") + } + + @Test + fun testAReplyTheContractDoesNotDeclareIsAFault() { + + startNcs { replied("""{"something": "else"}""") } + + val evaluated = evaluate("bessj") + val faults = evaluated.fitness.coveredTargets().filter { idMapper.isFault(it) } + + assertEquals(1, faults.size, coveredIds(evaluated).toString()) + assertTrue(idMapper.isSpecifiedFault(faults.single(), ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY)) + assertNull(results(evaluated).single().getReplyMessage()) + } + + @Test + fun testSilenceAfterAPromisedReplyIsAFault() { + + startNcs { silence(waited = 5000) } + + val evaluated = evaluate("bessj") + val covered = coveredIds(evaluated) + val faults = evaluated.fitness.coveredTargets().filter { idMapper.isFault(it) } + + assertTrue(covered.contains("ASYNCAPI_OUTCOME:NO_REPLY:bessj"), "$covered") + assertEquals(1, faults.size, "$covered") + assertTrue(idMapper.isSpecifiedFault(faults.single(), ExperimentalFaultCategory.ASYNCAPI_NO_REPLY)) + + val result = results(evaluated).single() + assertEquals(AsyncApiOutcome.NO_REPLY, result.getOutcome()) + assertEquals(5000L, result.getWaitedMs()) + //silence is a finding, not a broken setup: the test goes on + assertFalse(result.stopping) + } + + @Test + fun testAFireAndForgetOperationIsCoveredByBeingPublished() { + + start(EVENTS) { fireAndForget() } + + val evaluated = evaluate("publishEvent") + val covered = coveredIds(evaluated) + + assertTrue(covered.contains("ASYNCAPI_OUTCOME:PUBLISHED:publishEvent"), "$covered") + assertTrue(covered.none { IdMapper.isFault(it) }, "$covered") + assertEquals(AsyncApiOutcome.PUBLISHED, results(evaluated).single().getOutcome()) + + //no reply declared, so nothing to wait for + val dto = driver.published.single() + assertNull(dto.replyAddress) + assertNull(dto.replyTimeoutMs) + } + + @Test + fun testHeadersTravelAsAMapWithoutTheStampedOne() { + + start(EVENTS) { fireAndForget() } + + evaluate("publishEvent") + + val dto = driver.published.single() + + //the tenant header is the search's to vary; the correlation id is the driver's to stamp + assertEquals(setOf("tenant"), dto.headers.keys) + assertEquals(AsyncApiActionDto.CORRELATION_IN_HEADER, dto.correlationLocation) + assertEquals("/correlationId", dto.correlationPointer) + } + + @Test + fun testAMessageTheDriverCannotPublishStopsTheTest() { + + //a driver that cannot be reached: the remote controller reports that as no reply at all + startNcs { null } + + val evaluated = evaluate("bessj", "expint", "gammq") + + assertEquals(1, driver.published.size, "the test went on after a message that never left") + + val first = results(evaluated).first() + assertEquals(AsyncApiOutcome.PUBLISH_FAILED, first.getOutcome()) + assertTrue(first.stopping) + assertNotNull(first.getErrorMessage()) + + //a broken setup is not a finding about the service, so nothing is covered by it + assertTrue(coveredIds(evaluated).none { it.startsWith("ASYNCAPI") }, coveredIds(evaluated).toString()) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt new file mode 100644 index 0000000000..d5b216e74a --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt @@ -0,0 +1,83 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import com.webfuzzing.asyncapi.models.AsyncApiDocument +import com.webfuzzing.asyncapi.models.AsyncApiMessage +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class AsyncApiReplyClassifierTest { + + private val ncs: AsyncApiDocument = AsyncApiAccess.getAsyncApiFromResource("/asyncapi/sut/ncs-kafka.yaml") + + private val socket: AsyncApiDocument = + AsyncApiAccess.getAsyncApiFromResource("/asyncapi/artificial/websocket-reply.yaml") + + private fun repliesOf(document: AsyncApiDocument, operation: String): List = + document.replyMessagesOf(document.operations.getValue(operation)) + + private fun classify(document: AsyncApiDocument, operation: String, payload: String?): String? = + AsyncApiReplyClassifier.classify(payload, repliesOf(document, operation), document.componentSchemas)?.id + + @Test + fun testAResultIsToldFromAnError() { + + //both reply payloads are behind a $ref to a component schema + assertEquals("doubleResult", classify(ncs, "bessj", """{"resultAsDouble": 2.5}""")) + assertEquals("error", classify(ncs, "bessj", """{"error": {"code": 400, "message": "n must be >= 3"}}""")) + } + + @Test + fun testAReplyMatchingNoDeclaredMessageIsNotRecognised() { + + assertNull(classify(ncs, "bessj", """{"something": "else"}""")) + assertNull(classify(ncs, "bessj", """{"error": "not an object"}""")) + assertNull(classify(ncs, "bessj", """[1, 2, 3]""")) + } + + @Test + fun testTextThatIsNotJsonIsNotRecognised() { + + assertNull(classify(ncs, "bessj", "not json at all")) + assertNull(classify(ncs, "bessj", "")) + assertNull(classify(ncs, "bessj", null)) + } + + @Test + fun testAWholeNumberWrittenWithADecimalPointIsStillAnInteger() { + + //JSON Schema counts 3.0 as an integer, and services do write results that way + assertEquals("intResult", classify(ncs, "checkTriangle", """{"resultAsInt": 3.0}""")) + assertNull(classify(ncs, "checkTriangle", """{"resultAsInt": 3.5}""")) + } + + @Test + fun testTheMostSpecificMatchWinsWhenSeveralCouldFit() { + + /* + The socket's result messages declare no required fields, so an error payload also + fits them structurally. The error message requires its "error" field, which makes it + the more specific description, and so the one chosen. + */ + assertEquals("error", classify(socket, "recv_list_legs", """{"request_id": "r1", "error": {"code": 404}}""")) + assertEquals("listLegsResult", classify(socket, "recv_list_legs", """{"request_id": "r1", "legs": ["a", "b"]}""")) + } + + @Test + fun testATypeListIsHonoured() { + + //"leg" is declared as [object, "null"] + assertEquals("getLegResult", classify(socket, "recv_get_leg", """{"request_id": "r1", "leg": null}""")) + assertEquals("getLegResult", classify(socket, "recv_get_leg", """{"request_id": "r1", "leg": {"id": "x"}}""")) + assertNull(classify(socket, "recv_get_leg", """{"request_id": "r1", "leg": "not an object"}""")) + } + + @Test + fun testWhatItDoesNotUnderstandIsGivenTheBenefitOfTheDoubt() { + + //an array of legs with a non-string inside: items are checked, so this is rejected... + assertNull(classify(socket, "recv_list_legs", """{"legs": [1, 2]}""")) + //...but a format, a pattern or a bound is not read at all, so nothing is rejected on their account + assertEquals("doubleResult", classify(ncs, "bessj", """{"resultAsDouble": -1e308}""")) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSearchTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSearchTest.kt new file mode 100644 index 0000000000..7e294720dc --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSearchTest.kt @@ -0,0 +1,147 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.inject.AbstractModule +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.google.inject.util.Modules +import com.netflix.governator.guice.LifecycleInjector +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.BaseModule +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 +import org.evomaster.core.remote.service.RemoteController +import org.evomaster.core.search.algorithms.MioAlgorithm +import org.evomaster.core.search.service.IdMapper +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +/** + * A whole search, through the module Main binds, against a driver standing in for the NCS + * service over Kafka: the first time `--problemType ASYNCAPI` does everything but write tests. + */ +class AsyncApiSearchTest { + + companion object { + private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + private val NCS_OPERATIONS = setOf("checkTriangle", "bessj", "expint", "fisher", "gammq", "remainder") + + private val mapper = ObjectMapper() + } + + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + + /** + * Replies as the NCS service would, judging by the request alone: each operation has inputs + * it rejects, and answers those with the declared error message instead of a result. + */ + private fun ncsLike(call: AsyncApiActionDto): AsyncApiReplyDto { + + val request = try { + mapper.readTree(call.payload) + } catch (e: Exception) { + return replied(error("not JSON")) + } + + fun number(name: String) = request.get(name)?.takeIf { it.isNumber }?.asDouble() + + val a = number("a") + val b = number("b") + val c = number("c") + val n = number("n") + val m = number("m") + val x = number("x") + + val ok = when (call.operationId) { + "checkTriangle" -> a != null && b != null && c != null && a > 0 && b > 0 && c > 0 + "bessj" -> n != null && x != null && n >= 3 && n <= 1000 + "expint" -> n != null && x != null && n >= 0 && x >= 0 + "fisher" -> m != null && n != null && x != null && m in 1.0..1000.0 && n in 1.0..1000.0 + "gammq" -> a != null && x != null && a > 0 && x >= 0 + "remainder" -> a != null && b != null && b != 0.0 + else -> false + } + + if (!ok) { + return replied(error("rejected")) + } + + val integerResult = call.operationId == "checkTriangle" || call.operationId == "remainder" + + return replied(if (integerResult) """{"resultAsInt": 1}""" else """{"resultAsDouble": 0.5}""") + } + + private fun error(message: String) = """{"error": {"code": 400, "message": "$message"}}""" + + @Test + fun testASearchReachesBothDeclaredRepliesOfAnOperation() { + + val info = SutInfoDto().apply { + asyncApiProblem = AsyncApiProblemDto().apply { schemaText = AsyncApiAccess.readFromResource(NCS) } + defaultOutputFormat = SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 + } + val driver = FakeAsyncApiDriver(info) { ncsLike(it) } + + //MIO is asked for explicitly, since it is what is instantiated below and tracking follows the option + val args = arrayOf( + "--seed=42", + "--problemType=ASYNCAPI", + "--blackBox=true", + "--algorithm=MIO", + "--createTests=false", + "--stoppingCriterion=ACTION_EVALUATIONS", + "--maxEvaluations=100", + "--maxTestSize=3", + "--useTimeInFeedbackSampling=false" + ) + + val fake = object : AbstractModule() { + override fun configure() { + bind(RemoteController::class.java).toInstance(driver) + } + } + + val injector = LifecycleInjector.builder() + .withModules(listOf(BaseModule(args), Modules.override(AsyncApiModule()).with(fake))) + .build().createInjector() + + val mio = injector.getInstance(Key.get(object : TypeLiteral>() {})) + val solution = mio.search() + + val idMapper = injector.getInstance(IdMapper::class.java) + val covered = solution.overall.coveredTargets().map { idMapper.getDescriptiveId(it) }.toSet() + + /* + Every operation rejects some inputs, and a hundred evaluations are plenty for the + search to meet both the result and the error of at least one of them. That the two + are distinct targets is the point of the whole exercise; which operations get there + first depends on the seed, so no particular one is asked for. + */ + val bothRepliesReached = NCS_OPERATIONS.filter { op -> + covered.contains("ASYNCAPI_REPLY:error:$op") + && (covered.contains("ASYNCAPI_REPLY:doubleResult:$op") || covered.contains("ASYNCAPI_REPLY:intResult:$op")) + } + assertTrue(bothRepliesReached.isNotEmpty(), "no operation reached both of its replies: $covered") + + //every operation was published to and answered + NCS_OPERATIONS.forEach { assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:$it"), "$it missing in $covered") } + + //the stand-in always answers with a declared message, so nothing here is a fault + assertTrue(covered.none { IdMapper.isFault(it) }, "$covered") + + assertTrue(solution.individuals.isNotEmpty()) + assertTrue(driver.published.size >= 50, "only ${driver.published.size} messages published") + assertEquals(driver.published.size, driver.published.map { it.correlationId }.toSet().size) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt new file mode 100644 index 0000000000..806c1a9f16 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt @@ -0,0 +1,76 @@ +package org.evomaster.core.problem.asyncapi.service + +import org.evomaster.client.java.controller.api.dto.ActionDto +import org.evomaster.client.java.controller.api.dto.ControllerInfoDto +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.TestResultsDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.problem.external.service.DummyController +import org.evomaster.core.remote.service.RemoteController + +/** + * A driver that publishes nothing: it records what it was asked to publish, and answers each + * message with whatever [answer] decides. No instrumentation, so it reports no code targets. + */ +class FakeAsyncApiDriver( + private val info: SutInfoDto, + private val answer: (AsyncApiActionDto) -> AsyncApiReplyDto? +) : RemoteController by DummyController() { + + companion object { + + fun replied(payload: String, correlationMatched: Boolean = true) = AsyncApiReplyDto().apply { + published = true + replyExpected = true + replyReceived = true + replyPayload = payload + this.correlationMatched = correlationMatched + waitedMs = 12 + } + + fun silence(waited: Long = 5000) = AsyncApiReplyDto().apply { + published = true + replyExpected = true + replyReceived = false + waitedMs = waited + } + + fun fireAndForget() = AsyncApiReplyDto().apply { + published = true + replyExpected = false + } + } + + /** + * Everything this driver was asked to publish, in order. + */ + val published: MutableList = mutableListOf() + + override fun checkConnection() {} + + override fun startSUT() = true + + override fun resetSUT() = true + + override fun startANewSearch() = true + + override fun getSutInfo() = info + + override fun getControllerInfo() = ControllerInfoDto() + + override fun registerNewAction(actionDto: ActionDto) = true + + override fun executeNewAsyncApiActionAndGetReply(actionDto: ActionDto): AsyncApiReplyDto? { + val call = actionDto.asyncApiCall + published.add(call) + return answer(call)?.apply { index = actionDto.index } + } + + override fun getTestResults( + ids: Set, + ignoreKillSwitch: Boolean, + fullyCovered: Boolean, + descriptiveIds: Boolean + ) = TestResultsDto() +} From bf7e1f559ad490e9750d329bec187435b8717aa3 Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Sat, 12 Sep 2026 20:03:32 -0300 Subject: [PATCH 3/9] AsyncAPI 3.x: the structure mutator keeps a test within bounds Review: with room for exactly two messages, a one-message test was mutated by removal, leaving a test that publishes nothing; and a test could never grow to the maximum the user allowed, only to one less. Both bounds are now stated as such, and AsyncApiStructureMutatorTest holds them. Seeding test cases is refused at start-up, like writing them, instead of failing inside the sampler. The whole-search suite is named after the class it exercises, AsyncApiModule, and the three suites that build the injector share how they do it. --- .../kotlin/org/evomaster/core/EMConfig.kt | 4 + .../service/AsyncApiStructureMutator.kt | 10 +- .../kotlin/org/evomaster/core/EMConfigTest.kt | 11 ++ .../service/AsyncApiBlackBoxFitnessTest.kt | 143 +++++++++++++--- ...ApiSearchTest.kt => AsyncApiModuleTest.kt} | 32 +--- .../service/AsyncApiReplyClassifierTest.kt | 157 ++++++++++++++++++ .../service/AsyncApiStructureMutatorTest.kt | 118 +++++++++++++ .../asyncapi/service/AsyncApiTestInjector.kt | 39 +++++ 8 files changed, 458 insertions(+), 56 deletions(-) rename core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/{AsyncApiSearchTest.kt => AsyncApiModuleTest.kt} (80%) create mode 100644 core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt create mode 100644 core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 95b9a0ab7d..2e306d2491 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -811,6 +811,10 @@ class EMConfig { " For the time being, run with '--createTests false' to only search for faults.") } + if (problemType == ProblemType.ASYNCAPI && seedTestCases) { + throw ConfigProblemException("Seeding test cases is not supported for AsyncAPI services yet") + } + if (problemType == ProblemType.RPC && createTests && (enablePureRPCTestGeneration || enableRPCAssertionWithInstance) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt index 0bb530c748..c3b237f2ae 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt @@ -42,7 +42,15 @@ class AsyncApiStructureMutator : ApiWsStructureMutator() { val size = individual.seeMainExecutableActions().size - if ((size + 1 < config.maxTestSize) && (size <= 1 || randomness.nextBoolean())) { + //a test keeps at least one message, and never grows past what the user allowed + val canAdd = size < config.maxTestSize + val canRemove = size > 1 + + if (!canAdd && !canRemove) { + return + } + + if (canAdd && (!canRemove || randomness.nextBoolean())) { val added = sampler.sampleRandomAction() individual.addAction(action = added) mutatedGenes?.addRemovedOrAddedByAction( diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 0696df06da..98cc8d3bd3 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -799,4 +799,15 @@ internal class EMConfigTest{ //so a run has to say it only wants the search EMConfig().updateProperties(parser.parse("--problemType", "ASYNCAPI", "--createTests", "false")) } + + @Test + fun testAsyncApiCannotSeedTestsYet(){ + + val parser = EMConfig.getOptionParser() + + assertThrows { + EMConfig().updateProperties(parser.parse( + "--problemType", "ASYNCAPI", "--createTests", "false", "--seedTestCases", "true", "--seedTestCasesPath", "seeds.json")) + } + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt index 2c0c5faba4..c24dba2139 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt @@ -1,18 +1,12 @@ package org.evomaster.core.problem.asyncapi.service import com.fasterxml.jackson.databind.ObjectMapper -import com.google.inject.AbstractModule import com.google.inject.Injector import com.google.inject.Key import com.google.inject.TypeLiteral -import com.google.inject.util.Modules -import com.netflix.governator.guice.LifecycleInjector import com.webfuzzing.asyncapi.access.AsyncApiAccess -import org.evomaster.client.java.controller.api.dto.SutInfoDto -import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto -import org.evomaster.core.BaseModule import org.evomaster.core.problem.asyncapi.data.AsyncApiAction import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual @@ -23,7 +17,6 @@ import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion. import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.problem.enterprise.SampleType import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 -import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.service.FitnessFunction import org.evomaster.core.search.service.IdMapper @@ -58,10 +51,16 @@ class AsyncApiBlackBoxFitnessTest { event: headers: type: object - required: [tenant] + required: [tenant, meta] properties: tenant: type: string + meta: + type: object + required: [v] + properties: + v: + type: integer correlationId: type: string correlationId: @@ -78,6 +77,41 @@ class AsyncApiBlackBoxFitnessTest { channel: ${'$'}ref: '#/channels/events' """.trimIndent() + + /** + * A request whose reply comes back wherever the request says, rather than on a channel + * the contract fixes. + */ + private val DYNAMIC_REPLY = """ + asyncapi: 3.0.0 + info: + title: Dynamic reply + version: 1.0.0 + channels: + requests: + address: app.requests + messages: + request: + headers: + type: object + properties: + replyTo: + type: string + payload: + type: object + required: [id] + properties: + id: + type: string + operations: + ask: + action: receive + channel: + ${'$'}ref: '#/channels/requests' + reply: + address: + location: '${'$'}message.header#/replyTo' + """.trimIndent() } private lateinit var injector: Injector @@ -93,23 +127,8 @@ class AsyncApiBlackBoxFitnessTest { private fun start(schemaText: String, answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) { - val info = SutInfoDto().apply { - asyncApiProblem = AsyncApiProblemDto().apply { this.schemaText = schemaText } - defaultOutputFormat = SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 - } - driver = FakeAsyncApiDriver(info, answer) - - val args = arrayOf("--seed=42", "--problemType=ASYNCAPI", "--blackBox=false", "--createTests=false") - - val fake = object : AbstractModule() { - override fun configure() { - bind(RemoteController::class.java).toInstance(driver) - } - } - - injector = LifecycleInjector.builder() - .withModules(listOf(BaseModule(args), Modules.override(AsyncApiModule()).with(fake))) - .build().createInjector() + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(schemaText), answer) + injector = AsyncApiTestInjector.create(driver, "--blackBox=false") sampler = injector.getInstance(AsyncApiSampler::class.java) fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) @@ -277,10 +296,80 @@ class AsyncApiBlackBoxFitnessTest { val dto = driver.published.single() - //the tenant header is the search's to vary; the correlation id is the driver's to stamp - assertEquals(setOf("tenant"), dto.headers.keys) + //the declared headers are the search's to vary; the correlation id is the driver's to stamp + assertEquals(setOf("tenant", "meta"), dto.headers.keys) assertEquals(AsyncApiActionDto.CORRELATION_IN_HEADER, dto.correlationLocation) assertEquals("/correlationId", dto.correlationPointer) + + //a header that is itself structured travels as its JSON + val meta = ObjectMapper().readTree(dto.headers.getValue("meta")) + assertTrue(meta.isObject && meta.has("v"), "meta header: ${dto.headers["meta"]}") + } + + @Test + fun testTheCorrelationIdMayBelongInThePayload() { + + start(AsyncApiAccess.readFromResource("/asyncapi/artificial/websocket-reply.yaml")) { + replied("""{"request_id": "r", "legs": []}""") + } + + evaluate("recv_list_legs") + + val dto = driver.published.single() + + //MQTT 3.1.1 and raw WebSocket have no metadata, so such documents carry the id inside the message + assertEquals(AsyncApiActionDto.CORRELATION_IN_PAYLOAD, dto.correlationLocation) + assertEquals("/request_id", dto.correlationPointer) + assertEquals("/v1/vsi", dto.address) + assertEquals("/v1/vsi", dto.replyAddress) + } + + @Test + fun testAReplyAddressAnnouncedAtRunTimeIsNotWaitedForYet() { + + start(DYNAMIC_REPLY) { fireAndForget() } + + val evaluated = evaluate("ask") + + //the contract does promise a reply, but there is nowhere fixed to wait for it + assertTrue((sampler.seeAvailableActions().single() as AsyncApiAction).expectsReply()) + val dto = driver.published.single() + assertNull(dto.replyAddress) + assertNull(dto.replyTimeoutMs) + + assertEquals(AsyncApiOutcome.PUBLISHED, results(evaluated).single().getOutcome()) + assertTrue(coveredIds(evaluated).none { IdMapper.isFault(it) }) + } + + @Test + fun testADriverThatCouldNotPublishSaysWhy() { + + startNcs { AsyncApiReplyDto().apply { published = false; errorMessage = "broker unreachable" } } + + val evaluated = evaluate("bessj", "expint") + + val first = results(evaluated).first() + assertEquals(AsyncApiOutcome.PUBLISH_FAILED, first.getOutcome()) + assertEquals("broker unreachable", first.getErrorMessage()) + assertTrue(first.stopping) + assertEquals(1, driver.published.size) + } + + @Test + fun testAReplyThatDidNotCarryTheCorrelationIdBackIsRecordedAsSuch() { + + startNcs { replied(DOUBLE_RESULT, correlationMatched = false) } + + val evaluated = evaluate("bessj") + + /* + From outside there is no telling a defect from a service that correlates by some + business key instead, so this is recorded, not judged: the reply still counts. + */ + val result = results(evaluated).single() + assertEquals(AsyncApiOutcome.REPLIED, result.getOutcome()) + assertEquals(false, result.getCorrelationMatched()) + assertTrue(coveredIds(evaluated).contains("ASYNCAPI_REPLY:doubleResult:bessj")) } @Test diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSearchTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt similarity index 80% rename from core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSearchTest.kt rename to core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt index 7e294720dc..c99bfd971d 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSearchTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt @@ -1,22 +1,14 @@ package org.evomaster.core.problem.asyncapi.service -import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper -import com.google.inject.AbstractModule import com.google.inject.Key import com.google.inject.TypeLiteral -import com.google.inject.util.Modules -import com.netflix.governator.guice.LifecycleInjector import com.webfuzzing.asyncapi.access.AsyncApiAccess -import org.evomaster.client.java.controller.api.dto.SutInfoDto -import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto -import org.evomaster.core.BaseModule import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 -import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.algorithms.MioAlgorithm import org.evomaster.core.search.service.IdMapper import org.junit.jupiter.api.Assertions.* @@ -27,7 +19,7 @@ import org.junit.jupiter.api.Test * A whole search, through the module Main binds, against a driver standing in for the NCS * service over Kafka: the first time `--problemType ASYNCAPI` does everything but write tests. */ -class AsyncApiSearchTest { +class AsyncApiModuleTest { companion object { private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" @@ -87,35 +79,19 @@ class AsyncApiSearchTest { @Test fun testASearchReachesBothDeclaredRepliesOfAnOperation() { - val info = SutInfoDto().apply { - asyncApiProblem = AsyncApiProblemDto().apply { schemaText = AsyncApiAccess.readFromResource(NCS) } - defaultOutputFormat = SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 - } - val driver = FakeAsyncApiDriver(info) { ncsLike(it) } + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { ncsLike(it) } //MIO is asked for explicitly, since it is what is instantiated below and tracking follows the option - val args = arrayOf( - "--seed=42", - "--problemType=ASYNCAPI", + val injector = AsyncApiTestInjector.create( + driver, "--blackBox=true", "--algorithm=MIO", - "--createTests=false", "--stoppingCriterion=ACTION_EVALUATIONS", "--maxEvaluations=100", "--maxTestSize=3", "--useTimeInFeedbackSampling=false" ) - val fake = object : AbstractModule() { - override fun configure() { - bind(RemoteController::class.java).toInstance(driver) - } - } - - val injector = LifecycleInjector.builder() - .withModules(listOf(BaseModule(args), Modules.override(AsyncApiModule()).with(fake))) - .build().createInjector() - val mio = injector.getInstance(Key.get(object : TypeLiteral>() {})) val solution = mio.search() diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt index d5b216e74a..bc3cffd9be 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt @@ -13,6 +13,88 @@ class AsyncApiReplyClassifierTest { private val socket: AsyncApiDocument = AsyncApiAccess.getAsyncApiFromResource("/asyncapi/artificial/websocket-reply.yaml") + /** + * Replies shaped by the parts of JSON Schema the corpus documents happen not to use. + */ + private val shapes: AsyncApiDocument = AsyncApiAccess.parseFromText( + """ + asyncapi: 3.0.0 + info: + title: Shapes + version: 1.0.0 + components: + messages: + pinned: + payload: + ${'$'}ref: '#/components/schemas/Pinned' + tagged: + payload: + type: object + required: [kind] + properties: + kind: + enum: [a, b] + either: + payload: + oneOf: + - ${'$'}ref: '#/components/schemas/Left' + - ${'$'}ref: '#/components/schemas/Right' + both: + payload: + allOf: + - ${'$'}ref: '#/components/schemas/Left' + - type: object + required: [extra] + scalar: + payload: + anyOf: + - type: string + - type: boolean + numbers: + payload: + type: array + items: + type: integer + flag: + payload: + type: boolean + exotic: + payload: + type: something-this-does-not-know + part: + payload: + ${'$'}ref: '#/components/schemas/Pinned/properties/kind' + schemas: + Pinned: + ${'$'}ref: '#/components/schemas/PinnedTarget' + PinnedTarget: + type: object + required: [kind] + properties: + kind: + const: pinned + Left: + type: object + required: [left] + properties: + left: + type: boolean + Right: + type: object + required: [right] + properties: + right: + type: string + """.trimIndent() + ) + + private fun classifyAmong(payload: String, vararg candidates: String): String? = + AsyncApiReplyClassifier.classify( + payload, + candidates.map { shapes.messages.getValue(it) }, + shapes.componentSchemas + )?.id + private fun repliesOf(document: AsyncApiDocument, operation: String): List = document.replyMessagesOf(document.operations.getValue(operation)) @@ -80,4 +162,79 @@ class AsyncApiReplyClassifierTest { //...but a format, a pattern or a bound is not read at all, so nothing is rejected on their account assertEquals("doubleResult", classify(ncs, "bessj", """{"resultAsDouble": -1e308}""")) } + + @Test + fun testAConstDiscriminatorIsReadThroughAChainOfReferences() { + + //the message points at a schema that is itself only a reference to the real one + assertEquals("pinned", classifyAmong("""{"kind": "pinned"}""", "pinned", "tagged")) + assertNull(classifyAmong("""{"kind": "other"}""", "pinned", "tagged")) + } + + @Test + fun testAnEnumDiscriminatorIsRead() { + + assertEquals("tagged", classifyAmong("""{"kind": "a"}""", "pinned", "tagged")) + assertEquals("tagged", classifyAmong("""{"kind": "b"}""", "pinned", "tagged")) + assertNull(classifyAmong("""{"kind": "z"}""", "pinned", "tagged")) + } + + @Test + fun testOneOfMatchesEitherBranch() { + + assertEquals("either", classifyAmong("""{"left": true}""", "either")) + assertEquals("either", classifyAmong("""{"right": "r"}""", "either")) + assertNull(classifyAmong("""{"neither": 1}""", "either")) + } + + @Test + fun testAllOfNeedsEveryBranch() { + + assertEquals("both", classifyAmong("""{"left": true, "extra": 1}""", "both")) + assertNull(classifyAmong("""{"left": true}""", "both")) + assertNull(classifyAmong("""{"extra": 1}""", "both")) + } + + @Test + fun testAnyOfMatchesAnyBranch() { + + assertEquals("scalar", classifyAmong("\"text\"", "scalar")) + assertEquals("scalar", classifyAmong("true", "scalar")) + assertNull(classifyAmong("1", "scalar")) + } + + @Test + fun testArrayItemsAreChecked() { + + assertEquals("numbers", classifyAmong("[1, 2, 3]", "numbers")) + assertEquals("numbers", classifyAmong("[]", "numbers")) + assertNull(classifyAmong("""[1, "two"]""", "numbers")) + assertNull(classifyAmong("""{"not": "an array"}""", "numbers")) + } + + @Test + fun testABooleanIsNotItsSpelling() { + + assertEquals("flag", classifyAmong("false", "flag")) + assertNull(classifyAmong("\"false\"", "flag")) + } + + @Test + fun testATypeItDoesNotKnowRejectsNothing() { + + assertEquals("exotic", classifyAmong("""{"anything": 1}""", "exotic")) + } + + @Test + fun testAReferenceIntoTheMiddleOfASchemaCannotBeJudgedSoItIsNotRejected() { + + /* + "#/components/schemas/Pinned/properties/kind" names a part of a schema, which is + legal and which the classifier does not follow. Rejecting on what it cannot read + would turn every such reply into a false fault, so it matches instead -- and loses + to anything specific that also matches. + */ + assertEquals("part", classifyAmong("""{"anything": 1}""", "part")) + assertEquals("pinned", classifyAmong("""{"kind": "pinned"}""", "part", "pinned")) + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt new file mode 100644 index 0000000000..cc8c7aa094 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt @@ -0,0 +1,118 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.mutator.StructureMutator +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class AsyncApiStructureMutatorTest { + + companion object { + private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + private val NCS_OPERATIONS = setOf("checkTriangle", "bessj", "expint", "fisher", "gammq", "remainder") + } + + private lateinit var sampler: AsyncApiSampler + private lateinit var mutator: AsyncApiStructureMutator + private lateinit var fitness: FitnessFunction + + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + + private fun start(maxTestSize: Int) { + + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { + replied("""{"resultAsDouble": 1.0}""") + } + val injector = AsyncApiTestInjector.create(driver, "--blackBox=false", "--maxTestSize=$maxTestSize") + + sampler = injector.getInstance(AsyncApiSampler::class.java) + mutator = injector.getInstance(StructureMutator::class.java) as AsyncApiStructureMutator + fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + } + + private fun evaluate(individual: AsyncApiIndividual): EvaluatedIndividual = + fitness.calculateCoverage(individual, modifiedSpec = null) ?: fail("the fitness gave up on the individual") + + /** + * Mutate the structure [times] over, each time from the previous result, and return the + * sizes seen along the way. + */ + private fun sizesAlong(times: Int): List { + + var evaluated = evaluate(sampler.sample(forceRandomSample = true)) + val sizes = mutableListOf() + + repeat(times) { + val copy = evaluated.individual.copy() as AsyncApiIndividual + mutator.mutateStructure(copy, evaluated, null, setOf()) + sizes.add(copy.seeMainExecutableActions().size) + evaluated = evaluate(copy) + } + + return sizes + } + + @Test + fun testATestKeepsAtLeastOneMessageAndNeverGrowsPastTheMaximum() { + + start(maxTestSize = 2) + + val sizes = sizesAlong(40) + + assertTrue(sizes.all { it in 1..2 }, "sizes seen: $sizes") + //with room for exactly two, every mutation has to go the other way from the last + assertEquals(setOf(1, 2), sizes.toSet(), "sizes seen: $sizes") + } + + @Test + fun testTheMaximumCanBeReached() { + + start(maxTestSize = 4) + + val sizes = sizesAlong(60) + + assertTrue(sizes.all { it in 1..4 }, "sizes seen: $sizes") + assertTrue(sizes.contains(4), "the search never got to publish four messages: $sizes") + } + + @Test + fun testAnAddedMessageIsOneTheDocumentDeclares() { + + start(maxTestSize = 5) + + var evaluated = evaluate(sampler.sample(forceRandomSample = true)) + + repeat(30) { + val copy = evaluated.individual.copy() as AsyncApiIndividual + mutator.mutateStructure(copy, evaluated, null, setOf()) + assertTrue(copy.seeMainExecutableActions().all { it.getName() in NCS_OPERATIONS }) + evaluated = evaluate(copy) + } + } + + @Test + fun testNothingChangesWhenOnlyOneMessageIsAllowed() { + + start(maxTestSize = 1) + + val evaluated = evaluate(sampler.sample(forceRandomSample = true)) + val before = evaluated.individual.seeMainExecutableActions().map { it.getName() } + + val copy = evaluated.individual.copy() as AsyncApiIndividual + mutator.mutateStructure(copy, evaluated, null, setOf()) + + assertEquals(before, copy.seeMainExecutableActions().map { it.getName() }) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt new file mode 100644 index 0000000000..d8feee1dcf --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt @@ -0,0 +1,39 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.AbstractModule +import com.google.inject.Injector +import com.google.inject.util.Modules +import com.netflix.governator.guice.LifecycleInjector +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto +import org.evomaster.core.BaseModule +import org.evomaster.core.remote.service.RemoteController + +/** + * The injector Main would build for an AsyncAPI search, with the driver replaced by a fake. + */ +object AsyncApiTestInjector { + + /** + * What a driver declares for a service whose document it hands over as text. + */ + fun sutInfo(schemaText: String): SutInfoDto = SutInfoDto().apply { + asyncApiProblem = AsyncApiProblemDto().apply { this.schemaText = schemaText } + defaultOutputFormat = SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 + } + + fun create(driver: FakeAsyncApiDriver, vararg options: String): Injector { + + val args = arrayOf("--seed=42", "--problemType=ASYNCAPI", "--createTests=false") + options + + val fake = object : AbstractModule() { + override fun configure() { + bind(RemoteController::class.java).toInstance(driver) + } + } + + return LifecycleInjector.builder() + .withModules(listOf(BaseModule(args), Modules.override(AsyncApiModule()).with(fake))) + .build().createInjector() + } +} From e18bac73793aeeb5c376a3cd66bb8cfa5b4f8db8 Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Sun, 13 Sep 2026 12:05:16 -0300 Subject: [PATCH 4/9] AsyncAPI 3.x: name the separators, spell the query parameter once Review: the ':' joining a target id and the '-' inside a correlation id were spelled out where used; they are constants now, and the two kinds of target id are built in one place each. The remote controller named its queryFromDatabase parameter four times over, once in the new call and three in the ones it mirrors; it is a constant now. The fake driver's one field comes before its companion. --- .../service/AsyncApiBlackBoxFitness.kt | 22 ++++++++++++++++--- .../service/RemoteControllerImplementation.kt | 14 ++++++++---- .../asyncapi/service/FakeAsyncApiDriver.kt | 10 ++++----- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt index a3231b4482..c999478ef1 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt @@ -50,9 +50,25 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { */ const val REPLY_TARGET_PREFIX = "ASYNCAPI_REPLY" + private const val TARGET_SEPARATOR = ":" + + private const val CORRELATION_SEPARATOR = "-" + private const val DEFAULT_CONTENT_TYPE = "application/json" private val mapper = ObjectMapper() + + /** + * The id of the target covered when publishing to [actionName] had [outcome]. + */ + fun outcomeTargetId(outcome: AsyncApiOutcome, actionName: String): String = + listOf(OUTCOME_TARGET_PREFIX, outcome.name, actionName).joinToString(TARGET_SEPARATOR) + + /** + * The id of the target covered when a reply to [actionName] was recognised as [messageId]. + */ + fun replyTargetId(messageId: String, actionName: String): String = + listOf(REPLY_TARGET_PREFIX, messageId, actionName).joinToString(TARGET_SEPARATOR) } @Inject @@ -168,7 +184,7 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { val name = action.getName() val outcome = result.getOutcome()!! - fv.updateTarget(idMapper.handleLocalTarget("$OUTCOME_TARGET_PREFIX:${outcome.name}:$name"), 1.0, index) + fv.updateTarget(idMapper.handleLocalTarget(outcomeTargetId(outcome, name)), 1.0, index) when (outcome) { @@ -204,7 +220,7 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { } result.setReplyMessage(recognised.id) - fv.updateTarget(idMapper.handleLocalTarget("$REPLY_TARGET_PREFIX:${recognised.id}:$name"), 1.0, index) + fv.updateTarget(idMapper.handleLocalTarget(replyTargetId(recognised.id, name)), 1.0, index) } /** @@ -233,7 +249,7 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { dto.contentType = message?.contentType ?: document.defaultContentType ?: DEFAULT_CONTENT_TYPE dto.headers = headersOf(action) - dto.correlationId = "$runId-${published++}" + dto.correlationId = runId + CORRELATION_SEPARATOR + published++ message?.correlationId?.let { dto.correlationLocation = if (it.source == AsyncApiCorrelationId.Source.HEADER) { AsyncApiActionDto.CORRELATION_IN_HEADER diff --git a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt index e850ecaebc..f8a1589160 100644 --- a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt +++ b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt @@ -36,6 +36,12 @@ import javax.ws.rs.core.Response class RemoteControllerImplementation() : RemoteController{ companion object { + + /** + * Query parameter telling the driver whether SQL heuristics are computed from what it + * reads back, rather than from what was inserted. + */ + private const val QUERY_FROM_DATABASE = "queryFromDatabase" val log: Logger = LoggerFactory.getLogger(RemoteControllerImplementation::class.java) } @@ -343,7 +349,7 @@ class RemoteControllerImplementation() : RemoteController{ .queryParam("killSwitch", !ignoreKillSwitch && config.killSwitch) .queryParam("fullyCovered", fullyCovered) .queryParam("descriptiveIds", descriptiveIds) - .queryParam("queryFromDatabase", !config.useInsertionForSqlHeuristics) + .queryParam(QUERY_FROM_DATABASE, !config.useInsertionForSqlHeuristics) .request(MediaType.APPLICATION_JSON_TYPE) .post(Entity.entity(ids, MediaType.APPLICATION_JSON_TYPE)) } @@ -398,7 +404,7 @@ class RemoteControllerImplementation() : RemoteController{ val response = makeHttpCall { getWebTarget() .path(ControllerConstants.NEW_ACTION) - .queryParam("queryFromDatabase", !config.useInsertionForSqlHeuristics) + .queryParam(QUERY_FROM_DATABASE, !config.useInsertionForSqlHeuristics) .request() .put(Entity.entity(actionDto, MediaType.APPLICATION_JSON_TYPE)) } @@ -417,7 +423,7 @@ class RemoteControllerImplementation() : RemoteController{ val response = makeHttpCall { getWebTarget() .path(ControllerConstants.NEW_ACTION) - .queryParam("queryFromDatabase", !config.useInsertionForSqlHeuristics) + .queryParam(QUERY_FROM_DATABASE, !config.useInsertionForSqlHeuristics) .request() .put(Entity.entity(actionDto, MediaType.APPLICATION_JSON_TYPE)) } @@ -491,7 +497,7 @@ class RemoteControllerImplementation() : RemoteController{ getWebTarget() .path(ControllerConstants.SCHEDULE_TASKS_COMMAND) // shall we set `killSwitch` as true? - .queryParam("queryFromDatabase", !config.useInsertionForSqlHeuristics) + .queryParam(QUERY_FROM_DATABASE, !config.useInsertionForSqlHeuristics) .request() .post(Entity.entity(invocationDto, MediaType.APPLICATION_JSON_TYPE)) } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt index 806c1a9f16..0d9ff66d70 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt @@ -18,6 +18,11 @@ class FakeAsyncApiDriver( private val answer: (AsyncApiActionDto) -> AsyncApiReplyDto? ) : RemoteController by DummyController() { + /** + * Everything this driver was asked to publish, in order. + */ + val published: MutableList = mutableListOf() + companion object { fun replied(payload: String, correlationMatched: Boolean = true) = AsyncApiReplyDto().apply { @@ -42,11 +47,6 @@ class FakeAsyncApiDriver( } } - /** - * Everything this driver was asked to publish, in order. - */ - val published: MutableList = mutableListOf() - override fun checkConnection() {} override fun startSUT() = true From d5c242a42bf561c6e657fc7de8bba25d50bed260 Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Wed, 16 Sep 2026 18:55:30 -0300 Subject: [PATCH 5/9] AsyncAPI 3.x: publish where the contract says, and report only real faults Review findings, in order of what they cost a run. The address a message is published to ignored protocol bindings. A Kafka channel usually names no address and carries its topic in its binding -- microcks.yaml, already a test resource here, is exactly that shape -- so messages went to the channel's name and reached nobody. The parser's effectiveAddress() exists for this and was never called; when a channel names no address at all, the binding's topic is now taken as the one destination the document actually gives. An address left holding {placeholders} is warned about rather than published to silently. Deciding whether a number is an integer asked Jackson for a BigDecimal, which throws for a value that overflows a double. Nothing between the classifier and the search loop catches it, so one odd reply ended the run. Three ways a valid reply was reported as a fault: a body that is empty or not JSON, a const or enum written 1.0 against a reply carrying 1, and a type or combinator this cannot read. A reply with nothing to match against is now no finding at all, numbers compare by value, and what cannot be read rejects nothing. A $ref that points into a schema is followed instead of matching everything, which had quietly disabled the undeclared-reply oracle for any operation written that way, and specificity counts through the combinators so the more specific message wins. Faults ignored isEnabledFaultCategory, so two experimental oracles fired on a default run and --disabledOracleCodes did nothing. They are recorded on the action result now, which is where the reports count faults from; before, a run covered fault targets while every report said zero. AsyncApiCallResult was the only action result not overriding matchedType. An inferred problem type no longer dies on the test-generation constraint: when the driver reports an AsyncAPI service, createTests is turned off with a warning instead of throwing after the SUT has already been started. The class is AsyncApiFitness: it is bound as the only fitness and reads white-box coverage, so BlackBox in its name said something untrue. --- .../main/kotlin/org/evomaster/core/Main.kt | 12 ++ .../asyncapi/data/AsyncApiCallResult.kt | 5 + .../problem/asyncapi/data/AsyncApiOutcome.kt | 3 - ...iBlackBoxFitness.kt => AsyncApiFitness.kt} | 191 +++++++++++++---- .../asyncapi/service/AsyncApiModule.kt | 4 +- .../service/AsyncApiReplyClassifier.kt | 197 ++++++++++++++---- ...xFitnessTest.kt => AsyncApiFitnessTest.kt} | 0 7 files changed, 317 insertions(+), 95 deletions(-) rename core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/{AsyncApiBlackBoxFitness.kt => AsyncApiFitness.kt} (59%) rename core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/{AsyncApiBlackBoxFitnessTest.kt => AsyncApiFitnessTest.kt} (100%) diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 8cf1eae007..9a0211a057 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -572,6 +572,18 @@ class Main { config.problemType = EMConfig.ProblemType.WEBFRONTEND } else if (info.asyncApiProblem != null) { config.problemType = EMConfig.ProblemType.ASYNCAPI + if (config.createTests) { + /* + There is no test writer for AsyncAPI yet, and the constraints checked + below refuse the combination. As the problem type was inferred rather + than asked for, turning test generation off is better than failing. + */ + LoggingUtil.uniqueUserWarn( + "The driver describes an AsyncAPI service, for which test generation is not" + + " available yet. Continuing with 'createTests' off." + ) + config.createTests = false + } } else { throw IllegalStateException("Can connect to the EM Driver, but cannot infer the 'problemType'") } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt index 1fdbbbd671..89ecc703b8 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt @@ -1,6 +1,7 @@ package org.evomaster.core.problem.asyncapi.data import org.evomaster.core.problem.enterprise.EnterpriseActionResult +import org.evomaster.core.search.action.Action /** * What happened when one message was published: the outcome, and the reply when there was one. @@ -23,6 +24,10 @@ class AsyncApiCallResult : EnterpriseActionResult { return AsyncApiCallResult(this) } + override fun matchedType(action: Action): Boolean { + return action is AsyncApiAction + } + fun setOutcome(outcome: AsyncApiOutcome) { addResultValue(OUTCOME, outcome.name) } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt index 7a05c00af5..758ecf2b62 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt @@ -2,9 +2,6 @@ package org.evomaster.core.problem.asyncapi.data /** * What came of publishing one message, as the driver reported it. - * - * Only [NO_REPLY] is a fault in itself. [PUBLISH_FAILED] is a broken setup rather than a finding - * about the service, and the other two are the service doing what its contract says. */ enum class AsyncApiOutcome { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt similarity index 59% rename from core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt rename to core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index c999478ef1..7cec812d28 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -3,6 +3,7 @@ package org.evomaster.core.problem.asyncapi.service import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.databind.ObjectMapper import com.google.inject.Inject +import com.webfuzzing.asyncapi.models.AsyncApiChannel import com.webfuzzing.asyncapi.models.AsyncApiCorrelationId import com.webfuzzing.asyncapi.models.AsyncApiReply import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto @@ -15,6 +16,7 @@ import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome import org.evomaster.core.problem.asyncapi.param.AsyncApiParam +import org.evomaster.core.problem.enterprise.DetectedFault import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.FitnessValue @@ -24,31 +26,24 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory /** - * Publishes the messages of a test through the driver and turns what comes back into targets. - * - * The targets are the AsyncAPI analogue of REST's `(status x endpoint)`: for every operation, - * what publishing to it was seen to do ([AsyncApiOutcome]), and, when a reply came back, which - * of the messages the contract declares for the reply it was recognised as. A contract that - * enumerates a result and an error thus gives the search two things to reach. - * - * Two outcomes are faults: a promised reply that never arrives, and a reply matching none of - * the declared messages. A message the driver could not publish is neither; it is a broken - * setup, and the test stops there. + * Publishes the messages of a test through the driver and turns what comes back into targets: + * for every operation what publishing to it did, and which of the replies the contract declares + * was recognised. */ -class AsyncApiBlackBoxFitness : ApiWsFitness() { +class AsyncApiFitness : ApiWsFitness() { companion object { - private val log: Logger = LoggerFactory.getLogger(AsyncApiBlackBoxFitness::class.java) + private val log: Logger = LoggerFactory.getLogger(AsyncApiFitness::class.java) /** * Prefix of the `(outcome x operation)` targets, written as PREFIX:OUTCOME:action. */ - const val OUTCOME_TARGET_PREFIX = "ASYNCAPI_OUTCOME" + private const val OUTCOME_TARGET_PREFIX = "ASYNCAPI_OUTCOME" /** * Prefix of the `(declared reply x operation)` targets, written as PREFIX:messageId:action. */ - const val REPLY_TARGET_PREFIX = "ASYNCAPI_REPLY" + private const val REPLY_TARGET_PREFIX = "ASYNCAPI_REPLY" private const val TARGET_SEPARATOR = ":" @@ -56,18 +51,23 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { private const val DEFAULT_CONTENT_TYPE = "application/json" + /** + * What a {placeholder} in a channel address starts with. + */ + private const val PARAMETER_OPENING = "{" + private val mapper = ObjectMapper() /** * The id of the target covered when publishing to [actionName] had [outcome]. */ - fun outcomeTargetId(outcome: AsyncApiOutcome, actionName: String): String = + private fun getOutcomeTargetId(outcome: AsyncApiOutcome, actionName: String): String = listOf(OUTCOME_TARGET_PREFIX, outcome.name, actionName).joinToString(TARGET_SEPARATOR) /** * The id of the target covered when a reply to [actionName] was recognised as [messageId]. */ - fun replyTargetId(messageId: String, actionName: String): String = + private fun getReplyTargetId(messageId: String, actionName: String): String = listOf(REPLY_TARGET_PREFIX, messageId, actionName).joinToString(TARGET_SEPARATOR) } @@ -156,13 +156,16 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { return false } - record(reply, result) - handleTargets(fv, action, result, index) + val outcome = record(reply, result) + handleTargets(fv, action, result, outcome, index) return true } - private fun record(reply: AsyncApiReplyDto, result: AsyncApiCallResult) { + /** + * Copy what the driver reported onto the result, and say what it amounts to. + */ + private fun record(reply: AsyncApiReplyDto, result: AsyncApiCallResult): AsyncApiOutcome { val outcome = when { !reply.replyExpected -> AsyncApiOutcome.PUBLISHED @@ -177,33 +180,71 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { reply.replyPayload?.let { result.setReplyPayload(it) } result.setCorrelationMatched(reply.correlationMatched) } + + return outcome } - private fun handleTargets(fv: FitnessValue, action: AsyncApiAction, result: AsyncApiCallResult, index: Int) { + private fun handleTargets( + fv: FitnessValue, + action: AsyncApiAction, + result: AsyncApiCallResult, + outcome: AsyncApiOutcome, + index: Int + ) { val name = action.getName() - val outcome = result.getOutcome()!! - fv.updateTarget(idMapper.handleLocalTarget(outcomeTargetId(outcome, name)), 1.0, index) + fv.updateTarget(idMapper.handleLocalTarget(getOutcomeTargetId(outcome, name)), 1.0, index) when (outcome) { AsyncApiOutcome.REPLIED -> handleReplyTargets(fv, action, result, index) - AsyncApiOutcome.NO_REPLY -> { - val fault = idMapper.getFaultDescriptiveId(ExperimentalFaultCategory.ASYNCAPI_NO_REPLY, name) - fv.updateTarget(idMapper.handleLocalTarget(fault), 1.0, index) - } + AsyncApiOutcome.NO_REPLY -> + handleFault(fv, result, ExperimentalFaultCategory.ASYNCAPI_NO_REPLY, name, index) + /* + Nothing more to aim at. PUBLISH_FAILED never reaches here -- publishing gives up + before this is called -- but the compiler wants every outcome named. + */ AsyncApiOutcome.PUBLISHED, AsyncApiOutcome.PUBLISH_FAILED -> Unit } } + /** + * Register a fault, unless the user has switched off the category it belongs to. + * + * It goes both on the fitness value, where the search can aim at it, and on the action + * result, which is where the reports count faults from. + */ + private fun handleFault( + fv: FitnessValue, + result: AsyncApiCallResult, + category: ExperimentalFaultCategory, + actionName: String, + index: Int + ) { + if (!config.isEnabledFaultCategory(category)) { + return + } + + val descriptiveId = idMapper.getFaultDescriptiveId(category, actionName) + fv.updateTarget(idMapper.handleLocalTarget(descriptiveId), 1.0, index) + result.addFault(DetectedFault(category, actionName, null)) + } + private fun handleReplyTargets(fv: FitnessValue, action: AsyncApiAction, result: AsyncApiCallResult, index: Int) { val name = action.getName() val document = asyncApiSampler.document - val operation = document.operations[action.operationId] ?: return + val operation = document.operations[action.operationId] + + if (operation == null) { + LoggingUtil.uniqueUserWarn( + "No operation '" + action.operationId + "' in the document, so its replies cannot be recognised" + ) + return + } val declared = document.replyMessagesOf(operation) if (declared.isEmpty()) { @@ -211,16 +252,26 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { return } - val recognised = AsyncApiReplyClassifier.classify(result.getReplyPayload(), declared, document.componentSchemas) + val payload = result.getReplyPayload() + + if (payload.isNullOrBlank()) { + /* + A reply did arrive, but carries no body to match against the contract -- an + acknowledgement, or a transport that answers in metadata. Nothing was declared + to be wrong, so this is not the undeclared-reply fault. + */ + return + } + + val recognised = AsyncApiReplyClassifier.classify(payload, declared, document.componentSchemas) if (recognised == null) { - val fault = idMapper.getFaultDescriptiveId(ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY, name) - fv.updateTarget(idMapper.handleLocalTarget(fault), 1.0, index) + handleFault(fv, result, ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY, name, index) return } result.setReplyMessage(recognised.id) - fv.updateTarget(idMapper.handleLocalTarget(replyTargetId(recognised.id, name)), 1.0, index) + fv.updateTarget(idMapper.handleLocalTarget(getReplyTargetId(recognised.id, name)), 1.0, index) } /** @@ -238,16 +289,12 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { dto.channelName = action.channelName dto.messageId = action.messageId - /* - A channel may declare no address, meaning it is decided at run time. The driver is - then given the channel's name and left to map it, being the one that knows the broker. - */ - dto.address = channel?.address ?: action.channelName + dto.address = getAddress(channel, action.channelName) dto.payload = action.parameters.firstOrNull { it.name == AsyncApiParam.PAYLOAD } ?.gene?.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = null) dto.contentType = message?.contentType ?: document.defaultContentType ?: DEFAULT_CONTENT_TYPE - dto.headers = headersOf(action) + dto.headers = LinkedHashMap(buildHeaders(action)) dto.correlationId = runId + CORRELATION_SEPARATOR + published++ message?.correlationId?.let { @@ -260,7 +307,7 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { } action.replyTemplate?.let { reply -> - dto.replyAddress = replyAddressOf(reply, action) + dto.replyAddress = getReplyAddress(reply, action) if (dto.replyAddress != null) { dto.replyTimeoutMs = config.asyncApiReplyTimeoutMs.toLong() } @@ -270,10 +317,14 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { } /** - * The headers gene as a map, by way of its own JSON printing, which is what knows which - * optional headers are on. + * The headers to publish alongside the body: key is the header name as the document declares + * it, value is what to send under it, as text. + * + * They are read back from the gene's own JSON printing, which is what knows which optional + * headers are on. A header whose value prints as JSON null is left out rather than sent as + * the text "null". */ - private fun headersOf(action: AsyncApiAction): MutableMap { + private fun buildHeaders(action: AsyncApiAction): Map { val headers = LinkedHashMap() @@ -284,13 +335,19 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { val node = try { mapper.readTree(json) - } catch (e: JsonProcessingException) { + } catch (e: Exception) { log.warn("The headers of '{}' did not print as JSON: {}", action.getName(), e.message) return headers } - if (node.isObject) { - node.fields().forEach { (name, value) -> + if (!node.isObject) { + log.warn("The headers of '{}' printed as {}, not as an object, so none are sent", + action.getName(), json) + return headers + } + + node.fields().forEach { (name, value) -> + if (!value.isNull) { headers[name] = if (value.isValueNode) value.asText() else value.toString() } } @@ -301,7 +358,7 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { /** * Where the driver should wait for the reply, or null when there is nowhere to wait yet. */ - private fun replyAddressOf(reply: AsyncApiReply, action: AsyncApiAction): String? { + private fun getReplyAddress(reply: AsyncApiReply, action: AsyncApiAction): String? { val channelName = reply.channelName @@ -319,6 +376,48 @@ class AsyncApiBlackBoxFitness : ApiWsFitness() { return null } - return asyncApiSampler.document.channels[channelName]?.address ?: channelName + return getAddress(asyncApiSampler.document.channels[channelName], channelName) + } + + /** + * Where a message on [channel] actually goes on the wire. + * + * Usually the channel's address, but a protocol binding may override it: the Kafka binding + * carries its own topic, and a document that uses one often declares no address at all. A + * channel that declares neither is decided at run time, so the driver is given the channel's + * name and left to map it, being the one that knows the broker. + * + * Publishing to the wrong destination is silent -- the messages simply reach nobody -- so it + * is worth taking the binding into account rather than assuming the address is the whole story. + */ + private fun getAddress(channel: AsyncApiChannel?, channelName: String): String { + + if (channel == null) { + return channelName + } + + val protocol = asyncApiSampler.document.serversOf(channel).firstOrNull()?.protocol + + /* + A channel that declares no address of its own is normal for Kafka, where the topic + lives in the binding. The protocol is only known when the document declares a server, + so when there is no address to keep, the binding's topic is taken whatever it says: + it is the one destination the document actually names. + */ + val address = channel.effectiveAddress(protocol) + ?: channel.bindings?.kafkaTopic + ?: return channelName + + if (address.contains(PARAMETER_OPENING)) { + /* + The address has {placeholders} backed by the channel's parameters, which are not + filled in yet. Publishing to it verbatim reaches nobody. + */ + LoggingUtil.uniqueUserWarn( + "The address of channel '" + channelName + "' has parameters that are not supported yet: " + address + ) + } + + return address } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt index cc709ee442..7ea148004d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt @@ -53,11 +53,11 @@ class AsyncApiModule : EnterpriseModule() { .asEagerSingleton() bind(object : TypeLiteral>() {}) - .to(AsyncApiBlackBoxFitness::class.java) + .to(AsyncApiFitness::class.java) .asEagerSingleton() bind(object : TypeLiteral>() {}) - .to(AsyncApiBlackBoxFitness::class.java) + .to(AsyncApiFitness::class.java) .asEagerSingleton() bind(object : TypeLiteral>() {}) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt index 8170bfccab..e37f96398d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt @@ -7,21 +7,17 @@ import com.webfuzzing.asyncapi.models.AsyncApiMessage import com.webfuzzing.asyncapi.resolver.AsyncApiRefResolver /** - * Recognises which of the messages a contract declares for a reply an observed reply is. + * Recognises which of the messages a contract declares for a reply an observed reply is, by + * matching its payload against each declared schema. * - * A reply channel often carries several messages -- a result and an error, say -- and telling - * them apart is what gives a black-box search distinct outcomes to cover. There are no status - * codes to read, so the payload is matched against each declared schema instead. - * - * The matching is structural and deliberately lenient: it checks the parts of JSON Schema that - * tell one message from another (type, required fields, const and enum discriminators, the - * combinators), and gives the benefit of the doubt on anything it does not understand. It is a - * classifier, not a validator: its job is to tell which declared message a reply is, not to - * find every way in which it deviates from its schema. + * It is a classifier rather than a validator: it reads what tells the declared messages apart + * and gives the benefit of the doubt on anything it cannot read, because a reply it fails to + * recognise is reported as a fault. */ object AsyncApiReplyClassifier { private const val REF = "\$ref" + private const val PATH_SEPARATOR = "/" private const val TYPE = "type" private const val PROPERTIES = "properties" private const val REQUIRED = "required" @@ -41,11 +37,15 @@ object AsyncApiReplyClassifier { private const val TYPE_NULL = "null" /** - * How far to follow references and nesting before giving up. A schema that refers to - * itself is legitimate, and the payload it describes is finite, so this is only reached by - * a cycle in the schema that the data never enters. + * How deep to descend into a schema before giving up and accepting what is left. Nesting and + * the combinators both count towards it, so a deeply combinated schema can reach it. + */ + private const val MAX_SCHEMA_DEPTH = 32 + + /** + * How many `$ref` hops to follow before deciding the references form a cycle. */ - private const val MAX_DEPTH = 32 + private const val MAX_REF_CHAIN = 32 private val mapper = ObjectMapper() @@ -70,10 +70,15 @@ object AsyncApiReplyClassifier { val node = try { mapper.readTree(payload) - } catch (e: JsonProcessingException) { + } catch (_: JsonProcessingException) { //not JSON, so it is none of the JSON-described messages return null - } ?: return null + } + + //an empty body parses to nothing at all, which is no message either + if (node == null || node.isMissingNode) { + return null + } return candidates .filter { it.payload != null && matches(node, it.payload, componentSchemas, 0) } @@ -82,34 +87,52 @@ object AsyncApiReplyClassifier { private fun matches(node: JsonNode, schema: JsonNode, schemas: Map, depth: Int): Boolean { - if (depth > MAX_DEPTH) { + if (depth >= MAX_SCHEMA_DEPTH) { return true } //a reference that cannot be followed is something this cannot judge, so it does not reject - val s = resolve(schema, schemas) ?: return true + val resolved = resolve(schema, schemas) ?: return true - if (!s.isObject) { + if (!resolved.isObject) { return true } - s.get(CONST)?.let { if (node != it) return false } + val const = resolved.get(CONST) + if (const != null && !sameValue(node, const)) { + return false + } - s.get(ENUM)?.let { allowed -> if (allowed.isArray && allowed.none { it == node }) return false } + val allowed = resolved.get(ENUM) + if (allowed != null && allowed.isArray && allowed.none { sameValue(node, it) }) { + return false + } - s.get(TYPE)?.let { if (!isOfType(node, it)) return false } + val type = resolved.get(TYPE) + if (type != null && !isOfType(node, type)) { + return false + } - s.get(ALL_OF)?.let { all -> if (all.any { !matches(node, it, schemas, depth + 1) }) return false } + val all = getBranches(resolved, ALL_OF) + if (all != null && all.any { !matches(node, it, schemas, depth + 1) }) { + return false + } - s.get(ANY_OF)?.let { any -> if (any.none { matches(node, it, schemas, depth + 1) }) return false } + val any = getBranches(resolved, ANY_OF) + if (any != null && any.none { matches(node, it, schemas, depth + 1) }) { + return false + } //oneOf is read as "at least one": exclusivity is a validator's concern, not a classifier's - s.get(ONE_OF)?.let { one -> if (one.none { matches(node, it, schemas, depth + 1) }) return false } + val one = getBranches(resolved, ONE_OF) + if (one != null && one.none { matches(node, it, schemas, depth + 1) }) { + return false + } if (node.isObject) { - s.get(REQUIRED)?.let { required -> if (required.any { !node.has(it.asText()) }) return false } + resolved.get(REQUIRED)?.let { required -> if (required.any { !node.has(it.asText()) }) return false } - s.get(PROPERTIES)?.fields()?.forEach { (name, property) -> + resolved.get(PROPERTIES)?.fields()?.forEach { (name, property) -> val value = node.get(name) if (value != null && !matches(value, property, schemas, depth + 1)) { return false @@ -118,7 +141,7 @@ object AsyncApiReplyClassifier { } if (node.isArray) { - s.get(ITEMS)?.let { items -> + resolved.get(ITEMS)?.let { items -> if (items.isObject && node.any { !matches(it, items, schemas, depth + 1) }) return false } } @@ -126,21 +149,61 @@ object AsyncApiReplyClassifier { return true } + /** + * The branches of a combinator, or null when it is not a usable list of them. A combinator + * written as anything but a non-empty array says nothing, and must not reject everything. + */ + private fun getBranches(schema: JsonNode, keyword: String): List? { + + val branches = schema.get(keyword) ?: return null + + return if (branches.isArray && !branches.isEmpty) branches.toList() else null + } + + /** + * Whether two JSON values are the same as JSON Schema counts sameness. Numbers compare by + * value, so that a schema written `const: 1.0` accepts a reply carrying `1`; Jackson's own + * equality would say those differ, being of different node types. + */ + private fun sameValue(node: JsonNode, other: JsonNode): Boolean { + + if (node.isNumber && other.isNumber && hasDecimalValue(node) && hasDecimalValue(other)) { + return node.decimalValue().compareTo(other.decimalValue()) == 0 + } + + return node == other + } + + /** + * Whether the number has a decimal value at all. Only a floating-point node can hold an + * infinity or a NaN, and asking those for a [java.math.BigDecimal] throws. + */ + private fun hasDecimalValue(node: JsonNode): Boolean { + return !(node.isDouble || node.isFloat) || node.doubleValue().isFinite() + } + /** * Whether [node] is of one of the types [type] names. JSON Schema writes it as one name or a * list of them, and counts a number with no fractional part as an integer. + * + * A `type` that is neither is not something this can read, so nothing is rejected on it. */ private fun isOfType(node: JsonNode, type: JsonNode): Boolean { - val names = if (type.isArray) type.map { it.asText() } else listOf(type.asText()) + val names = when { + type.isArray && !type.isEmpty -> type.map { it.asText() } + type.isTextual -> listOf(type.asText()) + else -> return true + } return names.any { name -> when (name) { TYPE_OBJECT -> node.isObject TYPE_ARRAY -> node.isArray TYPE_STRING -> node.isTextual - TYPE_INTEGER -> node.isIntegralNumber - || (node.isNumber && node.decimalValue().stripTrailingZeros().scale() <= 0) + //a whole number written with a decimal point is an integer; canConvertToExactIntegral + //also answers for a value too large to be a BigDecimal, where decimalValue() throws + TYPE_INTEGER -> node.isIntegralNumber || (node.isNumber && node.canConvertToExactIntegral()) TYPE_NUMBER -> node.isNumber TYPE_BOOLEAN -> node.isBoolean TYPE_NULL -> node.isNull @@ -150,33 +213,79 @@ object AsyncApiReplyClassifier { } /** - * The schema itself, once any chain of `$ref` to a component schema is followed. Null when - * a reference points at something other than a whole component schema. + * The schema itself, once any chain of `$ref` has been followed. Null when a reference leads + * nowhere, which is the one case this cannot judge. + * + * A pointer may go deeper than the schema it names, as in + * `#/components/schemas/Order/properties/item`, which is a legitimate way of saying "the + * shape of that one property". */ - private fun resolve(schema: JsonNode, schemas: Map): JsonNode? { + private fun resolve(schema: JsonNode, schemas: Map, depth: Int = 0): JsonNode? { + + if (depth >= MAX_REF_CHAIN) { + return null + } var current = schema - repeat(MAX_DEPTH) { + repeat(MAX_REF_CHAIN - depth) { + val ref = AsyncApiRefResolver.refOf(current) ?: return current - val key = AsyncApiRefResolver.refKey(ref, AsyncApiRefResolver.SCHEMA_PREFIX) ?: return null - current = schemas[key] ?: return null + val key = AsyncApiRefResolver.schemaKeyOf(ref) ?: return null + val target = schemas[key] ?: return null + + val pointer = ref.removePrefix(AsyncApiRefResolver.SCHEMA_PREFIX).substringAfter(PATH_SEPARATOR, "") + + if (pointer.isEmpty()) { + current = target + } else { + //the schema the pointer goes into may itself be a reference, so follow that first + val base = resolve(target, schemas, depth + 1) ?: return null + current = base.at(PATH_SEPARATOR + pointer) + if (current.isMissingNode) { + return null + } + } } return null } /** - * How many fields the schema pins down at its top level, which is what tells a specific - * message from a permissive one when both match. + * How many fields a schema pins down, which is what tells a specific message from a + * permissive one when a reply matches both. + * + * Counted through the combinators as well: a message that says what it requires inside an + * `allOf` is no less specific for having written it that way. */ - private fun specificity(schema: JsonNode, schemas: Map): Int { + private fun specificity(schema: JsonNode, schemas: Map, depth: Int = 0): Int { + + if (depth >= MAX_SCHEMA_DEPTH) { + return 0 + } - val s = resolve(schema, schemas) ?: return 0 + val resolved = resolve(schema, schemas) ?: return 0 - val required = s.get(REQUIRED)?.size() ?: 0 - val pinned = s.get(PROPERTIES)?.count { it.has(CONST) || it.has(ENUM) } ?: 0 + if (!resolved.isObject) { + return 0 + } + + val required = resolved.get(REQUIRED)?.size() ?: 0 + + val pinned = resolved.get(PROPERTIES)?.count { property -> + val target = resolve(property, schemas) + target != null && (target.has(CONST) || target.has(ENUM)) + } ?: 0 + + //every branch of an allOf has to hold, so all of them count + val fromAll = getBranches(resolved, ALL_OF) + ?.sumOf { specificity(it, schemas, depth + 1) } ?: 0 + + //only one branch of a choice has to hold, so it is worth what its weakest branch is + val fromChoice = listOf(ANY_OF, ONE_OF).sumOf { keyword -> + getBranches(resolved, keyword)?.minOf { specificity(it, schemas, depth + 1) } ?: 0 + } - return required + pinned + return required + pinned + fromAll + fromChoice } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt similarity index 100% rename from core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiBlackBoxFitnessTest.kt rename to core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt From 08e725ab5472493c10997b5a041c632aa2189341 Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Wed, 16 Sep 2026 18:55:30 -0300 Subject: [PATCH 6/9] AsyncAPI 3.x: less duplication, and an option description users can read The new remote-controller call was a copy of the RPC one; both now share the body that PUTs an action and reads the reply back. The structure mutator is a third copy of logic REST and RPC already share, marked with a TODO the way the GraphQL one marks the same debt. The reply-timeout description is published verbatim into options.md, where every neighbour is one clause; the reasoning behind the option belongs in the pull request, not in the user's terminal. --- .../kotlin/org/evomaster/core/EMConfig.kt | 3 +-- .../service/AsyncApiStructureMutator.kt | 5 +++++ .../service/RemoteControllerImplementation.kt | 19 +++++++++++++++++-- docs/options.md | 2 +- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 2e306d2491..625982338f 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2847,8 +2847,7 @@ class EMConfig { @Experimental @Cfg("When testing an AsyncAPI service, how long to wait for the reply to a published message before" + - " treating it as unanswered, in milliseconds. A slow service and a stuck one look the same from" + - " outside, so this is a tuning parameter with no equivalent in a synchronous protocol.") + " treating it as unanswered, in milliseconds.") @Min(1.0) var asyncApiReplyTimeoutMs = 5000 diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt index c3b237f2ae..df61cdec8f 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt @@ -14,6 +14,11 @@ import org.evomaster.core.search.service.mutator.MutatedGeneSpecification */ class AsyncApiStructureMutator : ApiWsStructureMutator() { + /* + TODO Most of this is the same as RPCStructureMutator, which in turn shares it with the + REST one. Should be pulled up into ApiWsStructureMutator rather than copied a third time. + */ + @Inject private lateinit var sampler: AsyncApiSampler diff --git a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt index f8a1589160..664f984bbc 100644 --- a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt +++ b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt @@ -419,6 +419,21 @@ class RemoteControllerImplementation() : RemoteController{ } override fun executeNewAsyncApiActionAndGetReply(actionDto: ActionDto): AsyncApiReplyDto? { + return executeNewAction( + actionDto, + object : GenericType>() {}, + "Failed to publish an AsyncAPI message") + } + + /** + * Hand one action to the driver to execute, and read back what it reports. Null when the + * driver could not be reached, or answered with an error. + */ + private fun executeNewAction( + actionDto: ActionDto, + type: GenericType>, + errorMessage: String + ): T? { val response = makeHttpCall { getWebTarget() @@ -428,9 +443,9 @@ class RemoteControllerImplementation() : RemoteController{ .put(Entity.entity(actionDto, MediaType.APPLICATION_JSON_TYPE)) } - val dto = getDtoFromResponse(response, object : GenericType>() {}) + val dto = getDtoFromResponse(response, type) - if (!checkResponse(response, dto, "Failed to publish an AsyncAPI message")) { + if (!checkResponse(response, dto, errorMessage)) { return null } diff --git a/docs/options.md b/docs/options.md index 7ab8e627d3..aaed228ad1 100644 --- a/docs/options.md +++ b/docs/options.md @@ -278,7 +278,7 @@ There are 3 types of options: |`aiResponseClassifierWarmup`| __Int__. Number of training iterations required to update classifier parameters. For example, in the Gaussian model this affects mean and variance updates. For neural network (NN) models, the warm-up should typically be larger than 1000. *Default value*: `100`.| |`appendToTargetHeuristicsFile`| __Boolean__. Whether should add to an existing target heuristics file, instead of replacing it. It is only used when processFormat is TARGET_HEURISTIC. *Default value*: `false`.| |`arazzoLocation`| __String__. arazzo location on disk. *Default value*: `""`.| -|`asyncApiReplyTimeoutMs`| __Int__. When testing an AsyncAPI service, how long to wait for the reply to a published message before treating it as unanswered, in milliseconds. A slow service and a stuck one look the same from outside, so this is a tuning parameter with no equivalent in a synchronous protocol. *Constraints*: `min=1.0`. *Default value*: `5000`.| +|`asyncApiReplyTimeoutMs`| __Int__. When testing an AsyncAPI service, how long to wait for the reply to a published message before treating it as unanswered, in milliseconds. *Constraints*: `min=1.0`. *Default value*: `5000`.| |`breederParentsMin`| __Int__. Breeder GA: minimum number of individuals in parents pool after truncation. *Constraints*: `min=2.0`. *Default value*: `2`.| |`breederTruncationFraction`| __Double__. Breeder GA: fraction of top individuals to keep in parents pool (truncation). *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`callbackURLHostname`| __String__. HTTP callback verifier hostname. Default is set to 'localhost'. If the SUT is running inside a container (i.e., Docker), 'localhost' will refer to the container. This can be used to change the hostname. *Default value*: `localhost`.| From 21f2015e10f808d86058f10ce880363d4b6f3f2a Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Wed, 16 Sep 2026 18:55:41 -0300 Subject: [PATCH 7/9] AsyncAPI 3.x: tests for what the review found The guards that keep a valid reply from being reported as a fault had no tests, which is why they could be wrong. Now covered: a reply with no body, an operation whose reply declares no message, a number too large to be a decimal, a const compared by value, a reference into a schema, and the most specific match found through a combinator. Also covered: the topic a protocol binding names, the configured reply timeout, and that faults stay quiet until experimental oracles are asked for -- each of which would otherwise pass with the code deleted. The structure mutator's tracking is exercised with a real specification rather than null, so the add/remove bookkeeping the archive reads is checked. The module test asserts the bindings Main resolves, since this module deliberately inherits none. The sampler suite resets the static gene cache its siblings all reset. --- .../asyncapi/service/AsyncApiFitnessTest.kt | 138 +++++++++++++++++- .../asyncapi/service/AsyncApiModuleTest.kt | 41 +++++- .../service/AsyncApiReplyClassifierTest.kt | 119 ++++++++++++++- .../asyncapi/service/AsyncApiSamplerTest.kt | 11 ++ .../service/AsyncApiStructureMutatorTest.kt | 43 +++++- .../asyncapi/service/AsyncApiTestInjector.kt | 10 ++ 6 files changed, 344 insertions(+), 18 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt index c24dba2139..9654a5c1c2 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt @@ -26,13 +26,19 @@ import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -class AsyncApiBlackBoxFitnessTest { +class AsyncApiFitnessTest { companion object { private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" + /** + * Both AsyncAPI fault categories are experimental, so nothing reports them until the + * user asks for experimental oracles. + */ + private const val EXPERIMENTAL_ORACLES = "--useExperimentalOracles=true" + private const val ERROR = """{"error": {"code": 400, "message": "n must be >= 3"}}""" /** @@ -125,18 +131,29 @@ class AsyncApiBlackBoxFitnessTest { RestActionBuilderV3.cleanCache() } - private fun start(schemaText: String, answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) { + private fun start( + schemaText: String, + vararg options: String, + answer: (AsyncApiActionDto) -> AsyncApiReplyDto? + ) { driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(schemaText), answer) - injector = AsyncApiTestInjector.create(driver, "--blackBox=false") + injector = AsyncApiTestInjector.create(driver, "--blackBox=false", *options) sampler = injector.getInstance(AsyncApiSampler::class.java) fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) idMapper = injector.getInstance(IdMapper::class.java) } - private fun startNcs(answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) = - start(AsyncApiAccess.readFromResource(NCS), answer) + private fun startNcs(vararg options: String, answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) = + start(AsyncApiAccess.readFromResource(NCS), *options, answer = answer) + + /** + * The faults an evaluation recorded on its action results, which is where the reports read + * them from -- as opposed to the targets the search aims at. + */ + private fun faultsOn(evaluated: EvaluatedIndividual) = + results(evaluated).flatMap { it.getFaults() } private fun individualOf(vararg names: String): AsyncApiIndividual { @@ -239,7 +256,7 @@ class AsyncApiBlackBoxFitnessTest { @Test fun testAReplyTheContractDoesNotDeclareIsAFault() { - startNcs { replied("""{"something": "else"}""") } + startNcs(EXPERIMENTAL_ORACLES) { replied("""{"something": "else"}""") } val evaluated = evaluate("bessj") val faults = evaluated.fitness.coveredTargets().filter { idMapper.isFault(it) } @@ -247,12 +264,17 @@ class AsyncApiBlackBoxFitnessTest { assertEquals(1, faults.size, coveredIds(evaluated).toString()) assertTrue(idMapper.isSpecifiedFault(faults.single(), ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY)) assertNull(results(evaluated).single().getReplyMessage()) + + //the reports count faults off the action result, not off the covered targets + assertEquals( + listOf(ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY), + faultsOn(evaluated).map { it.category }) } @Test fun testSilenceAfterAPromisedReplyIsAFault() { - startNcs { silence(waited = 5000) } + startNcs(EXPERIMENTAL_ORACLES) { silence(waited = 5000) } val evaluated = evaluate("bessj") val covered = coveredIds(evaluated) @@ -267,6 +289,10 @@ class AsyncApiBlackBoxFitnessTest { assertEquals(5000L, result.getWaitedMs()) //silence is a finding, not a broken setup: the test goes on assertFalse(result.stopping) + + assertEquals( + listOf(ExperimentalFaultCategory.ASYNCAPI_NO_REPLY), + faultsOn(evaluated).map { it.category }) } @Test @@ -390,4 +416,102 @@ class AsyncApiBlackBoxFitnessTest { //a broken setup is not a finding about the service, so nothing is covered by it assertTrue(coveredIds(evaluated).none { it.startsWith("ASYNCAPI") }, coveredIds(evaluated).toString()) } + + @Test + fun testFaultsAreNotReportedUntilExperimentalOraclesAreAskedFor() { + + //both AsyncAPI categories are experimental, and nothing experimental is on by default + startNcs { silence() } + + val evaluated = evaluate("bessj") + + assertTrue(coveredIds(evaluated).contains("ASYNCAPI_OUTCOME:NO_REPLY:bessj")) + assertTrue(evaluated.fitness.coveredTargets().none { idMapper.isFault(it) }) + assertTrue(faultsOn(evaluated).isEmpty()) + } + + @Test + fun testAReplyWithNoBodyIsNotAnUndeclaredReply() { + + /* + The contract declares what a reply may be, and one arrived carrying nothing to match + against it -- an acknowledgement, or a transport that answers in its metadata. + Nothing was declared to be wrong, so this is not the undeclared-reply fault. + */ + startNcs(EXPERIMENTAL_ORACLES) { replied(payload = "") } + + val evaluated = evaluate("bessj") + + assertEquals(AsyncApiOutcome.REPLIED, results(evaluated).single().getOutcome()) + assertTrue(evaluated.fitness.coveredTargets().none { idMapper.isFault(it) }, coveredIds(evaluated).toString()) + assertTrue(faultsOn(evaluated).isEmpty()) + } + + @Test + fun testAnOperationWhoseReplyDeclaresNoMessageIsNotAFault() { + + /* + A reply channel that carries no message: the contract promises an answer but does not + say what it looks like, so there is nothing to recognise and nothing to report. + */ + val document = """ + asyncapi: 3.0.0 + info: + title: Unspecified reply + version: 1.0.0 + channels: + requests: + address: app.requests + messages: + request: + payload: + type: object + required: [id] + properties: + id: + type: string + replies: + address: app.replies + operations: + ask: + action: receive + channel: + ${'$'}ref: '#/channels/requests' + reply: + channel: + ${'$'}ref: '#/channels/replies' + """.trimIndent() + + start(document, EXPERIMENTAL_ORACLES) { replied("""{"anything": 1}""") } + + val evaluated = evaluate("ask") + + assertEquals(AsyncApiOutcome.REPLIED, results(evaluated).single().getOutcome()) + assertNull(results(evaluated).single().getReplyMessage()) + assertTrue(evaluated.fitness.coveredTargets().none { idMapper.isFault(it) }, coveredIds(evaluated).toString()) + } + + @Test + fun testTheTopicOfAProtocolBindingOverridesTheChannelAddress() { + + /* + A Kafka channel often names no address of its own and puts the topic in its binding. + Publishing to the channel's name instead would reach nobody. + */ + start(AsyncApiAccess.readFromResource("/asyncapi/sut/microcks.yaml")) { fireAndForget() } + + evaluate("receivedServiceChanges") + + assertEquals("microcks-services-updates", driver.published.single().address) + } + + @Test + fun testTheReplyTimeoutIsTheConfiguredOne() { + + startNcs("--asyncApiReplyTimeoutMs=1234") { replied(DOUBLE_RESULT) } + + evaluate("bessj") + + assertEquals(1234L, driver.published.single().replyTimeoutMs) + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt index c99bfd971d..c3de0f653b 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt @@ -10,7 +10,18 @@ import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 import org.evomaster.core.search.algorithms.MioAlgorithm +import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.service.IdMapper +import org.evomaster.core.output.service.NoTestCaseWriter +import org.evomaster.core.output.service.TestCaseWriter +import org.evomaster.core.output.service.TestSuiteWriter +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied +import org.evomaster.core.search.service.Archive +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector +import org.evomaster.core.search.service.Minimizer +import org.evomaster.core.search.service.Sampler +import org.evomaster.core.search.service.mutator.StructureMutator import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -22,9 +33,9 @@ import org.junit.jupiter.api.Test class AsyncApiModuleTest { companion object { - private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + private const val NCS = AsyncApiTestInjector.NCS - private val NCS_OPERATIONS = setOf("checkTriangle", "bessj", "expint", "fisher", "gammq", "remainder") + private val NCS_OPERATIONS = AsyncApiTestInjector.NCS_OPERATIONS private val mapper = ObjectMapper() } @@ -120,4 +131,30 @@ class AsyncApiModuleTest { assertTrue(driver.published.size >= 50, "only ${driver.published.size} messages published") assertEquals(driver.published.size, driver.published.map { it.correlationId }.toSet().size) } + + @Test + fun testEverythingMainResolvesIsBound() { + + /* + The module deliberately does not inherit EnterpriseModule's bindings, so anything + Main asks the injector for has to be bound here. A missing one is invisible until a + real run reaches that line, which is usually right after the search. + */ + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { + replied("""{"resultAsInt": 1}""") + } + val injector = AsyncApiTestInjector.create(driver, "--blackBox=false") + + injector.getInstance(Key.get(object : TypeLiteral>() {})) + injector.getInstance(Key.get(object : TypeLiteral>() {})) + injector.getInstance(Key.get(object : TypeLiteral>() {})) + injector.getInstance(Key.get(object : TypeLiteral>() {})) + injector.getInstance(Archive::class.java) + injector.getInstance(StructureMutator::class.java) + injector.getInstance(TestSuiteWriter::class.java) + injector.getInstance(RemoteController::class.java) + + //no test writer for AsyncAPI yet, so the one that writes nothing + assertTrue(injector.getInstance(TestCaseWriter::class.java) is NoTestCaseWriter) + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt index bc3cffd9be..0861339023 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt @@ -226,15 +226,120 @@ class AsyncApiReplyClassifierTest { } @Test - fun testAReferenceIntoTheMiddleOfASchemaCannotBeJudgedSoItIsNotRejected() { + fun testAReferenceIntoTheMiddleOfASchemaIsFollowed() { /* - "#/components/schemas/Pinned/properties/kind" names a part of a schema, which is - legal and which the classifier does not follow. Rejecting on what it cannot read - would turn every such reply into a false fault, so it matches instead -- and loses - to anything specific that also matches. + "#/components/schemas/Pinned/properties/kind" names one property of a schema, which + is legal. Left unresolved it would match anything, and an operation whose only reply + is written that way could then never report an undeclared reply. */ - assertEquals("part", classifyAmong("""{"anything": 1}""", "part")) - assertEquals("pinned", classifyAmong("""{"kind": "pinned"}""", "part", "pinned")) + assertEquals("part", classifyAmong("\"pinned\"", "part")) + assertNull(classifyAmong("""{"anything": 1}""", "part")) + } + + @Test + fun testANumberIsComparedByValueNotByHowItIsWritten() { + + //JSON Schema counts 1 and 1.0 as the same number; Jackson's own equality does not + val versions = AsyncApiAccess.parseFromText( + """ + asyncapi: 3.0.0 + info: + title: Versions + version: 1.0.0 + components: + messages: + v1: + payload: + type: object + properties: + version: + const: 1.0 + """.trimIndent() + ) + + fun classifyVersion(payload: String) = AsyncApiReplyClassifier.classify( + payload, listOf(versions.messages.getValue("v1")), versions.componentSchemas)?.id + + assertEquals("v1", classifyVersion("""{"version": 1}""")) + assertEquals("v1", classifyVersion("""{"version": 1.0}""")) + assertNull(classifyVersion("""{"version": 2}""")) + } + + @Test + fun testANumberTooLargeToBeADecimalDoesNotCrash() { + + /* + A float that overflows a double becomes an infinity, which has no BigDecimal. Asking + it for one used to throw, and nothing between here and the search loop catches it. + */ + assertNull(classify(ncs, "checkTriangle", """{"resultAsInt": 1E+400}""")) + assertEquals("intResult", classify(ncs, "checkTriangle", """{"resultAsInt": 1E+30}""")) + } + + @Test + fun testTheMostSpecificMatchIsFoundThroughACombinator() { + + /* + A message that says what it requires inside an allOf is no less specific for having + written it that way, and must still win over a permissive one that also matches. + */ + val combined = AsyncApiAccess.parseFromText( + """ + asyncapi: 3.0.0 + info: + title: Combined + version: 1.0.0 + components: + messages: + generic: + payload: + type: object + required: [requestId] + detailed: + payload: + allOf: + - type: object + required: [requestId] + - type: object + required: [error] + """.trimIndent() + ) + + val both = listOf(combined.messages.getValue("generic"), combined.messages.getValue("detailed")) + + assertEquals( + "detailed", + AsyncApiReplyClassifier.classify( + """{"requestId": "r1", "error": {"code": 404}}""", both, combined.componentSchemas)?.id + ) + assertEquals( + "generic", + AsyncApiReplyClassifier.classify("""{"requestId": "r1"}""", both, combined.componentSchemas)?.id + ) + } + + @Test + fun testAnEmptyBodyIsNoMessageAtAll() { + + //an empty body parses to nothing, which must not be read as matching a permissive schema + val anything = AsyncApiAccess.parseFromText( + """ + asyncapi: 3.0.0 + info: + title: Permissive + version: 1.0.0 + components: + messages: + loose: + payload: + required: [id] + """.trimIndent() + ) + + assertNull( + AsyncApiReplyClassifier.classify( + "", listOf(anything.messages.getValue("loose")), anything.componentSchemas) + ) } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt index 5c75267638..35b79fe6fa 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt @@ -11,10 +11,12 @@ import org.evomaster.core.EMConfig import org.evomaster.core.problem.asyncapi.data.AsyncApiAction import org.evomaster.core.problem.external.service.DummyController import org.evomaster.core.remote.SutProblemException +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.service.WarningsAggregator import org.evomaster.core.search.warning.WarningCategory import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import java.nio.charset.StandardCharsets @@ -23,6 +25,15 @@ import java.nio.file.Path class AsyncApiSamplerTest { + /** + * RestActionBuilderV3 keeps its built genes in a static cache, so a test that did not clear + * it could be handed one built by another. + */ + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + companion object { private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt index cc8c7aa094..242c1d75dd 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt @@ -8,6 +8,7 @@ import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion. import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.mutator.MutatedGeneSpecification import org.evomaster.core.search.service.mutator.StructureMutator import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach @@ -16,9 +17,9 @@ import org.junit.jupiter.api.Test class AsyncApiStructureMutatorTest { companion object { - private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + private const val NCS = AsyncApiTestInjector.NCS - private val NCS_OPERATIONS = setOf("checkTriangle", "bessj", "expint", "fisher", "gammq", "remainder") + private val NCS_OPERATIONS = AsyncApiTestInjector.NCS_OPERATIONS } private lateinit var sampler: AsyncApiSampler @@ -102,6 +103,44 @@ class AsyncApiStructureMutatorTest { } } + @Test + fun testWhatWasAddedOrRemovedIsRecordedForTheSearch() { + + /* + The archive and the impact bookkeeping are fed from this specification, and they are + shared with the other problem types, so a wrong entry degrades the search silently. + */ + start(maxTestSize = 4) + + var evaluated = evaluate(sampler.sample(forceRandomSample = true)) + var added = 0 + var removed = 0 + + repeat(30) { + val before = evaluated.individual.seeMainExecutableActions().size + val copy = evaluated.individual.copy() as AsyncApiIndividual + val recorded = MutatedGeneSpecification() + + mutator.mutateStructure(copy, evaluated, recorded, setOf()) + + val after = copy.seeMainExecutableActions().size + val grew = after > before + + /* + One entry per top gene of the action that moved, so what matters is that there is + at least one and that they all say the same thing. + */ + val types = recorded.mutatedGenes.map { it.type }.toSet() + assertEquals(setOf(if (grew) MutatedGeneSpecification.MutatedType.ADD else MutatedGeneSpecification.MutatedType.REMOVE), types, "size went $before -> $after") + + if (grew) added++ else removed++ + evaluated = evaluate(copy) + } + + //both directions were taken, so neither assertion above passed only by never happening + assertTrue(added > 0 && removed > 0, "added=$added removed=$removed") + } + @Test fun testNothingChangesWhenOnlyOneMessageIsAllowed() { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt index d8feee1dcf..fc0eecdc56 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt @@ -14,6 +14,16 @@ import org.evomaster.core.remote.service.RemoteController */ object AsyncApiTestInjector { + /** + * The NCS document, the corpus fixture these suites drive most of their cases from. + */ + const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + /** + * The operations NCS declares, all of them publishable. + */ + val NCS_OPERATIONS = setOf("checkTriangle", "bessj", "expint", "fisher", "gammq", "remainder") + /** * What a driver declares for a service whose document it hands over as text. */ From 95aa9d97b35e551928b9837e6fb658162eae3f7c Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Thu, 17 Sep 2026 14:06:07 -0300 Subject: [PATCH 8/9] AsyncAPI 3.x: decide what an absent value from the driver means The reply DTO no longer uses primitives, so a field the driver never set arrives as null rather than as a default. Kotlin maps those to platform types, so reading them as before would still compile and throw at run time. Each is now read deliberately. Whether the message was published has no safe default, so an absent answer is treated as a failure and said so in the error message, apart from a driver that answered false. Whether a reply was expected or arrived is read as not having happened when unset, which is what a fire-and-forget driver leaves blank. Correlation is the one that changes behaviour: a driver that does not track it at all reported false, which reads as the service having failed to echo the id back. That is a claim about the SUT the driver never made, so it is now recorded only when the driver actually looked. --- .../asyncapi/service/AsyncApiFitness.kt | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index 7cec812d28..5e9daafe81 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -144,14 +144,14 @@ class AsyncApiFitness : ApiWsFitness() { val reply = rc.executeNewAsyncApiActionAndGetReply(dto) - if (reply == null || !reply.published) { + if (reply == null || reply.published != true) { /* Not a finding about the service: the driver could not put the message on the - wire, or could not be reached at all. Nothing published after this point would - mean anything, so the test stops here, and no target is registered for it. + wire, could not be reached at all, or did not say. Nothing published after this + point would mean anything, so the test stops here, and no target is registered. */ result.setOutcome(AsyncApiOutcome.PUBLISH_FAILED) - result.setErrorMessage(reply?.errorMessage ?: "No response from the driver") + result.setErrorMessage(describeFailure(reply)) result.stopping = true return false } @@ -167,9 +167,14 @@ class AsyncApiFitness : ApiWsFitness() { */ private fun record(reply: AsyncApiReplyDto, result: AsyncApiCallResult): AsyncApiOutcome { + /* + Every flag here may be absent: they are boxed so that a driver which did not set one + can be told from a driver that set it to false. What was not said is read as not + having happened. + */ val outcome = when { - !reply.replyExpected -> AsyncApiOutcome.PUBLISHED - reply.replyReceived -> AsyncApiOutcome.REPLIED + reply.replyExpected != true -> AsyncApiOutcome.PUBLISHED + reply.replyReceived == true -> AsyncApiOutcome.REPLIED else -> AsyncApiOutcome.NO_REPLY } @@ -178,7 +183,12 @@ class AsyncApiFitness : ApiWsFitness() { if (outcome == AsyncApiOutcome.REPLIED) { reply.replyPayload?.let { result.setReplyPayload(it) } - result.setCorrelationMatched(reply.correlationMatched) + /* + Only when the driver actually checked. A driver that does not track correlation + says nothing here, which must not be recorded as the service having failed to + echo the id back. + */ + reply.correlationMatched?.let { result.setCorrelationMatched(it) } } return outcome @@ -211,6 +221,23 @@ class AsyncApiFitness : ApiWsFitness() { } } + /** + * Why a message did not go out, as far as can be told from what came back. + */ + private fun describeFailure(reply: AsyncApiReplyDto?): String { + + if (reply == null) { + return "No response from the driver" + } + + return reply.errorMessage + ?: if (reply.published == null) { + "The driver did not report whether the message was published" + } else { + "The driver could not publish the message" + } + } + /** * Register a fault, unless the user has switched off the category it belongs to. * From b96189728ff0b7c7142e791f4535439c26fadb81 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 18 Sep 2026 18:02:38 -0300 Subject: [PATCH 9/9] AsyncAPI 3.x: an unrecognised reply is a target, not only a fault Turning the experimental oracles off used to cost coverage rather than only reporting. A reply matching none of the declared messages registered nothing beyond the gated fault call, so it was indistinguishable from a recognised one and could be dropped when the solution was minimised. It now covers a target of its own, registered before the fault and independent of whether it is enabled, which is how a 500 is already handled for REST. Also in this round: - the correlation id prefix no longer comes from the seeded generator, which made a run repeated under the same seed reuse the very ids it exists to tell apart - an outcome that cannot reach target handling now says so by throwing rather than by a comment, and the branch stays exhaustive so that a newly added outcome fails to compile instead of falling through - why there is no separate black-box fitness, and why the reply classifier is written by hand rather than delegated to the validator already on the classpath, which reads draft-04 and so does not know const --- .../asyncapi/service/AsyncApiFitness.kt | 46 ++++++++++++++++--- .../service/AsyncApiReplyClassifier.kt | 11 ++++- .../asyncapi/service/AsyncApiFitnessTest.kt | 20 ++++++++ 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index 5e9daafe81..a5c520d636 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -6,6 +6,7 @@ import com.google.inject.Inject import com.webfuzzing.asyncapi.models.AsyncApiChannel import com.webfuzzing.asyncapi.models.AsyncApiCorrelationId import com.webfuzzing.asyncapi.models.AsyncApiReply +import java.util.UUID import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto import org.evomaster.core.database.sql.SqlAction @@ -45,6 +46,12 @@ class AsyncApiFitness : ApiWsFitness() { */ private const val REPLY_TARGET_PREFIX = "ASYNCAPI_REPLY" + /** + * Prefix of the targets covered when a reply matched no declared message, written as + * PREFIX:action. + */ + private const val UNRECOGNISED_REPLY_TARGET_PREFIX = "ASYNCAPI_UNRECOGNISED_REPLY" + private const val TARGET_SEPARATOR = ":" private const val CORRELATION_SEPARATOR = "-" @@ -69,16 +76,25 @@ class AsyncApiFitness : ApiWsFitness() { */ private fun getReplyTargetId(messageId: String, actionName: String): String = listOf(REPLY_TARGET_PREFIX, messageId, actionName).joinToString(TARGET_SEPARATOR) + + /** + * The id of the target covered when a reply to [actionName] matched no declared message. + */ + private fun getUnrecognisedReplyTargetId(actionName: String): String = + listOf(UNRECOGNISED_REPLY_TARGET_PREFIX, actionName).joinToString(TARGET_SEPARATOR) } @Inject private lateinit var asyncApiSampler: AsyncApiSampler /** - * Tells this run's correlation ids from those of an earlier run against the same broker. - * Drawn from [randomness], so that a seeded run is reproducible. + * Tells this run's correlation ids from those of an earlier run against the same broker, so + * that a reply left over from a previous run is not read as an answer to this one. + * + * Deliberately not drawn from [randomness]: a run repeated under the same seed would reuse + * the ids it used before, which is the one case this exists to tell apart. */ - private val runId: String by lazy { Integer.toHexString(randomness.nextInt()) } + private val runId: String = UUID.randomUUID().toString().take(8) /** * How many messages this run has published, which is what makes each correlation id unique. @@ -110,6 +126,13 @@ class AsyncApiFitness : ApiWsFitness() { } } + /* + There is no separate black-box fitness here, unlike REST and GraphQL: AsyncAPI publishes + through the driver in both modes, which is what EMConfig.usesDriver() states. What + changes is only what this call adds on top of the outcome and reply targets above: an + instrumented SUT reports the lines and branches the messages reached, an uninstrumented + one reports none. + */ val dto = updateFitnessAfterEvaluation(targets, allTargets, fullyCovered, descriptiveIds, individual, fv) ?: return null handleExtra(dto, fv) @@ -213,11 +236,15 @@ class AsyncApiFitness : ApiWsFitness() { AsyncApiOutcome.NO_REPLY -> handleFault(fv, result, ExperimentalFaultCategory.ASYNCAPI_NO_REPLY, name, index) + AsyncApiOutcome.PUBLISHED -> Unit + /* - Nothing more to aim at. PUBLISH_FAILED never reaches here -- publishing gives up - before this is called -- but the compiler wants every outcome named. + Publishing gives up before this is called, so getting here means the caller + changed. There is no `else` branch on purpose: a newly added outcome should fail + to compile here rather than fall through unnoticed at run time. */ - AsyncApiOutcome.PUBLISHED, AsyncApiOutcome.PUBLISH_FAILED -> Unit + AsyncApiOutcome.PUBLISH_FAILED -> + throw IllegalStateException("A message that was not published reached target handling") } } @@ -293,6 +320,13 @@ class AsyncApiFitness : ApiWsFitness() { val recognised = AsyncApiReplyClassifier.classify(payload, declared, document.componentSchemas) if (recognised == null) { + /* + Registered apart from the fault, and not subject to it being enabled: a reply that + matches nothing the contract declares is a behaviour worth reaching whether or not + it is also reported as a fault. Without this the search would have nothing to aim + at here, since no declared message was matched either. + */ + fv.updateTarget(idMapper.handleLocalTarget(getUnrecognisedReplyTargetId(name)), 1.0, index) handleFault(fv, result, ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY, name, index) return } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt index e37f96398d..ead7eebbb8 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt @@ -12,7 +12,16 @@ import com.webfuzzing.asyncapi.resolver.AsyncApiRefResolver * * It is a classifier rather than a validator: it reads what tells the declared messages apart * and gives the benefit of the doubt on anything it cannot read, because a reply it fails to - * recognise is reported as a fault. + * recognise is reported as a fault. Nothing else here has to do this -- REST is told which + * response applies by the status code, GraphQL by its `errors` field, and RPC by the driver -- + * so there was no existing answer to borrow. + * + * The matching is done by hand rather than with a schema validator. The one already on the + * classpath, pulled in by swagger-request-validator, understands draft-04 only, and so does not + * know `const`: the very keyword AsyncAPI documents lean on to tell message variants apart, and + * the reason [org.evomaster.core.problem.asyncapi.builder.AsyncApiGeneBuilder] has to rewrite it + * before the gene builder sees it. A validator that reads a modern draft would replace most of + * this, at the cost of a new dependency. */ object AsyncApiReplyClassifier { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt index 9654a5c1c2..7f03d0b60a 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt @@ -263,6 +263,7 @@ class AsyncApiFitnessTest { assertEquals(1, faults.size, coveredIds(evaluated).toString()) assertTrue(idMapper.isSpecifiedFault(faults.single(), ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY)) + assertTrue(coveredIds(evaluated).contains("ASYNCAPI_UNRECOGNISED_REPLY:bessj")) assertNull(results(evaluated).single().getReplyMessage()) //the reports count faults off the action result, not off the covered targets @@ -417,6 +418,25 @@ class AsyncApiFitnessTest { assertTrue(coveredIds(evaluated).none { it.startsWith("ASYNCAPI") }, coveredIds(evaluated).toString()) } + @Test + fun testAnUnrecognisedReplyIsStillATargetWithoutExperimentalOracles() { + + /* + The fault is experimental, but the behaviour it reports is not: a reply matching none + of the declared messages is somewhere the search should still be able to aim, or + turning the oracle off would quietly cost coverage rather than only reporting. + */ + startNcs { replied("""{"something": "else"}""") } + + val evaluated = evaluate("bessj") + + assertTrue( + coveredIds(evaluated).contains("ASYNCAPI_UNRECOGNISED_REPLY:bessj"), + coveredIds(evaluated).toString()) + assertTrue(evaluated.fitness.coveredTargets().none { idMapper.isFault(it) }) + assertTrue(faultsOn(evaluated).isEmpty()) + } + @Test fun testFaultsAreNotReportedUntilExperimentalOraclesAreAskedFor() {