diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 9015d09803..625982338f 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -806,6 +806,15 @@ class EMConfig { throw ConfigProblemException("When using the seedTestCases option, you must specify the file path of the test cases with the seedTestCasesPath option") } + if (problemType == ProblemType.ASYNCAPI && createTests) { + throw ConfigProblemException("Test generation for AsyncAPI services is not available yet." + + " For the time being, run with '--createTests false' to only search for faults.") + } + + if (problemType == ProblemType.ASYNCAPI && seedTestCases) { + throw ConfigProblemException("Seeding test cases is not supported for AsyncAPI services yet") + } + if (problemType == ProblemType.RPC && createTests && (enablePureRPCTestGeneration || enableRPCAssertionWithInstance) @@ -2836,6 +2845,12 @@ class EMConfig { @Cfg("Whether to enable extra targets for responses, e.g., regarding nullable response, having extra targets for whether it is null") var enableRPCExtraResponseTargets = true + @Experimental + @Cfg("When testing an AsyncAPI service, how long to wait for the reply to a published message before" + + " treating it as unanswered, in milliseconds.") + @Min(1.0) + var asyncApiReplyTimeoutMs = 5000 + @Cfg("Whether to enable customized responses indicating business logic") var enableRPCCustomizedResponseTargets = true diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 435b6fc8ad..9a0211a057 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -20,6 +20,7 @@ import org.evomaster.core.output.TestSuiteSplitter import org.evomaster.core.output.clustering.SplitResult import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.AsyncApiModule import org.evomaster.core.problem.enterprise.service.WFCReportWriter import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHttpWsResponseHandler import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler @@ -571,6 +572,18 @@ class Main { config.problemType = EMConfig.ProblemType.WEBFRONTEND } else if (info.asyncApiProblem != null) { config.problemType = EMConfig.ProblemType.ASYNCAPI + if (config.createTests) { + /* + There is no test writer for AsyncAPI yet, and the constraints checked + below refuse the combination. As the problem type was inferred rather + than asked for, turning test generation off is better than failing. + */ + LoggingUtil.uniqueUserWarn( + "The driver describes an AsyncAPI service, for which test generation is not" + + " available yet. Continuing with 'createTests' off." + ) + config.createTests = false + } } else { throw IllegalStateException("Can connect to the EM Driver, but cannot infer the 'problemType'") } @@ -627,8 +640,8 @@ class Main { } EMConfig.ProblemType.ASYNCAPI -> { - //the sampler, fitness and module for it are being added one at a time - throw IllegalStateException("AsyncAPI is not wired into the search yet") + //one module for both modes: the driver is needed either way, see EMConfig.usesDriver + AsyncApiModule() } //this should never happen, unless we add new type and forget to add it here diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt new file mode 100644 index 0000000000..89ecc703b8 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt @@ -0,0 +1,63 @@ +package org.evomaster.core.problem.asyncapi.data + +import org.evomaster.core.problem.enterprise.EnterpriseActionResult +import org.evomaster.core.search.action.Action + +/** + * What happened when one message was published: the outcome, and the reply when there was one. + */ +class AsyncApiCallResult : EnterpriseActionResult { + + companion object { + const val OUTCOME = "OUTCOME" + const val REPLY_PAYLOAD = "REPLY_PAYLOAD" + const val REPLY_MESSAGE = "REPLY_MESSAGE" + const val CORRELATION_MATCHED = "CORRELATION_MATCHED" + const val WAITED_MS = "WAITED_MS" + } + + constructor(sourceLocalId: String, stopping: Boolean = false) : super(sourceLocalId, stopping) + + private constructor(other: AsyncApiCallResult) : super(other) + + override fun copy(): AsyncApiCallResult { + return AsyncApiCallResult(this) + } + + override fun matchedType(action: Action): Boolean { + return action is AsyncApiAction + } + + fun setOutcome(outcome: AsyncApiOutcome) { + addResultValue(OUTCOME, outcome.name) + } + + fun getOutcome(): AsyncApiOutcome? = getResultValue(OUTCOME)?.let { AsyncApiOutcome.valueOf(it) } + + fun setReplyPayload(payload: String) { + addResultValue(REPLY_PAYLOAD, payload) + } + + fun getReplyPayload(): String? = getResultValue(REPLY_PAYLOAD) + + /** + * Which of the messages the contract declares for the reply this one was recognised as. + */ + fun setReplyMessage(messageId: String) { + addResultValue(REPLY_MESSAGE, messageId) + } + + fun getReplyMessage(): String? = getResultValue(REPLY_MESSAGE) + + fun setCorrelationMatched(matched: Boolean) { + addResultValue(CORRELATION_MATCHED, matched.toString()) + } + + fun getCorrelationMatched(): Boolean? = getResultValue(CORRELATION_MATCHED)?.toBoolean() + + fun setWaitedMs(ms: Long) { + addResultValue(WAITED_MS, ms.toString()) + } + + fun getWaitedMs(): Long? = getResultValue(WAITED_MS)?.toLong() +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt new file mode 100644 index 0000000000..758ecf2b62 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiOutcome.kt @@ -0,0 +1,27 @@ +package org.evomaster.core.problem.asyncapi.data + +/** + * What came of publishing one message, as the driver reported it. + */ +enum class AsyncApiOutcome { + + /** + * Published, and no reply was expected. + */ + PUBLISHED, + + /** + * Published, and a reply arrived that answers it. + */ + REPLIED, + + /** + * Published, a reply was expected, and none arrived within the time waited. + */ + NO_REPLY, + + /** + * The driver could not put the message on the wire at all. + */ + PUBLISH_FAILED +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt new file mode 100644 index 0000000000..a5c520d636 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -0,0 +1,484 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.inject.Inject +import com.webfuzzing.asyncapi.models.AsyncApiChannel +import com.webfuzzing.asyncapi.models.AsyncApiCorrelationId +import com.webfuzzing.asyncapi.models.AsyncApiReply +import java.util.UUID +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.database.sql.SqlAction +import org.evomaster.core.logging.LoggingUtil +import org.evomaster.core.problem.api.service.ApiWsFitness +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome +import org.evomaster.core.problem.asyncapi.param.AsyncApiParam +import org.evomaster.core.problem.enterprise.DetectedFault +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.FitnessValue +import org.evomaster.core.search.action.ActionResult +import org.evomaster.core.search.gene.utils.GeneUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * Publishes the messages of a test through the driver and turns what comes back into targets: + * for every operation what publishing to it did, and which of the replies the contract declares + * was recognised. + */ +class AsyncApiFitness : ApiWsFitness() { + + companion object { + private val log: Logger = LoggerFactory.getLogger(AsyncApiFitness::class.java) + + /** + * Prefix of the `(outcome x operation)` targets, written as PREFIX:OUTCOME:action. + */ + private const val OUTCOME_TARGET_PREFIX = "ASYNCAPI_OUTCOME" + + /** + * Prefix of the `(declared reply x operation)` targets, written as PREFIX:messageId:action. + */ + private const val REPLY_TARGET_PREFIX = "ASYNCAPI_REPLY" + + /** + * Prefix of the targets covered when a reply matched no declared message, written as + * PREFIX:action. + */ + private const val UNRECOGNISED_REPLY_TARGET_PREFIX = "ASYNCAPI_UNRECOGNISED_REPLY" + + private const val TARGET_SEPARATOR = ":" + + private const val CORRELATION_SEPARATOR = "-" + + private const val DEFAULT_CONTENT_TYPE = "application/json" + + /** + * What a {placeholder} in a channel address starts with. + */ + private const val PARAMETER_OPENING = "{" + + private val mapper = ObjectMapper() + + /** + * The id of the target covered when publishing to [actionName] had [outcome]. + */ + private fun getOutcomeTargetId(outcome: AsyncApiOutcome, actionName: String): String = + listOf(OUTCOME_TARGET_PREFIX, outcome.name, actionName).joinToString(TARGET_SEPARATOR) + + /** + * The id of the target covered when a reply to [actionName] was recognised as [messageId]. + */ + private fun getReplyTargetId(messageId: String, actionName: String): String = + listOf(REPLY_TARGET_PREFIX, messageId, actionName).joinToString(TARGET_SEPARATOR) + + /** + * The id of the target covered when a reply to [actionName] matched no declared message. + */ + private fun getUnrecognisedReplyTargetId(actionName: String): String = + listOf(UNRECOGNISED_REPLY_TARGET_PREFIX, actionName).joinToString(TARGET_SEPARATOR) + } + + @Inject + private lateinit var asyncApiSampler: AsyncApiSampler + + /** + * Tells this run's correlation ids from those of an earlier run against the same broker, so + * that a reply left over from a previous run is not read as an answer to this one. + * + * Deliberately not drawn from [randomness]: a run repeated under the same seed would reuse + * the ids it used before, which is the one case this exists to tell apart. + */ + private val runId: String = UUID.randomUUID().toString().take(8) + + /** + * How many messages this run has published, which is what makes each correlation id unique. + */ + private var published = 0L + + override fun doCalculateCoverage( + individual: AsyncApiIndividual, + targets: Set, + allTargets: Boolean, + fullyCovered: Boolean, + descriptiveIds: Boolean, + ): EvaluatedIndividual? { + + rc.resetSUT() + + val actionResults: MutableList = mutableListOf() + + doDbCalls(individual.seeInitializingActions().filterIsInstance(), actionResults = actionResults) + + val fv = FitnessValue(individual.size().toDouble()) + + val actions = individual.seeMainExecutableActions().filterIsInstance() + + for ((index, action) in actions.withIndex()) { + val ok = publish(action, index, actionResults, fv) + if (!ok) { + break + } + } + + /* + There is no separate black-box fitness here, unlike REST and GraphQL: AsyncAPI publishes + through the driver in both modes, which is what EMConfig.usesDriver() states. What + changes is only what this call adds on top of the outcome and reply targets above: an + instrumented SUT reports the lines and branches the messages reached, an uninstrumented + one reports none. + */ + val dto = updateFitnessAfterEvaluation(targets, allTargets, fullyCovered, descriptiveIds, individual, fv) + ?: return null + handleExtra(dto, fv) + + return EvaluatedIndividual( + fv, + individual.copy() as AsyncApiIndividual, + actionResults, + trackOperator = individual.trackOperator, + index = time.evaluatedIndividuals, + config = config + ) + } + + /** + * @return whether the message reached the wire, so that the test can go on + */ + private fun publish( + action: AsyncApiAction, + index: Int, + actionResults: MutableList, + fv: FitnessValue + ): Boolean { + + searchTimeController.waitForRateLimiter() + + val result = AsyncApiCallResult(action.getLocalId()) + actionResults.add(result) + + val dto = getActionDto(action, index) + dto.asyncApiCall = toDto(action) + + val reply = rc.executeNewAsyncApiActionAndGetReply(dto) + + if (reply == null || reply.published != true) { + /* + Not a finding about the service: the driver could not put the message on the + wire, could not be reached at all, or did not say. Nothing published after this + point would mean anything, so the test stops here, and no target is registered. + */ + result.setOutcome(AsyncApiOutcome.PUBLISH_FAILED) + result.setErrorMessage(describeFailure(reply)) + result.stopping = true + return false + } + + val outcome = record(reply, result) + handleTargets(fv, action, result, outcome, index) + + return true + } + + /** + * Copy what the driver reported onto the result, and say what it amounts to. + */ + private fun record(reply: AsyncApiReplyDto, result: AsyncApiCallResult): AsyncApiOutcome { + + /* + Every flag here may be absent: they are boxed so that a driver which did not set one + can be told from a driver that set it to false. What was not said is read as not + having happened. + */ + val outcome = when { + reply.replyExpected != true -> AsyncApiOutcome.PUBLISHED + reply.replyReceived == true -> AsyncApiOutcome.REPLIED + else -> AsyncApiOutcome.NO_REPLY + } + + result.setOutcome(outcome) + reply.waitedMs?.let { result.setWaitedMs(it) } + + if (outcome == AsyncApiOutcome.REPLIED) { + reply.replyPayload?.let { result.setReplyPayload(it) } + /* + Only when the driver actually checked. A driver that does not track correlation + says nothing here, which must not be recorded as the service having failed to + echo the id back. + */ + reply.correlationMatched?.let { result.setCorrelationMatched(it) } + } + + return outcome + } + + private fun handleTargets( + fv: FitnessValue, + action: AsyncApiAction, + result: AsyncApiCallResult, + outcome: AsyncApiOutcome, + index: Int + ) { + + val name = action.getName() + + fv.updateTarget(idMapper.handleLocalTarget(getOutcomeTargetId(outcome, name)), 1.0, index) + + when (outcome) { + + AsyncApiOutcome.REPLIED -> handleReplyTargets(fv, action, result, index) + + AsyncApiOutcome.NO_REPLY -> + handleFault(fv, result, ExperimentalFaultCategory.ASYNCAPI_NO_REPLY, name, index) + + AsyncApiOutcome.PUBLISHED -> Unit + + /* + Publishing gives up before this is called, so getting here means the caller + changed. There is no `else` branch on purpose: a newly added outcome should fail + to compile here rather than fall through unnoticed at run time. + */ + AsyncApiOutcome.PUBLISH_FAILED -> + throw IllegalStateException("A message that was not published reached target handling") + } + } + + /** + * Why a message did not go out, as far as can be told from what came back. + */ + private fun describeFailure(reply: AsyncApiReplyDto?): String { + + if (reply == null) { + return "No response from the driver" + } + + return reply.errorMessage + ?: if (reply.published == null) { + "The driver did not report whether the message was published" + } else { + "The driver could not publish the message" + } + } + + /** + * Register a fault, unless the user has switched off the category it belongs to. + * + * It goes both on the fitness value, where the search can aim at it, and on the action + * result, which is where the reports count faults from. + */ + private fun handleFault( + fv: FitnessValue, + result: AsyncApiCallResult, + category: ExperimentalFaultCategory, + actionName: String, + index: Int + ) { + if (!config.isEnabledFaultCategory(category)) { + return + } + + val descriptiveId = idMapper.getFaultDescriptiveId(category, actionName) + fv.updateTarget(idMapper.handleLocalTarget(descriptiveId), 1.0, index) + result.addFault(DetectedFault(category, actionName, null)) + } + + private fun handleReplyTargets(fv: FitnessValue, action: AsyncApiAction, result: AsyncApiCallResult, index: Int) { + + val name = action.getName() + val document = asyncApiSampler.document + val operation = document.operations[action.operationId] + + if (operation == null) { + LoggingUtil.uniqueUserWarn( + "No operation '" + action.operationId + "' in the document, so its replies cannot be recognised" + ) + return + } + val declared = document.replyMessagesOf(operation) + + if (declared.isEmpty()) { + //the contract says a reply comes, but not what it is: nothing to recognise it as + return + } + + val payload = result.getReplyPayload() + + if (payload.isNullOrBlank()) { + /* + A reply did arrive, but carries no body to match against the contract -- an + acknowledgement, or a transport that answers in metadata. Nothing was declared + to be wrong, so this is not the undeclared-reply fault. + */ + return + } + + val recognised = AsyncApiReplyClassifier.classify(payload, declared, document.componentSchemas) + + if (recognised == null) { + /* + Registered apart from the fault, and not subject to it being enabled: a reply that + matches nothing the contract declares is a behaviour worth reaching whether or not + it is also reported as a fault. Without this the search would have nothing to aim + at here, since no declared message was matched either. + */ + fv.updateTarget(idMapper.handleLocalTarget(getUnrecognisedReplyTargetId(name)), 1.0, index) + handleFault(fv, result, ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY, name, index) + return + } + + result.setReplyMessage(recognised.id) + fv.updateTarget(idMapper.handleLocalTarget(getReplyTargetId(recognised.id, name)), 1.0, index) + } + + /** + * Everything the driver needs to publish the message and wait for its reply, resolved + * against the document so that the driver never has to read it. + */ + private fun toDto(action: AsyncApiAction): AsyncApiActionDto { + + val document = asyncApiSampler.document + val message = document.messages[action.messageId] + val channel = document.channels[action.channelName] + + val dto = AsyncApiActionDto() + dto.operationId = action.operationId + dto.channelName = action.channelName + dto.messageId = action.messageId + + dto.address = getAddress(channel, action.channelName) + + dto.payload = action.parameters.firstOrNull { it.name == AsyncApiParam.PAYLOAD } + ?.gene?.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = null) + dto.contentType = message?.contentType ?: document.defaultContentType ?: DEFAULT_CONTENT_TYPE + dto.headers = LinkedHashMap(buildHeaders(action)) + + dto.correlationId = runId + CORRELATION_SEPARATOR + published++ + message?.correlationId?.let { + dto.correlationLocation = if (it.source == AsyncApiCorrelationId.Source.HEADER) { + AsyncApiActionDto.CORRELATION_IN_HEADER + } else { + AsyncApiActionDto.CORRELATION_IN_PAYLOAD + } + dto.correlationPointer = it.pointer + } + + action.replyTemplate?.let { reply -> + dto.replyAddress = getReplyAddress(reply, action) + if (dto.replyAddress != null) { + dto.replyTimeoutMs = config.asyncApiReplyTimeoutMs.toLong() + } + } + + return dto + } + + /** + * The headers to publish alongside the body: key is the header name as the document declares + * it, value is what to send under it, as text. + * + * They are read back from the gene's own JSON printing, which is what knows which optional + * headers are on. A header whose value prints as JSON null is left out rather than sent as + * the text "null". + */ + private fun buildHeaders(action: AsyncApiAction): Map { + + val headers = LinkedHashMap() + + val gene = action.parameters.firstOrNull { it.name == AsyncApiParam.HEADERS }?.gene + ?: return headers + + val json = gene.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = null) + + val node = try { + mapper.readTree(json) + } catch (e: Exception) { + log.warn("The headers of '{}' did not print as JSON: {}", action.getName(), e.message) + return headers + } + + if (!node.isObject) { + log.warn("The headers of '{}' printed as {}, not as an object, so none are sent", + action.getName(), json) + return headers + } + + node.fields().forEach { (name, value) -> + if (!value.isNull) { + headers[name] = if (value.isValueNode) value.asText() else value.toString() + } + } + + return headers + } + + /** + * Where the driver should wait for the reply, or null when there is nowhere to wait yet. + */ + private fun getReplyAddress(reply: AsyncApiReply, action: AsyncApiAction): String? { + + val channelName = reply.channelName + + if (channelName == null) { + /* + The reply address is announced inside the request (reply.address.location) rather + than fixed by the contract. Supporting that means minting an address and stamping + it into the message, which is not done yet; until then such an operation is + published without waiting. + */ + LoggingUtil.uniqueUserWarn( + "Operation '${action.operationId}' announces its reply address at run time, which is" + + " not supported yet: its replies will not be waited for" + ) + return null + } + + return getAddress(asyncApiSampler.document.channels[channelName], channelName) + } + + /** + * Where a message on [channel] actually goes on the wire. + * + * Usually the channel's address, but a protocol binding may override it: the Kafka binding + * carries its own topic, and a document that uses one often declares no address at all. A + * channel that declares neither is decided at run time, so the driver is given the channel's + * name and left to map it, being the one that knows the broker. + * + * Publishing to the wrong destination is silent -- the messages simply reach nobody -- so it + * is worth taking the binding into account rather than assuming the address is the whole story. + */ + private fun getAddress(channel: AsyncApiChannel?, channelName: String): String { + + if (channel == null) { + return channelName + } + + val protocol = asyncApiSampler.document.serversOf(channel).firstOrNull()?.protocol + + /* + A channel that declares no address of its own is normal for Kafka, where the topic + lives in the binding. The protocol is only known when the document declares a server, + so when there is no address to keep, the binding's topic is taken whatever it says: + it is the one destination the document actually names. + */ + val address = channel.effectiveAddress(protocol) + ?: channel.bindings?.kafkaTopic + ?: return channelName + + if (address.contains(PARAMETER_OPENING)) { + /* + The address has {placeholders} backed by the channel's parameters, which are not + filled in yet. Publishing to it verbatim reaches nobody. + */ + LoggingUtil.uniqueUserWarn( + "The address of channel '" + channelName + "' has parameters that are not supported yet: " + address + ) + } + + return address + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt new file mode 100644 index 0000000000..7ea148004d --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt @@ -0,0 +1,105 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.TypeLiteral +import org.evomaster.core.output.service.NoTestCaseWriter +import org.evomaster.core.output.service.TestCaseWriter +import org.evomaster.core.output.service.TestSuiteWriter +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.enterprise.service.EnterpriseModule +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler +import org.evomaster.core.remote.service.RemoteController +import org.evomaster.core.remote.service.RemoteControllerImplementation +import org.evomaster.core.search.service.Archive +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector +import org.evomaster.core.search.service.Minimizer +import org.evomaster.core.search.service.Sampler +import org.evomaster.core.search.service.mutator.Mutator +import org.evomaster.core.search.service.mutator.StandardMutator +import org.evomaster.core.search.service.mutator.StructureMutator + +/** + * The services a search over an AsyncAPI service is made of. + * + * One module serves both white-box and black-box mode, and the driver is bound unconditionally: + * it is what holds the connection to the broker, so it takes part either way (see + * [org.evomaster.core.EMConfig.usesDriver]). + * + * No test cases are written yet, which [org.evomaster.core.EMConfig] enforces by requiring + * `--createTests false`. + */ +class AsyncApiModule : EnterpriseModule() { + + override fun configure() { + + /* + No super.configure(): what EnterpriseModule binds there is REST-only, and the RPC, + GraphQL and Web modules leave it out the same way. + */ + + bind(object : TypeLiteral>() {}) + .to(AsyncApiSampler::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(AsyncApiSampler::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(AsyncApiSampler::class.java) + .asEagerSingleton() + + bind(AsyncApiSampler::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(AsyncApiFitness::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(AsyncApiFitness::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(object : TypeLiteral>() {}) + + bind(Archive::class.java) + .to(object : TypeLiteral>() {}) + + bind(RemoteController::class.java) + .to(RemoteControllerImplementation::class.java) + .asEagerSingleton() + + bind(object : TypeLiteral>() {}) + .to(object : TypeLiteral>() {}) + .asEagerSingleton() + + bind(StructureMutator::class.java) + .to(AsyncApiStructureMutator::class.java) + .asEagerSingleton() + + bind(TestCaseWriter::class.java) + .to(NoTestCaseWriter::class.java) + .asEagerSingleton() + + bind(TestSuiteWriter::class.java) + .asEagerSingleton() + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt new file mode 100644 index 0000000000..ead7eebbb8 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifier.kt @@ -0,0 +1,300 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.webfuzzing.asyncapi.models.AsyncApiMessage +import com.webfuzzing.asyncapi.resolver.AsyncApiRefResolver + +/** + * Recognises which of the messages a contract declares for a reply an observed reply is, by + * matching its payload against each declared schema. + * + * It is a classifier rather than a validator: it reads what tells the declared messages apart + * and gives the benefit of the doubt on anything it cannot read, because a reply it fails to + * recognise is reported as a fault. Nothing else here has to do this -- REST is told which + * response applies by the status code, GraphQL by its `errors` field, and RPC by the driver -- + * so there was no existing answer to borrow. + * + * The matching is done by hand rather than with a schema validator. The one already on the + * classpath, pulled in by swagger-request-validator, understands draft-04 only, and so does not + * know `const`: the very keyword AsyncAPI documents lean on to tell message variants apart, and + * the reason [org.evomaster.core.problem.asyncapi.builder.AsyncApiGeneBuilder] has to rewrite it + * before the gene builder sees it. A validator that reads a modern draft would replace most of + * this, at the cost of a new dependency. + */ +object AsyncApiReplyClassifier { + + private const val REF = "\$ref" + private const val PATH_SEPARATOR = "/" + private const val TYPE = "type" + private const val PROPERTIES = "properties" + private const val REQUIRED = "required" + private const val ITEMS = "items" + private const val CONST = "const" + private const val ENUM = "enum" + private const val ONE_OF = "oneOf" + private const val ANY_OF = "anyOf" + private const val ALL_OF = "allOf" + + private const val TYPE_OBJECT = "object" + private const val TYPE_ARRAY = "array" + private const val TYPE_STRING = "string" + private const val TYPE_INTEGER = "integer" + private const val TYPE_NUMBER = "number" + private const val TYPE_BOOLEAN = "boolean" + private const val TYPE_NULL = "null" + + /** + * How deep to descend into a schema before giving up and accepting what is left. Nesting and + * the combinators both count towards it, so a deeply combinated schema can reach it. + */ + private const val MAX_SCHEMA_DEPTH = 32 + + /** + * How many `$ref` hops to follow before deciding the references form a cycle. + */ + private const val MAX_REF_CHAIN = 32 + + private val mapper = ObjectMapper() + + /** + * The declared message [payload] is an instance of, or null when it is none of them. + * + * When more than one matches, the most specific wins: the one pinning down the most fields, + * through `required` and through `const`/`enum` properties. Ties go to the first declared. + * + * @param candidates the messages the contract declares for the reply, in declaration order + * @param componentSchemas where a `$ref` inside a payload schema is resolved + */ + fun classify( + payload: String?, + candidates: List, + componentSchemas: Map + ): AsyncApiMessage? { + + if (payload == null) { + return null + } + + val node = try { + mapper.readTree(payload) + } catch (_: JsonProcessingException) { + //not JSON, so it is none of the JSON-described messages + return null + } + + //an empty body parses to nothing at all, which is no message either + if (node == null || node.isMissingNode) { + return null + } + + return candidates + .filter { it.payload != null && matches(node, it.payload, componentSchemas, 0) } + .maxByOrNull { specificity(it.payload, componentSchemas) } + } + + private fun matches(node: JsonNode, schema: JsonNode, schemas: Map, depth: Int): Boolean { + + if (depth >= MAX_SCHEMA_DEPTH) { + return true + } + + //a reference that cannot be followed is something this cannot judge, so it does not reject + val resolved = resolve(schema, schemas) ?: return true + + if (!resolved.isObject) { + return true + } + + val const = resolved.get(CONST) + if (const != null && !sameValue(node, const)) { + return false + } + + val allowed = resolved.get(ENUM) + if (allowed != null && allowed.isArray && allowed.none { sameValue(node, it) }) { + return false + } + + val type = resolved.get(TYPE) + if (type != null && !isOfType(node, type)) { + return false + } + + val all = getBranches(resolved, ALL_OF) + if (all != null && all.any { !matches(node, it, schemas, depth + 1) }) { + return false + } + + val any = getBranches(resolved, ANY_OF) + if (any != null && any.none { matches(node, it, schemas, depth + 1) }) { + return false + } + + //oneOf is read as "at least one": exclusivity is a validator's concern, not a classifier's + val one = getBranches(resolved, ONE_OF) + if (one != null && one.none { matches(node, it, schemas, depth + 1) }) { + return false + } + + if (node.isObject) { + resolved.get(REQUIRED)?.let { required -> if (required.any { !node.has(it.asText()) }) return false } + + resolved.get(PROPERTIES)?.fields()?.forEach { (name, property) -> + val value = node.get(name) + if (value != null && !matches(value, property, schemas, depth + 1)) { + return false + } + } + } + + if (node.isArray) { + resolved.get(ITEMS)?.let { items -> + if (items.isObject && node.any { !matches(it, items, schemas, depth + 1) }) return false + } + } + + return true + } + + /** + * The branches of a combinator, or null when it is not a usable list of them. A combinator + * written as anything but a non-empty array says nothing, and must not reject everything. + */ + private fun getBranches(schema: JsonNode, keyword: String): List? { + + val branches = schema.get(keyword) ?: return null + + return if (branches.isArray && !branches.isEmpty) branches.toList() else null + } + + /** + * Whether two JSON values are the same as JSON Schema counts sameness. Numbers compare by + * value, so that a schema written `const: 1.0` accepts a reply carrying `1`; Jackson's own + * equality would say those differ, being of different node types. + */ + private fun sameValue(node: JsonNode, other: JsonNode): Boolean { + + if (node.isNumber && other.isNumber && hasDecimalValue(node) && hasDecimalValue(other)) { + return node.decimalValue().compareTo(other.decimalValue()) == 0 + } + + return node == other + } + + /** + * Whether the number has a decimal value at all. Only a floating-point node can hold an + * infinity or a NaN, and asking those for a [java.math.BigDecimal] throws. + */ + private fun hasDecimalValue(node: JsonNode): Boolean { + return !(node.isDouble || node.isFloat) || node.doubleValue().isFinite() + } + + /** + * Whether [node] is of one of the types [type] names. JSON Schema writes it as one name or a + * list of them, and counts a number with no fractional part as an integer. + * + * A `type` that is neither is not something this can read, so nothing is rejected on it. + */ + private fun isOfType(node: JsonNode, type: JsonNode): Boolean { + + val names = when { + type.isArray && !type.isEmpty -> type.map { it.asText() } + type.isTextual -> listOf(type.asText()) + else -> return true + } + + return names.any { name -> + when (name) { + TYPE_OBJECT -> node.isObject + TYPE_ARRAY -> node.isArray + TYPE_STRING -> node.isTextual + //a whole number written with a decimal point is an integer; canConvertToExactIntegral + //also answers for a value too large to be a BigDecimal, where decimalValue() throws + TYPE_INTEGER -> node.isIntegralNumber || (node.isNumber && node.canConvertToExactIntegral()) + TYPE_NUMBER -> node.isNumber + TYPE_BOOLEAN -> node.isBoolean + TYPE_NULL -> node.isNull + else -> true + } + } + } + + /** + * The schema itself, once any chain of `$ref` has been followed. Null when a reference leads + * nowhere, which is the one case this cannot judge. + * + * A pointer may go deeper than the schema it names, as in + * `#/components/schemas/Order/properties/item`, which is a legitimate way of saying "the + * shape of that one property". + */ + private fun resolve(schema: JsonNode, schemas: Map, depth: Int = 0): JsonNode? { + + if (depth >= MAX_REF_CHAIN) { + return null + } + + var current = schema + + repeat(MAX_REF_CHAIN - depth) { + + val ref = AsyncApiRefResolver.refOf(current) ?: return current + val key = AsyncApiRefResolver.schemaKeyOf(ref) ?: return null + val target = schemas[key] ?: return null + + val pointer = ref.removePrefix(AsyncApiRefResolver.SCHEMA_PREFIX).substringAfter(PATH_SEPARATOR, "") + + if (pointer.isEmpty()) { + current = target + } else { + //the schema the pointer goes into may itself be a reference, so follow that first + val base = resolve(target, schemas, depth + 1) ?: return null + current = base.at(PATH_SEPARATOR + pointer) + if (current.isMissingNode) { + return null + } + } + } + + return null + } + + /** + * How many fields a schema pins down, which is what tells a specific message from a + * permissive one when a reply matches both. + * + * Counted through the combinators as well: a message that says what it requires inside an + * `allOf` is no less specific for having written it that way. + */ + private fun specificity(schema: JsonNode, schemas: Map, depth: Int = 0): Int { + + if (depth >= MAX_SCHEMA_DEPTH) { + return 0 + } + + val resolved = resolve(schema, schemas) ?: return 0 + + if (!resolved.isObject) { + return 0 + } + + val required = resolved.get(REQUIRED)?.size() ?: 0 + + val pinned = resolved.get(PROPERTIES)?.count { property -> + val target = resolve(property, schemas) + target != null && (target.has(CONST) || target.has(ENUM)) + } ?: 0 + + //every branch of an allOf has to hold, so all of them count + val fromAll = getBranches(resolved, ALL_OF) + ?.sumOf { specificity(it, schemas, depth + 1) } ?: 0 + + //only one branch of a choice has to hold, so it is worth what its weakest branch is + val fromChoice = listOf(ANY_OF, ONE_OF).sumOf { keyword -> + getBranches(resolved, keyword)?.minOf { specificity(it, schemas, depth + 1) } ?: 0 + } + + return required + pinned + fromAll + fromChoice + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt index bbfea75981..6f494ccd26 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt @@ -41,6 +41,13 @@ class AsyncApiSampler : ApiWsSampler() { */ private val adHocInitialIndividuals: MutableList = mutableListOf() + /** + * The document the actions were built from. The fitness needs it again at execution time, + * to resolve addresses and to recognise which declared reply came back. + */ + lateinit var document: AsyncApiDocument + private set + /** * Start the service through the driver, read its document, and build one action per * publishable message. Anything the parser or the builder had to skip is reported. @@ -63,14 +70,14 @@ class AsyncApiSampler : ApiWsSampler() { val problem = infoDto.asyncApiProblem ?: throw SutProblemException("Missing problem definition object") - val schema = readSchema(problem) + document = readSchema(problem) val messages = AsyncApiActionBuilder.addActionsFromSchema( - schema, + document, actionCluster, AsyncApiGeneBuilder.options(config) ) - handleMessages(schema.warnings + messages) + handleMessages(document.warnings + messages) initSqlInfo(infoDto) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt new file mode 100644 index 0000000000..df61cdec8f --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutator.kt @@ -0,0 +1,89 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.Inject +import org.evomaster.core.database.sql.SqlInsertBuilder +import org.evomaster.core.problem.api.service.ApiWsStructureMutator +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.Individual +import org.evomaster.core.search.service.mutator.MutatedGeneSpecification + +/** + * Changes how many messages a test publishes: one more, drawn from the action templates, or one + * fewer. Which messages, and what they say, is left to the gene mutations. + */ +class AsyncApiStructureMutator : ApiWsStructureMutator() { + + /* + TODO Most of this is the same as RPCStructureMutator, which in turn shares it with the + REST one. Should be pulled up into ApiWsStructureMutator rather than copied a third time. + */ + + @Inject + private lateinit var sampler: AsyncApiSampler + + override fun mutateStructure( + individual: Individual, + evaluatedIndividual: EvaluatedIndividual<*>, + mutatedGenes: MutatedGeneSpecification?, + targets: Set + ) { + if (individual !is AsyncApiIndividual) { + throw IllegalArgumentException( + "Invalid: individual type to be mutated with AsyncApiStructureMutator should be" + + " AsyncApiIndividual but ${individual::class.java.simpleName}" + ) + } + + if (!individual.canMutateStructure()) return + if (config.maxTestSize == 1) return + + addOrRemoveAMessage(individual, mutatedGenes) + + if (config.trackingEnabled()) tag(individual, time.evaluatedIndividuals) + } + + private fun addOrRemoveAMessage(individual: AsyncApiIndividual, mutatedGenes: MutatedGeneSpecification?) { + + val size = individual.seeMainExecutableActions().size + + //a test keeps at least one message, and never grows past what the user allowed + val canAdd = size < config.maxTestSize + val canRemove = size > 1 + + if (!canAdd && !canRemove) { + return + } + + if (canAdd && (!canRemove || randomness.nextBoolean())) { + val added = sampler.sampleRandomAction() + individual.addAction(action = added) + mutatedGenes?.addRemovedOrAddedByAction( + added, + individual.seeFixedMainActions().indexOf(added), + null, + false, + size + ) + } else { + val chosen = randomness.choose(individual.seeMainActionComponents().indices) + val removed = individual.seeMainExecutableActions()[chosen] + mutatedGenes?.addRemovedOrAddedByAction( + removed, + individual.seeFixedMainActions().indexOf(removed), + null, + true, + size + ) + individual.removeAction(chosen) + } + } + + override fun addInitializingActions(individual: EvaluatedIndividual<*>, mutatedGenes: MutatedGeneSpecification?) { + addInitializingActions(individual, mutatedGenes, sampler) + } + + override fun getSqlInsertBuilder(): SqlInsertBuilder? { + return sampler.sqlInsertBuilder + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt index ad4b452e90..4cf81db2e3 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt @@ -11,6 +11,13 @@ enum class ExperimentalFaultCategory( //9xx for experimental, work-in-progress oracles + //AsyncAPI + // the contract promised a reply, and none arrived within the time waited + ASYNCAPI_NO_REPLY(950, "No Reply", "getsNoReply", + "TODO"), + // a reply arrived, but it matches none of the messages the contract says a reply can be + ASYNCAPI_UNDECLARED_REPLY(951, "Undeclared Reply", "repliesWithUndeclaredMessage", + "TODO"), //Implemented HTTP_TIMEOUT(960, "Request Timeout", "requestTimeout", "TODO"), diff --git a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteController.kt b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteController.kt index fb08a4b1c2..600a535931 100644 --- a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteController.kt +++ b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteController.kt @@ -1,6 +1,7 @@ package org.evomaster.core.remote.service import org.evomaster.client.java.controller.api.dto.* +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto import org.evomaster.client.java.controller.api.dto.problem.param.DeriveParamResponseDto import org.evomaster.client.java.controller.api.dto.problem.param.DerivedParamChangeReqDto import org.evomaster.core.scheduletask.ScheduleTaskExecutor @@ -51,6 +52,17 @@ interface RemoteController : DatabaseExecutor, ScheduleTaskExecutor { fun executeNewRPCActionAndGetResponse(actionDto: ActionDto) : ActionResponseDto? + /** + * Have the driver publish the message in [ActionDto.asyncApiCall] and, when one is + * expected, wait for its reply. Null when the driver could not be reached, or refused. + * + * Only a driver for an AsyncAPI service can do this, which is why the default throws + * rather than every other controller having to say it cannot. + */ + fun executeNewAsyncApiActionAndGetReply(actionDto: ActionDto) : AsyncApiReplyDto? { + throw IllegalStateException("This controller does not publish AsyncAPI messages") + } + fun postSearchAction(postSearchActionDto: PostSearchActionDto) : Boolean fun registerNewAction(actionDto: ActionDto) : Boolean diff --git a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt index 06e025ad1c..664f984bbc 100644 --- a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt +++ b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt @@ -5,6 +5,7 @@ import com.google.inject.Inject import org.evomaster.client.java.controller.api.ControllerConstants import org.evomaster.client.java.controller.api.dto.* import org.evomaster.client.java.controller.api.dto.database.operations.* +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto import org.evomaster.client.java.controller.api.dto.problem.param.DeriveParamResponseDto import org.evomaster.client.java.controller.api.dto.problem.param.DerivedParamChangeReqDto import org.evomaster.client.java.controller.api.dto.problem.rpc.ScheduleTaskInvocationsDto @@ -35,6 +36,12 @@ import javax.ws.rs.core.Response class RemoteControllerImplementation() : RemoteController{ companion object { + + /** + * Query parameter telling the driver whether SQL heuristics are computed from what it + * reads back, rather than from what was inserted. + */ + private const val QUERY_FROM_DATABASE = "queryFromDatabase" val log: Logger = LoggerFactory.getLogger(RemoteControllerImplementation::class.java) } @@ -342,7 +349,7 @@ class RemoteControllerImplementation() : RemoteController{ .queryParam("killSwitch", !ignoreKillSwitch && config.killSwitch) .queryParam("fullyCovered", fullyCovered) .queryParam("descriptiveIds", descriptiveIds) - .queryParam("queryFromDatabase", !config.useInsertionForSqlHeuristics) + .queryParam(QUERY_FROM_DATABASE, !config.useInsertionForSqlHeuristics) .request(MediaType.APPLICATION_JSON_TYPE) .post(Entity.entity(ids, MediaType.APPLICATION_JSON_TYPE)) } @@ -397,7 +404,7 @@ class RemoteControllerImplementation() : RemoteController{ val response = makeHttpCall { getWebTarget() .path(ControllerConstants.NEW_ACTION) - .queryParam("queryFromDatabase", !config.useInsertionForSqlHeuristics) + .queryParam(QUERY_FROM_DATABASE, !config.useInsertionForSqlHeuristics) .request() .put(Entity.entity(actionDto, MediaType.APPLICATION_JSON_TYPE)) } @@ -411,6 +418,40 @@ class RemoteControllerImplementation() : RemoteController{ return dto?.data } + override fun executeNewAsyncApiActionAndGetReply(actionDto: ActionDto): AsyncApiReplyDto? { + return executeNewAction( + actionDto, + object : GenericType>() {}, + "Failed to publish an AsyncAPI message") + } + + /** + * Hand one action to the driver to execute, and read back what it reports. Null when the + * driver could not be reached, or answered with an error. + */ + private fun executeNewAction( + actionDto: ActionDto, + type: GenericType>, + errorMessage: String + ): T? { + + val response = makeHttpCall { + getWebTarget() + .path(ControllerConstants.NEW_ACTION) + .queryParam(QUERY_FROM_DATABASE, !config.useInsertionForSqlHeuristics) + .request() + .put(Entity.entity(actionDto, MediaType.APPLICATION_JSON_TYPE)) + } + + val dto = getDtoFromResponse(response, type) + + if (!checkResponse(response, dto, errorMessage)) { + return null + } + + return dto?.data + } + /** * process post actions after search based on [postSearchActionDto] */ @@ -471,7 +512,7 @@ class RemoteControllerImplementation() : RemoteController{ getWebTarget() .path(ControllerConstants.SCHEDULE_TASKS_COMMAND) // shall we set `killSwitch` as true? - .queryParam("queryFromDatabase", !config.useInsertionForSqlHeuristics) + .queryParam(QUERY_FROM_DATABASE, !config.useInsertionForSqlHeuristics) .request() .post(Entity.entity(invocationDto, MediaType.APPLICATION_JSON_TYPE)) } diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 4a67d5de98..98cc8d3bd3 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -777,11 +777,37 @@ internal class EMConfigTest{ connection to the broker even when the service itself is a black box. */ val asyncApi = EMConfig() - asyncApi.updateProperties(parser.parse("--$blackBox", "true", "--problemType", "ASYNCAPI")) + asyncApi.updateProperties(parser.parse("--$blackBox", "true", "--problemType", "ASYNCAPI", "--createTests", "false")) assertTrue(asyncApi.usesDriver()) val whiteBox = EMConfig() whiteBox.updateProperties(parser.parse("--$blackBox", "false")) assertTrue(whiteBox.usesDriver()) } + + @Test + fun testAsyncApiCannotWriteTestsYet(){ + + val parser = EMConfig.getOptionParser() + + //createTests is on by default, and there is no test writer for AsyncAPI yet + val e = assertThrows { + EMConfig().updateProperties(parser.parse("--problemType", "ASYNCAPI")) + } + assertTrue(e.message!!.contains("createTests"), e.message) + + //so a run has to say it only wants the search + EMConfig().updateProperties(parser.parse("--problemType", "ASYNCAPI", "--createTests", "false")) + } + + @Test + fun testAsyncApiCannotSeedTestsYet(){ + + val parser = EMConfig.getOptionParser() + + assertThrows { + EMConfig().updateProperties(parser.parse( + "--problemType", "ASYNCAPI", "--createTests", "false", "--seedTestCases", "true", "--seedTestCasesPath", "seeds.json")) + } + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt new file mode 100644 index 0000000000..7f03d0b60a --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt @@ -0,0 +1,537 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.inject.Injector +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.fireAndForget +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.silence +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.core.problem.enterprise.SampleType +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.IdMapper +import org.evomaster.core.search.service.Randomness +import org.evomaster.core.search.service.SearchGlobalState +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class AsyncApiFitnessTest { + + companion object { + private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" + + /** + * Both AsyncAPI fault categories are experimental, so nothing reports them until the + * user asks for experimental oracles. + */ + private const val EXPERIMENTAL_ORACLES = "--useExperimentalOracles=true" + + private const val ERROR = """{"error": {"code": 400, "message": "n must be >= 3"}}""" + + /** + * One operation that expects nothing back, and one whose message carries a header + * besides the one the correlation id is stamped into. + */ + private val EVENTS = """ + asyncapi: 3.0.0 + info: + title: Events + version: 1.0.0 + channels: + events: + address: app.events + messages: + event: + headers: + type: object + required: [tenant, meta] + properties: + tenant: + type: string + meta: + type: object + required: [v] + properties: + v: + type: integer + correlationId: + type: string + correlationId: + location: '${'$'}message.header#/correlationId' + payload: + type: object + required: [id] + properties: + id: + type: string + operations: + publishEvent: + action: receive + channel: + ${'$'}ref: '#/channels/events' + """.trimIndent() + + /** + * A request whose reply comes back wherever the request says, rather than on a channel + * the contract fixes. + */ + private val DYNAMIC_REPLY = """ + asyncapi: 3.0.0 + info: + title: Dynamic reply + version: 1.0.0 + channels: + requests: + address: app.requests + messages: + request: + headers: + type: object + properties: + replyTo: + type: string + payload: + type: object + required: [id] + properties: + id: + type: string + operations: + ask: + action: receive + channel: + ${'$'}ref: '#/channels/requests' + reply: + address: + location: '${'$'}message.header#/replyTo' + """.trimIndent() + } + + private lateinit var injector: Injector + private lateinit var driver: FakeAsyncApiDriver + private lateinit var sampler: AsyncApiSampler + private lateinit var fitness: FitnessFunction + private lateinit var idMapper: IdMapper + + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + + private fun start( + schemaText: String, + vararg options: String, + answer: (AsyncApiActionDto) -> AsyncApiReplyDto? + ) { + + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(schemaText), answer) + injector = AsyncApiTestInjector.create(driver, "--blackBox=false", *options) + + sampler = injector.getInstance(AsyncApiSampler::class.java) + fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + idMapper = injector.getInstance(IdMapper::class.java) + } + + private fun startNcs(vararg options: String, answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) = + start(AsyncApiAccess.readFromResource(NCS), *options, answer = answer) + + /** + * The faults an evaluation recorded on its action results, which is where the reports read + * them from -- as opposed to the targets the search aims at. + */ + private fun faultsOn(evaluated: EvaluatedIndividual) = + results(evaluated).flatMap { it.getFaults() } + + private fun individualOf(vararg names: String): AsyncApiIndividual { + + val randomness = injector.getInstance(Randomness::class.java) + + val actions = names.map { name -> + val template = sampler.seeAvailableActions().first { it.getName() == name } + (template.copy() as AsyncApiAction).apply { doInitialize(randomness) } + } + + return AsyncApiIndividual(SampleType.RANDOM, actions.toMutableList()).apply { + doGlobalInitialize(injector.getInstance(SearchGlobalState::class.java)) + } + } + + private fun evaluate(vararg names: String): EvaluatedIndividual = + fitness.calculateCoverage(individualOf(*names), modifiedSpec = null) + ?: fail("the fitness gave up on the individual") + + private fun coveredIds(evaluated: EvaluatedIndividual): Set = + evaluated.fitness.coveredTargets().map { idMapper.getDescriptiveId(it) }.toSet() + + private fun results(evaluated: EvaluatedIndividual): List = + evaluated.evaluatedMainActions().map { it.result as AsyncApiCallResult } + + @Test + fun testTheDriverIsToldEverythingItNeeds() { + + startNcs { replied(DOUBLE_RESULT) } + + evaluate("bessj") + + val dto = driver.published.single() + + assertEquals("bessj", dto.operationId) + assertEquals("bessjRequest", dto.channelName) + assertEquals("bessjRequest", dto.messageId) + //resolved from the document, so the driver never has to read it + assertEquals("ncs.bessj.request", dto.address) + assertEquals("application/json", dto.contentType) + assertEquals("ncs.bessj.reply", dto.replyAddress) + assertEquals(5000L, dto.replyTimeoutMs) + + val payload = ObjectMapper().readTree(dto.payload) + assertTrue(payload.isObject, "payload is not a JSON object: ${dto.payload}") + assertEquals(setOf("n", "x"), payload.fieldNames().asSequence().toSet()) + + assertEquals(AsyncApiActionDto.CORRELATION_IN_HEADER, dto.correlationLocation) + assertEquals("/correlationId", dto.correlationPointer) + assertFalse(dto.correlationId.isNullOrBlank()) + assertTrue(dto.headers.isEmpty()) + } + + @Test + fun testEachPublishedMessageGetsItsOwnCorrelationId() { + + startNcs { replied(DOUBLE_RESULT) } + + evaluate("bessj", "expint", "gammq") + evaluate("bessj", "expint", "gammq") + + val ids = driver.published.map { it.correlationId } + assertEquals(6, ids.size) + assertEquals(6, ids.toSet().size, "correlation ids repeat: $ids") + } + + @Test + fun testARecognisedReplyCoversTheOperationAndTheDeclaredMessage() { + + startNcs { replied(DOUBLE_RESULT) } + + val evaluated = evaluate("bessj") + val covered = coveredIds(evaluated) + + assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:bessj"), "$covered") + assertTrue(covered.contains("ASYNCAPI_REPLY:doubleResult:bessj"), "$covered") + assertTrue(covered.none { IdMapper.isFault(it) }, "$covered") + + val result = results(evaluated).single() + assertEquals(AsyncApiOutcome.REPLIED, result.getOutcome()) + assertEquals("doubleResult", result.getReplyMessage()) + assertEquals(DOUBLE_RESULT, result.getReplyPayload()) + assertEquals(true, result.getCorrelationMatched()) + assertFalse(result.stopping) + } + + @Test + fun testTheErrorReplyIsADifferentTargetFromTheResult() { + + startNcs { replied(ERROR) } + + val covered = coveredIds(evaluate("bessj")) + + assertTrue(covered.contains("ASYNCAPI_REPLY:error:bessj"), "$covered") + assertFalse(covered.contains("ASYNCAPI_REPLY:doubleResult:bessj")) + //an error reply the contract declares is the service behaving, not a fault + assertTrue(covered.none { IdMapper.isFault(it) }, "$covered") + } + + @Test + fun testAReplyTheContractDoesNotDeclareIsAFault() { + + startNcs(EXPERIMENTAL_ORACLES) { replied("""{"something": "else"}""") } + + val evaluated = evaluate("bessj") + val faults = evaluated.fitness.coveredTargets().filter { idMapper.isFault(it) } + + assertEquals(1, faults.size, coveredIds(evaluated).toString()) + assertTrue(idMapper.isSpecifiedFault(faults.single(), ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY)) + assertTrue(coveredIds(evaluated).contains("ASYNCAPI_UNRECOGNISED_REPLY:bessj")) + assertNull(results(evaluated).single().getReplyMessage()) + + //the reports count faults off the action result, not off the covered targets + assertEquals( + listOf(ExperimentalFaultCategory.ASYNCAPI_UNDECLARED_REPLY), + faultsOn(evaluated).map { it.category }) + } + + @Test + fun testSilenceAfterAPromisedReplyIsAFault() { + + startNcs(EXPERIMENTAL_ORACLES) { silence(waited = 5000) } + + val evaluated = evaluate("bessj") + val covered = coveredIds(evaluated) + val faults = evaluated.fitness.coveredTargets().filter { idMapper.isFault(it) } + + assertTrue(covered.contains("ASYNCAPI_OUTCOME:NO_REPLY:bessj"), "$covered") + assertEquals(1, faults.size, "$covered") + assertTrue(idMapper.isSpecifiedFault(faults.single(), ExperimentalFaultCategory.ASYNCAPI_NO_REPLY)) + + val result = results(evaluated).single() + assertEquals(AsyncApiOutcome.NO_REPLY, result.getOutcome()) + assertEquals(5000L, result.getWaitedMs()) + //silence is a finding, not a broken setup: the test goes on + assertFalse(result.stopping) + + assertEquals( + listOf(ExperimentalFaultCategory.ASYNCAPI_NO_REPLY), + faultsOn(evaluated).map { it.category }) + } + + @Test + fun testAFireAndForgetOperationIsCoveredByBeingPublished() { + + start(EVENTS) { fireAndForget() } + + val evaluated = evaluate("publishEvent") + val covered = coveredIds(evaluated) + + assertTrue(covered.contains("ASYNCAPI_OUTCOME:PUBLISHED:publishEvent"), "$covered") + assertTrue(covered.none { IdMapper.isFault(it) }, "$covered") + assertEquals(AsyncApiOutcome.PUBLISHED, results(evaluated).single().getOutcome()) + + //no reply declared, so nothing to wait for + val dto = driver.published.single() + assertNull(dto.replyAddress) + assertNull(dto.replyTimeoutMs) + } + + @Test + fun testHeadersTravelAsAMapWithoutTheStampedOne() { + + start(EVENTS) { fireAndForget() } + + evaluate("publishEvent") + + val dto = driver.published.single() + + //the declared headers are the search's to vary; the correlation id is the driver's to stamp + assertEquals(setOf("tenant", "meta"), dto.headers.keys) + assertEquals(AsyncApiActionDto.CORRELATION_IN_HEADER, dto.correlationLocation) + assertEquals("/correlationId", dto.correlationPointer) + + //a header that is itself structured travels as its JSON + val meta = ObjectMapper().readTree(dto.headers.getValue("meta")) + assertTrue(meta.isObject && meta.has("v"), "meta header: ${dto.headers["meta"]}") + } + + @Test + fun testTheCorrelationIdMayBelongInThePayload() { + + start(AsyncApiAccess.readFromResource("/asyncapi/artificial/websocket-reply.yaml")) { + replied("""{"request_id": "r", "legs": []}""") + } + + evaluate("recv_list_legs") + + val dto = driver.published.single() + + //MQTT 3.1.1 and raw WebSocket have no metadata, so such documents carry the id inside the message + assertEquals(AsyncApiActionDto.CORRELATION_IN_PAYLOAD, dto.correlationLocation) + assertEquals("/request_id", dto.correlationPointer) + assertEquals("/v1/vsi", dto.address) + assertEquals("/v1/vsi", dto.replyAddress) + } + + @Test + fun testAReplyAddressAnnouncedAtRunTimeIsNotWaitedForYet() { + + start(DYNAMIC_REPLY) { fireAndForget() } + + val evaluated = evaluate("ask") + + //the contract does promise a reply, but there is nowhere fixed to wait for it + assertTrue((sampler.seeAvailableActions().single() as AsyncApiAction).expectsReply()) + val dto = driver.published.single() + assertNull(dto.replyAddress) + assertNull(dto.replyTimeoutMs) + + assertEquals(AsyncApiOutcome.PUBLISHED, results(evaluated).single().getOutcome()) + assertTrue(coveredIds(evaluated).none { IdMapper.isFault(it) }) + } + + @Test + fun testADriverThatCouldNotPublishSaysWhy() { + + startNcs { AsyncApiReplyDto().apply { published = false; errorMessage = "broker unreachable" } } + + val evaluated = evaluate("bessj", "expint") + + val first = results(evaluated).first() + assertEquals(AsyncApiOutcome.PUBLISH_FAILED, first.getOutcome()) + assertEquals("broker unreachable", first.getErrorMessage()) + assertTrue(first.stopping) + assertEquals(1, driver.published.size) + } + + @Test + fun testAReplyThatDidNotCarryTheCorrelationIdBackIsRecordedAsSuch() { + + startNcs { replied(DOUBLE_RESULT, correlationMatched = false) } + + val evaluated = evaluate("bessj") + + /* + From outside there is no telling a defect from a service that correlates by some + business key instead, so this is recorded, not judged: the reply still counts. + */ + val result = results(evaluated).single() + assertEquals(AsyncApiOutcome.REPLIED, result.getOutcome()) + assertEquals(false, result.getCorrelationMatched()) + assertTrue(coveredIds(evaluated).contains("ASYNCAPI_REPLY:doubleResult:bessj")) + } + + @Test + fun testAMessageTheDriverCannotPublishStopsTheTest() { + + //a driver that cannot be reached: the remote controller reports that as no reply at all + startNcs { null } + + val evaluated = evaluate("bessj", "expint", "gammq") + + assertEquals(1, driver.published.size, "the test went on after a message that never left") + + val first = results(evaluated).first() + assertEquals(AsyncApiOutcome.PUBLISH_FAILED, first.getOutcome()) + assertTrue(first.stopping) + assertNotNull(first.getErrorMessage()) + + //a broken setup is not a finding about the service, so nothing is covered by it + assertTrue(coveredIds(evaluated).none { it.startsWith("ASYNCAPI") }, coveredIds(evaluated).toString()) + } + + @Test + fun testAnUnrecognisedReplyIsStillATargetWithoutExperimentalOracles() { + + /* + The fault is experimental, but the behaviour it reports is not: a reply matching none + of the declared messages is somewhere the search should still be able to aim, or + turning the oracle off would quietly cost coverage rather than only reporting. + */ + startNcs { replied("""{"something": "else"}""") } + + val evaluated = evaluate("bessj") + + assertTrue( + coveredIds(evaluated).contains("ASYNCAPI_UNRECOGNISED_REPLY:bessj"), + coveredIds(evaluated).toString()) + assertTrue(evaluated.fitness.coveredTargets().none { idMapper.isFault(it) }) + assertTrue(faultsOn(evaluated).isEmpty()) + } + + @Test + fun testFaultsAreNotReportedUntilExperimentalOraclesAreAskedFor() { + + //both AsyncAPI categories are experimental, and nothing experimental is on by default + startNcs { silence() } + + val evaluated = evaluate("bessj") + + assertTrue(coveredIds(evaluated).contains("ASYNCAPI_OUTCOME:NO_REPLY:bessj")) + assertTrue(evaluated.fitness.coveredTargets().none { idMapper.isFault(it) }) + assertTrue(faultsOn(evaluated).isEmpty()) + } + + @Test + fun testAReplyWithNoBodyIsNotAnUndeclaredReply() { + + /* + The contract declares what a reply may be, and one arrived carrying nothing to match + against it -- an acknowledgement, or a transport that answers in its metadata. + Nothing was declared to be wrong, so this is not the undeclared-reply fault. + */ + startNcs(EXPERIMENTAL_ORACLES) { replied(payload = "") } + + val evaluated = evaluate("bessj") + + assertEquals(AsyncApiOutcome.REPLIED, results(evaluated).single().getOutcome()) + assertTrue(evaluated.fitness.coveredTargets().none { idMapper.isFault(it) }, coveredIds(evaluated).toString()) + assertTrue(faultsOn(evaluated).isEmpty()) + } + + @Test + fun testAnOperationWhoseReplyDeclaresNoMessageIsNotAFault() { + + /* + A reply channel that carries no message: the contract promises an answer but does not + say what it looks like, so there is nothing to recognise and nothing to report. + */ + val document = """ + asyncapi: 3.0.0 + info: + title: Unspecified reply + version: 1.0.0 + channels: + requests: + address: app.requests + messages: + request: + payload: + type: object + required: [id] + properties: + id: + type: string + replies: + address: app.replies + operations: + ask: + action: receive + channel: + ${'$'}ref: '#/channels/requests' + reply: + channel: + ${'$'}ref: '#/channels/replies' + """.trimIndent() + + start(document, EXPERIMENTAL_ORACLES) { replied("""{"anything": 1}""") } + + val evaluated = evaluate("ask") + + assertEquals(AsyncApiOutcome.REPLIED, results(evaluated).single().getOutcome()) + assertNull(results(evaluated).single().getReplyMessage()) + assertTrue(evaluated.fitness.coveredTargets().none { idMapper.isFault(it) }, coveredIds(evaluated).toString()) + } + + @Test + fun testTheTopicOfAProtocolBindingOverridesTheChannelAddress() { + + /* + A Kafka channel often names no address of its own and puts the topic in its binding. + Publishing to the channel's name instead would reach nobody. + */ + start(AsyncApiAccess.readFromResource("/asyncapi/sut/microcks.yaml")) { fireAndForget() } + + evaluate("receivedServiceChanges") + + assertEquals("microcks-services-updates", driver.published.single().address) + } + + @Test + fun testTheReplyTimeoutIsTheConfiguredOne() { + + startNcs("--asyncApiReplyTimeoutMs=1234") { replied(DOUBLE_RESULT) } + + evaluate("bessj") + + assertEquals(1234L, driver.published.single().replyTimeoutMs) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt new file mode 100644 index 0000000000..c3de0f653b --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt @@ -0,0 +1,160 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 +import org.evomaster.core.search.algorithms.MioAlgorithm +import org.evomaster.core.remote.service.RemoteController +import org.evomaster.core.search.service.IdMapper +import org.evomaster.core.output.service.NoTestCaseWriter +import org.evomaster.core.output.service.TestCaseWriter +import org.evomaster.core.output.service.TestSuiteWriter +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied +import org.evomaster.core.search.service.Archive +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector +import org.evomaster.core.search.service.Minimizer +import org.evomaster.core.search.service.Sampler +import org.evomaster.core.search.service.mutator.StructureMutator +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +/** + * A whole search, through the module Main binds, against a driver standing in for the NCS + * service over Kafka: the first time `--problemType ASYNCAPI` does everything but write tests. + */ +class AsyncApiModuleTest { + + companion object { + private const val NCS = AsyncApiTestInjector.NCS + + private val NCS_OPERATIONS = AsyncApiTestInjector.NCS_OPERATIONS + + private val mapper = ObjectMapper() + } + + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + + /** + * Replies as the NCS service would, judging by the request alone: each operation has inputs + * it rejects, and answers those with the declared error message instead of a result. + */ + private fun ncsLike(call: AsyncApiActionDto): AsyncApiReplyDto { + + val request = try { + mapper.readTree(call.payload) + } catch (e: Exception) { + return replied(error("not JSON")) + } + + fun number(name: String) = request.get(name)?.takeIf { it.isNumber }?.asDouble() + + val a = number("a") + val b = number("b") + val c = number("c") + val n = number("n") + val m = number("m") + val x = number("x") + + val ok = when (call.operationId) { + "checkTriangle" -> a != null && b != null && c != null && a > 0 && b > 0 && c > 0 + "bessj" -> n != null && x != null && n >= 3 && n <= 1000 + "expint" -> n != null && x != null && n >= 0 && x >= 0 + "fisher" -> m != null && n != null && x != null && m in 1.0..1000.0 && n in 1.0..1000.0 + "gammq" -> a != null && x != null && a > 0 && x >= 0 + "remainder" -> a != null && b != null && b != 0.0 + else -> false + } + + if (!ok) { + return replied(error("rejected")) + } + + val integerResult = call.operationId == "checkTriangle" || call.operationId == "remainder" + + return replied(if (integerResult) """{"resultAsInt": 1}""" else """{"resultAsDouble": 0.5}""") + } + + private fun error(message: String) = """{"error": {"code": 400, "message": "$message"}}""" + + @Test + fun testASearchReachesBothDeclaredRepliesOfAnOperation() { + + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { ncsLike(it) } + + //MIO is asked for explicitly, since it is what is instantiated below and tracking follows the option + val injector = AsyncApiTestInjector.create( + driver, + "--blackBox=true", + "--algorithm=MIO", + "--stoppingCriterion=ACTION_EVALUATIONS", + "--maxEvaluations=100", + "--maxTestSize=3", + "--useTimeInFeedbackSampling=false" + ) + + val mio = injector.getInstance(Key.get(object : TypeLiteral>() {})) + val solution = mio.search() + + val idMapper = injector.getInstance(IdMapper::class.java) + val covered = solution.overall.coveredTargets().map { idMapper.getDescriptiveId(it) }.toSet() + + /* + Every operation rejects some inputs, and a hundred evaluations are plenty for the + search to meet both the result and the error of at least one of them. That the two + are distinct targets is the point of the whole exercise; which operations get there + first depends on the seed, so no particular one is asked for. + */ + val bothRepliesReached = NCS_OPERATIONS.filter { op -> + covered.contains("ASYNCAPI_REPLY:error:$op") + && (covered.contains("ASYNCAPI_REPLY:doubleResult:$op") || covered.contains("ASYNCAPI_REPLY:intResult:$op")) + } + assertTrue(bothRepliesReached.isNotEmpty(), "no operation reached both of its replies: $covered") + + //every operation was published to and answered + NCS_OPERATIONS.forEach { assertTrue(covered.contains("ASYNCAPI_OUTCOME:REPLIED:$it"), "$it missing in $covered") } + + //the stand-in always answers with a declared message, so nothing here is a fault + assertTrue(covered.none { IdMapper.isFault(it) }, "$covered") + + assertTrue(solution.individuals.isNotEmpty()) + assertTrue(driver.published.size >= 50, "only ${driver.published.size} messages published") + assertEquals(driver.published.size, driver.published.map { it.correlationId }.toSet().size) + } + + @Test + fun testEverythingMainResolvesIsBound() { + + /* + The module deliberately does not inherit EnterpriseModule's bindings, so anything + Main asks the injector for has to be bound here. A missing one is invisible until a + real run reaches that line, which is usually right after the search. + */ + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { + replied("""{"resultAsInt": 1}""") + } + val injector = AsyncApiTestInjector.create(driver, "--blackBox=false") + + injector.getInstance(Key.get(object : TypeLiteral>() {})) + injector.getInstance(Key.get(object : TypeLiteral>() {})) + injector.getInstance(Key.get(object : TypeLiteral>() {})) + injector.getInstance(Key.get(object : TypeLiteral>() {})) + injector.getInstance(Archive::class.java) + injector.getInstance(StructureMutator::class.java) + injector.getInstance(TestSuiteWriter::class.java) + injector.getInstance(RemoteController::class.java) + + //no test writer for AsyncAPI yet, so the one that writes nothing + assertTrue(injector.getInstance(TestCaseWriter::class.java) is NoTestCaseWriter) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt new file mode 100644 index 0000000000..0861339023 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiReplyClassifierTest.kt @@ -0,0 +1,345 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import com.webfuzzing.asyncapi.models.AsyncApiDocument +import com.webfuzzing.asyncapi.models.AsyncApiMessage +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class AsyncApiReplyClassifierTest { + + private val ncs: AsyncApiDocument = AsyncApiAccess.getAsyncApiFromResource("/asyncapi/sut/ncs-kafka.yaml") + + private val socket: AsyncApiDocument = + AsyncApiAccess.getAsyncApiFromResource("/asyncapi/artificial/websocket-reply.yaml") + + /** + * Replies shaped by the parts of JSON Schema the corpus documents happen not to use. + */ + private val shapes: AsyncApiDocument = AsyncApiAccess.parseFromText( + """ + asyncapi: 3.0.0 + info: + title: Shapes + version: 1.0.0 + components: + messages: + pinned: + payload: + ${'$'}ref: '#/components/schemas/Pinned' + tagged: + payload: + type: object + required: [kind] + properties: + kind: + enum: [a, b] + either: + payload: + oneOf: + - ${'$'}ref: '#/components/schemas/Left' + - ${'$'}ref: '#/components/schemas/Right' + both: + payload: + allOf: + - ${'$'}ref: '#/components/schemas/Left' + - type: object + required: [extra] + scalar: + payload: + anyOf: + - type: string + - type: boolean + numbers: + payload: + type: array + items: + type: integer + flag: + payload: + type: boolean + exotic: + payload: + type: something-this-does-not-know + part: + payload: + ${'$'}ref: '#/components/schemas/Pinned/properties/kind' + schemas: + Pinned: + ${'$'}ref: '#/components/schemas/PinnedTarget' + PinnedTarget: + type: object + required: [kind] + properties: + kind: + const: pinned + Left: + type: object + required: [left] + properties: + left: + type: boolean + Right: + type: object + required: [right] + properties: + right: + type: string + """.trimIndent() + ) + + private fun classifyAmong(payload: String, vararg candidates: String): String? = + AsyncApiReplyClassifier.classify( + payload, + candidates.map { shapes.messages.getValue(it) }, + shapes.componentSchemas + )?.id + + private fun repliesOf(document: AsyncApiDocument, operation: String): List = + document.replyMessagesOf(document.operations.getValue(operation)) + + private fun classify(document: AsyncApiDocument, operation: String, payload: String?): String? = + AsyncApiReplyClassifier.classify(payload, repliesOf(document, operation), document.componentSchemas)?.id + + @Test + fun testAResultIsToldFromAnError() { + + //both reply payloads are behind a $ref to a component schema + assertEquals("doubleResult", classify(ncs, "bessj", """{"resultAsDouble": 2.5}""")) + assertEquals("error", classify(ncs, "bessj", """{"error": {"code": 400, "message": "n must be >= 3"}}""")) + } + + @Test + fun testAReplyMatchingNoDeclaredMessageIsNotRecognised() { + + assertNull(classify(ncs, "bessj", """{"something": "else"}""")) + assertNull(classify(ncs, "bessj", """{"error": "not an object"}""")) + assertNull(classify(ncs, "bessj", """[1, 2, 3]""")) + } + + @Test + fun testTextThatIsNotJsonIsNotRecognised() { + + assertNull(classify(ncs, "bessj", "not json at all")) + assertNull(classify(ncs, "bessj", "")) + assertNull(classify(ncs, "bessj", null)) + } + + @Test + fun testAWholeNumberWrittenWithADecimalPointIsStillAnInteger() { + + //JSON Schema counts 3.0 as an integer, and services do write results that way + assertEquals("intResult", classify(ncs, "checkTriangle", """{"resultAsInt": 3.0}""")) + assertNull(classify(ncs, "checkTriangle", """{"resultAsInt": 3.5}""")) + } + + @Test + fun testTheMostSpecificMatchWinsWhenSeveralCouldFit() { + + /* + The socket's result messages declare no required fields, so an error payload also + fits them structurally. The error message requires its "error" field, which makes it + the more specific description, and so the one chosen. + */ + assertEquals("error", classify(socket, "recv_list_legs", """{"request_id": "r1", "error": {"code": 404}}""")) + assertEquals("listLegsResult", classify(socket, "recv_list_legs", """{"request_id": "r1", "legs": ["a", "b"]}""")) + } + + @Test + fun testATypeListIsHonoured() { + + //"leg" is declared as [object, "null"] + assertEquals("getLegResult", classify(socket, "recv_get_leg", """{"request_id": "r1", "leg": null}""")) + assertEquals("getLegResult", classify(socket, "recv_get_leg", """{"request_id": "r1", "leg": {"id": "x"}}""")) + assertNull(classify(socket, "recv_get_leg", """{"request_id": "r1", "leg": "not an object"}""")) + } + + @Test + fun testWhatItDoesNotUnderstandIsGivenTheBenefitOfTheDoubt() { + + //an array of legs with a non-string inside: items are checked, so this is rejected... + assertNull(classify(socket, "recv_list_legs", """{"legs": [1, 2]}""")) + //...but a format, a pattern or a bound is not read at all, so nothing is rejected on their account + assertEquals("doubleResult", classify(ncs, "bessj", """{"resultAsDouble": -1e308}""")) + } + + @Test + fun testAConstDiscriminatorIsReadThroughAChainOfReferences() { + + //the message points at a schema that is itself only a reference to the real one + assertEquals("pinned", classifyAmong("""{"kind": "pinned"}""", "pinned", "tagged")) + assertNull(classifyAmong("""{"kind": "other"}""", "pinned", "tagged")) + } + + @Test + fun testAnEnumDiscriminatorIsRead() { + + assertEquals("tagged", classifyAmong("""{"kind": "a"}""", "pinned", "tagged")) + assertEquals("tagged", classifyAmong("""{"kind": "b"}""", "pinned", "tagged")) + assertNull(classifyAmong("""{"kind": "z"}""", "pinned", "tagged")) + } + + @Test + fun testOneOfMatchesEitherBranch() { + + assertEquals("either", classifyAmong("""{"left": true}""", "either")) + assertEquals("either", classifyAmong("""{"right": "r"}""", "either")) + assertNull(classifyAmong("""{"neither": 1}""", "either")) + } + + @Test + fun testAllOfNeedsEveryBranch() { + + assertEquals("both", classifyAmong("""{"left": true, "extra": 1}""", "both")) + assertNull(classifyAmong("""{"left": true}""", "both")) + assertNull(classifyAmong("""{"extra": 1}""", "both")) + } + + @Test + fun testAnyOfMatchesAnyBranch() { + + assertEquals("scalar", classifyAmong("\"text\"", "scalar")) + assertEquals("scalar", classifyAmong("true", "scalar")) + assertNull(classifyAmong("1", "scalar")) + } + + @Test + fun testArrayItemsAreChecked() { + + assertEquals("numbers", classifyAmong("[1, 2, 3]", "numbers")) + assertEquals("numbers", classifyAmong("[]", "numbers")) + assertNull(classifyAmong("""[1, "two"]""", "numbers")) + assertNull(classifyAmong("""{"not": "an array"}""", "numbers")) + } + + @Test + fun testABooleanIsNotItsSpelling() { + + assertEquals("flag", classifyAmong("false", "flag")) + assertNull(classifyAmong("\"false\"", "flag")) + } + + @Test + fun testATypeItDoesNotKnowRejectsNothing() { + + assertEquals("exotic", classifyAmong("""{"anything": 1}""", "exotic")) + } + + @Test + fun testAReferenceIntoTheMiddleOfASchemaIsFollowed() { + + /* + "#/components/schemas/Pinned/properties/kind" names one property of a schema, which + is legal. Left unresolved it would match anything, and an operation whose only reply + is written that way could then never report an undeclared reply. + */ + assertEquals("part", classifyAmong("\"pinned\"", "part")) + assertNull(classifyAmong("""{"anything": 1}""", "part")) + } + + @Test + fun testANumberIsComparedByValueNotByHowItIsWritten() { + + //JSON Schema counts 1 and 1.0 as the same number; Jackson's own equality does not + val versions = AsyncApiAccess.parseFromText( + """ + asyncapi: 3.0.0 + info: + title: Versions + version: 1.0.0 + components: + messages: + v1: + payload: + type: object + properties: + version: + const: 1.0 + """.trimIndent() + ) + + fun classifyVersion(payload: String) = AsyncApiReplyClassifier.classify( + payload, listOf(versions.messages.getValue("v1")), versions.componentSchemas)?.id + + assertEquals("v1", classifyVersion("""{"version": 1}""")) + assertEquals("v1", classifyVersion("""{"version": 1.0}""")) + assertNull(classifyVersion("""{"version": 2}""")) + } + + @Test + fun testANumberTooLargeToBeADecimalDoesNotCrash() { + + /* + A float that overflows a double becomes an infinity, which has no BigDecimal. Asking + it for one used to throw, and nothing between here and the search loop catches it. + */ + assertNull(classify(ncs, "checkTriangle", """{"resultAsInt": 1E+400}""")) + assertEquals("intResult", classify(ncs, "checkTriangle", """{"resultAsInt": 1E+30}""")) + } + + @Test + fun testTheMostSpecificMatchIsFoundThroughACombinator() { + + /* + A message that says what it requires inside an allOf is no less specific for having + written it that way, and must still win over a permissive one that also matches. + */ + val combined = AsyncApiAccess.parseFromText( + """ + asyncapi: 3.0.0 + info: + title: Combined + version: 1.0.0 + components: + messages: + generic: + payload: + type: object + required: [requestId] + detailed: + payload: + allOf: + - type: object + required: [requestId] + - type: object + required: [error] + """.trimIndent() + ) + + val both = listOf(combined.messages.getValue("generic"), combined.messages.getValue("detailed")) + + assertEquals( + "detailed", + AsyncApiReplyClassifier.classify( + """{"requestId": "r1", "error": {"code": 404}}""", both, combined.componentSchemas)?.id + ) + assertEquals( + "generic", + AsyncApiReplyClassifier.classify("""{"requestId": "r1"}""", both, combined.componentSchemas)?.id + ) + } + + @Test + fun testAnEmptyBodyIsNoMessageAtAll() { + + //an empty body parses to nothing, which must not be read as matching a permissive schema + val anything = AsyncApiAccess.parseFromText( + """ + asyncapi: 3.0.0 + info: + title: Permissive + version: 1.0.0 + components: + messages: + loose: + payload: + required: [id] + """.trimIndent() + ) + + assertNull( + AsyncApiReplyClassifier.classify( + "", listOf(anything.messages.getValue("loose")), anything.componentSchemas) + ) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt index 6c225e8105..35b79fe6fa 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt @@ -11,10 +11,12 @@ import org.evomaster.core.EMConfig import org.evomaster.core.problem.asyncapi.data.AsyncApiAction import org.evomaster.core.problem.external.service.DummyController import org.evomaster.core.remote.SutProblemException +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.service.WarningsAggregator import org.evomaster.core.search.warning.WarningCategory import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import java.nio.charset.StandardCharsets @@ -23,6 +25,15 @@ import java.nio.file.Path class AsyncApiSamplerTest { + /** + * RestActionBuilderV3 keeps its built genes in a static cache, so a test that did not clear + * it could be handed one built by another. + */ + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + companion object { private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" @@ -49,7 +60,7 @@ class AsyncApiSamplerTest { private fun injector(info: SutInfoDto, starts: Boolean = true, vararg options: String): Injector { - val args = arrayOf("--seed=42", "--problemType=ASYNCAPI") + options + val args = arrayOf("--seed=42", "--problemType=ASYNCAPI", "--createTests=false") + options val modules = listOf(BaseModule(args), object : AbstractModule() { override fun configure() { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt new file mode 100644 index 0000000000..242c1d75dd --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiStructureMutatorTest.kt @@ -0,0 +1,157 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.mutator.MutatedGeneSpecification +import org.evomaster.core.search.service.mutator.StructureMutator +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class AsyncApiStructureMutatorTest { + + companion object { + private const val NCS = AsyncApiTestInjector.NCS + + private val NCS_OPERATIONS = AsyncApiTestInjector.NCS_OPERATIONS + } + + private lateinit var sampler: AsyncApiSampler + private lateinit var mutator: AsyncApiStructureMutator + private lateinit var fitness: FitnessFunction + + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + + private fun start(maxTestSize: Int) { + + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { + replied("""{"resultAsDouble": 1.0}""") + } + val injector = AsyncApiTestInjector.create(driver, "--blackBox=false", "--maxTestSize=$maxTestSize") + + sampler = injector.getInstance(AsyncApiSampler::class.java) + mutator = injector.getInstance(StructureMutator::class.java) as AsyncApiStructureMutator + fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + } + + private fun evaluate(individual: AsyncApiIndividual): EvaluatedIndividual = + fitness.calculateCoverage(individual, modifiedSpec = null) ?: fail("the fitness gave up on the individual") + + /** + * Mutate the structure [times] over, each time from the previous result, and return the + * sizes seen along the way. + */ + private fun sizesAlong(times: Int): List { + + var evaluated = evaluate(sampler.sample(forceRandomSample = true)) + val sizes = mutableListOf() + + repeat(times) { + val copy = evaluated.individual.copy() as AsyncApiIndividual + mutator.mutateStructure(copy, evaluated, null, setOf()) + sizes.add(copy.seeMainExecutableActions().size) + evaluated = evaluate(copy) + } + + return sizes + } + + @Test + fun testATestKeepsAtLeastOneMessageAndNeverGrowsPastTheMaximum() { + + start(maxTestSize = 2) + + val sizes = sizesAlong(40) + + assertTrue(sizes.all { it in 1..2 }, "sizes seen: $sizes") + //with room for exactly two, every mutation has to go the other way from the last + assertEquals(setOf(1, 2), sizes.toSet(), "sizes seen: $sizes") + } + + @Test + fun testTheMaximumCanBeReached() { + + start(maxTestSize = 4) + + val sizes = sizesAlong(60) + + assertTrue(sizes.all { it in 1..4 }, "sizes seen: $sizes") + assertTrue(sizes.contains(4), "the search never got to publish four messages: $sizes") + } + + @Test + fun testAnAddedMessageIsOneTheDocumentDeclares() { + + start(maxTestSize = 5) + + var evaluated = evaluate(sampler.sample(forceRandomSample = true)) + + repeat(30) { + val copy = evaluated.individual.copy() as AsyncApiIndividual + mutator.mutateStructure(copy, evaluated, null, setOf()) + assertTrue(copy.seeMainExecutableActions().all { it.getName() in NCS_OPERATIONS }) + evaluated = evaluate(copy) + } + } + + @Test + fun testWhatWasAddedOrRemovedIsRecordedForTheSearch() { + + /* + The archive and the impact bookkeeping are fed from this specification, and they are + shared with the other problem types, so a wrong entry degrades the search silently. + */ + start(maxTestSize = 4) + + var evaluated = evaluate(sampler.sample(forceRandomSample = true)) + var added = 0 + var removed = 0 + + repeat(30) { + val before = evaluated.individual.seeMainExecutableActions().size + val copy = evaluated.individual.copy() as AsyncApiIndividual + val recorded = MutatedGeneSpecification() + + mutator.mutateStructure(copy, evaluated, recorded, setOf()) + + val after = copy.seeMainExecutableActions().size + val grew = after > before + + /* + One entry per top gene of the action that moved, so what matters is that there is + at least one and that they all say the same thing. + */ + val types = recorded.mutatedGenes.map { it.type }.toSet() + assertEquals(setOf(if (grew) MutatedGeneSpecification.MutatedType.ADD else MutatedGeneSpecification.MutatedType.REMOVE), types, "size went $before -> $after") + + if (grew) added++ else removed++ + evaluated = evaluate(copy) + } + + //both directions were taken, so neither assertion above passed only by never happening + assertTrue(added > 0 && removed > 0, "added=$added removed=$removed") + } + + @Test + fun testNothingChangesWhenOnlyOneMessageIsAllowed() { + + start(maxTestSize = 1) + + val evaluated = evaluate(sampler.sample(forceRandomSample = true)) + val before = evaluated.individual.seeMainExecutableActions().map { it.getName() } + + val copy = evaluated.individual.copy() as AsyncApiIndividual + mutator.mutateStructure(copy, evaluated, null, setOf()) + + assertEquals(before, copy.seeMainExecutableActions().map { it.getName() }) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt new file mode 100644 index 0000000000..fc0eecdc56 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt @@ -0,0 +1,49 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.AbstractModule +import com.google.inject.Injector +import com.google.inject.util.Modules +import com.netflix.governator.guice.LifecycleInjector +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto +import org.evomaster.core.BaseModule +import org.evomaster.core.remote.service.RemoteController + +/** + * The injector Main would build for an AsyncAPI search, with the driver replaced by a fake. + */ +object AsyncApiTestInjector { + + /** + * The NCS document, the corpus fixture these suites drive most of their cases from. + */ + const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + /** + * The operations NCS declares, all of them publishable. + */ + val NCS_OPERATIONS = setOf("checkTriangle", "bessj", "expint", "fisher", "gammq", "remainder") + + /** + * What a driver declares for a service whose document it hands over as text. + */ + fun sutInfo(schemaText: String): SutInfoDto = SutInfoDto().apply { + asyncApiProblem = AsyncApiProblemDto().apply { this.schemaText = schemaText } + defaultOutputFormat = SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 + } + + fun create(driver: FakeAsyncApiDriver, vararg options: String): Injector { + + val args = arrayOf("--seed=42", "--problemType=ASYNCAPI", "--createTests=false") + options + + val fake = object : AbstractModule() { + override fun configure() { + bind(RemoteController::class.java).toInstance(driver) + } + } + + return LifecycleInjector.builder() + .withModules(listOf(BaseModule(args), Modules.override(AsyncApiModule()).with(fake))) + .build().createInjector() + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt new file mode 100644 index 0000000000..0d9ff66d70 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/FakeAsyncApiDriver.kt @@ -0,0 +1,76 @@ +package org.evomaster.core.problem.asyncapi.service + +import org.evomaster.client.java.controller.api.dto.ActionDto +import org.evomaster.client.java.controller.api.dto.ControllerInfoDto +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.TestResultsDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.problem.external.service.DummyController +import org.evomaster.core.remote.service.RemoteController + +/** + * A driver that publishes nothing: it records what it was asked to publish, and answers each + * message with whatever [answer] decides. No instrumentation, so it reports no code targets. + */ +class FakeAsyncApiDriver( + private val info: SutInfoDto, + private val answer: (AsyncApiActionDto) -> AsyncApiReplyDto? +) : RemoteController by DummyController() { + + /** + * Everything this driver was asked to publish, in order. + */ + val published: MutableList = mutableListOf() + + companion object { + + fun replied(payload: String, correlationMatched: Boolean = true) = AsyncApiReplyDto().apply { + published = true + replyExpected = true + replyReceived = true + replyPayload = payload + this.correlationMatched = correlationMatched + waitedMs = 12 + } + + fun silence(waited: Long = 5000) = AsyncApiReplyDto().apply { + published = true + replyExpected = true + replyReceived = false + waitedMs = waited + } + + fun fireAndForget() = AsyncApiReplyDto().apply { + published = true + replyExpected = false + } + } + + override fun checkConnection() {} + + override fun startSUT() = true + + override fun resetSUT() = true + + override fun startANewSearch() = true + + override fun getSutInfo() = info + + override fun getControllerInfo() = ControllerInfoDto() + + override fun registerNewAction(actionDto: ActionDto) = true + + override fun executeNewAsyncApiActionAndGetReply(actionDto: ActionDto): AsyncApiReplyDto? { + val call = actionDto.asyncApiCall + published.add(call) + return answer(call)?.apply { index = actionDto.index } + } + + override fun getTestResults( + ids: Set, + ignoreKillSwitch: Boolean, + fullyCovered: Boolean, + descriptiveIds: Boolean + ) = TestResultsDto() +} diff --git a/docs/options.md b/docs/options.md index ba38a4c4e5..aaed228ad1 100644 --- a/docs/options.md +++ b/docs/options.md @@ -278,6 +278,7 @@ There are 3 types of options: |`aiResponseClassifierWarmup`| __Int__. Number of training iterations required to update classifier parameters. For example, in the Gaussian model this affects mean and variance updates. For neural network (NN) models, the warm-up should typically be larger than 1000. *Default value*: `100`.| |`appendToTargetHeuristicsFile`| __Boolean__. Whether should add to an existing target heuristics file, instead of replacing it. It is only used when processFormat is TARGET_HEURISTIC. *Default value*: `false`.| |`arazzoLocation`| __String__. arazzo location on disk. *Default value*: `""`.| +|`asyncApiReplyTimeoutMs`| __Int__. When testing an AsyncAPI service, how long to wait for the reply to a published message before treating it as unanswered, in milliseconds. *Constraints*: `min=1.0`. *Default value*: `5000`.| |`breederParentsMin`| __Int__. Breeder GA: minimum number of individuals in parents pool after truncation. *Constraints*: `min=2.0`. *Default value*: `2`.| |`breederTruncationFraction`| __Double__. Breeder GA: fraction of top individuals to keep in parents pool (truncation). *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`callbackURLHostname`| __String__. HTTP callback verifier hostname. Default is set to 'localhost'. If the SUT is running inside a container (i.e., Docker), 'localhost' will refer to the container. This can be used to change the hostname. *Default value*: `localhost`.|