diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt new file mode 100644 index 0000000000..bbfea75981 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt @@ -0,0 +1,237 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import com.webfuzzing.asyncapi.models.AsyncApiDocument +import com.webfuzzing.asyncapi.parser.AsyncApiParsingException +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto +import org.evomaster.core.AnsiColor +import org.evomaster.core.logging.LoggingUtil +import org.evomaster.core.problem.api.service.ApiWsSampler +import org.evomaster.core.problem.asyncapi.builder.AsyncApiActionBuilder +import org.evomaster.core.problem.asyncapi.builder.AsyncApiGeneBuilder +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.enterprise.SampleType +import org.evomaster.core.remote.SutProblemException +import org.evomaster.core.search.tracer.Traceable +import org.evomaster.core.search.warning.GeneralWarning +import org.evomaster.core.search.warning.WarningCategory +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import javax.annotation.PostConstruct + +/** + * Samples test cases for an AsyncAPI service: sequences of messages to publish, drawn from the + * operations the document declares. + * + * The same sampler serves white-box and black-box mode. The document always arrives through the + * driver, since the driver is what holds the connection to the broker (see + * [org.evomaster.core.EMConfig.usesDriver]). + */ +class AsyncApiSampler : ApiWsSampler() { + + companion object { + private val log: Logger = LoggerFactory.getLogger(AsyncApiSampler::class.java) + } + + /** + * One individual per action, each publishing that one message. Sampled before anything + * else, so that every operation is tried once before the search starts combining them. + */ + private val adHocInitialIndividuals: MutableList = mutableListOf() + + /** + * Start the service through the driver, read its document, and build one action per + * publishable message. Anything the parser or the builder had to skip is reported. + */ + @PostConstruct + fun initialize() { + + log.debug("Initializing {}", AsyncApiSampler::class.simpleName) + + rc.checkConnection() + + val started = rc.startSUT() + if (!started) { + throw SutProblemException("Failed to start the system under test") + } + + val infoDto = rc.getSutInfo() + ?: throw SutProblemException("Failed to retrieve the info about the system under test") + + val problem = infoDto.asyncApiProblem + ?: throw SutProblemException("Missing problem definition object") + + val schema = readSchema(problem) + + val messages = AsyncApiActionBuilder.addActionsFromSchema( + schema, + actionCluster, + AsyncApiGeneBuilder.options(config) + ) + handleMessages(schema.warnings + messages) + + initSqlInfo(infoDto) + + initAdHocInitialIndividuals() + + if (config.seedTestCases) { + initSeededTests(infoDto) + } + + updateConfigBasedOnSutInfoDto(infoDto) + + log.debug("Done initializing {}", AsyncApiSampler::class.simpleName) + } + + /** + * The document, from wherever the driver said it is. The driver guarantees exactly one of + * the two is given. + */ + private fun readSchema(problem: AsyncApiProblemDto): AsyncApiDocument { + + try { + if (!problem.schemaLocation.isNullOrBlank()) { + return AsyncApiAccess.getAsyncApiFromLocation(problem.schemaLocation) + } + if (!problem.schemaText.isNullOrBlank()) { + return AsyncApiAccess.parseFromText(problem.schemaText) + } + } catch (e: AsyncApiParsingException) { + throw SutProblemException("Cannot read the AsyncAPI document: ${e.message}") + } + + throw SutProblemException("No info on the AsyncAPI document was provided") + } + + /* + TODO Line for line the same as AbstractRestSampler.handleMessages. Once a third sampler + needs it, hoist it to EnterpriseSampler. + */ + /** + * What the parser and the action builder had to skip. Reported to the user, and kept for + * the final report. + */ + private fun handleMessages(messages: List) { + + if (messages.isEmpty()) { + return + } + + LoggingUtil.getInfoLogger().warn( + AnsiColor.inRed( + "There are ${messages.size} detected issues when analyzing the AsyncAPI document." + + " These are not necessarily problems in the document, but possible (temporary)" + + " limitations of EvoMaster itself." + ) + ) + messages.forEachIndexed { index, s -> + LoggingUtil.getInfoLogger().warn(AnsiColor.inYellow("$index: $s")) + warningsAggregator.addWarning(GeneralWarning(WarningCategory.SCHEMA, s)) + } + } + + /** + * A test of one to `maxTestSize` messages, each a random action with fresh genes. + */ + override fun sampleAtRandom(): AsyncApiIndividual { + + val n = randomness.nextInt(1, getMaxTestSizeDuringSampler()) + val actions = (0 until n).map { sampleRandomAction() }.toMutableList() + + return createIndividual(SampleType.RANDOM, actions) + } + + /* + TODO Message examples (AsyncApiMessage.getExamples) are parsed and never read. Sampling + from them some of the time, as REST does with probRestExamples, would start the search + from payloads the author knows the service accepts. + */ + /** + * A copy of one of the action templates, chosen at random, with its genes initialized. + */ + fun sampleRandomAction(): AsyncApiAction { + + val action = randomness.choose(actionCluster).copy() as AsyncApiAction + action.doInitialize(randomness) + + return action + } + + /* + TODO No AsyncAPI-specific strategy yet, only the single-message individuals and then + random tests. The one that matters is chaining: a message whose payload needs an id + that only the reply to a previous message produces. + */ + /** + * The next single-message individual while any are left, then a random test. + */ + override fun smartSample(): AsyncApiIndividual { + + if (adHocInitialIndividuals.isNotEmpty()) { + return adHocInitialIndividuals.removeAt(adHocInitialIndividuals.size - 1) + } + + return sampleAtRandom() + } + + /** + * Whether single-message individuals are still waiting to be handed out. + */ + override fun hasSpecialInitForSmartSampler(): Boolean { + return adHocInitialIndividuals.isNotEmpty() && config.isEnabledSmartSampling() + } + + /** + * Prepare the single-message individuals again, so every operation is tried once more. + */ + override fun resetSpecialInit() { + initAdHocInitialIndividuals() + } + + /** + * Not supported yet: there is no format to read AsyncAPI test cases from. + */ + override fun initSeededTests(infoDto: SutInfoDto?) { + //TODO There is no format to read message-based test cases from yet, as Postman is for REST + throw IllegalStateException("Seeding test cases is not supported for AsyncAPI yet") + } + + /** + * One individual per action, publishing that one message. + */ + private fun initAdHocInitialIndividuals() { + + adHocInitialIndividuals.clear() + + actionCluster.values.forEach { template -> + val action = template.copy() as AsyncApiAction + action.doInitialize(randomness) + adHocInitialIndividuals.add(createIndividual(SampleType.SMART, mutableListOf(action))) + } + } + + /** + * Wrap [actions] as an individual ready for the search: tracked if tracking is on, and + * with its global state and local ids set. + * + * @param sampleType how the individual came to be, which the structure mutators read to + * choose how to change it + */ + private fun createIndividual( + sampleType: SampleType, + actions: MutableList + ): AsyncApiIndividual { + + val individual = AsyncApiIndividual( + sampleType = sampleType, + actions = actions, + trackOperator = if (config.trackingEnabled()) this else null, + index = if (config.trackingEnabled()) time.evaluatedIndividuals else Traceable.DEFAULT_INDEX + ) + individual.doGlobalInitialize(searchGlobalState) + + return individual + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt new file mode 100644 index 0000000000..6c225e8105 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt @@ -0,0 +1,227 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.AbstractModule +import com.google.inject.Injector +import com.netflix.governator.guice.LifecycleInjector +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto +import org.evomaster.core.BaseModule +import org.evomaster.core.EMConfig +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.external.service.DummyController +import org.evomaster.core.remote.SutProblemException +import org.evomaster.core.remote.service.RemoteController +import org.evomaster.core.search.service.WarningsAggregator +import org.evomaster.core.search.warning.WarningCategory +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path + +class AsyncApiSamplerTest { + + companion object { + private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + private val NCS_OPERATIONS = setOf("checkTriangle", "bessj", "expint", "fisher", "gammq", "remainder") + } + + /** + * A driver that only answers what the sampler asks at start-up: that the service is running, + * and where its AsyncAPI document is. + */ + private class FakeController( + private val info: SutInfoDto, + private val starts: Boolean = true + ) : RemoteController by DummyController() { + override fun checkConnection() {} + override fun startSUT() = starts + override fun getSutInfo() = info + } + + private fun sutInfo(configure: AsyncApiProblemDto.() -> Unit) = SutInfoDto().apply { + asyncApiProblem = AsyncApiProblemDto().apply(configure) + defaultOutputFormat = SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 + } + + private fun injector(info: SutInfoDto, starts: Boolean = true, vararg options: String): Injector { + + val args = arrayOf("--seed=42", "--problemType=ASYNCAPI") + options + + val modules = listOf(BaseModule(args), object : AbstractModule() { + override fun configure() { + bind(RemoteController::class.java).toInstance(FakeController(info, starts)) + bind(AsyncApiSampler::class.java).asEagerSingleton() + } + }) + + return LifecycleInjector.builder().withModules(modules).build().createInjector() + } + + private fun sampler(info: SutInfoDto, vararg options: String): AsyncApiSampler = + injector(info, true, *options).getInstance(AsyncApiSampler::class.java) + + /** + * Whatever the sampler threw while being created, unwrapped from what Guice and Governator + * wrap it in. + */ + private fun causesOfFailingToCreate(block: () -> Unit): List { + val e = assertThrows(Throwable::class.java) { block() } + return generateSequence(e) { it.cause }.toList() + } + + private fun ncsSampler(vararg options: String) = + sampler(sutInfo { schemaText = AsyncApiAccess.readFromResource(NCS) }, *options) + + @Test + fun testOneActionPerOperationTheDriverDescribes() { + + val sampler = ncsSampler("--blackBox=false") + + assertEquals(NCS_OPERATIONS, sampler.seeAvailableActions().map { it.getName() }.toSet()) + } + + @Test + fun testTheSameDocumentServesBlackBoxMode() { + + /* + No black-box branch in the sampler: the document arrives through the driver either + way, since the driver is what holds the connection to the broker. + */ + val sampler = ncsSampler("--blackBox=true") + + assertEquals(NCS_OPERATIONS.size, sampler.numberOfDistinctActions()) + } + + @Test + fun testTheDocumentCanBeFetchedFromWhereTheDriverSaysItIs(@TempDir dir: Path) { + + val file = dir.resolve("asyncapi.yaml") + Files.write(file, AsyncApiAccess.readFromResource(NCS).toByteArray(StandardCharsets.UTF_8)) + + val sampler = sampler(sutInfo { schemaLocation = file.toString() }, "--blackBox=false") + + assertEquals(NCS_OPERATIONS.size, sampler.numberOfDistinctActions()) + } + + @Test + fun testARandomIndividualPublishesBetweenOneAndMaxTestSizeMessages() { + + val sampler = ncsSampler("--blackBox=false", "--maxTestSize=4") + + repeat(50) { + val individual = sampler.sample(forceRandomSample = true) + val actions = individual.seeMainExecutableActions() + + assertTrue(actions.size in 1..4, "got ${actions.size} messages") + assertTrue(actions.all { it is AsyncApiAction }) + assertTrue(actions.all { it.isInitialized() }, "an action was sampled with uninitialized genes") + assertTrue(individual.seeInitializingActions().isEmpty()) + } + } + + @Test + fun testEveryOperationIsTriedOnceBeforeMessagesAreCombined() { + + val sampler = ncsSampler("--blackBox=false", "--probOfSmartSampling=1.0") + + val first = (1..NCS_OPERATIONS.size).map { + assertTrue(sampler.hasSpecialInit(), "ran out of single-message individuals early") + sampler.sample() + } + + //six individuals of one message each, covering the six operations + assertTrue(first.all { it.seeMainExecutableActions().size == 1 }) + assertEquals(NCS_OPERATIONS, first.map { it.seeMainExecutableActions().single().getName() }.toSet()) + + assertFalse(sampler.hasSpecialInit()) + } + + @Test + fun testADocumentThatIsNotThereIsAProblemWithTheSut(@TempDir dir: Path) { + + //the likeliest mistake in a driver: a path that is right on the author's machine only + val causes = causesOfFailingToCreate { + sampler(sutInfo { schemaLocation = dir.resolve("absent.yaml").toString() }, "--blackBox=false") + } + + assertTrue(causes.any { it is SutProblemException }, causes.joinToString { it.toString() }) + } + + @Test + fun testAServiceThatDoesNotStartIsAProblemWithTheSut() { + + val causes = causesOfFailingToCreate { + injector(sutInfo { schemaText = AsyncApiAccess.readFromResource(NCS) }, starts = false, "--blackBox=false") + } + + assertTrue(causes.any { it is SutProblemException && it.message!!.contains("start") }, causes.joinToString { it.toString() }) + } + + @Test + fun testADriverThatDeclaresNoAsyncApiServiceIsAProblemWithTheSut() { + + val causes = causesOfFailingToCreate { + sampler(SutInfoDto(), "--blackBox=false") + } + + assertTrue(causes.any { it is SutProblemException && it.message!!.contains("problem definition") }, causes.joinToString { it.toString() }) + } + + @Test + fun testWhatTheParserHadToSkipReachesTheUser() { + + //an operation whose reply names a channel the document never declares + val document = """ + asyncapi: 3.0.0 + info: + title: Skips + version: 1.0.0 + channels: + requests: + address: app.requests + messages: + request: + payload: + type: object + operations: + ask: + action: receive + channel: + ${'$'}ref: '#/channels/requests' + reply: + channel: + ${'$'}ref: '#/channels/nowhere' + """.trimIndent() + + val injector = injector(sutInfo { schemaText = document }, true, "--blackBox=false") + + //the operation is still there: one message that could not be resolved does not cost the document + assertEquals(1, injector.getInstance(AsyncApiSampler::class.java).numberOfDistinctActions()) + + //and what was skipped is on record for the final report + val warnings = injector.getInstance(WarningsAggregator::class.java).getWarnings() + assertTrue( + warnings.any { it.category == WarningCategory.SCHEMA && it.message.contains("nowhere") }, + warnings.joinToString { it.message } + ) + } + + @Test + fun testTheSingleMessageIndividualsComeBackAfterAReset() { + + val sampler = ncsSampler("--blackBox=false", "--probOfSmartSampling=1.0") + + repeat(NCS_OPERATIONS.size) { sampler.sample() } + assertFalse(sampler.hasSpecialInit()) + + sampler.resetSpecialInit() + + assertTrue(sampler.hasSpecialInit()) + val again = (1..NCS_OPERATIONS.size).map { sampler.sample() } + assertEquals(NCS_OPERATIONS, again.map { it.seeMainExecutableActions().single().getName() }.toSet()) + } +}