diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 5164c0b604..756b100b48 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2895,6 +2895,13 @@ class EMConfig { @Min(1.0) var asyncApiReplyTimeoutMs = 5000 + @Experimental + @Cfg("When testing an AsyncAPI service, the probability of publishing a message's declared example as it is," + + " rather than a value sampled from its schema. A service that silently drops what it does not recognise" + + " may never reply to sampled values, and the examples are what its author knows it accepts.") + @Probability(true) + var probAsyncApiExamples = 0.0 + @Cfg("Whether to enable customized responses indicating business logic") var enableRPCCustomizedResponseTargets = true diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/builder/AsyncApiGeneBuilder.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/builder/AsyncApiGeneBuilder.kt index 1be992ed8e..64ae694f48 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/builder/AsyncApiGeneBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/builder/AsyncApiGeneBuilder.kt @@ -56,6 +56,14 @@ object AsyncApiGeneBuilder { private const val TYPE_NUMBER = "number" private const val TYPE_BOOLEAN = "boolean" + /* + What this builder reads off a Message Example Object: the two parts a gene is built for, + and the name that lets the search keep a whole example together. + */ + private const val EXAMPLE_PAYLOAD = "payload" + private const val EXAMPLE_HEADERS = "headers" + private const val EXAMPLE_NAME = "name" + /** * The keywords whose value is literal data rather than a schema, so nothing inside them is * a keyword either. @@ -70,12 +78,21 @@ object AsyncApiGeneBuilder { /** * The genes for a message's payload, or null when it declares none. + * + * When the message declares examples and [options] ask for them, their payloads are offered + * as whole values beside the schema-derived genes, with the probability the options give. */ fun buildPayloadGene( schema: AsyncApiDocument, message: AsyncApiMessage, options: RestActionBuilderV3.Options - ): Gene? = build(message.payload, "${message.id}.payload", schema, options) + ): Gene? = build( + message.payload, + "${message.id}.payload", + schema, + options, + examplesOf(message, EXAMPLE_PAYLOAD) + ) /** * The genes for a message's headers, or null when it declares none. @@ -98,7 +115,48 @@ object AsyncApiGeneBuilder { schema: AsyncApiDocument, message: AsyncApiMessage, options: RestActionBuilderV3.Options - ): Gene? = build(withoutCorrelationId(message), "${message.id}.headers", schema, options) + ): Gene? = build( + withoutCorrelationId(message), + "${message.id}.headers", + schema, + options, + examplesOf(message, EXAMPLE_HEADERS).map { (headers, name) -> + Pair(withoutCorrelationField(headers, message), name) + } + ) + + /** + * The [part] of every example the message declares that has one, each with the example's + * name where it gives one. + * + * They go to the gene builder beside the schema, the way REST passes a parameter's + * examples, rather than written into it: that path never crosses the OpenAPI parser, which + * would keep only one, and draws no complaint on a scalar. The name is what lets the search + * pick a whole example consistently across fields, when asked to with probNamedExamples. + */ + private fun examplesOf(message: AsyncApiMessage, part: String): List> = + message.examples.mapNotNull { example -> + example.get(part) + ?.takeUnless { it.isNull } + ?.let { Pair(it, example.get(EXAMPLE_NAME)?.takeIf { n -> n.isTextual }?.asText()) } + } + + /** + * The example headers without the one the correlation id is stamped into, which the headers + * gene does not have either. + */ + private fun withoutCorrelationField(example: JsonNode, message: AsyncApiMessage): JsonNode { + + val correlation = message.correlationId + val field = correlation?.fieldName + + if (correlation == null || correlation.source != AsyncApiCorrelationId.Source.HEADER + || field == null || !example.isObject) { + return example + } + + return (example.deepCopy() as ObjectNode).apply { remove(field) } + } /** * The headers schema without the property the correlation id is stamped into. @@ -146,6 +204,9 @@ object AsyncApiGeneBuilder { /** * Options for building AsyncAPI payloads. * + * `probUseExamples` is the probability of publishing a message's declared example as it is; + * it is off unless the user asks, since it is a way of steering the search. + * * Note `invalidData = false`, which is not what REST does. That flag makes the builder add * a bogus "EVOMASTER" member to every enum, on purpose, to probe how a service handles a * value it never declared. In a message payload that backfires: an enum of one value is how @@ -160,6 +221,7 @@ object AsyncApiGeneBuilder { fun options(config: EMConfig) = RestActionBuilderV3.Options( enableConstraintHandling = config.enableSchemaConstraintHandling, invalidData = false, + probUseExamples = config.probAsyncApiExamples, usingWhiteBox = !config.blackBox, enableAdvancedFormats = config.enableAdvancedFormats, inferFormatFromNames = config.inferFormatFromNames @@ -169,7 +231,8 @@ object AsyncApiGeneBuilder { declared: JsonNode?, inlineName: String, schema: AsyncApiDocument, - options: RestActionBuilderV3.Options + options: RestActionBuilderV3.Options, + examples: List> ): Gene? { if (declared == null) { @@ -201,7 +264,7 @@ object AsyncApiGeneBuilder { } //the format createGeneForDTO expects: the name of the wanted schema, then all of them - return RestActionBuilderV3.createGeneForDTO(name, "\"$name\":$schemas", options) + return RestActionBuilderV3.createGeneForDTO(name, "\"$name\":$schemas", options, examples) } /** diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt index 6f494ccd26..2d470aaf68 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSampler.kt @@ -150,11 +150,6 @@ class AsyncApiSampler : ApiWsSampler() { return createIndividual(SampleType.RANDOM, actions) } - /* - TODO Message examples (AsyncApiMessage.getExamples) are parsed and never read. Sampling - from them some of the time, as REST does with probRestExamples, would start the search - from payloads the author knows the service accepts. - */ /** * A copy of one of the action templates, chosen at random, with its genes initialized. */ diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/builder/RestActionBuilderV3.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/builder/RestActionBuilderV3.kt index 711ca26b10..5a67a35b87 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/builder/RestActionBuilderV3.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/builder/RestActionBuilderV3.kt @@ -293,12 +293,17 @@ object RestActionBuilderV3 { * @throws IllegalArgumentException if the provided 'name' is not found at the beginning of 'allSchemas'. * @throws IllegalStateException if the schema with the specified 'name' cannot be found in 'allSchemas'. * + * @param examples values to offer for the wanted schema as a whole, each with an optional + * name, beside the genes derived from it. They take the same path as a REST + * parameter's examples, so they never go through the OpenAPI parser, and as + * for those, none is offered unless [Options.probUseExamples] gives it a chance. * @see Gene * @see Options */ fun createGeneForDTO(dtoSchemaName: String, allSchemas: String, - options: Options + options: Options, + examples: List> = listOf() ) : Gene{ if(!allSchemas.startsWith("\"$dtoSchemaName\"")){ throw IllegalArgumentException("Invalid name $dtoSchemaName for schema $allSchemas") @@ -309,7 +314,14 @@ object RestActionBuilderV3 { val schemas = getMapStringFromSchemas(allSchemasValue) val dtoSchema = schemas[dtoSchemaName] ?: throw IllegalStateException("cannot find the schema with $dtoSchemaName from $allSchemas") - if(dtoCache.containsKey(dtoSchema)){ + //gated here rather than left to the assembler, which would wrap them in a choice weighted at zero + val offered = if (options.probUseExamples > 0) examples else listOf() + + /* + The cache is keyed on the schema text alone, so it can only serve a call that brings + no examples: two callers may share a schema and still offer different examples for it. + */ + if(offered.isEmpty() && dtoCache.containsKey(dtoSchema)){ return dtoCache[dtoSchema]!!.copy() } @@ -328,7 +340,10 @@ object RestActionBuilderV3 { val currentSchema = SchemaOpenAPI(schema,swagger.openAPI, SchemaLocation.MEMORY) val schemaHolder = RestSchema(currentSchema) + var wanted: Gene? = null + schemas.forEach { (t, u) -> + val forThisOne = if (t == dtoSchemaName) offered else listOf() val gene = getGene(t, swagger.openAPI.components.schemas[t]!!, schemaHolder, @@ -336,11 +351,17 @@ object RestActionBuilderV3 { ArrayDeque(), t, options, + examples = forThisOne, messages = mutableListOf()) - dtoCache[u] = gene + if (forThisOne.isEmpty()) { + dtoCache[u] = gene + } + if (t == dtoSchemaName) { + wanted = gene + } } - return dtoCache[dtoSchema]!!.copy() + return (wanted ?: throw IllegalStateException("No gene was built for $dtoSchemaName")).copy() } /** diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/builder/AsyncApiGeneBuilderTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/builder/AsyncApiGeneBuilderTest.kt index 5086ecaf4a..7e0c8e94d1 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/builder/AsyncApiGeneBuilderTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/builder/AsyncApiGeneBuilderTest.kt @@ -9,11 +9,14 @@ import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.search.gene.collection.ArrayGene import org.evomaster.core.search.gene.collection.EnumGene +import org.evomaster.core.search.gene.interfaces.UserExamplesGene import org.evomaster.core.search.gene.numeric.DoubleGene import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.search.gene.wrapper.ChoiceGene import org.evomaster.core.search.gene.wrapper.OptionalGene +import org.evomaster.core.search.gene.utils.GeneUtils +import org.evomaster.core.search.service.Randomness import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -443,4 +446,209 @@ class AsyncApiGeneBuilderTest { ) } } + + // ------------------------------------------------------------------ message examples + + /** + * Options asking for the example every time, so that what is published is deterministic. + */ + private val alwaysExamples = AsyncApiGeneBuilder.options(EMConfig().apply { probAsyncApiExamples = 1.0 }) + + private fun printed(gene: Gene): String { + gene.doInitialize(Randomness().apply { updateSeed(42) }) + return gene.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = null) + } + + private fun document(text: String): AsyncApiDocument = AsyncApiAccess.parseFromText(text.trimIndent()) + + @Test + fun testAMessageExampleIsOfferedAsAWholePayload() { + + val schema = AsyncApiAccess.getAsyncApiFromResource("/asyncapi/sut/scalar.yaml") + val message = schema.messages.getValue("PlanetCreated") + + val gene = AsyncApiGeneBuilder.buildPayloadGene(schema, message, alwaysExamples)!! + + //a choice between the example and the schema-derived genes, as REST offers schema examples + assertTrue(gene is ChoiceGene<*>, gene::class.simpleName) + + //asked for every time, what goes out is the payload the author wrote, whole + val json = printed(gene) + assertTrue(json.contains("evt_1234567890"), json) + assertTrue(json.contains("\"Mars\""), json) + assertTrue(json.contains("\"CO2\""), json) + } + + @Test + fun testExamplesAreNotUsedUnlessAskedFor() { + + //the default options leave the probability at zero, so nothing changes shape + val gene = payloadOf("/asyncapi/sut/scalar.yaml", "PlanetCreated") + + assertTrue(gene is ObjectGene, gene::class.simpleName) + } + + @Test + fun testAnExampleForANonObjectPayloadIsOfferedToo() { + + /* + Passed beside the schema rather than written into it, an example draws no complaint + from the gene builder about 'example' on a scalar, so a scalar payload gets the same + choice an object does. + */ + val schema = document(""" + asyncapi: 3.0.0 + info: + title: Heartbeat + version: 1.0.0 + components: + messages: + beat: + payload: + type: integer + examples: + - payload: 42 + """) + + val gene = AsyncApiGeneBuilder.buildPayloadGene(schema, schema.messages.getValue("beat"), alwaysExamples)!! + + assertTrue(gene is ChoiceGene<*>, gene::class.simpleName) + assertEquals("42", printed(gene)) + } + + @Test + fun testAHeadersExampleLeavesOutTheStampedCorrelationId() { + + val schema = document(""" + asyncapi: 3.0.0 + info: + title: Headers + version: 1.0.0 + components: + messages: + order: + headers: + type: object + required: [tenant, correlationId] + properties: + tenant: + type: string + correlationId: + type: string + correlationId: + location: '${'$'}message.header#/correlationId' + payload: + type: object + examples: + - headers: + tenant: acme + correlationId: abc-123 + payload: {} + """) + + val gene = AsyncApiGeneBuilder.buildHeadersGene(schema, schema.messages.getValue("order"), alwaysExamples)!! + + //the tenant comes from the example; the correlation id is the driver's to stamp, so it is not there to copy + val json = printed(gene) + assertTrue(json.contains("\"acme\""), json) + assertFalse(json.contains("abc-123"), json) + assertFalse(json.contains("correlationId"), json) + } + + @Test + fun testEveryExampleIsOffered() { + + //both are there to choose from, not only the first + val schema = document(""" + asyncapi: 3.0.0 + info: + title: Two examples + version: 1.0.0 + components: + messages: + greeting: + payload: + type: object + required: [text] + properties: + text: + type: string + examples: + - payload: + text: first + - payload: + text: second + """) + + val gene = AsyncApiGeneBuilder.buildPayloadGene(schema, schema.messages.getValue("greeting"), alwaysExamples)!! + + //each example pins the field to its own value, so every value the field can take is one of them + val offered = gene.flatView() + .filterIsInstance>() + .filter { it.name == "text" } + .flatMap { it.values } + .toSet() + + assertEquals(setOf("first", "second"), offered) + } + + @Test + fun testAnExampleKeepsItsName() { + + //the name is what the sampler keys on to keep a whole example together across fields + val schema = AsyncApiAccess.getAsyncApiFromResource("/asyncapi/sut/scalar.yaml") + val message = schema.messages.getValue("PlanetCreated") + + val gene = AsyncApiGeneBuilder.buildPayloadGene(schema, message, alwaysExamples)!! + + val named = gene.flatView() + .filterIsInstance() + .filter { it.isUsedForExamples() } + .flatMap { it.getAvailableExampleNames() } + + assertEquals(listOf("Mars Discovery"), named) + } + + @Test + fun testTwoMessagesSharingASchemaKeepTheirOwnExamples() { + + /* + The gene builder caches what it builds by schema text. Two messages may well share a + schema, a request and its reply say, and each must still get the example it declares + rather than whichever was built first. + */ + val schema = document(""" + asyncapi: 3.0.0 + info: + title: Shared + version: 1.0.0 + components: + schemas: + Note: + type: object + required: [text] + properties: + text: + type: string + messages: + first: + payload: + ${'$'}ref: '#/components/schemas/Note' + examples: + - payload: + text: from-first + second: + payload: + ${'$'}ref: '#/components/schemas/Note' + examples: + - payload: + text: from-second + """) + + val first = printed(AsyncApiGeneBuilder.buildPayloadGene(schema, schema.messages.getValue("first"), alwaysExamples)!!) + val second = printed(AsyncApiGeneBuilder.buildPayloadGene(schema, schema.messages.getValue("second"), alwaysExamples)!!) + + assertTrue(first.contains("from-first"), first) + assertTrue(second.contains("from-second"), second) + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt index 35b79fe6fa..a3b8569f58 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt @@ -9,6 +9,9 @@ import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto import org.evomaster.core.BaseModule import org.evomaster.core.EMConfig import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.param.AsyncApiParam +import org.evomaster.core.search.gene.utils.GeneUtils +import org.evomaster.core.search.service.Randomness import org.evomaster.core.problem.external.service.DummyController import org.evomaster.core.remote.SutProblemException import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 @@ -235,4 +238,39 @@ class AsyncApiSamplerTest { val again = (1..NCS_OPERATIONS.size).map { sampler.sample() } assertEquals(NCS_OPERATIONS, again.map { it.seeMainExecutableActions().single().getName() }.toSet()) } + + @Test + fun testExamplesSeedTheSearchWhenAskedFor() { + + val scalar = AsyncApiAccess.readFromResource("/asyncapi/sut/scalar.yaml") + val sampler = sampler(sutInfo { schemaText = scalar }, "--blackBox=false", "--probAsyncApiExamples=1.0") + + //an action whose message declares an example + val action = sampler.seeAvailableActions() + .map { it as AsyncApiAction } + .first { sampler.document.messages.getValue(it.messageId).examples.isNotEmpty() } + val example = sampler.document.messages.getValue(action.messageId).examples.first().get("payload") + val anExampleValue = example.fields().asSequence().map { it.value }.first { it.isTextual }.asText() + + //sampled the way the search does, the payload is the author's example + val sampled = (action.copy() as AsyncApiAction).apply { doInitialize(Randomness().apply { updateSeed(7) }) } + val payload = sampled.parameters.first { it.name == AsyncApiParam.PAYLOAD }.gene + .getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = null) + + assertTrue(payload.contains(anExampleValue), "expected '$anExampleValue' in $payload") + } + + @Test + fun testANamedExampleReachesTheAction() { + + //what the sampler's named-example pass keys on, so a whole example can be kept together + val scalar = AsyncApiAccess.readFromResource("/asyncapi/sut/scalar.yaml") + val sampler = sampler(sutInfo { schemaText = scalar }, "--blackBox=false", "--probAsyncApiExamples=1.0") + + val named = sampler.seeAvailableActions() + .map { it as AsyncApiAction } + .flatMap { it.getNamedExamples().keys } + + assertTrue(named.contains("Mars Discovery"), named.toString()) + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestActionBuilderV3Test.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestActionBuilderV3Test.kt index 1831a96b45..65c114ea02 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestActionBuilderV3Test.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestActionBuilderV3Test.kt @@ -39,6 +39,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource import java.time.format.DateTimeFormatter +import org.evomaster.core.search.gene.interfaces.UserExamplesGene class RestActionBuilderV3Test{ @@ -284,6 +285,40 @@ class RestActionBuilderV3Test{ } } + @Test + fun testExamplesPassedBesideADtoSchema() { + + RestActionBuilderV3.cleanCache() + + val name = "Note" + val dtoSchema = """ + "$name": { + "$name": { + "type": "object", + "required": ["text"], + "properties": { "text": { "type": "string" } } + } + } + """.trimIndent() + val example = ObjectMapper().readTree("""{"text": "hello"}""") + val options = RestActionBuilderV3.Options(probUseExamples = 0.5) + + val withExample = RestActionBuilderV3.createGeneForDTO(name, dtoSchema, options, listOf(Pair(example, "greeting"))) + assertTrue(withExample is ChoiceGene<*>, withExample::class.simpleName) + val named = withExample.flatView() + .filterIsInstance() + .filter { it.isUsedForExamples() } + .flatMap { it.getAvailableExampleNames() } + assertEquals(listOf("greeting"), named) + + //the cache is keyed on the schema alone, so neither call may be handed the other's gene + val without = RestActionBuilderV3.createGeneForDTO(name, dtoSchema, options) + assertTrue(without is ObjectGene, without::class.simpleName) + + val again = RestActionBuilderV3.createGeneForDTO(name, dtoSchema, options, listOf(Pair(example, "greeting"))) + assertTrue(again is ChoiceGene<*>, again::class.simpleName) + } + @ParameterizedTest @ValueSource(booleans = [true, false]) fun testPropertiesWithAdditionalProperties(enableConstraintHandling : Boolean){ diff --git a/docs/options.md b/docs/options.md index d6c2badc4d..b2a2d1b601 100644 --- a/docs/options.md +++ b/docs/options.md @@ -348,6 +348,7 @@ There are 3 types of options: |`mutationTargetsSelectionStrategy`| __Enum__. Specify a strategy to select targets for evaluating mutation. *Valid values*: `FIRST_NOT_COVERED_TARGET, EXPANDED_UPDATED_NOT_COVERED_TARGET, UPDATED_NOT_COVERED_TARGET`. *Default value*: `FIRST_NOT_COVERED_TARGET`.| |`onePlusLambdaLambdaOffspringSize`| __Int__. 1+(λ,λ) GA: number of offspring (λ) per generation. *Constraints*: `min=1.0`. *Default value*: `4`.| |`prematureStopStrategy`| __Enum__. Specify how 'improvement' is defined: either any kind of improvement even if partial (ANY), or at least one new target is fully covered (NEW). *Valid values*: `ANY, NEW`. *Default value*: `NEW`.| +|`probAsyncApiExamples`| __Double__. When testing an AsyncAPI service, the probability of publishing a message's declared example as it is, rather than a value sampled from its schema. A service that silently drops what it does not recognise may never reply to sampled values, and the examples are what its author knows it accepts. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| |`probOfArazzoSampling`| __Double__. Probability of controlling the creation of Arazzo individuals. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`probOfHandlingLength`| __Double__. Specify a probability of applying length handling. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| |`probOfHarvestingResponsesFromActualExternalServices`| __Double__. a probability of harvesting actual responses from external services as seeds. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|