From c33de90803a73b6a6ec94832ca9ac33a4c344ecb Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Thu, 24 Sep 2026 12:58:29 -0300 Subject: [PATCH 01/20] AsyncAPI 3.x: write tests that publish with the transport's own client A generated test cannot go through the driver: there would be no test without EvoMaster running. It cannot use a client from the core either, since a broker has no universal one and the transport code belongs in drivers. So the driver renders the publish and await as source lines while the search runs, and the writer pastes them, which is what RPC already does for an endpoint invocation. The core asks for them on the action, naming the language and the variable the reply has to land in, so two actions in one test cannot collide. It asks only when a test will be written. Around the pasted lines it adds what the driver cannot know: the operation and its outcome, that a promised reply arrived, and which declared message the classifier recognised it as. A driver that renders nothing produces a test that says so rather than an empty one. With this, --problemType ASYNCAPI no longer refuses to write tests. What replaces that refusal is a narrower one: only Java and Kotlin, because the lines being pasted are the driver's. For the same reason black-box no longer defaults the format to Python for AsyncAPI, which would have made an ordinary run fail at start-up: it has a driver in both modes, so the format comes from the driver as it does in white-box. The suite splitter read a status code off every result, which would have thrown for the first AsyncAPI individual written. It now branches on the result type, counting a message that went out and was answered as a success. --- .../problem/asyncapi/AsyncApiActionDto.java | 19 ++ .../problem/asyncapi/AsyncApiReplyDto.java | 17 ++ .../kotlin/org/evomaster/core/EMConfig.kt | 15 +- .../main/kotlin/org/evomaster/core/Main.kt | 16 -- .../core/output/TestSuiteSplitter.kt | 15 +- .../AsyncApiActionTestCaseNamingStrategy.kt | 70 ++++++++ .../naming/TestCaseNamingStrategyFactory.kt | 2 + .../output/service/AsyncApiTestCaseWriter.kt | 146 ++++++++++++++++ .../asyncapi/data/AsyncApiCallResult.kt | 24 +++ .../asyncapi/service/AsyncApiFitness.kt | 27 ++- .../asyncapi/service/AsyncApiModule.kt | 4 +- .../kotlin/org/evomaster/core/EMConfigTest.kt | 24 ++- .../asyncapi/service/AsyncApiModuleTest.kt | 6 +- .../service/AsyncApiTestCaseWriterTest.kt | 165 ++++++++++++++++++ .../asyncapi/service/AsyncApiTestInjector.kt | 9 +- 15 files changed, 522 insertions(+), 37 deletions(-) create mode 100644 core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt create mode 100644 core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt create mode 100644 core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java index 01e1e71d74..b2a26b55a2 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java @@ -1,5 +1,7 @@ package org.evomaster.client.java.controller.api.dto.problem.asyncapi; +import org.evomaster.client.java.controller.api.dto.SutInfoDto; + import java.util.LinkedHashMap; import java.util.Map; @@ -96,4 +98,21 @@ public enum CorrelationLocation { * Null when no reply is expected, as there is then nothing to wait for. */ public Long replyTimeoutMs; + + /** + * The language to render {@link AsyncApiReplyDto#testScript} in, or null when the core is + * not generating tests and the driver should not spend time rendering anything. + * + * A generated test talks to the broker with an ordinary client of the transport rather than + * through the driver, so the lines that publish and await have to come from the one side + * that knows the transport. Only Java and Kotlin are asked for. + */ + public SutInfoDto.OutputFormat outputFormat; + + /** + * The name the rendered lines must leave the reply payload in, as text, when a reply is + * expected. Named by the core so that two actions in one test cannot collide. Null when no + * script was asked for. + */ + public String replyVariable; } diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java index c36e0e0703..03adc1ab92 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java @@ -1,6 +1,7 @@ package org.evomaster.client.java.controller.api.dto.problem.asyncapi; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; /** @@ -86,4 +87,20 @@ public class AsyncApiReplyDto { * Why publishing failed, when it did. */ public String errorMessage; + + /** + * The publish and await this action just did, rendered as source lines for a generated test, + * in the language {@link AsyncApiActionDto#outputFormat} asked for. Null when none was asked + * for, or when the driver does not render them. + * + * A generated test talks to the broker directly rather than through the driver, so these + * lines stand up a client of the transport, publish the same message and read the reply. + * Only the driver can write them: it is the one side that knows the transport, and the core + * pastes them without reading them. + * + * Where a reply is expected, the lines must leave it in the variable the core named in + * {@link AsyncApiActionDto#replyVariable}, which is what the assertions the core appends are + * written against. + */ + public List testScript; } diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 756b100b48..bb1a782c68 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -673,7 +673,12 @@ class EMConfig { " 'problemType'. The system will default to RESTful API testing.") problemType = ProblemType.REST } - if (outputFormat == OutputFormat.DEFAULT) { + /* + AsyncAPI is the exception: it has a driver even in black-box, since there is no + universal client for a broker, so the format comes from the driver as it does in + white-box rather than from the black-box default. + */ + if (outputFormat == OutputFormat.DEFAULT && problemType != ProblemType.ASYNCAPI) { LoggingUtil.uniqueUserWarn("You are doing Black-Box testing, but you did not specify the" + " 'outputFormat'. The system will default to $defaultOutputFormatForBlackBox.") outputFormat = defaultOutputFormatForBlackBox @@ -816,9 +821,11 @@ class EMConfig { throw ConfigProblemException("When using the seedTestCases option, you must specify the file path of the test cases with the seedTestCasesPath option") } - if (problemType == ProblemType.ASYNCAPI && createTests) { - throw ConfigProblemException("Test generation for AsyncAPI services is not available yet." + - " For the time being, run with '--createTests false' to only search for faults.") + if (problemType == ProblemType.ASYNCAPI + && createTests + && outputFormat != OutputFormat.DEFAULT && !outputFormat.isJavaOrKotlin()) { + throw ConfigProblemException("Tests for an AsyncAPI service are written against the driver's own" + + " transport client, which is Java, so outputFormat only supports Java or Kotlin now") } if (problemType == ProblemType.ASYNCAPI && seedTestCases) { diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 99870b8101..8cf1eae007 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -572,22 +572,6 @@ class Main { config.problemType = EMConfig.ProblemType.WEBFRONTEND } else if (info.asyncApiProblem != null) { config.problemType = EMConfig.ProblemType.ASYNCAPI - if (config.createTests) { - /* - There is no test writer for AsyncAPI yet, and the constraints checked - below refuse the combination. As the problem type was inferred rather - than asked for, turning test generation off is better than failing. - - TODO remove this block once there is a test writer for AsyncAPI: the - combination will be valid then, and silently switching test generation - off would hide it. - */ - LoggingUtil.uniqueUserWarn( - "The driver describes an AsyncAPI service, for which test generation is not" + - " available yet. Continuing with 'createTests' off." - ) - config.createTests = false - } } else { throw IllegalStateException("Can connect to the EM Driver, but cannot infer the 'problemType'") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteSplitter.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteSplitter.kt index 98dfbfb736..cc40bca155 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteSplitter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteSplitter.kt @@ -6,6 +6,8 @@ import org.evomaster.core.output.service.PartialOracles import org.evomaster.core.problem.graphql.GraphQlCallResult import org.evomaster.core.problem.httpws.HttpWsCallResult import org.evomaster.core.problem.rpc.RPCCallResult +import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult +import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome import org.evomaster.core.problem.rpc.RPCIndividual import org.evomaster.core.search.* import com.google.gson.* @@ -122,8 +124,17 @@ object TestSuiteSplitter { && ind.evaluatedMainActions().all { ac -> //TODO generic per type - val code = (ac.result as HttpWsCallResult).getStatusCode() - (code != null && code < 400) + when (val r = ac.result) { + is HttpWsCallResult -> r.getStatusCode().let { it != null && it < 400 } + /* + A message that went out, and was answered when an answer was + promised. There is no status code to read here. + */ + is AsyncApiCallResult -> r.getOutcome().let { + it == AsyncApiOutcome.PUBLISHED || it == AsyncApiOutcome.REPLIED + } + else -> true + } } }.toMutableList() diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt new file mode 100644 index 0000000000..3e1af9540c --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt @@ -0,0 +1,70 @@ +package org.evomaster.core.output.naming + +import org.evomaster.core.output.TestWriterUtils.safeVariableName +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult +import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.Solution +import org.evomaster.core.search.action.EvaluatedAction + +/** + * Names a test after the operation it publishes to and what came of it. + * + * There is no status code to name an outcome with, so the name carries the outcome itself, and + * the declared message a reply was recognised as where there is one: two tests over the same + * operation are usually told apart by which reply variant they reached. + */ +open class AsyncApiActionTestCaseNamingStrategy( + solution: Solution<*>, + languageConventionFormatter: LanguageConventionFormatter, + maxTestCaseNameLength: Int, +) : ActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) { + + override fun expandName( + individual: EvaluatedIndividual<*>, + nameTokens: MutableList, + ambiguitySolvers: List + ): String { + val evaluatedAction = individual.evaluatedMainActions().last() + val action = evaluatedAction.action as AsyncApiAction + var remainingNameChars = maxTestCaseNameLength - namePrefixChars() + + remainingNameChars = addNameTokensIfAllowed( + nameTokens, + listOf(on, safeVariableName(action.operationId)), + remainingNameChars + ) + addResult(individual, nameTokens, remainingNameChars) + + return formatName(nameTokens) + } + + override fun resolveAmbiguities(duplicatedIndividuals: Set>): Map, String> { + return emptyMap() + } + + override fun addActionResult( + evaluatedAction: EvaluatedAction, + nameTokens: MutableList, + remainingNameChars: Int + ): Int { + val result = evaluatedAction.result as AsyncApiCallResult + + val candidateTokens = when (result.getOutcome()) { + AsyncApiOutcome.REPLIED -> { + val message = result.getReplyMessage() + if (message != null) { + mutableListOf(returns, safeVariableName(message)) + } else { + mutableListOf(returns, success) + } + } + AsyncApiOutcome.PUBLISHED -> mutableListOf("published") + AsyncApiOutcome.NO_REPLY -> mutableListOf("getsNoReply") + AsyncApiOutcome.PUBLISH_FAILED, null -> mutableListOf(error) + } + + return addNameTokensIfAllowed(nameTokens, candidateTokens, remainingNameChars) + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt index 2be4e0b11a..6401f56ebd 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt @@ -7,6 +7,7 @@ import org.evomaster.core.output.naming.rest.RestActionTestCaseNamingStrategy import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.problem.graphql.GraphQLIndividual import org.evomaster.core.problem.rest.data.RestIndividual +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual import org.evomaster.core.problem.rpc.RPCIndividual import org.evomaster.core.problem.webfrontend.WebIndividual import org.evomaster.core.search.Solution @@ -44,6 +45,7 @@ class TestCaseNamingStrategyFactory( individuals.any { it.individual is RestIndividual } -> RestActionTestCaseNamingStrategy(solution, languageConventionFormatter, nameWithQueryParameters, maxTestCaseNameLength) individuals.any { it.individual is GraphQLIndividual } -> GraphQLActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) individuals.any { it.individual is RPCIndividual } -> RPCActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) + individuals.any { it.individual is AsyncApiIndividual } -> AsyncApiActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) individuals.any { it.individual is WebIndividual } -> { log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") NamingHelperNumberedTestCaseNamingStrategy(solution) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt new file mode 100644 index 0000000000..7728c09755 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt @@ -0,0 +1,146 @@ +package org.evomaster.core.output.service + +import org.evomaster.core.output.Lines +import org.evomaster.core.output.TestCase +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.action.Action +import org.evomaster.core.search.action.ActionResult +import java.nio.file.Path + +/** + * Writes a test for an AsyncAPI service. + * + * Unlike REST, there is no universal client to call: a message goes out over whichever broker + * the contract names, so the lines that publish it and read the reply are the driver's, rendered + * while the search ran and pasted here. This follows RPC, whose driver renders an invocation the + * same way, and it is what keeps a generated test standing on its own: it talks to the broker + * with an ordinary client of that transport, and needs no EvoMaster driver at run time. + * + * What the core adds around those lines is what only it knows: which outcome the action had, + * which of the declared reply messages the reply was recognised as, and that a reply promised by + * the contract did arrive. + */ +class AsyncApiTestCaseWriter : ApiTestCaseWriter() { + + override fun handleTestInitialization( + lines: Lines, + baseUrlOfSut: String, + ind: EvaluatedIndividual<*>, + sqlInsertionVars: MutableList>, + mongoInsertionVars: MutableList>, + redisInsertionVars: MutableList>, + dynamoDbInsertionVars: MutableList>, + testName: String + ) { + super.handleTestInitialization( + lines, + baseUrlOfSut, + ind, + sqlInsertionVars, + mongoInsertionVars, + redisInsertionVars, + dynamoDbInsertionVars, + testName + ) + } + + override fun handleActionCalls( + lines: Lines, + baseUrlOfSut: String, + ind: EvaluatedIndividual<*>, + sqlInsertionVars: MutableList>, + mongoInsertionVars: MutableList>, + redisInsertionVars: MutableList>, + dynamoDbInsertionVars: MutableList>, + testCaseName: String, + testSuitePath: Path? + ) { + if (ind.individual !is AsyncApiIndividual) { + return + } + + ind.evaluatedMainActions().forEachIndexed { index, evaluated -> + lines.addEmpty() + addActionLines( + evaluated.action, + index, + testCaseName, + lines, + evaluated.result, + testSuitePath, + baseUrlOfSut + ) + } + } + + override fun addActionLinesPerType( + action: Action, + index: Int, + testCaseName: String, + lines: Lines, + result: ActionResult, + testSuitePath: Path?, + baseUrlOfSut: String + ) { + val call = action as? AsyncApiAction + ?: throw IllegalStateException("Not an AsyncAPI action: ${action::class.java.simpleName}") + val res = result as? AsyncApiCallResult + ?: throw IllegalStateException("Not an AsyncAPI result: ${result::class.java.simpleName}") + + lines.addSingleCommentLine("${call.operationId}: ${res.getOutcome() ?: "not executed"}") + + val script = res.getTestScript() + + if (script.isEmpty()) { + /* + Nothing can be written that would reach the service: publishing needs a client of + the transport, and only the driver has one. Said plainly rather than left to a + reader wondering why a test does nothing. + */ + lines.addSingleCommentLine( + "The driver rendered no lines for this message, so there is nothing to publish here." + ) + lines.addSingleCommentLine( + "See SutController.executeAsyncApiAction for what a driver fills in." + ) + return + } + + script.forEach { lines.add(it) } + + addReplyAssertions(lines, res) + } + + /** + * What the core can say about the reply, on top of whatever the driver's own lines assert. + */ + private fun addReplyAssertions(lines: Lines, res: AsyncApiCallResult) { + + if (res.getOutcome() != AsyncApiOutcome.REPLIED) { + return + } + + val variable = res.getReplyVariableName() ?: return + + lines.add("assertNotNull($variable)") + lines.appendSemicolon() + + res.getReplyMessage()?.let { + lines.addSingleCommentLine("recognised as the declared message '$it'") + } + } + + /** + * A message that could not be published stops the test, and the search already recorded why, + * so there is no exception for a generated test to expect. + */ + override fun shouldFailIfExceptionNotThrown(result: ActionResult) = false + + override fun addTestCommentBlock(lines: Lines, test: TestCase) { + //the per-action comments carry the outcome, which is what there is to say + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt index 89ecc703b8..7015b71889 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt @@ -14,6 +14,14 @@ class AsyncApiCallResult : EnterpriseActionResult { const val REPLY_MESSAGE = "REPLY_MESSAGE" const val CORRELATION_MATCHED = "CORRELATION_MATCHED" const val WAITED_MS = "WAITED_MS" + const val TEST_SCRIPT = "TEST_SCRIPT" + const val REPLY_VARIABLE = "REPLY_VARIABLE" + + /** + * What the script's lines are joined with when stored, and split on when read back. + * A result holds strings, and a line of source never contains this. + */ + private const val SCRIPT_SEPARATOR = "\n" } constructor(sourceLocalId: String, stopping: Boolean = false) : super(sourceLocalId, stopping) @@ -60,4 +68,20 @@ class AsyncApiCallResult : EnterpriseActionResult { } fun getWaitedMs(): Long? = getResultValue(WAITED_MS)?.toLong() + + /** + * The lines the driver rendered for a generated test, or empty when it rendered none. + */ + fun setTestScript(lines: List) { + addResultValue(TEST_SCRIPT, lines.joinToString(SCRIPT_SEPARATOR)) + } + + fun getTestScript(): List = + getResultValue(TEST_SCRIPT)?.split(SCRIPT_SEPARATOR) ?: listOf() + + fun setReplyVariableName(name: String) { + addResultValue(REPLY_VARIABLE, name) + } + + fun getReplyVariableName(): String? = getResultValue(REPLY_VARIABLE) } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index 349f786dac..d987f14c13 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -6,6 +6,7 @@ import com.google.inject.Inject import com.webfuzzing.asyncapi.models.AsyncApiChannel import com.webfuzzing.asyncapi.models.AsyncApiCorrelationId import com.webfuzzing.asyncapi.models.AsyncApiReply +import org.evomaster.client.java.controller.api.dto.SutInfoDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto import org.evomaster.core.database.sql.SqlAction @@ -56,6 +57,12 @@ class AsyncApiFitness : ApiWsFitness() { private const val CORRELATION_SEPARATOR = "-" + /** + * What the variable holding a reply is called in a generated test, before the action's + * index. Named by the core so that two actions in one test cannot collide. + */ + private const val REPLY_VARIABLE_PREFIX = "asyncApiReply_" + private const val DEFAULT_CONTENT_TYPE = "application/json" /** @@ -166,7 +173,7 @@ class AsyncApiFitness : ApiWsFitness() { actionResults.add(result) val dto = getActionDto(action, index) - dto.asyncApiCall = toDto(action) + dto.asyncApiCall = toDto(action, index) val reply = rc.executeNewAsyncApiActionAndGetReply(dto) @@ -183,6 +190,9 @@ class AsyncApiFitness : ApiWsFitness() { } val outcome = record(reply, result) + if (dto.asyncApiCall.replyVariable != null) { + result.setReplyVariableName(dto.asyncApiCall.replyVariable) + } handleTargets(fv, action, result, outcome, index) return true @@ -207,6 +217,9 @@ class AsyncApiFitness : ApiWsFitness() { result.setOutcome(outcome) reply.waitedMs?.let { result.setWaitedMs(it) } + //what the generated test will publish with, when the driver rendered it + reply.testScript?.takeIf { it.isNotEmpty() }?.let { result.setTestScript(it) } + if (outcome == AsyncApiOutcome.REPLIED) { reply.replyPayload?.let { result.setReplyPayload(it) } /* @@ -342,7 +355,7 @@ class AsyncApiFitness : ApiWsFitness() { * Everything the driver needs to publish the message and wait for its reply, resolved * against the document so that the driver never has to read it. */ - private fun toDto(action: AsyncApiAction): AsyncApiActionDto { + private fun toDto(action: AsyncApiAction, index: Int): AsyncApiActionDto { val document = asyncApiSampler.document val message = document.messages[action.messageId] @@ -360,6 +373,16 @@ class AsyncApiFitness : ApiWsFitness() { dto.contentType = message?.contentType ?: document.defaultContentType ?: DEFAULT_CONTENT_TYPE dto.headers = LinkedHashMap(buildHeaders(action)) + if (config.createTests) { + /* + A generated test publishes with a client of the transport, which only the driver + has, so it renders those lines while the search runs. Not asked for when no test + will be written, so a driver need not spend time on it. + */ + dto.outputFormat = SutInfoDto.OutputFormat.valueOf(config.outputFormat.toString()) + dto.replyVariable = REPLY_VARIABLE_PREFIX + index + } + dto.correlationId = runId + CORRELATION_SEPARATOR + published++ message?.correlationId?.let { dto.correlationLocation = if (it.source == AsyncApiCorrelationId.Source.HEADER) { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt index 7ea148004d..1446283249 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt @@ -1,7 +1,7 @@ package org.evomaster.core.problem.asyncapi.service import com.google.inject.TypeLiteral -import org.evomaster.core.output.service.NoTestCaseWriter +import org.evomaster.core.output.service.AsyncApiTestCaseWriter import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual @@ -96,7 +96,7 @@ class AsyncApiModule : EnterpriseModule() { .asEagerSingleton() bind(TestCaseWriter::class.java) - .to(NoTestCaseWriter::class.java) + .to(AsyncApiTestCaseWriter::class.java) .asEagerSingleton() bind(TestSuiteWriter::class.java) diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 98cc8d3bd3..42c26534b2 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -786,18 +786,28 @@ internal class EMConfigTest{ } @Test - fun testAsyncApiCannotWriteTestsYet(){ + fun testAsyncApiWritesTestsOnlyInJavaOrKotlin(){ - val parser = EMConfig.getOptionParser() + //a parser remembers what it last parsed, so each case gets its own + fun parse(vararg args: String) = + EMConfig().updateProperties(EMConfig.getOptionParser().parse(*args)) - //createTests is on by default, and there is no test writer for AsyncAPI yet + /* + The lines that publish are rendered by the driver, which is Java, so there is no + other language they could be pasted into. + */ val e = assertThrows { - EMConfig().updateProperties(parser.parse("--problemType", "ASYNCAPI")) + parse("--problemType", "ASYNCAPI", "--outputFormat", "PYTHON_UNITTEST") } - assertTrue(e.message!!.contains("createTests"), e.message) + assertTrue(e.message!!.contains("outputFormat"), e.message) + + //the default, and the two it can write + parse("--problemType", "ASYNCAPI") + parse("--problemType", "ASYNCAPI", "--outputFormat", "JAVA_JUNIT_5") + parse("--problemType", "ASYNCAPI", "--outputFormat", "KOTLIN_JUNIT_5") - //so a run has to say it only wants the search - EMConfig().updateProperties(parser.parse("--problemType", "ASYNCAPI", "--createTests", "false")) + //and a run that only wants the search is still allowed + parse("--problemType", "ASYNCAPI", "--createTests", "false") } @Test diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt index c3de0f653b..91b6e57cc6 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt @@ -12,7 +12,7 @@ import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 import org.evomaster.core.search.algorithms.MioAlgorithm import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.service.IdMapper -import org.evomaster.core.output.service.NoTestCaseWriter +import org.evomaster.core.output.service.AsyncApiTestCaseWriter import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied @@ -154,7 +154,7 @@ class AsyncApiModuleTest { injector.getInstance(TestSuiteWriter::class.java) injector.getInstance(RemoteController::class.java) - //no test writer for AsyncAPI yet, so the one that writes nothing - assertTrue(injector.getInstance(TestCaseWriter::class.java) is NoTestCaseWriter) + //the writer that pastes the driver's own publish-and-await lines + assertTrue(injector.getInstance(TestCaseWriter::class.java) is AsyncApiTestCaseWriter) } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt new file mode 100644 index 0000000000..630caba13f --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -0,0 +1,165 @@ +package org.evomaster.core.problem.asyncapi.service + +import com.google.inject.Injector +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.SutInfoDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.output.TestCase +import org.evomaster.core.output.service.TestCaseWriter +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.enterprise.SampleType +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.Randomness +import org.evomaster.core.search.service.SearchGlobalState +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +/** + * What a generated test looks like. The lines that publish are the driver's, so the fake one + * renders a stand-in for them here, exactly as a Kafka driver would render a producer. + */ +class AsyncApiTestCaseWriterTest { + + companion object { + private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" + + /** + * What a driver hands over: lines that stand up a client, publish, and read the reply + * into a variable it names. + */ + private fun withScript(payload: String, dto: AsyncApiActionDto): AsyncApiReplyDto = + FakeAsyncApiDriver.replied(payload).apply { + //named off the variable the core supplied, which is unique per action + val v = dto.replyVariable + testScript = listOf( + "val producer_$v = connectProducer(\"localhost:9092\")", + "producer_$v.send(recordFor(\"${dto.address}\"))", + "val $v = pollUntilCorrelated(\"${dto.replyAddress}\")" + ) + } + } + + private lateinit var injector: Injector + private lateinit var driver: FakeAsyncApiDriver + private lateinit var sampler: AsyncApiSampler + private lateinit var fitness: FitnessFunction + + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + + private fun start(answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) { + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS)), answer) + injector = AsyncApiTestInjector.create( + driver, + "--blackBox=false", + //the core only asks a driver to render lines when a test will be written + "--createTests=true", + "--outputFormat=KOTLIN_JUNIT_5" + ) + sampler = injector.getInstance(AsyncApiSampler::class.java) + fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + } + + private fun evaluate(vararg names: String): EvaluatedIndividual { + val randomness = injector.getInstance(Randomness::class.java) + val actions = names.map { name -> + val template = sampler.seeAvailableActions().first { it.getName() == name } + (template.copy() as AsyncApiAction).apply { doInitialize(randomness) } + } + val individual = AsyncApiIndividual(SampleType.RANDOM, actions.toMutableList()).apply { + doGlobalInitialize(injector.getInstance(SearchGlobalState::class.java)) + } + return fitness.calculateCoverage(individual, modifiedSpec = null) + ?: fail("the fitness gave up on the individual") + } + + /** + * The generated body for one test, as the suite writer would ask for it. + */ + private fun bodyOf(evaluated: EvaluatedIndividual): String { + val writer = injector.getInstance(TestCaseWriter::class.java) + return writer.convertToCompilableTestCode(TestCase(evaluated, "test_0"), "baseUrlOfSut").toString() + } + + @Test + fun testTheDriversLinesAreWhatPublishes() { + + start { withScript(DOUBLE_RESULT, it) } + + val body = bodyOf(evaluate("bessj")) + + //the driver's own client code, pasted as it was rendered + assertTrue(body.contains("connectProducer(\"localhost:9092\")"), body) + assertTrue(body.contains("pollUntilCorrelated"), body) + + //and what only the core knows about the reply + assertTrue(body.contains("assertNotNull(asyncApiReply_0)"), body) + assertTrue(body.contains("bessj"), body) + } + + @Test + fun testEachActionKeepsItsOwnLines() { + + start { withScript(DOUBLE_RESULT, it) } + + val body = bodyOf(evaluate("bessj", "expint")) + + //the core names the variables, so two actions in one test cannot collide + assertTrue(body.contains("assertNotNull(asyncApiReply_0)"), body) + assertTrue(body.contains("assertNotNull(asyncApiReply_1)"), body) + assertTrue(body.contains("producer_asyncApiReply_0"), body) + assertTrue(body.contains("producer_asyncApiReply_1"), body) + } + + @Test + fun testTheDriverIsToldWhichLanguageToRenderAndWhereToLeaveTheReply() { + + start { withScript(DOUBLE_RESULT, it) } + evaluate("bessj") + + val asked = driver.published.single() + assertEquals(SutInfoDto.OutputFormat.KOTLIN_JUNIT_5, asked.outputFormat) + assertEquals("asyncApiReply_0", asked.replyVariable) + } + + @Test + fun testADriverThatRendersNothingSaysSoRatherThanWritingAnEmptyTest() { + + //a driver that publishes but renders no lines: there is nothing the core could write + start { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } + + val body = bodyOf(evaluate("bessj")) + + assertTrue(body.contains("rendered no lines"), body) + assertTrue(body.contains("executeAsyncApiAction"), body) + assertFalse(body.contains("assertNotNull"), body) + } + + @Test + fun testAMessageThatWentUnansweredIsWrittenWithoutAReplyAssertion() { + + start { + FakeAsyncApiDriver.silence().apply { + testScript = listOf("publishAndWait()") + } + } + + val body = bodyOf(evaluate("bessj")) + + assertTrue(body.contains("publishAndWait()"), body) + assertTrue(body.contains("NO_REPLY"), body) + //nothing arrived, so there is nothing to assert on + assertFalse(body.contains("assertNotNull"), body) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt index fc0eecdc56..a0dab84872 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt @@ -34,7 +34,14 @@ object AsyncApiTestInjector { fun create(driver: FakeAsyncApiDriver, vararg options: String): Injector { - val args = arrayOf("--seed=42", "--problemType=ASYNCAPI", "--createTests=false") + options + /* + Most suites here search without writing tests, but one is about what gets written, + and the option cannot be given twice. + */ + val defaults = listOf("--seed=42", "--problemType=ASYNCAPI", "--createTests=false") + .filterNot { d -> options.any { it.substringBefore('=') == d.substringBefore('=') } } + + val args = (defaults + options).toTypedArray() val fake = object : AbstractModule() { override fun configure() { From 3cb016b0235858ca4f06f6c98c0e918e6239cd0a Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 25 Sep 2026 17:47:47 -0300 Subject: [PATCH 02/20] AsyncAPI 3.x: write the Kafka client into the test, from the contract REST writes its own RestAssured calls rather than asking the driver for them, and Kafka can be written the same way: the document names the broker on the server, the topic on the channel or its binding, and the field the correlation id rides in. Nothing else is needed to publish the same message again. So a Kafka test now stands up a producer and a consumer of its own. It seeks to the end of the reply topic before publishing, so a reply older than the test is not taken for an answer to it, mints a fresh id each run and matches it on the way back. Core writes the text and links nothing: the dependency lands on the generated suite, as rest-assured does for REST. The driver's own rendering stays, and wins where it is present. It is what a transport the contract cannot describe needs: one whose correlation id rides inside the service's own message layout, where the document says nothing about where to put it. Neither available, and the test says so rather than being silently empty. What the search resolved is kept on the result, so none of it is worked out a second time here. --- .../output/service/AsyncApiTestCaseWriter.kt | 44 ++- .../output/service/KafkaTestClientEmitter.kt | 271 ++++++++++++++++++ .../asyncapi/data/AsyncApiCallResult.kt | 53 ++++ .../asyncapi/service/AsyncApiFitness.kt | 50 ++++ .../service/AsyncApiTestCaseWriterTest.kt | 75 ++++- 5 files changed, 476 insertions(+), 17 deletions(-) create mode 100644 core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt index 7728c09755..6b122c3fc7 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt @@ -94,23 +94,41 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("${call.operationId}: ${res.getOutcome() ?: "not executed"}") val script = res.getTestScript() + val variable = res.getReplyVariableName() - if (script.isEmpty()) { + when { /* - Nothing can be written that would reach the service: publishing needs a client of - the transport, and only the driver has one. Said plainly rather than left to a - reader wondering why a test does nothing. + The driver rendered the lines itself, which is how a transport the contract does + not describe well enough gets written: one whose correlation id rides inside the + service's own message layout, or one nothing here knows at all. */ - lines.addSingleCommentLine( - "The driver rendered no lines for this message, so there is nothing to publish here." - ) - lines.addSingleCommentLine( - "See SutController.executeAsyncApiAction for what a driver fills in." - ) - return - } + script.isNotEmpty() -> script.forEach { lines.add(it) } - script.forEach { lines.add(it) } + /* + Kafka is written from the contract, as REST writes its RestAssured calls: the + document named the broker, the topics and the header the id rides in, so nothing + else has to. + */ + variable != null && KafkaTestClientEmitter.canEmit(res) -> + KafkaTestClientEmitter.emit(lines, res, variable, index, format) + + else -> { + /* + Nothing can be written that would reach the service. Said plainly rather than + left to a reader wondering why a test does nothing. + */ + lines.addSingleCommentLine( + "No lines could be written to publish this message: the transport is not one" + ) + lines.addSingleCommentLine( + "written from the contract, and the driver rendered none. See" + ) + lines.addSingleCommentLine( + "SutController.executeAsyncApiAction for what a driver can fill in." + ) + return + } + } addReplyAssertions(lines, res) } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt new file mode 100644 index 0000000000..401732070c --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt @@ -0,0 +1,271 @@ +package org.evomaster.core.output.service + +import com.fasterxml.jackson.databind.ObjectMapper +import org.evomaster.core.output.Lines +import org.evomaster.core.output.OutputFormat +import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult + +/** + * Writes the Kafka client code that republishes one message, for a generated test. + * + * This is the AsyncAPI counterpart of the RestAssured calls the REST writer emits: the test + * talks to the broker itself, with an ordinary `kafka-clients` producer and consumer, and needs + * no EvoMaster driver at run time. Everything it needs was resolved from the contract while the + * search ran and kept on the result, so nothing is read from the document again here. + * + * Only Kafka is written this way. A transport that carries its correlation id inside the + * service's own message layout cannot be emitted from the contract alone, because the contract + * does not describe that layout; there, the driver renders the lines instead. + * + * The emitted code is deliberately self-contained per action rather than calling a shared + * helper: a test that can be read top to bottom is worth more than a short one, and it keeps a + * suite that never publishes to Kafka from carrying the dependency. + */ +object KafkaTestClientEmitter { + + /** + * The protocols this can write. The document names them on the server. + */ + private val KAFKA_PROTOCOLS = setOf("kafka", "kafka-secure") + + private const val DEFAULT_POLL_MS = 200L + + private val mapper = ObjectMapper() + + /** + * Whether this result carries enough, and the right transport, to be written as Kafka code. + */ + fun canEmit(result: AsyncApiCallResult): Boolean = + result.getProtocol()?.lowercase() in KAFKA_PROTOCOLS + && !result.getBroker().isNullOrBlank() + && !result.getAddress().isNullOrBlank() + + /** + * Publish the message again and, when one is expected, wait for the reply that answers it. + * + * @param variable what the reply payload is left in, named by the core + * @param index of the action, so that two in one test do not collide + */ + fun emit(lines: Lines, result: AsyncApiCallResult, variable: String, index: Int, format: OutputFormat) { + + val suffix = "_$index" + val props = "asyncApiProps$suffix" + val producer = "asyncApiProducer$suffix" + val consumer = "asyncApiConsumer$suffix" + val record = "asyncApiRecord$suffix" + val parts = "asyncApiPartitions$suffix" + val cid = "asyncApiCid$suffix" + val deadline = "asyncApiDeadline$suffix" + + val broker = quoted(result.getBroker()!!, format) + val topic = quoted(result.getAddress()!!, format) + val replyTopic = result.getReplyAddress() + val timeout = result.getReplyTimeoutMs() ?: 0L + + declareProperties(lines, props, broker, format) + + //a fresh id per run: a reply to an earlier run's message is not an answer to this one + assign(lines, cid, "java.util.UUID.randomUUID().toString()", "String", format) + + val serializer = "org.apache.kafka.common.serialization.String" + newInstance( + lines, producer, + "org.apache.kafka.clients.producer.KafkaProducer", + "org.apache.kafka.clients.producer.KafkaProducer<>($props, new ${serializer}Serializer(), new ${serializer}Serializer())", + "org.apache.kafka.clients.producer.KafkaProducer($props, ${serializer}Serializer(), ${serializer}Serializer())", + format + ) + + if (replyTopic != null) { + newInstance( + lines, consumer, + "org.apache.kafka.clients.consumer.KafkaConsumer", + "org.apache.kafka.clients.consumer.KafkaConsumer<>($props, new ${serializer}Deserializer(), new ${serializer}Deserializer())", + "org.apache.kafka.clients.consumer.KafkaConsumer($props, ${serializer}Deserializer(), ${serializer}Deserializer())", + format + ) + seekToEnd(lines, consumer, parts, quoted(replyTopic, format), format) + } + + buildRecord(lines, record, topic, result, format) + stampCorrelation(lines, record, cid, result, format) + + statement(lines, "$producer.send($record).get()", format) + + if (replyTopic == null) { + close(lines, producer, format) + //nothing is expected back, so the variable the core named stays unset + declareReply(lines, variable, format) + return + } + + declareReply(lines, variable, format) + awaitReply(lines, consumer, variable, cid, deadline, timeout, result, format) + close(lines, producer, format) + close(lines, consumer, format) + } + + private fun declareProperties(lines: Lines, name: String, broker: String, format: OutputFormat) { + newInstance(lines, name, "java.util.Properties", "java.util.Properties()", "java.util.Properties()", format) + statement(lines, "$name.put(\"bootstrap.servers\", $broker)", format) + //read only what is published from here on, never what was already on the topic + statement(lines, "$name.put(\"auto.offset.reset\", \"latest\")", format) + statement(lines, "$name.put(\"enable.auto.commit\", \"false\")", format) + } + + private fun seekToEnd(lines: Lines, consumer: String, parts: String, topic: String, format: OutputFormat) { + + val expr = "$consumer.partitionsFor($topic).stream()" + + ".map(p -> new org.apache.kafka.common.TopicPartition(p.topic(), p.partition()))" + + ".collect(java.util.stream.Collectors.toList())" + val kotlinExpr = "$consumer.partitionsFor($topic)" + + ".map { org.apache.kafka.common.TopicPartition(it.topic(), it.partition()) }" + + when { + format.isJava() -> lines.add("java.util.List $parts = $expr;") + format.isKotlin() -> lines.add("val $parts = $kotlinExpr") + } + + statement(lines, "$consumer.assign($parts)", format) + statement(lines, "$consumer.seekToEnd($parts)", format) + //seekToEnd is lazy, so make it take effect before anything is published + statement(lines, "$consumer.poll(java.time.Duration.ZERO)", format) + } + + private fun buildRecord(lines: Lines, name: String, topic: String, result: AsyncApiCallResult, format: OutputFormat) { + + val payload = result.getPayload()?.let { quoted(it, format) } ?: nullLiteral(format) + + newInstance( + lines, name, + "org.apache.kafka.clients.producer.ProducerRecord", + "org.apache.kafka.clients.producer.ProducerRecord<>($topic, $payload)", + "org.apache.kafka.clients.producer.ProducerRecord($topic, $payload)", + format + ) + + headersOf(result).forEach { (key, value) -> + statement(lines, "$name.headers().add(${quoted(key, format)}, ${bytesOf(quoted(value, format), format)})", format) + } + } + + private fun stampCorrelation(lines: Lines, record: String, cid: String, result: AsyncApiCallResult, format: OutputFormat) { + val header = result.getCorrelationHeader() ?: return + statement(lines, "$record.headers().add(${quoted(header, format)}, ${bytesOf(cid, format)})", format) + } + + private fun awaitReply( + lines: Lines, + consumer: String, + variable: String, + cid: String, + deadline: String, + timeout: Long, + result: AsyncApiCallResult, + format: OutputFormat + ) { + val header = result.getCorrelationHeader() + + when { + format.isJava() -> lines.add("long $deadline = System.currentTimeMillis() + $timeout;") + format.isKotlin() -> lines.add("val $deadline = System.currentTimeMillis() + $timeout") + } + + lines.add("while ($variable == null && System.currentTimeMillis() < $deadline) {") + lines.indented { + val loop = if (format.isJava()) { + "for (org.apache.kafka.clients.consumer.ConsumerRecord r : " + + "$consumer.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {" + } else { + "for (r in $consumer.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {" + } + lines.add(loop) + lines.indented { + if (header == null) { + //no correlation declared, so the first thing on the reply topic is taken + statement(lines, "$variable = r.value()", format) + } else { + val readHeader = "r.headers().lastHeader(${quoted(header, format)})" + when { + format.isJava() -> lines.add("org.apache.kafka.common.header.Header h = $readHeader;") + format.isKotlin() -> lines.add("val h = $readHeader") + } + lines.add("if (h != null && $cid.equals(new String(h.value(), java.nio.charset.StandardCharsets.UTF_8))) {" + .let { if (format.isKotlin()) "if (h != null && $cid == String(h.value(), java.nio.charset.StandardCharsets.UTF_8)) {" else it }) + lines.indented { statement(lines, "$variable = r.value()", format) } + lines.add("}") + } + } + lines.add("}") + } + lines.add("}") + } + + private fun headersOf(result: AsyncApiCallResult): Map { + val json = result.getHeadersAsJson() ?: return mapOf() + return try { + val node = mapper.readTree(json) + node.fieldNames().asSequence().associateWith { node.get(it).asText() } + } catch (e: Exception) { + mapOf() + } + } + + private fun declareReply(lines: Lines, variable: String, format: OutputFormat) { + when { + format.isJava() -> lines.add("String $variable = null;") + format.isKotlin() -> lines.add("var $variable: String? = null") + } + } + + private fun newInstance( + lines: Lines, + name: String, + javaType: String, + javaCtor: String, + kotlinCtor: String, + format: OutputFormat + ) { + when { + format.isJava() -> lines.add("$javaType $name = new $javaCtor;") + format.isKotlin() -> lines.add("val $name = $kotlinCtor") + } + } + + private fun assign(lines: Lines, name: String, expr: String, javaType: String, format: OutputFormat) { + when { + format.isJava() -> lines.add("$javaType $name = $expr;") + format.isKotlin() -> lines.add("val $name = $expr") + } + } + + private fun statement(lines: Lines, code: String, format: OutputFormat) { + lines.add(code) + if (format.isJava()) { + lines.append(";") + } + } + + private fun close(lines: Lines, name: String, format: OutputFormat) = + statement(lines, "$name.close()", format) + + private fun bytesOf(expr: String, format: OutputFormat) = + "$expr.toByteArray(java.nio.charset.StandardCharsets.UTF_8)" + .let { if (format.isJava()) "$expr.getBytes(java.nio.charset.StandardCharsets.UTF_8)" else it } + + private fun nullLiteral(format: OutputFormat) = "null" + + /** + * A string as a literal of the target language, with what would end it escaped. + */ + private fun quoted(value: String, format: OutputFormat): String { + val escaped = value + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + .let { if (format.isKotlin()) it.replace("$", "\\$") else it } + return "\"$escaped\"" + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt index 7015b71889..6e5a7c455d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt @@ -15,6 +15,19 @@ class AsyncApiCallResult : EnterpriseActionResult { const val CORRELATION_MATCHED = "CORRELATION_MATCHED" const val WAITED_MS = "WAITED_MS" const val TEST_SCRIPT = "TEST_SCRIPT" + + /* + What was published, kept so that a generated test can publish the same thing without + resolving the document a second time. Only recorded when tests are being written. + */ + const val BROKER = "BROKER" + const val PROTOCOL = "PROTOCOL" + const val ADDRESS = "ADDRESS" + const val REPLY_ADDRESS = "REPLY_ADDRESS" + const val PAYLOAD = "PAYLOAD" + const val HEADERS = "HEADERS" + const val CORRELATION_HEADER = "CORRELATION_HEADER" + const val REPLY_TIMEOUT_MS = "REPLY_TIMEOUT_MS" const val REPLY_VARIABLE = "REPLY_VARIABLE" /** @@ -84,4 +97,44 @@ class AsyncApiCallResult : EnterpriseActionResult { } fun getReplyVariableName(): String? = getResultValue(REPLY_VARIABLE) + + /** + * Everything a generated test needs to publish this message again, as the search resolved it + * from the document. Headers are a JSON object, so that one string holds them all. + */ + fun setPublished( + broker: String?, + protocol: String?, + address: String?, + replyAddress: String?, + payload: String?, + headersAsJson: String, + correlationHeader: String?, + replyTimeoutMs: Long? + ) { + broker?.let { addResultValue(BROKER, it) } + protocol?.let { addResultValue(PROTOCOL, it) } + address?.let { addResultValue(ADDRESS, it) } + replyAddress?.let { addResultValue(REPLY_ADDRESS, it) } + payload?.let { addResultValue(PAYLOAD, it) } + addResultValue(HEADERS, headersAsJson) + correlationHeader?.let { addResultValue(CORRELATION_HEADER, it) } + replyTimeoutMs?.let { addResultValue(REPLY_TIMEOUT_MS, it.toString()) } + } + + fun getBroker(): String? = getResultValue(BROKER) + + fun getProtocol(): String? = getResultValue(PROTOCOL) + + fun getAddress(): String? = getResultValue(ADDRESS) + + fun getReplyAddress(): String? = getResultValue(REPLY_ADDRESS) + + fun getPayload(): String? = getResultValue(PAYLOAD) + + fun getHeadersAsJson(): String? = getResultValue(HEADERS) + + fun getCorrelationHeader(): String? = getResultValue(CORRELATION_HEADER) + + fun getReplyTimeoutMs(): Long? = getResultValue(REPLY_TIMEOUT_MS)?.toLong() } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index d987f14c13..d13c448e28 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -57,6 +57,16 @@ class AsyncApiFitness : ApiWsFitness() { private const val CORRELATION_SEPARATOR = "-" + /** + * Separator inside a JSON Pointer, whose last segment names the field. + */ + private const val PATH_SEPARATOR = "/" + + /** + * What a correlation id's header is called when the document declares no field for it. + */ + private const val DEFAULT_CORRELATION_HEADER = "correlationId" + /** * What the variable holding a reply is called in a generated test, before the action's * index. Named by the core so that two actions in one test cannot collide. @@ -192,6 +202,7 @@ class AsyncApiFitness : ApiWsFitness() { val outcome = record(reply, result) if (dto.asyncApiCall.replyVariable != null) { result.setReplyVariableName(dto.asyncApiCall.replyVariable) + recordWhatWasPublished(action, dto.asyncApiCall, result) } handleTargets(fv, action, result, outcome, index) @@ -411,6 +422,45 @@ class AsyncApiFitness : ApiWsFitness() { * headers are on. A header whose value prints as JSON null is left out rather than sent as * the text "null". */ + /** + * Keep on the result what a generated test needs to publish this message again, so that the + * writer does not resolve the document a second time. Only when a test will be written. + */ + private fun recordWhatWasPublished( + action: AsyncApiAction, + call: AsyncApiActionDto, + result: AsyncApiCallResult + ) { + val document = asyncApiSampler.document + val channel = document.channels[action.channelName] + val server = channel?.let { document.serversOf(it).firstOrNull() } + + result.setPublished( + broker = server?.host, + protocol = server?.protocol, + address = call.address, + replyAddress = call.replyAddress, + payload = call.payload, + headersAsJson = mapper.writeValueAsString(call.headers ?: emptyMap()), + correlationHeader = correlationHeaderOf(call), + replyTimeoutMs = call.replyTimeoutMs + ) + } + + /** + * The header a correlation id rides in, when it rides in one. The document names the field + * with a pointer, and its last segment is the header's name. + */ + private fun correlationHeaderOf(call: AsyncApiActionDto): String? { + + if (call.correlationLocation != AsyncApiActionDto.CorrelationLocation.HEADER) { + return null + } + + return call.correlationPointer?.substringAfterLast(PATH_SEPARATOR)?.takeIf { it.isNotBlank() } + ?: DEFAULT_CORRELATION_HEADER + } + private fun buildHeaders(action: AsyncApiAction): Map { val headers = LinkedHashMap() diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index 630caba13f..de59ad30a2 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -134,16 +134,83 @@ class AsyncApiTestCaseWriterTest { } @Test - fun testADriverThatRendersNothingSaysSoRatherThanWritingAnEmptyTest() { + fun testKafkaIsWrittenFromTheContractWhenTheDriverRendersNothing() { - //a driver that publishes but renders no lines: there is nothing the core could write + /* + The document names the broker, the topics and the header the correlation id rides in, + so the test can publish with an ordinary client and no driver has to render anything. + */ start { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } val body = bodyOf(evaluate("bessj")) - assertTrue(body.contains("rendered no lines"), body) + //a real producer and consumer, against the broker the document names + assertTrue(body.contains("KafkaProducer"), body) + assertTrue(body.contains("KafkaConsumer"), body) + assertTrue(body.contains("\"bootstrap.servers\", \"localhost:9092\""), body) + assertTrue(body.contains("ProducerRecord(\"ncs.bessj.request\""), body) + + //a reply older than this publish is not an answer to it + assertTrue(body.contains("seekToEnd"), body) + + //a fresh id each run, stamped where the document says and matched on the way back + assertTrue(body.contains("UUID.randomUUID()"), body) + assertTrue(body.contains("lastHeader(\"correlationId\")"), body) + + assertTrue(body.contains("assertNotNull(asyncApiReply_0)"), body) + } + + @Test + fun testATransportTheContractDoesNotDescribeSaysSoRatherThanWritingAnEmptyTest() { + + /* + A socket carries its correlation id inside the service's own message layout, which the + contract does not describe, so nothing here can write the client code. Only a driver + can, and this one renders nothing. + */ + val socket = """ + asyncapi: 3.0.0 + info: + title: Socket + version: 1.0.0 + servers: + live: + host: localhost:8080 + protocol: ws + channels: + requests: + address: /requests + servers: + - ${'$'}ref: '#/servers/live' + messages: + request: + payload: + type: object + properties: + id: + type: string + operations: + ask: + action: receive + channel: + ${'$'}ref: '#/channels/requests' + """.trimIndent() + + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(socket)) { FakeAsyncApiDriver.fireAndForget() } + injector = AsyncApiTestInjector.create( + driver, + "--blackBox=false", + "--createTests=true", + "--outputFormat=KOTLIN_JUNIT_5" + ) + sampler = injector.getInstance(AsyncApiSampler::class.java) + fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + + val body = bodyOf(evaluate("ask")) + + assertTrue(body.contains("not one"), body) assertTrue(body.contains("executeAsyncApiAction"), body) - assertFalse(body.contains("assertNotNull"), body) + assertFalse(body.contains("KafkaProducer"), body) } @Test From 5c2e8731c9fb8b6881914650074613bc99b02b20 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 25 Sep 2026 17:53:51 -0300 Subject: [PATCH 03/20] AsyncAPI 3.x: name the generated variables and tests as the others do The REST writer calls its variables res_0 and body_0, so there is no reason for these to carry an asyncApi prefix on every one. They are now producer_0, consumer_0, record_0, correlationId_0 and reply_0, keyed off the reply variable the core names rather than off the action index, so the locals of one action always agree with the reply it leaves behind. A test name read verb, target, result in REST, and ours was missing the verb: publish_on__returns_. The two outcomes that are not a declared reply say what they are rather than naming an enum constant. --- .../AsyncApiActionTestCaseNamingStrategy.kt | 13 +++++++-- .../output/service/AsyncApiTestCaseWriter.kt | 2 +- .../output/service/KafkaTestClientEmitter.kt | 29 +++++++++++-------- .../asyncapi/service/AsyncApiFitness.kt | 2 +- .../service/AsyncApiTestCaseWriterTest.kt | 14 ++++----- 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt index 3e1af9540c..8f694f70e4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt @@ -21,6 +21,8 @@ open class AsyncApiActionTestCaseNamingStrategy( maxTestCaseNameLength: Int, ) : ActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) { + private val publish = "publish" + override fun expandName( individual: EvaluatedIndividual<*>, nameTokens: MutableList, @@ -30,9 +32,13 @@ open class AsyncApiActionTestCaseNamingStrategy( val action = evaluatedAction.action as AsyncApiAction var remainingNameChars = maxTestCaseNameLength - namePrefixChars() + /* + Shaped like REST's, which reads verb then target then result. What a tester does to + an operation is publish to it, whichever direction the document declares. + */ remainingNameChars = addNameTokensIfAllowed( nameTokens, - listOf(on, safeVariableName(action.operationId)), + listOf(publish, on, safeVariableName(action.operationId)), remainingNameChars ) addResult(individual, nameTokens, remainingNameChars) @@ -60,8 +66,9 @@ open class AsyncApiActionTestCaseNamingStrategy( mutableListOf(returns, success) } } - AsyncApiOutcome.PUBLISHED -> mutableListOf("published") - AsyncApiOutcome.NO_REPLY -> mutableListOf("getsNoReply") + //nothing is declared to come back, so reaching the broker is the whole outcome + AsyncApiOutcome.PUBLISHED -> mutableListOf(returns, "nothing") + AsyncApiOutcome.NO_REPLY -> mutableListOf("gets", "no", "reply") AsyncApiOutcome.PUBLISH_FAILED, null -> mutableListOf(error) } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt index 6b122c3fc7..520fa693c3 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt @@ -110,7 +110,7 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { else has to. */ variable != null && KafkaTestClientEmitter.canEmit(res) -> - KafkaTestClientEmitter.emit(lines, res, variable, index, format) + KafkaTestClientEmitter.emit(lines, res, variable, format) else -> { /* diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt index 401732070c..e35822d977 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt @@ -43,19 +43,24 @@ object KafkaTestClientEmitter { /** * Publish the message again and, when one is expected, wait for the reply that answers it. * - * @param variable what the reply payload is left in, named by the core - * @param index of the action, so that two in one test do not collide + * @param variable what the reply payload is left in, named by the core, and what the + * other names here are keyed off so two actions cannot collide */ - fun emit(lines: Lines, result: AsyncApiCallResult, variable: String, index: Int, format: OutputFormat) { - - val suffix = "_$index" - val props = "asyncApiProps$suffix" - val producer = "asyncApiProducer$suffix" - val consumer = "asyncApiConsumer$suffix" - val record = "asyncApiRecord$suffix" - val parts = "asyncApiPartitions$suffix" - val cid = "asyncApiCid$suffix" - val deadline = "asyncApiDeadline$suffix" + fun emit(lines: Lines, result: AsyncApiCallResult, variable: String, format: OutputFormat) { + + /* + Taken from the variable the core named rather than from the index, so the locals of + one action always agree with the reply it leaves behind, whatever numbered it. + */ + val suffix = "_" + variable.substringAfterLast('_') + + val props = "kafka$suffix" + val producer = "producer$suffix" + val consumer = "consumer$suffix" + val record = "record$suffix" + val parts = "partitions$suffix" + val cid = "correlationId$suffix" + val deadline = "deadline$suffix" val broker = quoted(result.getBroker()!!, format) val topic = quoted(result.getAddress()!!, format) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index d13c448e28..cfca8ded9b 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -71,7 +71,7 @@ class AsyncApiFitness : ApiWsFitness() { * What the variable holding a reply is called in a generated test, before the action's * index. Named by the core so that two actions in one test cannot collide. */ - private const val REPLY_VARIABLE_PREFIX = "asyncApiReply_" + private const val REPLY_VARIABLE_PREFIX = "reply_" private const val DEFAULT_CONTENT_TYPE = "application/json" diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index de59ad30a2..003b4f765b 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -104,7 +104,7 @@ class AsyncApiTestCaseWriterTest { assertTrue(body.contains("pollUntilCorrelated"), body) //and what only the core knows about the reply - assertTrue(body.contains("assertNotNull(asyncApiReply_0)"), body) + assertTrue(body.contains("assertNotNull(reply_0)"), body) assertTrue(body.contains("bessj"), body) } @@ -116,10 +116,10 @@ class AsyncApiTestCaseWriterTest { val body = bodyOf(evaluate("bessj", "expint")) //the core names the variables, so two actions in one test cannot collide - assertTrue(body.contains("assertNotNull(asyncApiReply_0)"), body) - assertTrue(body.contains("assertNotNull(asyncApiReply_1)"), body) - assertTrue(body.contains("producer_asyncApiReply_0"), body) - assertTrue(body.contains("producer_asyncApiReply_1"), body) + assertTrue(body.contains("assertNotNull(reply_0)"), body) + assertTrue(body.contains("assertNotNull(reply_1)"), body) + assertTrue(body.contains("producer_reply_0"), body) + assertTrue(body.contains("producer_reply_1"), body) } @Test @@ -130,7 +130,7 @@ class AsyncApiTestCaseWriterTest { val asked = driver.published.single() assertEquals(SutInfoDto.OutputFormat.KOTLIN_JUNIT_5, asked.outputFormat) - assertEquals("asyncApiReply_0", asked.replyVariable) + assertEquals("reply_0", asked.replyVariable) } @Test @@ -157,7 +157,7 @@ class AsyncApiTestCaseWriterTest { assertTrue(body.contains("UUID.randomUUID()"), body) assertTrue(body.contains("lastHeader(\"correlationId\")"), body) - assertTrue(body.contains("assertNotNull(asyncApiReply_0)"), body) + assertTrue(body.contains("assertNotNull(reply_0)"), body) } @Test From e2571667030f323a98c260301346855a5d64f764 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 25 Sep 2026 18:33:30 -0300 Subject: [PATCH 04/20] AsyncAPI 3.x: one line per action, and the Kafka scaffolding written once A REST action is a two-line RestAssured call. Publishing to a broker and correlating a reply needs properties, a producer, a consumer, a seek, a send and a poll loop, and repeating that per action buried the test in scaffolding: eighteen lines to say what a reader wanted in one. It is a method now, written once for the suite, so an action is the call and its assertion. It follows the SSRF assertion helper, which is emitted into the same part of the class and gated the same way, on whether the solution has anything that uses it. A suite that publishes over a transport the contract cannot describe, where the driver renders the lines instead, is not made to carry a Kafka dependency it never calls. The new hook takes the solution, which addExtraStaticVariables does not, because whether a member is needed is a property of the whole suite rather than of one action. --- .../output/service/AsyncApiTestCaseWriter.kt | 18 + .../output/service/KafkaTestClientEmitter.kt | 360 ++++++++---------- .../core/output/service/TestCaseWriter.kt | 10 + .../core/output/service/TestSuiteWriter.kt | 2 + .../service/AsyncApiTestCaseWriterTest.kt | 53 ++- 5 files changed, 236 insertions(+), 207 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt index 520fa693c3..16fb38f35d 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt @@ -7,6 +7,7 @@ import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.Solution import org.evomaster.core.search.action.Action import org.evomaster.core.search.action.ActionResult import java.nio.file.Path @@ -158,6 +159,23 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { */ override fun shouldFailIfExceptionNotThrown(result: ActionResult) = false + /** + * The helper the Kafka tests call, written once and only when something in the suite calls + * it: a suite that publishes over another transport must not be made to carry a Kafka + * dependency it never uses. + */ + override fun addExtraClassMembers(lines: Lines, solution: Solution<*>) { + + val needsKafka = solution.individuals + .flatMap { it.evaluatedMainActions() } + .mapNotNull { it.result as? AsyncApiCallResult } + .any { it.getTestScript().isEmpty() && KafkaTestClientEmitter.canEmit(it) } + + if (needsKafka) { + KafkaTestClientEmitter.emitHelper(lines, format) + } + } + override fun addTestCommentBlock(lines: Lines, test: TestCase) { //the per-action comments carry the outcome, which is what there is to say } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt index e35822d977..2d2692a440 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt @@ -30,6 +30,15 @@ object KafkaTestClientEmitter { private const val DEFAULT_POLL_MS = 200L + /** + * What the helper the tests call is named. + */ + const val HELPER_NAME = "publishAndAwaitReply" + + private const val STRING_SERDE = "org.apache.kafka.common.serialization" + + private const val UTF_8 = "java.nio.charset.StandardCharsets.UTF_8" + private val mapper = ObjectMapper() /** @@ -41,173 +50,176 @@ object KafkaTestClientEmitter { && !result.getAddress().isNullOrBlank() /** - * Publish the message again and, when one is expected, wait for the reply that answers it. - * - * @param variable what the reply payload is left in, named by the core, and what the - * other names here are keyed off so two actions cannot collide + * The call one action becomes, once [emitHelper] has written what it calls. */ fun emit(lines: Lines, result: AsyncApiCallResult, variable: String, format: OutputFormat) { - /* - Taken from the variable the core named rather than from the index, so the locals of - one action always agree with the reply it leaves behind, whatever numbered it. - */ - val suffix = "_" + variable.substringAfterLast('_') - - val props = "kafka$suffix" - val producer = "producer$suffix" - val consumer = "consumer$suffix" - val record = "record$suffix" - val parts = "partitions$suffix" - val cid = "correlationId$suffix" - val deadline = "deadline$suffix" - - val broker = quoted(result.getBroker()!!, format) - val topic = quoted(result.getAddress()!!, format) - val replyTopic = result.getReplyAddress() - val timeout = result.getReplyTimeoutMs() ?: 0L - - declareProperties(lines, props, broker, format) - - //a fresh id per run: a reply to an earlier run's message is not an answer to this one - assign(lines, cid, "java.util.UUID.randomUUID().toString()", "String", format) - - val serializer = "org.apache.kafka.common.serialization.String" - newInstance( - lines, producer, - "org.apache.kafka.clients.producer.KafkaProducer", - "org.apache.kafka.clients.producer.KafkaProducer<>($props, new ${serializer}Serializer(), new ${serializer}Serializer())", - "org.apache.kafka.clients.producer.KafkaProducer($props, ${serializer}Serializer(), ${serializer}Serializer())", - format - ) - - if (replyTopic != null) { - newInstance( - lines, consumer, - "org.apache.kafka.clients.consumer.KafkaConsumer", - "org.apache.kafka.clients.consumer.KafkaConsumer<>($props, new ${serializer}Deserializer(), new ${serializer}Deserializer())", - "org.apache.kafka.clients.consumer.KafkaConsumer($props, ${serializer}Deserializer(), ${serializer}Deserializer())", - format - ) - seekToEnd(lines, consumer, parts, quoted(replyTopic, format), format) - } - - buildRecord(lines, record, topic, result, format) - stampCorrelation(lines, record, cid, result, format) - - statement(lines, "$producer.send($record).get()", format) - - if (replyTopic == null) { - close(lines, producer, format) - //nothing is expected back, so the variable the core named stays unset - declareReply(lines, variable, format) - return - } - - declareReply(lines, variable, format) - awaitReply(lines, consumer, variable, cid, deadline, timeout, result, format) - close(lines, producer, format) - close(lines, consumer, format) - } - - private fun declareProperties(lines: Lines, name: String, broker: String, format: OutputFormat) { - newInstance(lines, name, "java.util.Properties", "java.util.Properties()", "java.util.Properties()", format) - statement(lines, "$name.put(\"bootstrap.servers\", $broker)", format) - //read only what is published from here on, never what was already on the topic - statement(lines, "$name.put(\"auto.offset.reset\", \"latest\")", format) - statement(lines, "$name.put(\"enable.auto.commit\", \"false\")", format) - } - - private fun seekToEnd(lines: Lines, consumer: String, parts: String, topic: String, format: OutputFormat) { - - val expr = "$consumer.partitionsFor($topic).stream()" + - ".map(p -> new org.apache.kafka.common.TopicPartition(p.topic(), p.partition()))" + - ".collect(java.util.stream.Collectors.toList())" - val kotlinExpr = "$consumer.partitionsFor($topic)" + - ".map { org.apache.kafka.common.TopicPartition(it.topic(), it.partition()) }" - - when { - format.isJava() -> lines.add("java.util.List $parts = $expr;") - format.isKotlin() -> lines.add("val $parts = $kotlinExpr") - } - - statement(lines, "$consumer.assign($parts)", format) - statement(lines, "$consumer.seekToEnd($parts)", format) - //seekToEnd is lazy, so make it take effect before anything is published - statement(lines, "$consumer.poll(java.time.Duration.ZERO)", format) - } - - private fun buildRecord(lines: Lines, name: String, topic: String, result: AsyncApiCallResult, format: OutputFormat) { - - val payload = result.getPayload()?.let { quoted(it, format) } ?: nullLiteral(format) - - newInstance( - lines, name, - "org.apache.kafka.clients.producer.ProducerRecord", - "org.apache.kafka.clients.producer.ProducerRecord<>($topic, $payload)", - "org.apache.kafka.clients.producer.ProducerRecord($topic, $payload)", - format + val args = mutableListOf( + quoted(result.getBroker()!!, format), + quoted(result.getAddress()!!, format), + result.getReplyAddress()?.let { quoted(it, format) } ?: "null", + result.getPayload()?.let { quoted(it, format) } ?: "null", + result.getCorrelationHeader()?.let { quoted(it, format) } ?: "null", + (result.getReplyTimeoutMs() ?: 0L).toString() + if (format.isJava()) "L" else "L" ) + //the message's own headers, flattened so that one call still fits on one line headersOf(result).forEach { (key, value) -> - statement(lines, "$name.headers().add(${quoted(key, format)}, ${bytesOf(quoted(value, format), format)})", format) + args.add(quoted(key, format)) + args.add(quoted(value, format)) } - } - - private fun stampCorrelation(lines: Lines, record: String, cid: String, result: AsyncApiCallResult, format: OutputFormat) { - val header = result.getCorrelationHeader() ?: return - statement(lines, "$record.headers().add(${quoted(header, format)}, ${bytesOf(cid, format)})", format) - } - private fun awaitReply( - lines: Lines, - consumer: String, - variable: String, - cid: String, - deadline: String, - timeout: Long, - result: AsyncApiCallResult, - format: OutputFormat - ) { - val header = result.getCorrelationHeader() + val call = "$HELPER_NAME(${args.joinToString(", ")})" when { - format.isJava() -> lines.add("long $deadline = System.currentTimeMillis() + $timeout;") - format.isKotlin() -> lines.add("val $deadline = System.currentTimeMillis() + $timeout") + format.isJava() -> lines.add("String $variable = $call;") + format.isKotlin() -> lines.add("val $variable = $call") } + } - lines.add("while ($variable == null && System.currentTimeMillis() < $deadline) {") - lines.indented { - val loop = if (format.isJava()) { - "for (org.apache.kafka.clients.consumer.ConsumerRecord r : " + - "$consumer.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {" - } else { - "for (r in $consumer.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {" - } - lines.add(loop) - lines.indented { - if (header == null) { - //no correlation declared, so the first thing on the reply topic is taken - statement(lines, "$variable = r.value()", format) - } else { - val readHeader = "r.headers().lastHeader(${quoted(header, format)})" - when { - format.isJava() -> lines.add("org.apache.kafka.common.header.Header h = $readHeader;") - format.isKotlin() -> lines.add("val h = $readHeader") - } - lines.add("if (h != null && $cid.equals(new String(h.value(), java.nio.charset.StandardCharsets.UTF_8))) {" - .let { if (format.isKotlin()) "if (h != null && $cid == String(h.value(), java.nio.charset.StandardCharsets.UTF_8)) {" else it }) - lines.indented { statement(lines, "$variable = r.value()", format) } - lines.add("}") - } - } - lines.add("}") - } - lines.add("}") + /** + * Write the helper the tests call, once for the suite. + * + * It is a method rather than lines repeated in every test because the scaffolding a broker + * needs is long and identical each time, and a test that reads in one line is worth more + * than one that spells out a producer. + */ + fun emitHelper(lines: Lines, format: OutputFormat) { + val body = if (format.isJava()) javaHelper() else kotlinHelper() + lines.addEmpty() + body.forEach { lines.add(it) } + lines.addEmpty() } + private fun javaHelper(): List = listOf( + "/**", + " * Publish one message and, when a reply is declared, wait for the one that answers it.", + " *", + " * The consumer seeks to the end of the reply topic before anything is published, so a", + " * reply left over from an earlier run is never taken for an answer to this one, and a", + " * fresh correlation id is minted per call for the same reason.", + " */", + "private static String $HELPER_NAME(String broker, String topic, String replyTopic, String payload,", + " String correlationHeader, long timeoutMs, String... headerPairs) throws Exception {", + " java.util.Properties props = new java.util.Properties();", + " props.put(\"bootstrap.servers\", broker);", + " props.put(\"auto.offset.reset\", \"latest\");", + " props.put(\"enable.auto.commit\", \"false\");", + " String correlationId = java.util.UUID.randomUUID().toString();", + " org.apache.kafka.clients.consumer.KafkaConsumer consumer = null;", + " if (replyTopic != null) {", + " consumer = new org.apache.kafka.clients.consumer.KafkaConsumer<>(props,", + " new $STRING_SERDE.StringDeserializer(), new $STRING_SERDE.StringDeserializer());", + " java.util.List parts = new java.util.ArrayList<>();", + " for (org.apache.kafka.common.PartitionInfo p : consumer.partitionsFor(replyTopic)) {", + " parts.add(new org.apache.kafka.common.TopicPartition(p.topic(), p.partition()));", + " }", + " consumer.assign(parts);", + " consumer.seekToEnd(parts);", + " consumer.poll(java.time.Duration.ZERO);", + " }", + " org.apache.kafka.clients.producer.KafkaProducer producer =", + " new org.apache.kafka.clients.producer.KafkaProducer<>(props,", + " new $STRING_SERDE.StringSerializer(), new $STRING_SERDE.StringSerializer());", + " org.apache.kafka.clients.producer.ProducerRecord record =", + " new org.apache.kafka.clients.producer.ProducerRecord<>(topic, payload);", + " for (int i = 0; i + 1 < headerPairs.length; i += 2) {", + " record.headers().add(headerPairs[i], headerPairs[i + 1].getBytes($UTF_8));", + " }", + " if (correlationHeader != null) {", + " record.headers().add(correlationHeader, correlationId.getBytes($UTF_8));", + " }", + " producer.send(record).get();", + " producer.close();", + " if (consumer == null) {", + " return null;", + " }", + " String reply = null;", + " long deadline = System.currentTimeMillis() + timeoutMs;", + " while (reply == null && System.currentTimeMillis() < deadline) {", + " for (org.apache.kafka.clients.consumer.ConsumerRecord r :", + " consumer.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {", + " if (correlationHeader == null) {", + " reply = r.value();", + " continue;", + " }", + " org.apache.kafka.common.header.Header h = r.headers().lastHeader(correlationHeader);", + " if (h != null && correlationId.equals(new String(h.value(), $UTF_8))) {", + " reply = r.value();", + " }", + " }", + " }", + " consumer.close();", + " return reply;", + "}" + ) + + private fun kotlinHelper(): List = listOf( + "/**", + " * Publish one message and, when a reply is declared, wait for the one that answers it.", + " *", + " * The consumer seeks to the end of the reply topic before anything is published, so a", + " * reply left over from an earlier run is never taken for an answer to this one, and a", + " * fresh correlation id is minted per call for the same reason.", + " */", + "private fun $HELPER_NAME(broker: String, topic: String, replyTopic: String?, payload: String?,", + " correlationHeader: String?, timeoutMs: Long, vararg headerPairs: String): String? {", + " val props = java.util.Properties()", + " props.put(\"bootstrap.servers\", broker)", + " props.put(\"auto.offset.reset\", \"latest\")", + " props.put(\"enable.auto.commit\", \"false\")", + " val correlationId = java.util.UUID.randomUUID().toString()", + " var consumer: org.apache.kafka.clients.consumer.KafkaConsumer? = null", + " if (replyTopic != null) {", + " consumer = org.apache.kafka.clients.consumer.KafkaConsumer(props,", + " $STRING_SERDE.StringDeserializer(), $STRING_SERDE.StringDeserializer())", + " val parts = consumer.partitionsFor(replyTopic)", + " .map { org.apache.kafka.common.TopicPartition(it.topic(), it.partition()) }", + " consumer.assign(parts)", + " consumer.seekToEnd(parts)", + " consumer.poll(java.time.Duration.ZERO)", + " }", + " val producer = org.apache.kafka.clients.producer.KafkaProducer(props,", + " $STRING_SERDE.StringSerializer(), $STRING_SERDE.StringSerializer())", + " val record = org.apache.kafka.clients.producer.ProducerRecord(topic, payload)", + " var i = 0", + " while (i + 1 < headerPairs.size) {", + " record.headers().add(headerPairs[i], headerPairs[i + 1].toByteArray($UTF_8))", + " i += 2", + " }", + " if (correlationHeader != null) {", + " record.headers().add(correlationHeader, correlationId.toByteArray($UTF_8))", + " }", + " producer.send(record).get()", + " producer.close()", + " if (consumer == null) {", + " return null", + " }", + " var reply: String? = null", + " val deadline = System.currentTimeMillis() + timeoutMs", + " while (reply == null && System.currentTimeMillis() < deadline) {", + " for (r in consumer.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {", + " if (correlationHeader == null) {", + " reply = r.value()", + " continue", + " }", + " val h = r.headers().lastHeader(correlationHeader)", + " if (h != null && correlationId == String(h.value(), $UTF_8)) {", + " reply = r.value()", + " }", + " }", + " }", + " consumer.close()", + " return reply", + "}" + ) + /** + * The message's own headers, as the search sent them. + */ private fun headersOf(result: AsyncApiCallResult): Map { + val json = result.getHeadersAsJson() ?: return mapOf() + return try { val node = mapper.readTree(json) node.fieldNames().asSequence().associateWith { node.get(it).asText() } @@ -216,54 +228,11 @@ object KafkaTestClientEmitter { } } - private fun declareReply(lines: Lines, variable: String, format: OutputFormat) { - when { - format.isJava() -> lines.add("String $variable = null;") - format.isKotlin() -> lines.add("var $variable: String? = null") - } - } - - private fun newInstance( - lines: Lines, - name: String, - javaType: String, - javaCtor: String, - kotlinCtor: String, - format: OutputFormat - ) { - when { - format.isJava() -> lines.add("$javaType $name = new $javaCtor;") - format.isKotlin() -> lines.add("val $name = $kotlinCtor") - } - } - - private fun assign(lines: Lines, name: String, expr: String, javaType: String, format: OutputFormat) { - when { - format.isJava() -> lines.add("$javaType $name = $expr;") - format.isKotlin() -> lines.add("val $name = $expr") - } - } - - private fun statement(lines: Lines, code: String, format: OutputFormat) { - lines.add(code) - if (format.isJava()) { - lines.append(";") - } - } - - private fun close(lines: Lines, name: String, format: OutputFormat) = - statement(lines, "$name.close()", format) - - private fun bytesOf(expr: String, format: OutputFormat) = - "$expr.toByteArray(java.nio.charset.StandardCharsets.UTF_8)" - .let { if (format.isJava()) "$expr.getBytes(java.nio.charset.StandardCharsets.UTF_8)" else it } - - private fun nullLiteral(format: OutputFormat) = "null" - /** * A string as a literal of the target language, with what would end it escaped. */ private fun quoted(value: String, format: OutputFormat): String { + val escaped = value .replace("\\", "\\\\") .replace("\"", "\\\"") @@ -271,6 +240,7 @@ object KafkaTestClientEmitter { .replace("\r", "\\r") .replace("\t", "\\t") .let { if (format.isKotlin()) it.replace("$", "\\$") else it } + return "\"$escaped\"" } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 7ff07f095d..816ca6780a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -17,6 +17,7 @@ import org.evomaster.core.problem.httpws.HttpWsCallResult import org.evomaster.core.search.action.Action import org.evomaster.core.search.action.ActionResult import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.Solution import org.slf4j.LoggerFactory import java.nio.file.Files import java.nio.file.Path @@ -337,6 +338,15 @@ abstract class TestCaseWriter { */ open fun addExtraInitStatement(lines: Lines) {} + /** + * Add members to the test class that a problem type needs, such as a helper the tests call. + * + * Unlike [addExtraStaticVariables] this is given the whole solution, so that a member is + * written only when something in the suite actually uses it, and a suite that does not is + * left without the dependency it would carry. + */ + open fun addExtraClassMembers(lines: Lines, solution: Solution<*>) {} + protected fun addActionInTryCatch( call: Action, index: Int, diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 63f1fa41dd..431c1b2297 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -779,6 +779,8 @@ class TestSuiteWriter { testCaseWriter.addExtraStaticVariables(lines) + testCaseWriter.addExtraClassMembers(lines, solution) + // if (config.expectationsActive) { // if (config.outputFormat.isJavaOrKotlin()) { // //TODO JS and C# diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index 003b4f765b..594b2e8324 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -8,6 +8,11 @@ import org.evomaster.client.java.controller.api.dto.SutInfoDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto import org.evomaster.core.output.TestCase +import org.evomaster.core.output.Lines +import org.evomaster.core.output.OutputFormat +import org.evomaster.core.output.Termination +import org.evomaster.core.output.service.KafkaTestClientEmitter +import org.evomaster.core.search.Solution import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.problem.asyncapi.data.AsyncApiAction import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual @@ -84,6 +89,17 @@ class AsyncApiTestCaseWriterTest { ?: fail("the fitness gave up on the individual") } + /** + * The members the writer adds to the test class for a solution holding this one test. + */ + private fun membersOf(evaluated: EvaluatedIndividual): String { + val writer = injector.getInstance(TestCaseWriter::class.java) + val solution = Solution(mutableListOf(evaluated), "Prefix", "Suffix", Termination.NONE, listOf(), listOf()) + val lines = Lines(OutputFormat.KOTLIN_JUNIT_5) + writer.addExtraClassMembers(lines, solution) + return lines.toString() + } + /** * The generated body for one test, as the suite writer would ask for it. */ @@ -142,22 +158,35 @@ class AsyncApiTestCaseWriterTest { */ start { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } - val body = bodyOf(evaluate("bessj")) + val evaluated = evaluate("bessj") + val body = bodyOf(evaluated) - //a real producer and consumer, against the broker the document names - assertTrue(body.contains("KafkaProducer"), body) - assertTrue(body.contains("KafkaConsumer"), body) - assertTrue(body.contains("\"bootstrap.servers\", \"localhost:9092\""), body) - assertTrue(body.contains("ProducerRecord(\"ncs.bessj.request\""), body) + //one line per action: everything it needs came off the contract + assertTrue(body.contains("${KafkaTestClientEmitter.HELPER_NAME}("), body) + assertTrue(body.contains("\"localhost:9092\""), body) + assertTrue(body.contains("\"ncs.bessj.request\""), body) + assertTrue(body.contains("\"ncs.bessj.reply\""), body) + assertTrue(body.contains("\"correlationId\""), body) + assertTrue(body.contains("assertNotNull(reply_0)"), body) + + //and what it calls is written once for the suite + val members = membersOf(evaluated) + assertTrue(members.contains("KafkaProducer"), members) + assertTrue(members.contains("KafkaConsumer"), members) + //a reply older than the test is not an answer to it, and each run stamps its own id + assertTrue(members.contains("seekToEnd"), members) + assertTrue(members.contains("UUID.randomUUID()"), members) + } - //a reply older than this publish is not an answer to it - assertTrue(body.contains("seekToEnd"), body) + @Test + fun testTheKafkaHelperIsLeftOutWhenNothingCallsIt() { - //a fresh id each run, stamped where the document says and matched on the way back - assertTrue(body.contains("UUID.randomUUID()"), body) - assertTrue(body.contains("lastHeader(\"correlationId\")"), body) + //a suite that never publishes over Kafka must not be made to carry the dependency + start { FakeAsyncApiDriver.replied(DOUBLE_RESULT).apply { testScript = listOf("publishSomehow()") } } - assertTrue(body.contains("assertNotNull(reply_0)"), body) + val members = membersOf(evaluate("bessj")) + + assertFalse(members.contains("KafkaProducer"), members) } @Test From 17b074ca6949561a5f391734922ff889d0997f10 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Sun, 27 Sep 2026 14:55:06 -0300 Subject: [PATCH 05/20] AsyncAPI 3.x: a test for the names, and why two things differ Every other problem type's naming strategy has a unit test asserting the exact strings it produces; this one had none. It does now, including that silence is named the same whether or not the experimental oracles are on. It is named by the outcome when they are off and by the fault's label when they are on, and a test's name must not change with a flag that is only about reporting. Two places deliberately depart from what the other writers do, and now say so where a reader meets them. The reply variable is not minted from the counter that gives REST res_0 and body_0, because the name has to be settled while the search runs, being part of what a driver renders against; the action's index is the only number in hand there. And the emitted Kafka code names its types in full instead of adding imports, because the suite's imports are decided before it is known whether anything will publish over Kafka, and adding them always would make every AsyncAPI suite need the dependency. --- .../AsyncApiActionTestCaseNamingStrategy.kt | 5 + .../output/service/KafkaTestClientEmitter.kt | 8 +- .../asyncapi/service/AsyncApiFitness.kt | 5 + .../AsyncApiActionNamingStrategyTest.kt | 138 ++++++++++++++++++ 4 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt index 8f694f70e4..0fd01c8d24 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt @@ -68,6 +68,11 @@ open class AsyncApiActionTestCaseNamingStrategy( } //nothing is declared to come back, so reaching the broker is the whole outcome AsyncApiOutcome.PUBLISHED -> mutableListOf(returns, "nothing") + /* + Worded to match the fault's own label, since a detected fault names the test + instead of this. With the experimental oracles off there is no fault and this is + what names it, and a name must not change with a flag that is about reporting. + */ AsyncApiOutcome.NO_REPLY -> mutableListOf("gets", "no", "reply") AsyncApiOutcome.PUBLISH_FAILED, null -> mutableListOf(error) } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt index 2d2692a440..def257a971 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt @@ -17,9 +17,11 @@ import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult * service's own message layout cannot be emitted from the contract alone, because the contract * does not describe that layout; there, the driver renders the lines instead. * - * The emitted code is deliberately self-contained per action rather than calling a shared - * helper: a test that can be read top to bottom is worth more than a short one, and it keeps a - * suite that never publishes to Kafka from carrying the dependency. + * Everything emitted here names its types in full rather than adding imports to the suite, which + * is how the REST writer reaches the one method RestAssured does not expose. The suite's imports + * are decided before it is known whether anything will publish over Kafka, and adding them + * unconditionally would make every AsyncAPI suite need the dependency, including one that only + * ever talks to a socket through a driver. */ object KafkaTestClientEmitter { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index cfca8ded9b..f8ac9ebaee 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -70,6 +70,11 @@ class AsyncApiFitness : ApiWsFitness() { /** * What the variable holding a reply is called in a generated test, before the action's * index. Named by the core so that two actions in one test cannot collide. + * + * Not minted the way the writer mints `res_0` and `body_0`, from a counter it resets per + * test: the name has to be settled here, while the search runs, because it is part of + * what a driver is asked to render against. The action's index is the only number in + * hand at that point, and it is unique within the test being executed. */ private const val REPLY_VARIABLE_PREFIX = "reply_" diff --git a/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt b/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt new file mode 100644 index 0000000000..56fe92c5a2 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt @@ -0,0 +1,138 @@ +package org.evomaster.core.output.naming + +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.output.OutputFormat +import org.evomaster.core.output.Termination +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.AsyncApiSampler +import org.evomaster.core.problem.asyncapi.service.AsyncApiTestInjector +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver +import org.evomaster.core.problem.enterprise.SampleType +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 +import org.evomaster.core.search.Solution +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.Randomness +import org.evomaster.core.search.service.SearchGlobalState +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +/** + * What a test over an AsyncAPI service is called. There is no status code to name an outcome + * with, so the name carries the outcome itself, and the declared message a reply was recognised + * as where there is one. + */ +class AsyncApiActionNamingStrategyTest { + + companion object { + private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" + + private val outputFormat = OutputFormat.KOTLIN_JUNIT_5 + + private val formatter = LanguageConventionFormatter(outputFormat) + + private const val MAX_NAME_LENGTH = 80 + } + + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + + @Test + fun testAReplyIsNamedAfterTheDeclaredMessageItWasRecognisedAs() { + + val names = namesOf({ FakeAsyncApiDriver.replied(DOUBLE_RESULT) }, "bessj") + + assertEquals(listOf("test_0_publishOnBessjReturnsDoubleResult"), names) + } + + @Test + fun testAnOperationThatExpectsNothingBackSaysSo() { + + val names = namesOf({ FakeAsyncApiDriver.fireAndForget() }, "bessj") + + assertEquals(listOf("test_0_publishOnBessjReturnsNothing"), names) + } + + @Test + fun testSilenceIsNamedTheSameWhetherOrNotItIsReportedAsAFault() { + + /* + With the experimental oracles off there is no fault, so the outcome names the test; + with them on the fault label does. Both have to read the same, or the name of a test + would change with a flag that is about reporting. + */ + val withoutOracles = namesOf({ FakeAsyncApiDriver.silence() }, "bessj") + val withOracles = namesOf({ FakeAsyncApiDriver.silence() }, "bessj", "--useExperimentalOracles=true") + + assertEquals(listOf("test_0_publishOnBessjGetsNoReply"), withoutOracles) + assertEquals(withoutOracles, withOracles) + } + + @Test + fun testTheOperationIsKeptWhenTheOutcomeNoLongerFits() { + + //tokens are dropped whole, left to right, so the operation outlives the outcome + val solution = solutionOf({ FakeAsyncApiDriver.replied(DOUBLE_RESULT) }, "bessj") + + val roomForBoth = AsyncApiActionTestCaseNamingStrategy(solution, formatter, MAX_NAME_LENGTH).getTestCases() + val roomForTheOperation = AsyncApiActionTestCaseNamingStrategy(solution, formatter, 25).getTestCases() + val roomForNeither = AsyncApiActionTestCaseNamingStrategy(solution, formatter, 10).getTestCases() + + assertEquals("test_0_publishOnBessjReturnsDoubleResult", roomForBoth[0].name) + assertEquals("test_0_publishOnBessj", roomForTheOperation[0].name) + assertEquals("test_0", roomForNeither[0].name) + } + + private fun namesOf( + answer: (AsyncApiActionDto) -> AsyncApiReplyDto?, + operation: String, + vararg options: String + ): List { + val solution = solutionOf(answer, operation, options = options) + return AsyncApiActionTestCaseNamingStrategy(solution, formatter, MAX_NAME_LENGTH) + .getTestCases() + .map { it.name } + } + + private fun solutionOf( + answer: (AsyncApiActionDto) -> AsyncApiReplyDto?, + operation: String, + options: Array = arrayOf() + ): Solution { + + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS)), answer) + val injector = AsyncApiTestInjector.create(driver, "--blackBox=false", *options) + + val sampler = injector.getInstance(AsyncApiSampler::class.java) + val fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + val randomness = injector.getInstance(Randomness::class.java) + + val template = sampler.seeAvailableActions().first { it.getName() == operation } + val actions = listOf((template.copy() as AsyncApiAction).apply { doInitialize(randomness) }) + + val individual = AsyncApiIndividual(SampleType.RANDOM, actions.toMutableList()).apply { + doGlobalInitialize(injector.getInstance(SearchGlobalState::class.java)) + } + + val evaluated = fitness.calculateCoverage(individual, modifiedSpec = null) + ?: throw IllegalStateException("the fitness gave up on the individual") + + return Solution( + mutableListOf(evaluated), + "suitePrefix", + "suiteSuffix", + Termination.NONE, + listOf(), + listOf() + ) + } +} From 7b11521cf9a2829ef802b7245d690ca28ce72b6e Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Tue, 29 Sep 2026 23:18:01 -0300 Subject: [PATCH 06/20] AsyncAPI 3.x: the test reaches the broker, and asserts on the reply What the search published is now kept on the result -- broker, topics, the header the correlation id rode in, the payload -- so the writer has the whole call in hand and never reads the document again. A suite asks the driver where each server is, the way it asks for baseUrlOfSut, and keeps the answer in a variable per server. A broker started on a random port is then reachable from the generated test, which the document's own address would not be. The default keeps the address the document gave, so a black-box suite, which has no driver to ask, is unchanged. The reply is asserted on the way REST asserts on a response: the same enableBasicAssertions, the same fieldsToSkipInAssertions, the same size and collection limits, the same tolerance on doubles. Nothing is predicted from the schema; every assertion is a regression on what came back. Kafka is written in Java and Python as well as Kotlin, so the suite can be generated in any of the three the rest of EvoMaster offers. The fault line of a test's comment block moves up into TestCaseWriter, which is what the TODO beside it asked for, and the AsyncAPI block now lists its calls with their outcome where REST lists them with their status code. --- .../client/java/controller/SutHandler.java | 16 + .../kotlin/org/evomaster/core/EMConfig.kt | 13 +- .../AsyncApiActionTestCaseNamingStrategy.kt | 8 +- .../output/service/AsyncApiReplyAssertions.kt | 312 ++++++++++++++++++ .../output/service/AsyncApiTestCaseWriter.kt | 126 ++++++- .../output/service/KafkaTestClientEmitter.kt | 116 ++++++- .../core/output/service/RestTestCaseWriter.kt | 19 +- .../core/output/service/TestCaseWriter.kt | 29 ++ .../core/output/service/TestSuiteWriter.kt | 7 + .../core/output/sorting/SortingHelper.kt | 19 ++ .../asyncapi/data/AsyncApiCallResult.kt | 9 + .../asyncapi/service/AsyncApiFitness.kt | 25 +- .../kotlin/org/evomaster/core/EMConfigTest.kt | 12 +- .../compiler/CompilerForTestGenerated.kt | 58 +++- .../service/AsyncApiTestCaseWriterTest.kt | 169 +++++++++- 15 files changed, 867 insertions(+), 71 deletions(-) create mode 100644 core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java index 3990f3c0eb..3d31b43e8c 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java @@ -147,6 +147,22 @@ default void setupForGeneratedTest(){} */ default Object executeRPCEndpoint(String json) throws Exception {return null;} + /** + *

+ * Where the AsyncAPI server called {@code serverName} can actually be reached, now. + *

+ * + * The document gives an address, but it is written for the deployment the author had in + * mind. A system started for testing is often somewhere else: a broker in a container binds + * a port chosen when it starts, and a different one on the next run. A generated test asks + * this rather than carrying an address that was only ever true once, in the same way it + * takes the base URL of a REST system from {@link #startSut()}. + * + * @param serverName the key of the server in the document + * @return where to reach it, or null to fall back to what the document says + */ + default String getAsyncApiServerAddress(String serverName) {return null;} + /** *

* execute an RPC endpoint with evomaster driver diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index bb1a782c68..37a6f3de8f 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -823,9 +823,11 @@ class EMConfig { if (problemType == ProblemType.ASYNCAPI && createTests - && outputFormat != OutputFormat.DEFAULT && !outputFormat.isJavaOrKotlin()) { - throw ConfigProblemException("Tests for an AsyncAPI service are written against the driver's own" + - " transport client, which is Java, so outputFormat only supports Java or Kotlin now") + && outputFormat != OutputFormat.DEFAULT + && !outputFormat.isJavaOrKotlin() && !outputFormat.isPython()) { + throw ConfigProblemException("Tests for an AsyncAPI service publish with a client of the transport," + + " and one is written only for Java, Kotlin and Python so far, so outputFormat supports" + + " only those") } if (problemType == ProblemType.ASYNCAPI && seedTestCases) { @@ -1350,7 +1352,10 @@ class EMConfig { @Important(2.0) @Cfg("Specify in which format the tests should be outputted." + " If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used." + - " On the other hand, for black-box testing it will default to a predefined type (e.g., Python).") + " On the other hand, for black-box testing it will default to a predefined type (e.g., Python)." + + " AsyncAPI is the exception on both counts: it has a driver in either mode, so the format comes from" + + " there, and only Java, Kotlin and Python can be generated, as the tests publish with a client of the" + + " transport.") var outputFormat = OutputFormat.DEFAULT @Important(2.1) diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt index 0fd01c8d24..bd1f42b7cc 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/AsyncApiActionTestCaseNamingStrategy.kt @@ -63,7 +63,13 @@ open class AsyncApiActionTestCaseNamingStrategy( if (message != null) { mutableListOf(returns, safeVariableName(message)) } else { - mutableListOf(returns, success) + /* + A reply arrived that matches none of the declared messages. Worded to + match the fault's own label, for the same reason as below: with the + experimental oracles off this names the test, with them on the fault + does, and the two must read alike. + */ + mutableListOf("replies", "with", "undeclared", "message") } } //nothing is declared to come back, so reaching the broker is the whole outcome diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt new file mode 100644 index 0000000000..b428685660 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt @@ -0,0 +1,312 @@ +package org.evomaster.core.output.service + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import org.evomaster.core.EMConfig +import org.evomaster.core.output.Lines +import org.evomaster.core.output.OutputFormat + +/** + * Writes assertions over the reply a message drew, from the reply the search actually saw. + * + * These are regression assertions, the same kind the REST writer makes on a response body: they + * say what came back, not what should. They honour the same rules, because the reasons are the + * same ones. A field that changes between runs makes a test flaky whatever the protocol, so the + * fields REST skips are skipped here; a collection is asserted only to the configured depth; a + * value that cannot be printed safely becomes a comment. + * + * The reply is text, not a response object, so unlike REST there is no fluent client to hang the + * assertions off. They are made against the tree it parses into. + */ +object AsyncApiReplyAssertions { + + private val mapper = ObjectMapper() + + /** + * Fields whose value is usually different on the next run, so asserting them would make the + * test flaky. The same list the REST writer keeps, for the same reason. + */ + private val ALWAYS_SKIPPED = listOf("id", "timestamp", "self") + + /** + * Content that must not be asserted on even when its field is fine: it carries something + * that differs between runs or environments. + */ + private val HTML_ENTITY = "&[a-zA-Z]+;|&#\\d+;".toRegex() + + private val HOST_AND_PORT = """\w+:\d{4,5}""".toRegex() + + /** + * Write what can be said about [payload], into [variable] as the test holds it. + * + * @param bodyVariable what the parsed reply is called, when one is needed + */ + fun emit( + lines: Lines, + payload: String?, + variable: String, + bodyVariable: String, + config: EMConfig, + format: OutputFormat + ) { + if (!config.enableBasicAssertions) { + return + } + + if (payload.isNullOrBlank()) { + //an acknowledgement, or a transport that answers in metadata + assertion(lines, "$variable.isNullOrEmpty()", "$variable == null || $variable.isEmpty()", format) + return + } + + if (payload.length > config.maxResponseByteSize) { + lines.addSingleCommentLine( + "the reply was larger than the threshold of ${config.maxResponseByteSize} bytes," + + " so no assertion is made on it" + ) + return + } + + val tree = try { + mapper.readTree(payload) + } catch (e: Exception) { + null + } + + if (tree == null || tree.isMissingNode) { + //not JSON: all that can be said is what it contained + if (printable(payload)) { + contains(lines, variable, payload.trim(), format) + } + return + } + + parse(lines, variable, bodyVariable, format) + onNode(lines, tree, bodyVariable, "", config, format) + } + + private fun parse(lines: Lines, variable: String, bodyVariable: String, format: OutputFormat) { + val read = "com.fasterxml.jackson.databind.ObjectMapper().readTree($variable)" + when { + format.isJava() -> lines.add( + "com.fasterxml.jackson.databind.JsonNode $bodyVariable = " + + "new com.fasterxml.jackson.databind.ObjectMapper().readTree($variable);" + ) + format.isKotlin() -> lines.add("val $bodyVariable = $read") + format.isPython() -> lines.add("$bodyVariable = json.loads($variable)") + } + } + + /** + * Everything that can be said about one node, and about what is under it. + */ + private fun onNode( + lines: Lines, + node: JsonNode, + path: String, + fieldPath: String, + config: EMConfig, + format: OutputFormat + ) { + when { + node.isObject -> onObject(lines, node, path, fieldPath, config, format) + node.isArray -> onArray(lines, node, path, fieldPath, config, format) + else -> onValue(lines, node, path, fieldPath, config, format) + } + } + + private fun onObject( + lines: Lines, + node: JsonNode, + path: String, + fieldPath: String, + config: EMConfig, + format: OutputFormat + ) { + if (node.isEmpty) { + val java = if (format.isPython()) "len($path) == 0" else "$path.isEmpty()" + assertion(lines, if (format.isKotlin()) "$path.isEmpty" else java, java, format) + return + } + + node.fieldNames().asSequence().toList().forEach { name -> + + val child = node.get(name) + val here = if (fieldPath.isEmpty()) name else "$fieldPath.$name" + + /* + The field is still asserted to be there even when its value is not asserted: + that it is present is stable, which is what tells one declared message from + another, and it is the value that changes between runs. + */ + has(lines, path, name, format) + + if (skip(name, config)) { + lines.addSingleCommentLine("$here is not asserted on, as its value changes between runs") + return@forEach + } + + onNode(lines, child, child(path, name, format), here, config, format) + } + } + + private fun onArray( + lines: Lines, + node: JsonNode, + path: String, + fieldPath: String, + config: EMConfig, + format: OutputFormat + ) { + equals(lines, node.size().toString(), if (format.isPython()) "len($path)" else "$path.size()", format) + + val limit = if (config.maxAssertionForDataInCollection >= 0) { + config.maxAssertionForDataInCollection + } else { + node.size() + } + + node.take(limit).forEachIndexed { i, child -> + onNode(lines, child, index(path, i, format), "$fieldPath[$i]", config, format) + } + + if (node.size() > limit) { + lines.addSingleCommentLine( + "the remaining ${node.size() - limit} elements are not asserted on;" + + " the limit is maxAssertionForDataInCollection" + ) + } + } + + private fun onValue( + lines: Lines, + node: JsonNode, + path: String, + fieldPath: String, + config: EMConfig, + format: OutputFormat + ) { + when { + node.isNull -> { + val java = if (format.isPython()) "$path is None" else "$path.isNull()" + assertion(lines, if (format.isKotlin()) "$path.isNull" else java, java, format) + } + + node.isBoolean -> equals( + lines, + if (format.isPython()) node.asBoolean().toString().replaceFirstChar { it.uppercase() } + else node.asBoolean().toString(), + value(path, "asBoolean()", format), + format + ) + + node.isIntegralNumber -> { + val v = node.asLong() + val suffix = if (format.isJava() && (v > Int.MAX_VALUE || v < Int.MIN_VALUE)) "L" else "" + equals(lines, "$v$suffix", value(path, "asLong()", format), format) + } + + /* + A double is compared within a tolerance, as the REST writer does: the same + computation can land a bit apart once it has been through text. + */ + node.isNumber -> delta(lines, node.asDouble().toString(), value(path, "asDouble()", format), format) + + node.isTextual -> { + val text = node.asText() + if (printable(text)) { + equals(lines, quoted(text, format), value(path, "asText()", format), format) + } else { + lines.addSingleCommentLine( + "$fieldPath is not asserted on, as its value is not stable between runs" + ) + } + } + } + } + + /** + * Whether a value can be asserted on without making the test flaky. The REST writer's rule: + * something that says it was logged, an HTML entity, or what looks like a host and port has + * a good chance of differing on the next run or on another machine. + */ + private fun printable(content: String) = + content != "null" + && !content.contains("logged") + && !HTML_ENTITY.containsMatchIn(content) + && !HOST_AND_PORT.containsMatchIn(content) + + private fun skip(fieldName: String, config: EMConfig): Boolean { + + val field = fieldName.lowercase() + + val configured = config.fieldsToSkipInAssertions + .split(",") + .map { it.trim().lowercase() } + .filter { it.isNotEmpty() } + + return ALWAYS_SKIPPED.contains(field) || configured.contains(field) + } + + private fun has(lines: Lines, path: String, field: String, format: OutputFormat) { + val expr = if (format.isPython()) "\"$field\" in $path" else "$path.has(\"$field\")" + assertion(lines, expr, expr, format) + } + + private fun child(path: String, name: String, format: OutputFormat) = + if (format.isPython()) "$path[\"$name\"]" else "$path.get(\"$name\")" + + private fun index(path: String, i: Int, format: OutputFormat) = + if (format.isPython()) "$path[$i]" else "$path.get($i)" + + /** + * Reading a value out: on the JVM the tree is asked for it as a type, in Python it is the + * value already. + */ + private fun value(path: String, asType: String, format: OutputFormat) = + if (format.isPython()) path else "$path.$asType" + + private fun assertion(lines: Lines, kotlin: String, java: String, format: OutputFormat) { + when { + format.isJava() -> lines.add("assertTrue($java);") + format.isKotlin() -> lines.add("assertTrue($kotlin)") + format.isPython() -> lines.add("assert $kotlin") + } + } + + private fun equals(lines: Lines, expected: String, actual: String, format: OutputFormat) { + when { + format.isJava() -> lines.add("assertEquals($expected, $actual);") + format.isKotlin() -> lines.add("assertEquals($expected, $actual)") + format.isPython() -> lines.add("assert $actual == $expected") + } + } + + private fun delta(lines: Lines, expected: String, actual: String, format: OutputFormat) { + when { + format.isJava() -> lines.add("assertEquals($expected, $actual, 0.001);") + format.isKotlin() -> lines.add("assertEquals($expected, $actual, 0.001)") + format.isPython() -> lines.add("assert abs($actual - $expected) < 0.001") + } + } + + private fun contains(lines: Lines, variable: String, text: String, format: OutputFormat) { + val expected = quoted(text, format) + when { + format.isJava() -> lines.add("assertTrue($variable.contains($expected));") + format.isKotlin() -> lines.add("assertTrue($variable!!.contains($expected))") + format.isPython() -> lines.add("assert $expected in $variable") + } + } + + private fun quoted(value: String, format: OutputFormat): String { + val escaped = value + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + .let { if (format.isKotlin()) it.replace("$", "\\$") else it } + return "\"$escaped\"" + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt index 16fb38f35d..a6271ca1e0 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt @@ -1,6 +1,7 @@ package org.evomaster.core.output.service import org.evomaster.core.output.Lines +import org.evomaster.core.output.TestWriterUtils import org.evomaster.core.output.TestCase import org.evomaster.core.problem.asyncapi.data.AsyncApiAction import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult @@ -111,7 +112,7 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { else has to. */ variable != null && KafkaTestClientEmitter.canEmit(res) -> - KafkaTestClientEmitter.emit(lines, res, variable, format) + KafkaTestClientEmitter.emit(lines, res, variable, brokerOf(res), format) else -> { /* @@ -151,6 +152,21 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { res.getReplyMessage()?.let { lines.addSingleCommentLine("recognised as the declared message '$it'") } + + /* + What came back, asserted the way REST asserts a response body: from what was actually + observed, not from what the contract promises. Which of the declared messages a reply + is turns on the fields it carries, so these assertions are also what keeps a test + honest about the variant its own name claims. + */ + AsyncApiReplyAssertions.emit( + lines, + res.getReplyPayload(), + variable, + bodyVariableFor(variable), + config, + format + ) } /** @@ -159,24 +175,122 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { */ override fun shouldFailIfExceptionNotThrown(result: ActionResult) = false + /** + * The servers this suite publishes to, by the name the document gives them, each with the + * address the document declares. Filled in while the class members are written, and read + * again when the init statements are, which run after them. + */ + private val servers = LinkedHashMap() + /** * The helper the Kafka tests call, written once and only when something in the suite calls * it: a suite that publishes over another transport must not be made to carry a Kafka * dependency it never uses. + * + * The address of each server is written here too, when the suite has a driver to ask. The + * document's address is where the author deployed it, which is not where a system started + * for testing is: a broker in a container answers on a port chosen at start-up. This is the + * same reason the base URL of a REST system is taken from the driver rather than the + * document. */ override fun addExtraClassMembers(lines: Lines, solution: Solution<*>) { - val needsKafka = solution.individuals + val kafkaResults = solution.individuals .flatMap { it.evaluatedMainActions() } .mapNotNull { it.result as? AsyncApiCallResult } - .any { it.getTestScript().isEmpty() && KafkaTestClientEmitter.canEmit(it) } + .filter { it.getTestScript().isEmpty() && KafkaTestClientEmitter.canEmit(it) } + + if (kafkaResults.isEmpty()) { + return + } + + KafkaTestClientEmitter.emitHelper(lines, format) + + if (!hasDriver()) { + return + } + + servers.clear() + kafkaResults.forEach { r -> + val name = r.getServerName() ?: return@forEach + servers.putIfAbsent(name, r.getBroker() ?: "") + } + + servers.keys.forEach { name -> + when { + format.isJava() -> lines.add("private static String ${variableFor(name)};") + format.isKotlin() -> lines.add("private lateinit var ${variableFor(name)}: String") + } + } + lines.addEmpty() + } + + /** + * Ask the driver where each server is, falling back to the document when it does not say, + * which is what a driver that never overrode the method does. + */ + override fun addExtraInitStatement(lines: Lines) { - if (needsKafka) { - KafkaTestClientEmitter.emitHelper(lines, format) + servers.forEach { (name, declared) -> + val variable = variableFor(name) + val ask = "${TestSuiteWriter.controller}.getAsyncApiServerAddress(\"$name\")" + val fallback = KafkaTestClientEmitter.brokerLiteral(declared, format) + + when { + format.isJava() -> lines.add("$variable = $ask != null ? $ask : $fallback;") + format.isKotlin() -> lines.add("$variable = $ask ?: $fallback") + } } } + /** + * Where a message goes, as the generated test sees it: the variable the suite filled in from + * the driver, or the document's own address when there is no driver to ask. + */ + private fun brokerOf(result: AsyncApiCallResult): String { + + val name = result.getServerName() + + if (hasDriver() && name != null && servers.containsKey(name)) { + return variableFor(name) + } + + return KafkaTestClientEmitter.brokerLiteral(result.getBroker() ?: "", format) + } + + /** + * What the parsed reply is called, beside the reply it was parsed from. + */ + private fun bodyVariableFor(replyVariable: String) = + "body_" + replyVariable.substringAfterLast('_') + + private fun variableFor(serverName: String) = + KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX + TestWriterUtils.safeVariableName(serverName) + + /** + * Whether the generated suite has a driver in it at all. A black-box suite does not, so + * there is nothing to ask and the document's address is all there is. + */ + private fun hasDriver() = !config.blackBox || config.bbExperiments + override fun addTestCommentBlock(lines: Lines, test: TestCase) { - //the per-action comments carry the outcome, which is what there is to say + + val actions = test.test.evaluatedMainActions() + + /* + What REST puts above a test, with the outcome where it puts the status code: there is + no code to report, and the outcome is the same kind of one-word summary of what came + back. + */ + if (actions.isNotEmpty()) { + lines.addBlockCommentLine("Calls:") + actions.forEachIndexed { index, evaluated -> + val outcome = (evaluated.result as? AsyncApiCallResult)?.getOutcome() ?: "not executed" + val prefix = if (actions.size == 1) "" else "${index + 1} - " + lines.addBlockCommentLine("$prefix($outcome) ${evaluated.action.getName()}") + } + } + + addFaultsCommentLine(lines, test) } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt index def257a971..ed33247358 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt @@ -37,12 +37,29 @@ object KafkaTestClientEmitter { */ const val HELPER_NAME = "publishAndAwaitReply" + /** + * What the variable holding a server's address is called in a generated test, before the + * server's own name. + */ + const val SERVER_VARIABLE_PREFIX = "asyncApiServer_" + private const val STRING_SERDE = "org.apache.kafka.common.serialization" private const val UTF_8 = "java.nio.charset.StandardCharsets.UTF_8" private val mapper = ObjectMapper() + /** + * The document's own address for the server, as a literal of the target language. + */ + /** + * What stands for an absent value in the target language. + */ + private fun nothing(format: OutputFormat) = if (format.isPython()) "None" else "null" + + fun brokerLiteral(address: String, format: OutputFormat): String = + quoted(address, format) + /** * Whether this result carries enough, and the right transport, to be written as Kafka code. */ @@ -53,16 +70,27 @@ object KafkaTestClientEmitter { /** * The call one action becomes, once [emitHelper] has written what it calls. + * + * @param variable what the reply payload is left in, named by the core, and what the other + * names here are keyed off so two actions cannot collide + * @param broker where to publish, as an expression: the name of a variable the suite + * filled in from the driver, or a literal taken from the document */ - fun emit(lines: Lines, result: AsyncApiCallResult, variable: String, format: OutputFormat) { + fun emit( + lines: Lines, + result: AsyncApiCallResult, + variable: String, + broker: String, + format: OutputFormat + ) { val args = mutableListOf( - quoted(result.getBroker()!!, format), + broker, quoted(result.getAddress()!!, format), - result.getReplyAddress()?.let { quoted(it, format) } ?: "null", - result.getPayload()?.let { quoted(it, format) } ?: "null", - result.getCorrelationHeader()?.let { quoted(it, format) } ?: "null", - (result.getReplyTimeoutMs() ?: 0L).toString() + if (format.isJava()) "L" else "L" + result.getReplyAddress()?.let { quoted(it, format) } ?: nothing(format), + result.getPayload()?.let { quoted(it, format) } ?: nothing(format), + result.getCorrelationHeader()?.let { quoted(it, format) } ?: nothing(format), + (result.getReplyTimeoutMs() ?: 0L).toString() + if (format.isPython()) "" else "L" ) //the message's own headers, flattened so that one call still fits on one line @@ -76,6 +104,7 @@ object KafkaTestClientEmitter { when { format.isJava() -> lines.add("String $variable = $call;") format.isKotlin() -> lines.add("val $variable = $call") + format.isPython() -> lines.add("$variable = self.$call") } } @@ -87,7 +116,11 @@ object KafkaTestClientEmitter { * than one that spells out a producer. */ fun emitHelper(lines: Lines, format: OutputFormat) { - val body = if (format.isJava()) javaHelper() else kotlinHelper() + val body = when { + format.isJava() -> javaHelper() + format.isPython() -> pythonHelper() + else -> kotlinHelper() + } lines.addEmpty() body.forEach { lines.add(it) } lines.addEmpty() @@ -118,13 +151,17 @@ object KafkaTestClientEmitter { " }", " consumer.assign(parts);", " consumer.seekToEnd(parts);", - " consumer.poll(java.time.Duration.ZERO);", + " //seekToEnd is lazy: asking for the position is what makes it take effect now", + " if (!parts.isEmpty()) {", + " consumer.position(parts.get(0));", + " }", " }", " org.apache.kafka.clients.producer.KafkaProducer producer =", " new org.apache.kafka.clients.producer.KafkaProducer<>(props,", " new $STRING_SERDE.StringSerializer(), new $STRING_SERDE.StringSerializer());", + " //a message with no payload of its own publishes an empty body", " org.apache.kafka.clients.producer.ProducerRecord record =", - " new org.apache.kafka.clients.producer.ProducerRecord<>(topic, payload);", + " new org.apache.kafka.clients.producer.ProducerRecord<>(topic, payload == null ? \"\" : payload);", " for (int i = 0; i + 1 < headerPairs.length; i += 2) {", " record.headers().add(headerPairs[i], headerPairs[i + 1].getBytes($UTF_8));", " }", @@ -156,6 +193,59 @@ object KafkaTestClientEmitter { "}" ) + /** + * The Python form, against kafka-python. A Python suite has no driver to ask, since the + * controller is Java, so it always publishes to the address the document declares. + */ + private fun pythonHelper(): List = listOf( + "@staticmethod", + "def $HELPER_NAME(broker, topic, reply_topic, payload, correlation_header, timeout_ms, *header_pairs):", + " \"\"\"", + " Publish one message and, when a reply is declared, wait for the one that answers it.", + "", + " The consumer seeks to the end of the reply topic before anything is published, so a", + " reply left over from an earlier run is never taken for an answer to this one, and a", + " fresh correlation id is minted per call for the same reason.", + " \"\"\"", + " correlation_id = str(uuid.uuid4())", + " consumer = None", + " if reply_topic is not None:", + " consumer = kafka.KafkaConsumer(bootstrap_servers=broker,", + " auto_offset_reset='latest', enable_auto_commit=False)", + " partitions = [kafka.TopicPartition(reply_topic, p)", + " for p in (consumer.partitions_for_topic(reply_topic) or [])]", + " consumer.assign(partitions)", + " consumer.seek_to_end()", + " producer = kafka.KafkaProducer(bootstrap_servers=broker)", + " headers = []", + " i = 0", + " while i + 1 < len(header_pairs):", + " headers.append((header_pairs[i], header_pairs[i + 1].encode('utf-8')))", + " i += 2", + " if correlation_header is not None:", + " headers.append((correlation_header, correlation_id.encode('utf-8')))", + " #a message with no payload of its own publishes an empty body", + " body = (payload if payload is not None else '').encode('utf-8')", + " producer.send(topic, value=body, headers=headers)", + " producer.flush()", + " producer.close()", + " if consumer is None:", + " return None", + " reply = None", + " deadline = time.time() + (timeout_ms / 1000.0)", + " while reply is None and time.time() < deadline:", + " for _, records in consumer.poll(timeout_ms=$DEFAULT_POLL_MS).items():", + " for r in records:", + " if correlation_header is None:", + " reply = r.value.decode('utf-8')", + " continue", + " for key, value in (r.headers or []):", + " if key == correlation_header and value.decode('utf-8') == correlation_id:", + " reply = r.value.decode('utf-8')", + " consumer.close()", + " return reply" + ) + private fun kotlinHelper(): List = listOf( "/**", " * Publish one message and, when a reply is declared, wait for the one that answers it.", @@ -179,11 +269,15 @@ object KafkaTestClientEmitter { " .map { org.apache.kafka.common.TopicPartition(it.topic(), it.partition()) }", " consumer.assign(parts)", " consumer.seekToEnd(parts)", - " consumer.poll(java.time.Duration.ZERO)", + " //seekToEnd is lazy: asking for the position is what makes it take effect now", + " if (parts.isNotEmpty()) {", + " consumer.position(parts[0])", + " }", " }", " val producer = org.apache.kafka.clients.producer.KafkaProducer(props,", " $STRING_SERDE.StringSerializer(), $STRING_SERDE.StringSerializer())", - " val record = org.apache.kafka.clients.producer.ProducerRecord(topic, payload)", + " //a message with no payload of its own publishes an empty body", + " val record = org.apache.kafka.clients.producer.ProducerRecord(topic, payload ?: \"\")", " var i = 0", " while (i + 1 < headerPairs.size) {", " record.headers().add(headerPairs[i], headerPairs[i + 1].toByteArray($UTF_8))", diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 160fb5f683..0d17c7d76a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -7,7 +7,6 @@ import org.evomaster.core.output.SqlWriter import org.evomaster.core.output.TestCase import org.evomaster.core.output.TestWriterUtils import org.evomaster.core.problem.api.param.Param -import org.evomaster.core.problem.enterprise.EnterpriseActionResult import org.evomaster.core.problem.httpws.HttpWsAction import org.evomaster.core.problem.httpws.HttpWsCallResult import org.evomaster.core.problem.rest.data.RestCallAction @@ -499,24 +498,8 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { } } - //TODO move up when adding test comments to other problem types as well //faults - val faults = ea.map { it.result } - .filterIsInstance() - .flatMap { it.getFaults() } - if(faults.isNotEmpty()){ - if(faults.size == 1){ - lines.addBlockCommentLine("Found 1 potential fault of type-code ${faults.first().category.code}") - } else { - val codes = faults.asSequence().map { it.category.code }.toSet().toList().sorted() - val codeInfo = if (codes.size == 1) { - " of type-code ${codes[0]}" - } else { - ". Type-codes: ${codes.joinToString(", ")}" - } - lines.addBlockCommentLine("Found ${faults.size} potential faults$codeInfo") - } - } + addFaultsCommentLine(lines, test) //examples val examples = getAllUsedExamples(ind.individual as RestIndividual) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 816ca6780a..80cdf3d012 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -82,6 +82,35 @@ abstract class TestCaseWriter { protected abstract fun addTestCommentBlock(lines: Lines, test: TestCase) + /** + * The faults found in a test, as a line of its comment block. Here rather than in one writer + * because a fault is a fault whatever the protocol was, and is worth saying the same way. + */ + protected fun addFaultsCommentLine(lines: Lines, test: TestCase) { + + val faults = test.test.evaluatedMainActions() + .map { it.result } + .filterIsInstance() + .flatMap { it.getFaults() } + + if (faults.isEmpty()) { + return + } + + if (faults.size == 1) { + lines.addBlockCommentLine("Found 1 potential fault of type-code ${faults.first().category.code}") + return + } + + val codes = faults.asSequence().map { it.category.code }.toSet().toList().sorted() + val codeInfo = if (codes.size == 1) { + " of type-code ${codes[0]}" + } else { + ". Type-codes: ${codes.joinToString(", ")}" + } + lines.addBlockCommentLine("Found ${faults.size} potential faults$codeInfo") + } + /** * Compute Playwright fixtures to destructure in the test signature. * For REST-only generation we currently need only the `request` fixture. diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 431c1b2297..b4e68dda6c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -595,6 +595,13 @@ class TestSuiteWriter { lines.add("import unittest") lines.add("import requests") + if (config.problemType == EMConfig.ProblemType.ASYNCAPI) { + //what the helper that publishes and awaits a reply uses + lines.add("import time") + lines.add("import uuid") + lines.add("import kafka") + } + if(config.sqli){ lines.add("import time") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt index 7245928412..5e4c57edf6 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt @@ -8,6 +8,9 @@ import org.evomaster.core.problem.httpws.HttpWsCallResult import org.evomaster.core.problem.rest.data.RestCallAction import org.evomaster.core.problem.rest.data.RestIndividual import org.evomaster.core.problem.rpc.RPCCallAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual import org.evomaster.core.problem.rpc.RPCIndividual import org.evomaster.core.problem.webfrontend.WebIndividual import org.evomaster.core.search.EvaluatedIndividual @@ -106,6 +109,21 @@ class SortingHelper { (ind.evaluatedMainActions().last().action as GraphQLAction).parameters.size } + /** + * Groups the tests of one operation together, then orders them by what publishing did: a + * reply that was recognised comes before silence, which is the AsyncAPI stand-in for + * ordering REST by status code. + */ + private val asyncApiComparator: Comparator> = compareBy> { ind -> + (ind.evaluatedMainActions().last().action as AsyncApiAction).operationId + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().result as AsyncApiCallResult).getOutcome()?.ordinal ?: 0 + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().result as AsyncApiCallResult).getReplyMessage() ?: "" + } + private val rpcComparator: Comparator> = compareBy> { ind -> (ind.evaluatedMainActions().last().action as RPCCallAction).getSimpleClassName() } @@ -173,6 +191,7 @@ class SortingHelper { tests.any { it.individual is RestIndividual } -> restComparator tests.any { it.individual is GraphQLIndividual } -> graphQLComparator tests.any { it.individual is RPCIndividual } -> rpcComparator + tests.any { it.individual is AsyncApiIndividual } -> asyncApiComparator tests.any { it.individual is WebIndividual } -> { log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") statusCode diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt index 6e5a7c455d..faa0755e87 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt @@ -21,6 +21,7 @@ class AsyncApiCallResult : EnterpriseActionResult { resolving the document a second time. Only recorded when tests are being written. */ const val BROKER = "BROKER" + const val SERVER_NAME = "SERVER_NAME" const val PROTOCOL = "PROTOCOL" const val ADDRESS = "ADDRESS" const val REPLY_ADDRESS = "REPLY_ADDRESS" @@ -103,6 +104,7 @@ class AsyncApiCallResult : EnterpriseActionResult { * from the document. Headers are a JSON object, so that one string holds them all. */ fun setPublished( + serverName: String?, broker: String?, protocol: String?, address: String?, @@ -112,6 +114,7 @@ class AsyncApiCallResult : EnterpriseActionResult { correlationHeader: String?, replyTimeoutMs: Long? ) { + serverName?.let { addResultValue(SERVER_NAME, it) } broker?.let { addResultValue(BROKER, it) } protocol?.let { addResultValue(PROTOCOL, it) } address?.let { addResultValue(ADDRESS, it) } @@ -124,6 +127,12 @@ class AsyncApiCallResult : EnterpriseActionResult { fun getBroker(): String? = getResultValue(BROKER) + /** + * The key of the server in the document, which is what the driver is asked about when a + * generated test needs to know where that server is now. + */ + fun getServerName(): String? = getResultValue(SERVER_NAME) + fun getProtocol(): String? = getResultValue(PROTOCOL) fun getAddress(): String? = getResultValue(ADDRESS) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index f8ac9ebaee..6d4bb79f18 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -69,14 +69,16 @@ class AsyncApiFitness : ApiWsFitness() { /** * What the variable holding a reply is called in a generated test, before the action's - * index. Named by the core so that two actions in one test cannot collide. + * index, which keeps two actions in one test from colliding. The same `res_0` every + * other problem type writes, as this is the same thing: what came back. * - * Not minted the way the writer mints `res_0` and `body_0`, from a counter it resets per - * test: the name has to be settled here, while the search runs, because it is part of - * what a driver is asked to render against. The action's index is the only number in - * hand at that point, and it is unique within the test being executed. + * Named here rather than by the writer, which mints its own from a counter it resets per + * test, because the name has to be settled while the search runs: it is part of what a + * driver is asked to render against. RPC settles it in the same place, for the same + * reason. The action's index is the only number in hand there, and it is unique within + * the test being executed. */ - private const val REPLY_VARIABLE_PREFIX = "reply_" + private const val REPLY_VARIABLE_PREFIX = "res_" private const val DEFAULT_CONTENT_TYPE = "application/json" @@ -395,7 +397,15 @@ class AsyncApiFitness : ApiWsFitness() { has, so it renders those lines while the search runs. Not asked for when no test will be written, so a driver need not spend time on it. */ - dto.outputFormat = SutInfoDto.OutputFormat.valueOf(config.outputFormat.toString()) + /* + Only when the driver could render that language at all. It is Java, so it knows + nothing of the formats a JVM driver cannot produce, and asking would fail. The + variable is still named here either way: the core owns that name, whoever writes + the lines that assign to it. + */ + dto.outputFormat = SutInfoDto.OutputFormat.values() + .firstOrNull { it.name == config.outputFormat.name } + dto.replyVariable = REPLY_VARIABLE_PREFIX + index } @@ -441,6 +451,7 @@ class AsyncApiFitness : ApiWsFitness() { val server = channel?.let { document.serversOf(it).firstOrNull() } result.setPublished( + serverName = server?.name, broker = server?.host, protocol = server?.protocol, address = call.address, diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 42c26534b2..7ad2cb7c14 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -786,25 +786,27 @@ internal class EMConfigTest{ } @Test - fun testAsyncApiWritesTestsOnlyInJavaOrKotlin(){ + fun testAsyncApiWritesTestsOnlyInTheLanguagesItCanPublishFrom(){ //a parser remembers what it last parsed, so each case gets its own fun parse(vararg args: String) = EMConfig().updateProperties(EMConfig.getOptionParser().parse(*args)) /* - The lines that publish are rendered by the driver, which is Java, so there is no - other language they could be pasted into. + A test publishes with a client of the transport, and one is written only for these + languages so far. Anything else has nothing to publish with. */ val e = assertThrows { - parse("--problemType", "ASYNCAPI", "--outputFormat", "PYTHON_UNITTEST") + parse("--problemType", "ASYNCAPI", "--outputFormat", "JS_JEST") } assertTrue(e.message!!.contains("outputFormat"), e.message) - //the default, and the two it can write + //the default, and the three it can write parse("--problemType", "ASYNCAPI") parse("--problemType", "ASYNCAPI", "--outputFormat", "JAVA_JUNIT_5") parse("--problemType", "ASYNCAPI", "--outputFormat", "KOTLIN_JUNIT_5") + //Python output is black-box only, which EvoMaster checks separately + parse("--problemType", "ASYNCAPI", "--outputFormat", "PYTHON_UNITTEST", "--blackBox", "true") //and a run that only wants the search is still allowed parse("--problemType", "ASYNCAPI", "--createTests", "false") diff --git a/core/src/test/kotlin/org/evomaster/core/output/compiler/CompilerForTestGenerated.kt b/core/src/test/kotlin/org/evomaster/core/output/compiler/CompilerForTestGenerated.kt index 566fbe7e83..ab2118e69f 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/compiler/CompilerForTestGenerated.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/compiler/CompilerForTestGenerated.kt @@ -8,6 +8,9 @@ import org.jetbrains.kotlin.cli.common.messages.PrintingMessageCollector import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler import org.jetbrains.kotlin.config.JvmTarget import org.jetbrains.kotlin.config.Services +import javax.tools.DiagnosticCollector +import javax.tools.JavaFileObject +import javax.tools.ToolProvider import java.io.ByteArrayOutputStream import java.io.File import java.io.PrintStream @@ -28,7 +31,8 @@ object CompilerForTestGenerated{ source.deleteRecursively() source.mkdirs() - val target = source.toPath().resolve(Paths.get("$testClassName.kt")).toFile() + val extension = if (format.isJava()) "java" else "kt" + val target = source.toPath().resolve(Paths.get("$testClassName.$extension")).toFile() target.createNewFile() target.writeText(code) @@ -48,22 +52,52 @@ object CompilerForTestGenerated{ destination: File ) { - /* - Handling Java in JDK is doable (can do same as done in EvoSuite), but bit - messy/unclean (see all issues with tools.jar). - In JDK 11 should be easier (but haven't verified it), and invalidate how done - in JKD 8. - - TODO: so, we will support Java only once upgraded to JDK 11 - */ - - when(format){ - OutputFormat.KOTLIN_JUNIT_5 -> compileKotlin(source, destination) + when { + format.isKotlin() -> compileKotlin(source, destination) + /* + This used to wait on the move to JDK 11, since doing it on 8 meant dealing with + tools.jar. The compiler has been part of the platform since then, so it is just + asked for. + */ + format.isJava() -> compileJava(source, destination) else -> throw IllegalStateException("Format $format not supported yet for compilation checks") } } + private fun compileJava(source: File, destination: File){ + + val compiler = ToolProvider.getSystemJavaCompiler() + ?: throw IllegalStateException("No Java compiler available: a JDK is needed, not a JRE") + + val files = source.walkTopDown().filter { it.isFile && it.name.endsWith(".java") }.toList() + + if (files.isEmpty()) { + throw IllegalStateException("No Java sources to compile under " + source.absolutePath) + } + + destination.mkdirs() + + val diagnostics = DiagnosticCollector() + + compiler.getStandardFileManager(diagnostics, null, null).use { manager -> + + val units = manager.getJavaFileObjectsFromFiles(files) + + val options = listOf( + "-classpath", System.getProperty("java.class.path"), + "-d", destination.absolutePath + ) + + val ok = compiler.getTask(null, manager, diagnostics, options, null, units).call() + + if (ok != true) { + val errors = diagnostics.diagnostics.joinToString(System.lineSeparator()) + throw RuntimeException("Failed to compile class. Error:" + System.lineSeparator() + errors) + } + } + } + private fun compileKotlin(source: File, destination: File){ //collect error messages to a buffer, and print them only if compilation fails diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index 594b2e8324..0475ebf2a7 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -13,6 +13,8 @@ import org.evomaster.core.output.OutputFormat import org.evomaster.core.output.Termination import org.evomaster.core.output.service.KafkaTestClientEmitter import org.evomaster.core.search.Solution +import org.evomaster.core.EMConfig +import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.problem.asyncapi.data.AsyncApiAction import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual @@ -37,6 +39,57 @@ class AsyncApiTestCaseWriterTest { private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" + /** + * Every line any other suite opens with, taken verbatim from what the writer produces + * for a suite with no tests in it (see TestSuiteWriterTest). The class name and the + * driver's own name are left out, as those are all that may differ. + */ + private val SCAFFOLDING_EVERY_SUITE_HAS = listOf( + "/**", + "* LICENSE DISCLAIMER", + "* This file has been generated by EvoMaster.", + "* The content of this file is not subject to the license of EvoMaster itself, i.e., LGPL.", + "* This generated software (i.e., the test suite in this file) can be freely used, modified,", + "* and distributed as you see fit without any restrictions.", + "*/", + "import org.junit.jupiter.api.AfterAll", + "import org.junit.jupiter.api.BeforeAll", + "import org.junit.jupiter.api.BeforeEach", + "import org.junit.jupiter.api.Test", + "import org.junit.jupiter.api.Timeout", + "import org.junit.jupiter.api.Assertions.*", + "import java.util.List", + "import org.evomaster.test.utils.EMTestUtils.*", + "import org.evomaster.client.java.controller.SutHandler", + "import java.math.BigDecimal", + "import java.math.BigInteger", + "import org.hamcrest.Matchers", + "import org.hamcrest.Matchers.*", + "import org.evomaster.client.java.controller.contentMatchers.NumberMatcher.*", + "import org.evomaster.client.java.controller.contentMatchers.StringMatcher.*", + "import org.evomaster.client.java.controller.contentMatchers.SubStringMatcher.*", + " companion object {", + " private lateinit var baseUrlOfSut: String", + " @BeforeAll", + " @JvmStatic", + " fun initClass() {", + " controller.setupForGeneratedTest()", + " baseUrlOfSut = controller.startSut()", + " controller.registerOrExecuteInitSqlCommandsIfNeeded()", + " assertNotNull(baseUrlOfSut)", + " }", + " @AfterAll", + " @JvmStatic", + " fun tearDown() {", + " controller.stopSut()", + " }", + " }", + " @BeforeEach", + " fun initTest() {", + " controller.resetStateOfSUT()", + " }" + ) + /** * What a driver hands over: lines that stand up a client, publish, and read the reply * into a variable it names. @@ -120,7 +173,7 @@ class AsyncApiTestCaseWriterTest { assertTrue(body.contains("pollUntilCorrelated"), body) //and what only the core knows about the reply - assertTrue(body.contains("assertNotNull(reply_0)"), body) + assertTrue(body.contains("assertNotNull(res_0)"), body) assertTrue(body.contains("bessj"), body) } @@ -132,10 +185,10 @@ class AsyncApiTestCaseWriterTest { val body = bodyOf(evaluate("bessj", "expint")) //the core names the variables, so two actions in one test cannot collide - assertTrue(body.contains("assertNotNull(reply_0)"), body) - assertTrue(body.contains("assertNotNull(reply_1)"), body) - assertTrue(body.contains("producer_reply_0"), body) - assertTrue(body.contains("producer_reply_1"), body) + assertTrue(body.contains("assertNotNull(res_0)"), body) + assertTrue(body.contains("assertNotNull(res_1)"), body) + assertTrue(body.contains("producer_res_0"), body) + assertTrue(body.contains("producer_res_1"), body) } @Test @@ -146,7 +199,7 @@ class AsyncApiTestCaseWriterTest { val asked = driver.published.single() assertEquals(SutInfoDto.OutputFormat.KOTLIN_JUNIT_5, asked.outputFormat) - assertEquals("reply_0", asked.replyVariable) + assertEquals("res_0", asked.replyVariable) } @Test @@ -167,7 +220,7 @@ class AsyncApiTestCaseWriterTest { assertTrue(body.contains("\"ncs.bessj.request\""), body) assertTrue(body.contains("\"ncs.bessj.reply\""), body) assertTrue(body.contains("\"correlationId\""), body) - assertTrue(body.contains("assertNotNull(reply_0)"), body) + assertTrue(body.contains("assertNotNull(res_0)"), body) //and what it calls is written once for the suite val members = membersOf(evaluated) @@ -178,6 +231,108 @@ class AsyncApiTestCaseWriterTest { assertTrue(members.contains("UUID.randomUUID()"), members) } + @Test + fun testThePythonSuiteIsValidPython() { + + /* + A Python suite has no driver to ask where the broker is, since the controller is + Java, so it publishes to the address the document declares. What matters here is + that what comes out parses: indentation is part of the language, so a helper written + as text can break it in a way no JVM format can. + */ + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { + FakeAsyncApiDriver.replied(DOUBLE_RESULT) + } + injector = AsyncApiTestInjector.create( + driver, + //Python output is for black-box only, which is also why its suite has no driver to ask + "--blackBox=true", + "--createTests=true", + "--outputFormat=PYTHON_UNITTEST" + ) + sampler = injector.getInstance(AsyncApiSampler::class.java) + fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + + val evaluated = evaluate("bessj") + val body = bodyOf(evaluated) + val members = membersOf(evaluated) + + //the call, and the reply read as a dict rather than a tree of nodes + assertTrue(body.contains("${KafkaTestClientEmitter.HELPER_NAME}("), body) + assertTrue(body.contains("json.loads(res_0)"), body) + assertTrue(body.contains("\"resultAsDouble\" in body_0"), body) + assertTrue(body.contains("assert "), body) + + //and the helper it calls + assertTrue(members.contains("kafka.KafkaProducer"), members) + assertTrue(members.contains("seek_to_end"), members) + + //what comes out has to be parseable Python, indentation included + val module = (members.trimEnd().lines() + body.trimEnd().lines()) + .joinToString("\n") { " " + it } + val file = java.io.File.createTempFile("asyncapi_suite", ".py") + file.writeText("import json\nimport time\nimport uuid\nimport kafka\n\n\nclass Suite:\n$module\n") + + val process = ProcessBuilder("python3", "-m", "py_compile", file.absolutePath) + .redirectErrorStream(true) + .start() + val output = process.inputStream.bufferedReader().readText() + val code = process.waitFor() + + file.delete() + assertEquals(0, code, "the generated Python does not parse:\n$output\n\n$module") + } + + @Test + fun testTheWholeSuiteHasTheSameScaffoldingAsAnyOther() { + + /* + The class, its lifecycle and the driver it starts come from the shared suite writer, + so an AsyncAPI suite has to look like any other: only what a test does inside its own + body is ours. Written to disk and read back, rather than asserted on fragments. + */ + val folder = java.nio.file.Files.createTempDirectory("asyncapi_suite").toFile() + + start { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } + + val config = injector.getInstance(EMConfig::class.java) + config.outputFolder = folder.absolutePath + config.outputFilePrefix = "AsyncApiSuite" + config.outputFileSuffix = "" + + val evaluated = evaluate("bessj") + val solution = Solution(mutableListOf(evaluated), "AsyncApiSuite", "", Termination.NONE, listOf(), listOf()) + + injector.getInstance(TestSuiteWriter::class.java) + .writeTests(solution, FakeAsyncApiDriver::class.qualifiedName!!, null) + + val written = folder.walkTopDown().first { it.isFile && it.name.endsWith(".kt") }.readText() + + val actual = written.lines().map { it.trimEnd() } + + /* + Every line another suite opens with has to be here, in the same order and with the + same indentation. Extra lines in between are allowed: that is where what a suite of + this problem type adds goes. + */ + var from = 0 + SCAFFOLDING_EVERY_SUITE_HAS.forEach { line -> + val at = actual.subList(from, actual.size).indexOf(line) + assertTrue(at >= 0, "missing, or out of order: '" + line + "'\n\n" + written) + from += at + 1 + } + + //the class and the driver it holds, which only differ by their names + assertTrue(actual.contains("internal class AsyncApiSuite {"), written) + assertTrue(actual.any { it.startsWith(" private val controller : SutHandler = ") }, written) + + //and ours on top of it, not instead of it + assertTrue(written.contains("asyncApiServer_kafka = controller.getAsyncApiServerAddress"), written) + assertTrue(written.contains("publishOnBessj"), written) + + folder.deleteRecursively() + } + @Test fun testTheKafkaHelperIsLeftOutWhenNothingCallsIt() { From ceed7c93159fdfff663a06a3e1f41c9fa3a6c30d Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Tue, 29 Sep 2026 23:18:15 -0300 Subject: [PATCH 07/20] AsyncAPI 3.x: document what a driver and a generated suite need write_driver gains the AsyncAPI section: what a driver has to answer for a message-driven service, and that one is needed even in black-box mode, which is the opposite of every other problem type and so worth saying twice. library_dependencies gains the two a generated Kafka suite compiles against, and options.md is regenerated for outputFormat, which now accepts the languages a Kafka suite can be written in. --- docs/blackbox.md | 4 +++- docs/library_dependencies.md | 16 ++++++++++++++++ docs/options.md | 2 +- docs/write_driver.md | 31 +++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/docs/blackbox.md b/docs/blackbox.md index 9cad18785f..5719b125e4 100644 --- a/docs/blackbox.md +++ b/docs/blackbox.md @@ -31,7 +31,9 @@ The command is doing the following: The executable `evomaster.jar` must be either [downloaded](download.md) or [built from source](build.md) (in which case it will end up under the `core/target/` folder of the cloned repository). Alternatively, you can just run `evomaster` if it was installed via `pip`. -* `--blackBox true`: technically, a redundant parameter, as black-box testing is the default mode since version 6.0.0. It can be omitted. +* `--blackBox true`: technically, a redundant parameter, as black-box testing is the default mode since version 6.0.0. +Note that AsyncAPI services still need an [_EvoMaster Driver_](write_driver.md) in this mode, as there is no +universal client for a message broker to point the fuzzer at. It can be omitted. * `--schema ...`: URL of where the OpenAPI/Swagger schema is. The location of the API will be inferred from this schema (e.g., from `host` and `servers` tags). If such info is missing, then the API is assumed to be on same host as the schema. If needed, the API host location can be changed with the optional `--base` (which overrides what specified in the schema). * `--outputFormat JAVA_JUNIT_4`: must specify how the tests will be generated, e.g., in Java using JUnit 4 in this case. Note: the language of the generated tests is not necessarily related diff --git a/docs/library_dependencies.md b/docs/library_dependencies.md index 70f85214f0..151eea0605 100644 --- a/docs/library_dependencies.md +++ b/docs/library_dependencies.md @@ -78,6 +78,22 @@ For white-box testing, you need as well to import the client controller library: as explained in the documentation for writing [white-box drivers](write_driver.md). +When fuzzing an **AsyncAPI** service, the generated tests publish with a client of the transport rather +than through the _EvoMaster Driver_, so that client library is needed as well. For Kafka that is: + +``` + + org.apache.kafka + kafka-clients + USE_LATEST_VERSION + test + +``` + +The assertions those tests make on a reply read it as JSON, so `jackson-databind` is needed too. It is +already part of `evomaster-client-java-dependencies`, whereas the transport client is not, since which one +is needed depends on the transport the service speaks. + ## NodeJS (e.g., JavaScript) Generated tests for block-box testing requires different libraries, such as Jest and SuperAgent. diff --git a/docs/options.md b/docs/options.md index b2a2d1b601..6f9bf0eb66 100644 --- a/docs/options.md +++ b/docs/options.md @@ -42,7 +42,7 @@ There are 3 types of options: |`configPath`| __String__. File path for file with configuration settings. Supported formats are YAML and TOML. When EvoMaster starts, it will read such file and import all configurations from it. *Constraints*: `regex .*\.(yml\|yaml\|toml)`. *Default value*: `em.yaml`.| |`outputFilePrefix`| __String__. The name prefix of generated file(s) with the test cases, without file type extension. In JVM languages, if the name contains '.', folders will be created to represent the given package structure. Also, in JVM languages, should not use '-' in the file name, as not valid symbol for class identifiers. This prefix be combined with the outputFileSuffix to combined the final name. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `EvoMaster`.| |`outputFileSuffix`| __String__. The name suffix for the generated file(s), to be added before the file type extension. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `Test`.| -|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| +|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). AsyncAPI is the exception on both counts: it has a driver in either mode, so the format comes from there, and only Java, Kotlin and Python can be generated, as the tests publish with a client of the transport. *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| |`testTimeout`| __Int__. Enforce timeout (in seconds) in the generated tests. This feature might not be supported in all frameworks. If 0 or negative, the timeout is not applied. *Default value*: `60`.| |`ratePerMinute`| __Int__. Rate limiter, of how many actions to do per minute. For example, when making HTTP calls towards an external service, might want to limit the number of calls to avoid bombarding such service (which could end up becoming equivalent to a DoS attack). A value of zero or negative means that no limiter is applied. This is needed only for black-box testing of remote services. Note that, evan without this parameter, EvoMaster will still respect the Retry-After given back in 429 responses. *Default value*: `0`.| |`header0`| __String__. In black-box testing, we still need to deal with authentication of the HTTP requests. With this parameter it is possible to specify a HTTP header that is going to be added to most requests. This should be provided in the form _name:value_. If more than 1 header is needed, use as well the other options _header1_ and _header2_. *Constraints*: `regex (.+:.+)\|(^$)`. *Default value*: `""`.| diff --git a/docs/write_driver.md b/docs/write_driver.md index a8c0722c1b..36c1e0d8dc 100644 --- a/docs/write_driver.md +++ b/docs/write_driver.md @@ -2,6 +2,8 @@ To generate tests for [white-box testing](whitebox.md), you need an _EvoMaster Driver_ up and running before executing `evomaster.jar`. +One kind of API needs a driver in _black-box_ mode as well: AsyncAPI services, where there is no universal +client for a message broker to point the fuzzer at. See [AsyncAPI](#asyncapi) below. These drivers have to be built manually for each system under test (SUT). See the [EMB repository](https://github.com/WebFuzzing/EMB) for a set of existing SUTs with drivers. @@ -359,6 +361,35 @@ To test a GraphQL API, in the the `getProblemInfo()`, you need to return an inst Here, you need to specify the endpoint of where the GraphQL API can be accessed. Default value is `/graphql`. Note: must be able to do an _introspective query_ on such API to fetch its schema. If this is disabled for security reasons, _EvoMaster_ will fail. +## AsyncAPI + +To test an AsyncAPI service, in the `getProblemInfo()` you need to return an instance of the +`AsyncApiProblem` class, built from the AsyncAPI document itself, either its text or where to find it. +Only AsyncAPI 3.x is supported, as it is the first version able to declare the reply to an operation, and +that reply is what makes a service observable from outside. + +Unlike REST and GraphQL, **a driver is needed in black-box mode too**. There is one HTTP client for every +REST API, but no one client for every broker: the transport may be Kafka, AMQP, MQTT or a socket, and they +share nothing. So publishing a message is the driver's job, by overriding `executeAsyncApiAction`. What is +asked of it is only to put the message on the wire and report what came back, never to judge it: deciding +what an outcome means is the fuzzer's job, so that it means the same thing whatever the transport. + +Two things are worth knowing when writing one: + +- **A reply published before the action was is never an answer to it.** Only the driver can ensure that, + by seeking to the end of the reply destination before publishing, or by a fresh subscription. The fuzzer + does not vary its correlation ids between runs to compensate, because a run repeated under the same seed + has to behave the same way. +- **Where a server actually is may not be what the document says.** The document gives the address of the + deployment its author had in mind; a system started for testing is often elsewhere, and a broker in a + container binds a port chosen at start-up. Override `getAsyncApiServerAddress(String serverName)` to say + where it is now. Generated tests ask for it there, in the same way they take the base URL of a REST + system from `startSut()`. Returning `null`, which is the default, means the document's own address is + used. + +Generated tests publish with a client of the transport rather than through the driver, so the suite needs +that client library on its classpath: see [library dependencies](library_dependencies.md). + ## RPC APIs To test RPC APIs, in the the `getProblemInfo()`, you need to return an instance of the From 6e90d719c3acc87b40f14d3aaac84620adb7de07 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 2 Oct 2026 15:04:36 -0300 Subject: [PATCH 08/20] AsyncAPI 3.x: ask the driver for an address once, not twice Java has no elvis, and the ternary the Java suite was written with named the call on both sides of it. A driver that works the address out, rather than keeping one, did that work twice every time a suite started. Two statements instead, which is also what the generated code reads like by hand. --- .../output/service/AsyncApiTestCaseWriter.kt | 13 +++++- .../service/AsyncApiTestCaseWriterTest.kt | 42 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt index a6271ca1e0..4c41a41ce8 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt @@ -237,7 +237,18 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { val fallback = KafkaTestClientEmitter.brokerLiteral(declared, format) when { - format.isJava() -> lines.add("$variable = $ask != null ? $ask : $fallback;") + /* + Written over two statements rather than a ternary, so that a driver which + computes the address, rather than keeping one, is asked for it only once. + */ + format.isJava() -> { + lines.add("$variable = $ask;") + lines.add("if ($variable == null) {") + lines.indented { + lines.add("$variable = $fallback;") + } + lines.add("}") + } format.isKotlin() -> lines.add("$variable = $ask ?: $fallback") } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index 0475ebf2a7..4affe1cc00 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -333,6 +333,48 @@ class AsyncApiTestCaseWriterTest { folder.deleteRecursively() } + @Test + fun testTheDriverIsAskedForAnAddressOnlyOnce() { + + /* + Java has no elvis, and the obvious ternary would name the call on both sides of it. + A driver that works the address out, rather than keeping one, would then do that + work twice every time a suite starts. + */ + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { + FakeAsyncApiDriver.replied(DOUBLE_RESULT) + } + injector = AsyncApiTestInjector.create( + driver, + "--blackBox=false", + "--createTests=true", + "--outputFormat=JAVA_JUNIT_5" + ) + sampler = injector.getInstance(AsyncApiSampler::class.java) + fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + + val evaluated = evaluate("bessj") + val writer = injector.getInstance(TestCaseWriter::class.java) + val solution = Solution(mutableListOf(evaluated), "Prefix", "Suffix", Termination.NONE, listOf(), listOf()) + + //declaring the members is what tells the writer which servers there are to ask about + writer.addExtraClassMembers(Lines(OutputFormat.JAVA_JUNIT_5), solution) + + val lines = Lines(OutputFormat.JAVA_JUNIT_5) + writer.addExtraInitStatement(lines) + val init = lines.toString() + + assertEquals(1, init.split("getAsyncApiServerAddress").size - 1, init) + + assertEquals( + "asyncApiServer_kafka = controller.getAsyncApiServerAddress(\"kafka\");\n" + + "if (asyncApiServer_kafka == null) {\n" + + " asyncApiServer_kafka = \"localhost:9092\";\n" + + "}", + init.trimEnd() + ) + } + @Test fun testTheKafkaHelperIsLeftOutWhenNothingCallsIt() { From 19208d534115754f9a4c75fc36a56e5b57d5afb2 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 2 Oct 2026 17:44:32 -0300 Subject: [PATCH 09/20] AsyncAPI 3.x: say which half of the run the driver is for The note on outputFormat read as though a driver were involved in the tests as well. It is not: a generated Kafka test publishes with a client of the transport, and the driver renders the publishing lines only for a transport the document cannot describe. What a driver does in either mode is publish while the search runs, which is also why a DEFAULT format is resolved from it rather than from the black-box default. And it could not resolve to Python even so: SutInfoDto.OutputFormat has no entry for it, so a Python suite has to be asked for with the option. Said where the resolution happens, and in the option's own text. The black-box page had the same gap, and the sentence had been put between "--blackBox true" and the "It" that referred to it. --- .../main/kotlin/org/evomaster/core/EMConfig.kt | 17 +++++++++++------ docs/blackbox.md | 7 ++++--- docs/options.md | 2 +- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 37a6f3de8f..624d1c883c 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -674,9 +674,12 @@ class EMConfig { problemType = ProblemType.REST } /* - AsyncAPI is the exception: it has a driver even in black-box, since there is no - universal client for a broker, so the format comes from the driver as it does in - white-box rather than from the black-box default. + AsyncAPI is the exception: a driver publishes its messages even in black-box, since + there is no universal client for a broker, so the format is resolved from the driver + as it is in white-box rather than from the black-box default. + + A driver can only name a format SutInfoDto.OutputFormat has, which Python is not, so + a Python suite has to be asked for with this option. */ if (outputFormat == OutputFormat.DEFAULT && problemType != ProblemType.ASYNCAPI) { LoggingUtil.uniqueUserWarn("You are doing Black-Box testing, but you did not specify the" + @@ -1353,9 +1356,11 @@ class EMConfig { @Cfg("Specify in which format the tests should be outputted." + " If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used." + " On the other hand, for black-box testing it will default to a predefined type (e.g., Python)." + - " AsyncAPI is the exception on both counts: it has a driver in either mode, so the format comes from" + - " there, and only Java, Kotlin and Python can be generated, as the tests publish with a client of the" + - " transport.") + " AsyncAPI is the exception on both counts: a driver is what publishes during the search, in either" + + " mode, so a `DEFAULT` is resolved from the driver there as well rather than from the black-box" + + " default. Only Java, Kotlin and Python can be generated, as a generated test publishes with a client" + + " of the transport rather than through the driver, and only those three have one written for them so" + + " far. Python has to be asked for here explicitly, as a driver has no way of naming it.") var outputFormat = OutputFormat.DEFAULT @Important(2.1) diff --git a/docs/blackbox.md b/docs/blackbox.md index 5719b125e4..8f515147ce 100644 --- a/docs/blackbox.md +++ b/docs/blackbox.md @@ -31,9 +31,10 @@ The command is doing the following: The executable `evomaster.jar` must be either [downloaded](download.md) or [built from source](build.md) (in which case it will end up under the `core/target/` folder of the cloned repository). Alternatively, you can just run `evomaster` if it was installed via `pip`. -* `--blackBox true`: technically, a redundant parameter, as black-box testing is the default mode since version 6.0.0. -Note that AsyncAPI services still need an [_EvoMaster Driver_](write_driver.md) in this mode, as there is no -universal client for a message broker to point the fuzzer at. It can be omitted. +* `--blackBox true`: technically, a redundant parameter, as black-box testing is the default mode since version 6.0.0. It can be omitted. +Note that AsyncAPI services still need an [_EvoMaster Driver_](write_driver.md) in this mode: there is no universal +client for a message broker to publish through while the search runs. The generated tests do not need one, as they +publish with a client of the transport themselves. * `--schema ...`: URL of where the OpenAPI/Swagger schema is. The location of the API will be inferred from this schema (e.g., from `host` and `servers` tags). If such info is missing, then the API is assumed to be on same host as the schema. If needed, the API host location can be changed with the optional `--base` (which overrides what specified in the schema). * `--outputFormat JAVA_JUNIT_4`: must specify how the tests will be generated, e.g., in Java using JUnit 4 in this case. Note: the language of the generated tests is not necessarily related diff --git a/docs/options.md b/docs/options.md index 6f9bf0eb66..8946898ec3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -42,7 +42,7 @@ There are 3 types of options: |`configPath`| __String__. File path for file with configuration settings. Supported formats are YAML and TOML. When EvoMaster starts, it will read such file and import all configurations from it. *Constraints*: `regex .*\.(yml\|yaml\|toml)`. *Default value*: `em.yaml`.| |`outputFilePrefix`| __String__. The name prefix of generated file(s) with the test cases, without file type extension. In JVM languages, if the name contains '.', folders will be created to represent the given package structure. Also, in JVM languages, should not use '-' in the file name, as not valid symbol for class identifiers. This prefix be combined with the outputFileSuffix to combined the final name. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `EvoMaster`.| |`outputFileSuffix`| __String__. The name suffix for the generated file(s), to be added before the file type extension. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `Test`.| -|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). AsyncAPI is the exception on both counts: it has a driver in either mode, so the format comes from there, and only Java, Kotlin and Python can be generated, as the tests publish with a client of the transport. *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| +|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). AsyncAPI is the exception on both counts: a driver is what publishes during the search, in either mode, so a `DEFAULT` is resolved from the driver there as well rather than from the black-box default. Only Java, Kotlin and Python can be generated, as a generated test publishes with a client of the transport rather than through the driver, and only those three have one written for them so far. Python has to be asked for here explicitly, as a driver has no way of naming it. *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| |`testTimeout`| __Int__. Enforce timeout (in seconds) in the generated tests. This feature might not be supported in all frameworks. If 0 or negative, the timeout is not applied. *Default value*: `60`.| |`ratePerMinute`| __Int__. Rate limiter, of how many actions to do per minute. For example, when making HTTP calls towards an external service, might want to limit the number of calls to avoid bombarding such service (which could end up becoming equivalent to a DoS attack). A value of zero or negative means that no limiter is applied. This is needed only for black-box testing of remote services. Note that, evan without this parameter, EvoMaster will still respect the Retry-After given back in 429 responses. *Default value*: `0`.| |`header0`| __String__. In black-box testing, we still need to deal with authentication of the HTTP requests. With this parameter it is possible to specify a HTTP header that is going to be added to most requests. This should be provided in the form _name:value_. If more than 1 header is needed, use as well the other options _header1_ and _header2_. *Constraints*: `regex (.+:.+)\|(^$)`. *Default value*: `""`.| From 3dfe6cf85a73389a8ddc6507ad79c1fcfc659035 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 2 Oct 2026 18:27:01 -0300 Subject: [PATCH 10/20] AsyncAPI 3.x: a suite that runs, not only one that parses The Python half was only ever checked by py_compile, and three things that parse perfectly were waiting there. assertNotNull is a JUnit name and no Python suite defines one. The blank-reply assertion handed Python Kotlin's isNullOrEmpty. And the server address was declared only for the JVM formats while the call that uses it was written for all three, so a Python suite with a driver named a variable that was never there. The driver is now asked only where a suite can hold one, and the Kafka imports follow the helper rather than every AsyncAPI Python suite, which is what the comment beside them already promised. One writer writes every suite of a run. Clearing the servers it found came after two early returns, so a suite with nothing to publish over Kafka kept the last one's and assigned to fields it had not declared. Cleared first now. Two server names that differ only where an identifier cannot, "a-b" and "a.b", collapsed onto one variable and declared it twice; the first keeps it and the rest fall back to the address the document gave. A field name is a literal too. They went in unescaped, so a reply with a field called "$ref" -- ordinary JSON -- wrote Kotlin that reads it as a template and does not compile. Values already went through the quoting helper; names and server names do now as well. A number outside a double's range parses as an infinity, which is not a literal in any of the three, so it is left to a comment. Kafka is only written when the correlation id rides in a header. Where the document puts it in the payload, the recorded payload is the one the search composed, without it: that test would publish a message nothing could answer and then take whatever was on the reply destination for its answer. Left to the driver, as the class comment always said it should be. The emitted client now closes what it opens, in all three languages, rather than on the happy path only, and a reply destination with no partitions is treated as nothing to wait for instead of being polled, which throws. A format with no client written for it throws where it used to quietly write Kotlin into a file of another language. --- .../output/service/AsyncApiReplyAssertions.kt | 38 ++- .../output/service/AsyncApiTestCaseWriter.kt | 84 ++--- .../output/service/KafkaTestClientEmitter.kt | 320 +++++++++++------- .../core/output/service/TestSuiteWriter.kt | 10 +- .../core/output/sorting/SortingHelper.kt | 8 +- 5 files changed, 281 insertions(+), 179 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt index b428685660..7063a8dd3a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt @@ -12,8 +12,9 @@ import org.evomaster.core.output.OutputFormat * These are regression assertions, the same kind the REST writer makes on a response body: they * say what came back, not what should. They honour the same rules, because the reasons are the * same ones. A field that changes between runs makes a test flaky whatever the protocol, so the - * fields REST skips are skipped here; a collection is asserted only to the configured depth; a - * value that cannot be printed safely becomes a comment. + * fields REST skips are skipped here; only the first few elements of a collection are asserted, + * as maxAssertionForDataInCollection says; a value that cannot be printed safely becomes a + * comment, and a whole payload that cannot becomes one too. * * The reply is text, not a response object, so unlike REST there is no fluent client to hang the * assertions off. They are made against the tree it parses into. @@ -55,7 +56,9 @@ object AsyncApiReplyAssertions { if (payload.isNullOrBlank()) { //an acknowledgement, or a transport that answers in metadata - assertion(lines, "$variable.isNullOrEmpty()", "$variable == null || $variable.isEmpty()", format) + val java = "$variable == null || $variable.isEmpty()" + val kotlin = if (format.isPython()) "$variable is None or len($variable) == 0" else "$variable.isNullOrEmpty()" + assertion(lines, kotlin, java, format) return } @@ -77,6 +80,10 @@ object AsyncApiReplyAssertions { //not JSON: all that can be said is what it contained if (printable(payload)) { contains(lines, variable, payload.trim(), format) + } else { + lines.addSingleCommentLine( + "the reply is not asserted on, as its value is not stable between runs" + ) } return } @@ -210,7 +217,20 @@ object AsyncApiReplyAssertions { A double is compared within a tolerance, as the REST writer does: the same computation can land a bit apart once it has been through text. */ - node.isNumber -> delta(lines, node.asDouble().toString(), value(path, "asDouble()", format), format) + node.isNumber -> { + val v = node.asDouble() + /* + A JSON number out of a double's range parses as an infinity, which no target + language writes that way. Nothing true can be said of it in a literal. + */ + if (v.isFinite()) { + delta(lines, v.toString(), value(path, "asDouble()", format), format) + } else { + lines.addSingleCommentLine( + "$fieldPath is not asserted on, as it is outside the range of a double" + ) + } + } node.isTextual -> { val text = node.asText() @@ -249,12 +269,18 @@ object AsyncApiReplyAssertions { } private fun has(lines: Lines, path: String, field: String, format: OutputFormat) { - val expr = if (format.isPython()) "\"$field\" in $path" else "$path.has(\"$field\")" + val name = quoted(field, format) + val expr = if (format.isPython()) "$name in $path" else "$path.has($name)" assertion(lines, expr, expr, format) } + /* + A field name is a literal of the target language as much as a value is, and a document + can call a field anything JSON allows: a name holding a quote, or one starting with '$', + which Kotlin would otherwise read as the start of a template. + */ private fun child(path: String, name: String, format: OutputFormat) = - if (format.isPython()) "$path[\"$name\"]" else "$path.get(\"$name\")" + if (format.isPython()) "$path[${quoted(name, format)}]" else "$path.get(${quoted(name, format)})" private fun index(path: String, i: Int, format: OutputFormat) = if (format.isPython()) "$path[$i]" else "$path.get($i)" diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt index 4c41a41ce8..cb3f321126 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt @@ -17,10 +17,12 @@ import java.nio.file.Path * Writes a test for an AsyncAPI service. * * Unlike REST, there is no universal client to call: a message goes out over whichever broker - * the contract names, so the lines that publish it and read the reply are the driver's, rendered - * while the search ran and pasted here. This follows RPC, whose driver renders an invocation the - * same way, and it is what keeps a generated test standing on its own: it talks to the broker - * with an ordinary client of that transport, and needs no EvoMaster driver at run time. + * the contract names. For Kafka the contract says enough -- the server, the topics and the + * header the correlation id rides in -- so the lines that publish and await are written here + * from it. For a transport it does not describe, the driver renders those lines while the search + * runs and they are pasted verbatim, which is how RPC does it. Either way the generated test + * stands on its own: it talks to the broker with an ordinary client of that transport, and needs + * no EvoMaster driver at run time. * * What the core adds around those lines is what only it knows: which outcome the action had, * which of the declared reply messages the reply was recognised as, and that a reply promised by @@ -28,28 +30,6 @@ import java.nio.file.Path */ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { - override fun handleTestInitialization( - lines: Lines, - baseUrlOfSut: String, - ind: EvaluatedIndividual<*>, - sqlInsertionVars: MutableList>, - mongoInsertionVars: MutableList>, - redisInsertionVars: MutableList>, - dynamoDbInsertionVars: MutableList>, - testName: String - ) { - super.handleTestInitialization( - lines, - baseUrlOfSut, - ind, - sqlInsertionVars, - mongoInsertionVars, - redisInsertionVars, - dynamoDbInsertionVars, - testName - ) - } - override fun handleActionCalls( lines: Lines, baseUrlOfSut: String, @@ -104,7 +84,7 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { not describe well enough gets written: one whose correlation id rides inside the service's own message layout, or one nothing here knows at all. */ - script.isNotEmpty() -> script.forEach { lines.add(it) } + script.any { it.isNotBlank() } -> script.forEach { lines.add(it) } /* Kafka is written from the contract, as REST writes its RestAssured calls: the @@ -146,8 +126,12 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { val variable = res.getReplyVariableName() ?: return - lines.add("assertNotNull($variable)") - lines.appendSemicolon() + if (format.isPython()) { + lines.add("assert $variable is not None") + } else { + lines.add("assertNotNull($variable)") + lines.appendSemicolon() + } res.getReplyMessage()?.let { lines.addSingleCommentLine("recognised as the declared message '$it'") @@ -195,10 +179,14 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { */ override fun addExtraClassMembers(lines: Lines, solution: Solution<*>) { - val kafkaResults = solution.individuals - .flatMap { it.evaluatedMainActions() } - .mapNotNull { it.result as? AsyncApiCallResult } - .filter { it.getTestScript().isEmpty() && KafkaTestClientEmitter.canEmit(it) } + val kafkaResults = KafkaTestClientEmitter.resultsIn(solution) + + /* + Before anything may return: one writer writes every suite of a run, so what the last + one found must not be left behind for the next, which would assign to a field it + never declared. + */ + servers.clear() if (kafkaResults.isEmpty()) { return @@ -206,13 +194,23 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { KafkaTestClientEmitter.emitHelper(lines, format) - if (!hasDriver()) { + if (!canAskDriver()) { return } - servers.clear() kafkaResults.forEach { r -> - val name = r.getServerName() ?: return@forEach + val name = r.getServerName() + if (name.isNullOrBlank()) { + return@forEach + } + /* + Two names that differ only where a variable cannot, say "a-b" and "a.b", would be + declared twice under one identifier. The first keeps the variable; the rest fall + back to the address the document gave them. + */ + if (servers.keys.any { it != name && variableFor(it) == variableFor(name) }) { + return@forEach + } servers.putIfAbsent(name, r.getBroker() ?: "") } @@ -233,7 +231,8 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { servers.forEach { (name, declared) -> val variable = variableFor(name) - val ask = "${TestSuiteWriter.controller}.getAsyncApiServerAddress(\"$name\")" + val ask = "${TestSuiteWriter.controller}.getAsyncApiServerAddress(" + + KafkaTestClientEmitter.brokerLiteral(name, format) + ")" val fallback = KafkaTestClientEmitter.brokerLiteral(declared, format) when { @@ -262,7 +261,7 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { val name = result.getServerName() - if (hasDriver() && name != null && servers.containsKey(name)) { + if (canAskDriver() && name != null && servers.containsKey(name)) { return variableFor(name) } @@ -279,10 +278,13 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX + TestWriterUtils.safeVariableName(serverName) /** - * Whether the generated suite has a driver in it at all. A black-box suite does not, so - * there is nothing to ask and the document's address is all there is. + * Whether the generated suite can ask a driver where a server is. + * + * It needs one in it at all, which a black-box suite has not, and it needs to be written in + * a language that can hold the driver: the controller is only declared for Java and Kotlin. + * Otherwise the document's address is all there is. */ - private fun hasDriver() = !config.blackBox || config.bbExperiments + private fun canAskDriver() = (!config.blackBox || config.bbExperiments) && format.isJavaOrKotlin() override fun addTestCommentBlock(lines: Lines, test: TestCase) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt index ed33247358..28c037c0d0 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt @@ -1,9 +1,13 @@ package org.evomaster.core.output.service import com.fasterxml.jackson.databind.ObjectMapper +import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.output.Lines import org.evomaster.core.output.OutputFormat import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult +import org.evomaster.core.search.Solution +import org.slf4j.Logger +import org.slf4j.LoggerFactory /** * Writes the Kafka client code that republishes one message, for a generated test. @@ -17,14 +21,17 @@ import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult * service's own message layout cannot be emitted from the contract alone, because the contract * does not describe that layout; there, the driver renders the lines instead. * - * Everything emitted here names its types in full rather than adding imports to the suite, which - * is how the REST writer reaches the one method RestAssured does not expose. The suite's imports - * are decided before it is known whether anything will publish over Kafka, and adding them - * unconditionally would make every AsyncAPI suite need the dependency, including one that only - * ever talks to a socket through a driver. + * On the JVM everything emitted here names its types in full rather than adding imports to the + * suite, which is how the REST writer reaches the one method RestAssured does not expose. The + * suite's imports are decided before it is known whether anything will publish over Kafka, and + * adding them unconditionally would make every AsyncAPI suite need the dependency, including one + * that only ever talks to a socket through a driver. Python has no such spelling, so its import + * is written with the helper instead, by [TestSuiteWriter], under the same condition. */ object KafkaTestClientEmitter { + private val log: Logger = LoggerFactory.getLogger(KafkaTestClientEmitter::class.java) + /** * The protocols this can write. The document names them on the server. */ @@ -49,24 +56,42 @@ object KafkaTestClientEmitter { private val mapper = ObjectMapper() - /** - * The document's own address for the server, as a literal of the target language. - */ /** * What stands for an absent value in the target language. */ private fun nothing(format: OutputFormat) = if (format.isPython()) "None" else "null" + /** + * A name or an address out of the document, as a literal of the target language. + */ fun brokerLiteral(address: String, format: OutputFormat): String = quoted(address, format) /** * Whether this result carries enough, and the right transport, to be written as Kafka code. + * + * A correlation id that rides inside the service's own message layout is the case this + * cannot write: the document says where in the payload it goes, but the payload recorded + * here is the one the search composed, without it. A test written from that would publish a + * message no reply could answer, and -- having no id to match on -- would take whatever + * happened to be on the reply destination for its answer. Left to the driver instead. */ fun canEmit(result: AsyncApiCallResult): Boolean = result.getProtocol()?.lowercase() in KAFKA_PROTOCOLS && !result.getBroker().isNullOrBlank() && !result.getAddress().isNullOrBlank() + && (result.getReplyAddress().isNullOrBlank() || result.getCorrelationHeader() != null) + + /** + * The results of a solution this can write Kafka code for: the ones whose driver left the + * publishing to it. One predicate, so that what declares the helper, what imports it, and + * what calls it cannot disagree. + */ + fun resultsIn(solution: Solution<*>): List = + solution.individuals + .flatMap { it.evaluatedMainActions() } + .mapNotNull { it.result as? AsyncApiCallResult } + .filter { r -> r.getTestScript().none { it.isNotBlank() } && canEmit(r) } /** * The call one action becomes, once [emitHelper] has written what it calls. @@ -105,6 +130,7 @@ object KafkaTestClientEmitter { format.isJava() -> lines.add("String $variable = $call;") format.isKotlin() -> lines.add("val $variable = $call") format.isPython() -> lines.add("$variable = self.$call") + else -> throw IllegalStateException("No Kafka client is written for $format") } } @@ -119,7 +145,8 @@ object KafkaTestClientEmitter { val body = when { format.isJava() -> javaHelper() format.isPython() -> pythonHelper() - else -> kotlinHelper() + format.isKotlin() -> kotlinHelper() + else -> throw IllegalStateException("No Kafka client is written for $format") } lines.addEmpty() body.forEach { lines.add(it) } @@ -142,54 +169,68 @@ object KafkaTestClientEmitter { " props.put(\"enable.auto.commit\", \"false\");", " String correlationId = java.util.UUID.randomUUID().toString();", " org.apache.kafka.clients.consumer.KafkaConsumer consumer = null;", - " if (replyTopic != null) {", - " consumer = new org.apache.kafka.clients.consumer.KafkaConsumer<>(props,", - " new $STRING_SERDE.StringDeserializer(), new $STRING_SERDE.StringDeserializer());", - " java.util.List parts = new java.util.ArrayList<>();", - " for (org.apache.kafka.common.PartitionInfo p : consumer.partitionsFor(replyTopic)) {", - " parts.add(new org.apache.kafka.common.TopicPartition(p.topic(), p.partition()));", + " org.apache.kafka.clients.producer.KafkaProducer producer = null;", + " try {", + " if (replyTopic != null) {", + " consumer = new org.apache.kafka.clients.consumer.KafkaConsumer<>(props,", + " new $STRING_SERDE.StringDeserializer(), new $STRING_SERDE.StringDeserializer());", + " java.util.List parts = new java.util.ArrayList<>();", + " java.util.List info = consumer.partitionsFor(replyTopic);", + " for (org.apache.kafka.common.PartitionInfo p : info == null", + " ? java.util.Collections.emptyList() : info) {", + " parts.add(new org.apache.kafka.common.TopicPartition(p.topic(), p.partition()));", + " }", + " if (parts.isEmpty()) {", + " //the reply destination does not exist yet, so nothing can be waiting on it", + " consumer.close();", + " consumer = null;", + " } else {", + " consumer.assign(parts);", + " consumer.seekToEnd(parts);", + " //seekToEnd is lazy: asking for the position is what makes it take effect now", + " consumer.position(parts.get(0));", + " }", " }", - " consumer.assign(parts);", - " consumer.seekToEnd(parts);", - " //seekToEnd is lazy: asking for the position is what makes it take effect now", - " if (!parts.isEmpty()) {", - " consumer.position(parts.get(0));", + " producer = new org.apache.kafka.clients.producer.KafkaProducer<>(props,", + " new $STRING_SERDE.StringSerializer(), new $STRING_SERDE.StringSerializer());", + " //a message with no payload of its own publishes an empty body", + " org.apache.kafka.clients.producer.ProducerRecord record =", + " new org.apache.kafka.clients.producer.ProducerRecord<>(topic, payload == null ? \"\" : payload);", + " for (int i = 0; i + 1 < headerPairs.length; i += 2) {", + " record.headers().add(headerPairs[i], headerPairs[i + 1].getBytes($UTF_8));", " }", - " }", - " org.apache.kafka.clients.producer.KafkaProducer producer =", - " new org.apache.kafka.clients.producer.KafkaProducer<>(props,", - " new $STRING_SERDE.StringSerializer(), new $STRING_SERDE.StringSerializer());", - " //a message with no payload of its own publishes an empty body", - " org.apache.kafka.clients.producer.ProducerRecord record =", - " new org.apache.kafka.clients.producer.ProducerRecord<>(topic, payload == null ? \"\" : payload);", - " for (int i = 0; i + 1 < headerPairs.length; i += 2) {", - " record.headers().add(headerPairs[i], headerPairs[i + 1].getBytes($UTF_8));", - " }", - " if (correlationHeader != null) {", - " record.headers().add(correlationHeader, correlationId.getBytes($UTF_8));", - " }", - " producer.send(record).get();", - " producer.close();", - " if (consumer == null) {", - " return null;", - " }", - " String reply = null;", - " long deadline = System.currentTimeMillis() + timeoutMs;", - " while (reply == null && System.currentTimeMillis() < deadline) {", - " for (org.apache.kafka.clients.consumer.ConsumerRecord r :", - " consumer.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {", - " if (correlationHeader == null) {", - " reply = r.value();", - " continue;", - " }", - " org.apache.kafka.common.header.Header h = r.headers().lastHeader(correlationHeader);", - " if (h != null && correlationId.equals(new String(h.value(), $UTF_8))) {", - " reply = r.value();", + " if (correlationHeader != null) {", + " record.headers().add(correlationHeader, correlationId.getBytes($UTF_8));", + " }", + " producer.send(record).get();", + " if (consumer == null) {", + " return null;", + " }", + " String reply = null;", + " long deadline = System.currentTimeMillis() + timeoutMs;", + " while (reply == null && System.currentTimeMillis() < deadline) {", + " for (org.apache.kafka.clients.consumer.ConsumerRecord r :", + " consumer.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {", + " if (correlationHeader == null) {", + " reply = r.value();", + " continue;", + " }", + " org.apache.kafka.common.header.Header h = r.headers().lastHeader(correlationHeader);", + " if (h != null && correlationId.equals(new String(h.value(), $UTF_8))) {", + " reply = r.value();", + " }", " }", " }", + " return reply;", + " } finally {", + " //a suite runs many of these, so a client left open would outlive every one of them", + " if (producer != null) {", + " producer.close();", + " }", + " if (consumer != null) {", + " consumer.close();", + " }", " }", - " consumer.close();", - " return reply;", "}" ) @@ -209,41 +250,52 @@ object KafkaTestClientEmitter { " \"\"\"", " correlation_id = str(uuid.uuid4())", " consumer = None", - " if reply_topic is not None:", - " consumer = kafka.KafkaConsumer(bootstrap_servers=broker,", - " auto_offset_reset='latest', enable_auto_commit=False)", - " partitions = [kafka.TopicPartition(reply_topic, p)", - " for p in (consumer.partitions_for_topic(reply_topic) or [])]", - " consumer.assign(partitions)", - " consumer.seek_to_end()", - " producer = kafka.KafkaProducer(bootstrap_servers=broker)", - " headers = []", - " i = 0", - " while i + 1 < len(header_pairs):", - " headers.append((header_pairs[i], header_pairs[i + 1].encode('utf-8')))", - " i += 2", - " if correlation_header is not None:", - " headers.append((correlation_header, correlation_id.encode('utf-8')))", - " #a message with no payload of its own publishes an empty body", - " body = (payload if payload is not None else '').encode('utf-8')", - " producer.send(topic, value=body, headers=headers)", - " producer.flush()", - " producer.close()", - " if consumer is None:", - " return None", - " reply = None", - " deadline = time.time() + (timeout_ms / 1000.0)", - " while reply is None and time.time() < deadline:", - " for _, records in consumer.poll(timeout_ms=$DEFAULT_POLL_MS).items():", - " for r in records:", - " if correlation_header is None:", - " reply = r.value.decode('utf-8')", - " continue", - " for key, value in (r.headers or []):", - " if key == correlation_header and value.decode('utf-8') == correlation_id:", + " producer = None", + " try:", + " if reply_topic is not None:", + " consumer = kafka.KafkaConsumer(bootstrap_servers=broker,", + " auto_offset_reset='latest', enable_auto_commit=False)", + " partitions = [kafka.TopicPartition(reply_topic, p)", + " for p in (consumer.partitions_for_topic(reply_topic) or [])]", + " if not partitions:", + " #the reply destination does not exist yet, so nothing can be waiting on it", + " consumer.close()", + " consumer = None", + " else:", + " consumer.assign(partitions)", + " consumer.seek_to_end()", + " producer = kafka.KafkaProducer(bootstrap_servers=broker)", + " headers = []", + " i = 0", + " while i + 1 < len(header_pairs):", + " headers.append((header_pairs[i], header_pairs[i + 1].encode('utf-8')))", + " i += 2", + " if correlation_header is not None:", + " headers.append((correlation_header, correlation_id.encode('utf-8')))", + " #a message with no payload of its own publishes an empty body", + " body = (payload if payload is not None else '').encode('utf-8')", + " producer.send(topic, value=body, headers=headers)", + " producer.flush()", + " if consumer is None:", + " return None", + " reply = None", + " deadline = time.time() + (timeout_ms / 1000.0)", + " while reply is None and time.time() < deadline:", + " for _, records in consumer.poll(timeout_ms=$DEFAULT_POLL_MS).items():", + " for r in records:", + " if correlation_header is None:", " reply = r.value.decode('utf-8')", - " consumer.close()", - " return reply" + " continue", + " for key, value in (r.headers or []):", + " if key == correlation_header and value.decode('utf-8') == correlation_id:", + " reply = r.value.decode('utf-8')", + " return reply", + " finally:", + " #a suite runs many of these, so a client left open would outlive every one of them", + " if producer is not None:", + " producer.close()", + " if consumer is not None:", + " consumer.close()" ) private fun kotlinHelper(): List = listOf( @@ -262,51 +314,60 @@ object KafkaTestClientEmitter { " props.put(\"enable.auto.commit\", \"false\")", " val correlationId = java.util.UUID.randomUUID().toString()", " var consumer: org.apache.kafka.clients.consumer.KafkaConsumer? = null", - " if (replyTopic != null) {", - " consumer = org.apache.kafka.clients.consumer.KafkaConsumer(props,", - " $STRING_SERDE.StringDeserializer(), $STRING_SERDE.StringDeserializer())", - " val parts = consumer.partitionsFor(replyTopic)", - " .map { org.apache.kafka.common.TopicPartition(it.topic(), it.partition()) }", - " consumer.assign(parts)", - " consumer.seekToEnd(parts)", - " //seekToEnd is lazy: asking for the position is what makes it take effect now", - " if (parts.isNotEmpty()) {", - " consumer.position(parts[0])", - " }", - " }", - " val producer = org.apache.kafka.clients.producer.KafkaProducer(props,", - " $STRING_SERDE.StringSerializer(), $STRING_SERDE.StringSerializer())", - " //a message with no payload of its own publishes an empty body", - " val record = org.apache.kafka.clients.producer.ProducerRecord(topic, payload ?: \"\")", - " var i = 0", - " while (i + 1 < headerPairs.size) {", - " record.headers().add(headerPairs[i], headerPairs[i + 1].toByteArray($UTF_8))", - " i += 2", - " }", - " if (correlationHeader != null) {", - " record.headers().add(correlationHeader, correlationId.toByteArray($UTF_8))", - " }", - " producer.send(record).get()", - " producer.close()", - " if (consumer == null) {", - " return null", - " }", - " var reply: String? = null", - " val deadline = System.currentTimeMillis() + timeoutMs", - " while (reply == null && System.currentTimeMillis() < deadline) {", - " for (r in consumer.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {", - " if (correlationHeader == null) {", - " reply = r.value()", - " continue", + " var producer: org.apache.kafka.clients.producer.KafkaProducer? = null", + " try {", + " if (replyTopic != null) {", + " val c = org.apache.kafka.clients.consumer.KafkaConsumer(props,", + " $STRING_SERDE.StringDeserializer(), $STRING_SERDE.StringDeserializer())", + " consumer = c", + " val parts = (c.partitionsFor(replyTopic) ?: listOf())", + " .map { org.apache.kafka.common.TopicPartition(it.topic(), it.partition()) }", + " if (parts.isEmpty()) {", + " //the reply destination does not exist yet, so nothing can be waiting on it", + " c.close()", + " consumer = null", + " } else {", + " c.assign(parts)", + " c.seekToEnd(parts)", + " //seekToEnd is lazy: asking for the position is what makes it take effect now", + " c.position(parts[0])", " }", - " val h = r.headers().lastHeader(correlationHeader)", - " if (h != null && correlationId == String(h.value(), $UTF_8)) {", - " reply = r.value()", + " }", + " val p = org.apache.kafka.clients.producer.KafkaProducer(props,", + " $STRING_SERDE.StringSerializer(), $STRING_SERDE.StringSerializer())", + " producer = p", + " //a message with no payload of its own publishes an empty body", + " val record = org.apache.kafka.clients.producer.ProducerRecord(topic, payload ?: \"\")", + " var i = 0", + " while (i + 1 < headerPairs.size) {", + " record.headers().add(headerPairs[i], headerPairs[i + 1].toByteArray($UTF_8))", + " i += 2", + " }", + " if (correlationHeader != null) {", + " record.headers().add(correlationHeader, correlationId.toByteArray($UTF_8))", + " }", + " p.send(record).get()", + " val c = consumer ?: return null", + " var reply: String? = null", + " val deadline = System.currentTimeMillis() + timeoutMs", + " while (reply == null && System.currentTimeMillis() < deadline) {", + " for (r in c.poll(java.time.Duration.ofMillis($DEFAULT_POLL_MS))) {", + " if (correlationHeader == null) {", + " reply = r.value()", + " continue", + " }", + " val h = r.headers().lastHeader(correlationHeader)", + " if (h != null && correlationId == String(h.value(), $UTF_8)) {", + " reply = r.value()", + " }", " }", " }", + " return reply", + " } finally {", + " //a suite runs many of these, so a client left open would outlive every one of them", + " producer?.close()", + " consumer?.close()", " }", - " consumer.close()", - " return reply", "}" ) /** @@ -320,6 +381,11 @@ object KafkaTestClientEmitter { val node = mapper.readTree(json) node.fieldNames().asSequence().associateWith { node.get(it).asText() } } catch (e: Exception) { + /* + The search recorded these, so this should not happen; said out loud rather than + quietly publishing a message without the headers the service may route on. + */ + LoggingUtil.uniqueWarn(log, "Could not read the headers recorded for a message: ${e.message}") mapOf() } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index b4e68dda6c..5b9180fabc 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -595,8 +595,14 @@ class TestSuiteWriter { lines.add("import unittest") lines.add("import requests") - if (config.problemType == EMConfig.ProblemType.ASYNCAPI) { - //what the helper that publishes and awaits a reply uses + /* + What the helper that publishes and awaits a reply uses, and only when something + in this suite calls it. Python cannot name a type without importing it, so a + suite that publishes over another transport would otherwise be made to carry a + Kafka dependency it never uses. + */ + if (config.problemType == EMConfig.ProblemType.ASYNCAPI + && KafkaTestClientEmitter.resultsIn(solution).isNotEmpty()) { lines.add("import time") lines.add("import uuid") lines.add("import kafka") diff --git a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt index 5e4c57edf6..795ae2cfa0 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt @@ -111,14 +111,16 @@ class SortingHelper { /** * Groups the tests of one operation together, then orders them by what publishing did: a - * reply that was recognised comes before silence, which is the AsyncAPI stand-in for - * ordering REST by status code. + * reply that arrived comes before silence, and within that by the declared message it was + * recognised as. This is the AsyncAPI stand-in for ordering REST by status code. */ private val asyncApiComparator: Comparator> = compareBy> { ind -> (ind.evaluatedMainActions().last().action as AsyncApiAction).operationId } .thenBy { ind -> - (ind.evaluatedMainActions().last().result as AsyncApiCallResult).getOutcome()?.ordinal ?: 0 + //an action with no outcome recorded never ran, so it belongs after the ones that did + (ind.evaluatedMainActions().last().result as AsyncApiCallResult).getOutcome()?.ordinal + ?: Int.MAX_VALUE } .thenBy { ind -> (ind.evaluatedMainActions().last().result as AsyncApiCallResult).getReplyMessage() ?: "" From c31a3033f6066601c3e295fbfc70f11cc10d6f4f Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 2 Oct 2026 18:27:16 -0300 Subject: [PATCH 11/20] AsyncAPI 3.x: check the format after the driver has named it The constraint that a suite can only be written in Java, Kotlin or Python was checked while the options were read, and skipped a DEFAULT because the driver had not been asked yet. For AsyncAPI that is the ordinary case in both modes, so a driver naming anything else reached the writer unchecked. The check moved to where it can be run twice, and is run again as soon as the driver's answer lands. Beside it, what the comments there claim. A driver cannot name Python at all, since the enum it answers with has no entry for one, and that is now said both where the format is resolved and in the option's own text. The DTO said the reply variable was null when no script was asked for, when it is set whenever tests are written. The module still said no tests were written yet. And getAsyncApiServerAddress said a generated test asks it, which only a suite that carries a driver can: a plain black-box one publishes to the address the document declares. A driver script of nothing but blank lines is no script, and was taken for one: the test published nothing and then asserted on a variable nothing had assigned. --- .../problem/asyncapi/AsyncApiActionDto.java | 14 +++++--- .../client/java/controller/SutHandler.java | 9 +++-- .../kotlin/org/evomaster/core/EMConfig.kt | 31 +++++++++++----- .../asyncapi/service/AsyncApiFitness.kt | 35 +++++++++---------- .../asyncapi/service/AsyncApiModule.kt | 4 +-- .../enterprise/service/EnterpriseSampler.kt | 2 ++ docs/options.md | 2 +- 7 files changed, 59 insertions(+), 38 deletions(-) diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java index b2a26b55a2..54da1e380a 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java @@ -100,19 +100,23 @@ public enum CorrelationLocation { public Long replyTimeoutMs; /** - * The language to render {@link AsyncApiReplyDto#testScript} in, or null when the core is - * not generating tests and the driver should not spend time rendering anything. + * The language to render {@link AsyncApiReplyDto#testScript} in, or null when no script is + * wanted: either the core is not generating tests, or it is generating a format this enum + * cannot name, which is what a Python run is. * * A generated test talks to the broker with an ordinary client of the transport rather than * through the driver, so the lines that publish and await have to come from the one side - * that knows the transport. Only Java and Kotlin are asked for. + * that knows the transport. Only Java and Kotlin are ever asked for. */ public SutInfoDto.OutputFormat outputFormat; /** * The name the rendered lines must leave the reply payload in, as text, when a reply is - * expected. Named by the core so that two actions in one test cannot collide. Null when no - * script was asked for. + * expected. Named by the core so that two actions in one test cannot collide. + * + * Set whenever tests are being generated, whether or not a script was asked for: the core + * owns the name, whoever ends up writing the lines that assign to it. Null when no tests + * are being generated. */ public String replyVariable; } diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java index 3d31b43e8c..0f189c395e 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java @@ -154,9 +154,12 @@ default void setupForGeneratedTest(){} * * The document gives an address, but it is written for the deployment the author had in * mind. A system started for testing is often somewhere else: a broker in a container binds - * a port chosen when it starts, and a different one on the next run. A generated test asks - * this rather than carrying an address that was only ever true once, in the same way it - * takes the base URL of a REST system from {@link #startSut()}. + * a port chosen when it starts, and a different one on the next run. + * + * A generated suite that carries the driver asks this when it starts, rather than holding an + * address that was only ever true once, in the same way it takes the base URL of a REST + * system from {@link #startSut()}. A plain black-box suite has no driver in it, so it + * publishes to the address the document declares, as it does when this returns null. * * @param serverName the key of the server in the document * @return where to reach it, or null to fall back to what the document says diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 624d1c883c..de4f4422fc 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -824,14 +824,7 @@ class EMConfig { throw ConfigProblemException("When using the seedTestCases option, you must specify the file path of the test cases with the seedTestCasesPath option") } - if (problemType == ProblemType.ASYNCAPI - && createTests - && outputFormat != OutputFormat.DEFAULT - && !outputFormat.isJavaOrKotlin() && !outputFormat.isPython()) { - throw ConfigProblemException("Tests for an AsyncAPI service publish with a client of the transport," + - " and one is written only for Java, Kotlin and Python so far, so outputFormat supports" + - " only those") - } + checkOutputFormatIsWritable() if (problemType == ProblemType.ASYNCAPI && seedTestCases) { throw ConfigProblemException("Seeding test cases is not supported for AsyncAPI services yet") @@ -1062,6 +1055,25 @@ class EMConfig { */ fun usesDriver() = !blackBox || bbExperiments || problemType == ProblemType.ASYNCAPI + /** + * Whether a test can be written at all in the chosen format. + * + * Checked here with the rest of the options, and again once a DEFAULT has been resolved from + * the driver, which happens after every check here has run. An AsyncAPI run in black-box + * mode takes that road as well, so this is the only place that sees what it ended up with. + */ + fun checkOutputFormatIsWritable() { + + if (problemType == ProblemType.ASYNCAPI + && createTests + && outputFormat != OutputFormat.DEFAULT + && !outputFormat.isJavaOrKotlin() && !outputFormat.isPython()) { + throw ConfigProblemException("Tests for an AsyncAPI service publish with a client of the transport," + + " and one is written only for Java, Kotlin and Python so far, so outputFormat supports" + + " only those. Was: $outputFormat") + } + } + fun activatedExperimentalFeatures(): List { val properties = getConfigurationProperties() @@ -1564,7 +1576,8 @@ class EMConfig { " If left to DEFAULT, the type will be inferred from the EM Driver." + " However, in case of ambiguities (e.g., the driver specifies more than one type)," + " then this field must be set with a specific type." + - " This is also the case for Black-Box testing where there is no EM Driver." + + " This is also the case for Black-Box testing, where there is usually no EM Driver" + + " (AsyncAPI is the exception: see outputFormat)." + " In this latter case, the system defaults to handle REST APIs.") var problemType = ProblemType.DEFAULT diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index 6d4bb79f18..261ea64ab0 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -236,7 +236,8 @@ class AsyncApiFitness : ApiWsFitness() { reply.waitedMs?.let { result.setWaitedMs(it) } //what the generated test will publish with, when the driver rendered it - reply.testScript?.takeIf { it.isNotEmpty() }?.let { result.setTestScript(it) } + //lines that are all blank are no script: the writer would publish nothing and assert on it + reply.testScript?.takeIf { lines -> lines.any { it.isNotBlank() } }?.let { result.setTestScript(it) } if (outcome == AsyncApiOutcome.REPLIED) { reply.replyPayload?.let { result.setReplyPayload(it) } @@ -393,15 +394,13 @@ class AsyncApiFitness : ApiWsFitness() { if (config.createTests) { /* - A generated test publishes with a client of the transport, which only the driver - has, so it renders those lines while the search runs. Not asked for when no test - will be written, so a driver need not spend time on it. - */ - /* - Only when the driver could render that language at all. It is Java, so it knows - nothing of the formats a JVM driver cannot produce, and asking would fail. The - variable is still named here either way: the core owns that name, whoever writes - the lines that assign to it. + A transport the contract cannot describe is published by lines the driver renders + while the search runs, so it is told which language to render them in. Not asked + for when no test will be written, so a driver need not spend time on it. + + SutInfoDto.OutputFormat is the driver's own enum and has no Python, so a Python + run asks for no script at all. The reply variable is named here either way: the + core owns that name, whoever writes the lines that assign to it. */ dto.outputFormat = SutInfoDto.OutputFormat.values() .firstOrNull { it.name == config.outputFormat.name } @@ -429,14 +428,6 @@ class AsyncApiFitness : ApiWsFitness() { return dto } - /** - * The headers to publish alongside the body: key is the header name as the document declares - * it, value is what to send under it, as text. - * - * They are read back from the gene's own JSON printing, which is what knows which optional - * headers are on. A header whose value prints as JSON null is left out rather than sent as - * the text "null". - */ /** * Keep on the result what a generated test needs to publish this message again, so that the * writer does not resolve the document a second time. Only when a test will be written. @@ -477,6 +468,14 @@ class AsyncApiFitness : ApiWsFitness() { ?: DEFAULT_CORRELATION_HEADER } + /** + * The headers to publish alongside the body: key is the header name as the document declares + * it, value is what to send under it, as text. + * + * They are read back from the gene's own JSON printing, which is what knows which optional + * headers are on. A header whose value prints as JSON null is left out rather than sent as + * the text "null". + */ private fun buildHeaders(action: AsyncApiAction): Map { val headers = LinkedHashMap() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt index 1446283249..13d16dae1c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModule.kt @@ -25,8 +25,8 @@ import org.evomaster.core.search.service.mutator.StructureMutator * it is what holds the connection to the broker, so it takes part either way (see * [org.evomaster.core.EMConfig.usesDriver]). * - * No test cases are written yet, which [org.evomaster.core.EMConfig] enforces by requiring - * `--createTests false`. + * The tests a run writes are the [org.evomaster.core.output.service.AsyncApiTestCaseWriter]'s, + * which publishes with a client of the transport rather than through the driver. */ class AsyncApiModule : EnterpriseModule() { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt index 61bd4adc81..e5a1ca556c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt @@ -89,6 +89,8 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { } catch (e: Exception) { throw SutProblemException("Failed to use test output format: " + infoDto.defaultOutputFormat) } + //what the driver asked for still has to be a format a test can be written in + config.checkOutputFormatIsWritable() } // only check this configuration if DB exists diff --git a/docs/options.md b/docs/options.md index 8946898ec3..6b9aba71a5 100644 --- a/docs/options.md +++ b/docs/options.md @@ -202,7 +202,7 @@ There are 3 types of options: |`probRestDefault`| __Double__. In REST, specify probability of using 'default' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.05`.| |`probRestExamples`| __Double__. In REST, specify probability of using 'example(s)' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.| |`probUseRestLinks`| __Double__. In REST, enable the supports of 'links' between resources defined in the OpenAPI schema, if any. When sampling a test case, if the last call has links, given this probability new calls are added for the link. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| -|`problemType`| __Enum__. The type of SUT we want to generate tests for, e.g., a RESTful API. If left to DEFAULT, the type will be inferred from the EM Driver. However, in case of ambiguities (e.g., the driver specifies more than one type), then this field must be set with a specific type. This is also the case for Black-Box testing where there is no EM Driver. In this latter case, the system defaults to handle REST APIs. *Valid values*: `DEFAULT, REST, GRAPHQL`. *Experimental values*: `RPC, WEBFRONTEND, MCP, ASYNCAPI`. *Default value*: `DEFAULT`.| +|`problemType`| __Enum__. The type of SUT we want to generate tests for, e.g., a RESTful API. If left to DEFAULT, the type will be inferred from the EM Driver. However, in case of ambiguities (e.g., the driver specifies more than one type), then this field must be set with a specific type. This is also the case for Black-Box testing, where there is usually no EM Driver (AsyncAPI is the exception: see outputFormat). In this latter case, the system defaults to handle REST APIs. *Valid values*: `DEFAULT, REST, GRAPHQL`. *Experimental values*: `RPC, WEBFRONTEND, MCP, ASYNCAPI`. *Default value*: `DEFAULT`.| |`processFiles`| __String__. Specify a folder to save results when a search monitor is enabled. *DEBUG option*. *Default value*: `process_data`.| |`processFormat`| __Enum__. Specify a format to save the process data. *DEBUG option*. *Valid values*: `JSON_ALL, TEST_IND, TARGET_TEST_IND, TARGET_HEURISTIC`. *Default value*: `JSON_ALL`.| |`processInterval`| __Double__. Specify how often to save results when a search monitor is enabled, and 0.0 presents to record all evaluated individual. *DEBUG option*. *Constraints*: `min=0.0, max=50.0`. *Default value*: `0.0`.| From dafcc50762fd18f2b81ce045788ba7a09588fdd0 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 2 Oct 2026 18:27:16 -0300 Subject: [PATCH 12/20] AsyncAPI 3.x: say which half of a run each page is about The driver publishes while the search runs; the tests it leaves behind do not use it. write_driver said a generated test asks the driver where a server is, without saying that only a suite carrying one can. blackbox put its note between a bullet and the "It" that referred to it, where Markdown reads it as part of that bullet anyway. library_dependencies listed what a JVM suite needs but not what a Python one does, which is kafka-python. The section heading now matches its neighbours, as does the name the page uses for the tool. --- docs/blackbox.md | 6 +++--- docs/library_dependencies.md | 7 ++++++- docs/write_driver.md | 24 +++++++++++++----------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/docs/blackbox.md b/docs/blackbox.md index 8f515147ce..b990fe25e1 100644 --- a/docs/blackbox.md +++ b/docs/blackbox.md @@ -32,9 +32,9 @@ The command is doing the following: or [built from source](build.md) (in which case it will end up under the `core/target/` folder of the cloned repository). Alternatively, you can just run `evomaster` if it was installed via `pip`. * `--blackBox true`: technically, a redundant parameter, as black-box testing is the default mode since version 6.0.0. It can be omitted. -Note that AsyncAPI services still need an [_EvoMaster Driver_](write_driver.md) in this mode: there is no universal -client for a message broker to publish through while the search runs. The generated tests do not need one, as they -publish with a client of the transport themselves. + Note that AsyncAPI services still need an [_EvoMaster Driver_](write_driver.md) in this mode: there is no + universal client for a message broker to publish through while the search runs. The generated tests do not + need one, as they publish with a client of the transport themselves. * `--schema ...`: URL of where the OpenAPI/Swagger schema is. The location of the API will be inferred from this schema (e.g., from `host` and `servers` tags). If such info is missing, then the API is assumed to be on same host as the schema. If needed, the API host location can be changed with the optional `--base` (which overrides what specified in the schema). * `--outputFormat JAVA_JUNIT_4`: must specify how the tests will be generated, e.g., in Java using JUnit 4 in this case. Note: the language of the generated tests is not necessarily related diff --git a/docs/library_dependencies.md b/docs/library_dependencies.md index 151eea0605..a72c2d3521 100644 --- a/docs/library_dependencies.md +++ b/docs/library_dependencies.md @@ -85,7 +85,7 @@ than through the _EvoMaster Driver_, so that client library is needed as well. F org.apache.kafka kafka-clients - USE_LATEST_VERSION + LATEST test ``` @@ -123,6 +123,11 @@ urllib3==1.26.5 requests==2.25.1 timeout-decorator==0.5.0 ``` +When fuzzing an **AsyncAPI** service over Kafka, the generated tests publish with `kafka-python`, so that +is needed as well: +``` +kafka-python==2.0.2 +``` These can then be saved in a `requirements.txt` file, and installed with: ``` diff --git a/docs/write_driver.md b/docs/write_driver.md index 36c1e0d8dc..b450df0221 100644 --- a/docs/write_driver.md +++ b/docs/write_driver.md @@ -2,11 +2,12 @@ To generate tests for [white-box testing](whitebox.md), you need an _EvoMaster Driver_ up and running before executing `evomaster.jar`. -One kind of API needs a driver in _black-box_ mode as well: AsyncAPI services, where there is no universal -client for a message broker to point the fuzzer at. See [AsyncAPI](#asyncapi) below. These drivers have to be built manually for each system under test (SUT). See the [EMB repository](https://github.com/WebFuzzing/EMB) for a set of existing SUTs with drivers. +One kind of API needs a driver in _black-box_ mode as well: AsyncAPI services, where there is no universal +client for a message broker for _EvoMaster_ to publish through. See [AsyncAPI Schema](#asyncapi-schema) below. + To build a client driver in Java (or any JVM language), you need to import the _EvoMaster_ Java client library. For example, in Maven: @@ -361,7 +362,7 @@ To test a GraphQL API, in the the `getProblemInfo()`, you need to return an inst Here, you need to specify the endpoint of where the GraphQL API can be accessed. Default value is `/graphql`. Note: must be able to do an _introspective query_ on such API to fetch its schema. If this is disabled for security reasons, _EvoMaster_ will fail. -## AsyncAPI +## AsyncAPI Schema To test an AsyncAPI service, in the `getProblemInfo()` you need to return an instance of the `AsyncApiProblem` class, built from the AsyncAPI document itself, either its text or where to find it. @@ -372,20 +373,21 @@ Unlike REST and GraphQL, **a driver is needed in black-box mode too**. There is REST API, but no one client for every broker: the transport may be Kafka, AMQP, MQTT or a socket, and they share nothing. So publishing a message is the driver's job, by overriding `executeAsyncApiAction`. What is asked of it is only to put the message on the wire and report what came back, never to judge it: deciding -what an outcome means is the fuzzer's job, so that it means the same thing whatever the transport. +what an outcome means is _EvoMaster_'s job, so that it means the same thing whatever the transport. Two things are worth knowing when writing one: -- **A reply published before the action was is never an answer to it.** Only the driver can ensure that, - by seeking to the end of the reply destination before publishing, or by a fresh subscription. The fuzzer - does not vary its correlation ids between runs to compensate, because a run repeated under the same seed - has to behave the same way. +- **A reply that was already on the destination before the action published is never an answer to it.** + Only the driver can ensure that, by seeking to the end of the reply destination before publishing, or by + a fresh subscription. _EvoMaster_ does not vary its correlation ids between runs to compensate, because a + run repeated under the same seed has to behave the same way. - **Where a server actually is may not be what the document says.** The document gives the address of the deployment its author had in mind; a system started for testing is often elsewhere, and a broker in a container binds a port chosen at start-up. Override `getAsyncApiServerAddress(String serverName)` to say - where it is now. Generated tests ask for it there, in the same way they take the base URL of a REST - system from `startSut()`. Returning `null`, which is the default, means the document's own address is - used. + where it is now. A generated suite that carries the driver -- white-box, or black-box with + `--bbExperiments` -- asks for it when it starts, in the same way it takes the base URL of a REST system + from `startSut()`. A plain black-box suite has no driver in it, so it publishes to the address the + document declares, as it does when this returns `null`, which is the default. Generated tests publish with a client of the transport rather than through the driver, so the suite needs that client library on its classpath: see [library dependencies](library_dependencies.md). From 788d6d77f76356bb077fa33c0dd3bea3e74c9fc5 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 2 Oct 2026 18:27:26 -0300 Subject: [PATCH 13/20] AsyncAPI 3.x: tests for what the suite writer emits AsyncApiReplyAssertions had three hundred lines and one payload shape reaching them; it now has a test of its own, driving every branch across the three languages, which is what caught the Python forms. The sorter had none at all, because sorting happens only when a suite is written and no unit test wrote one. The Java half of the compile step had none either, and only a run with a broker exercised it. Two existing tests were not checking what they claimed. The Kafka body test asked for the body before the class members, which is the reverse of what the suite writer does, so it ran with no servers known and pinned the black-box shape while configured white-box; it now runs in the suite writer's order and asserts the variable a white-box suite actually publishes to. And asserting that a body contains "bessj" could not fail, as the fake driver's own script renders the topic names. The name-with-or-without-the-fault test compared a name with itself if the fault stopped being reported, so it now checks the fault is there first. The Python test skips where there is no python3 rather than erroring, and the files both tests write are removed whatever happens. --- .../compiler/CompilerForTestGeneratedTest.kt | 37 ++++ .../AsyncApiActionNamingStrategyTest.kt | 20 +- .../service/AsyncApiReplyAssertionsTest.kt | 187 ++++++++++++++++++ .../output/sorting/AsyncApiSortingTest.kt | 100 ++++++++++ .../service/AsyncApiTestCaseWriterTest.kt | 62 ++++-- 5 files changed, 386 insertions(+), 20 deletions(-) create mode 100644 core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt create mode 100644 core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt diff --git a/core/src/test/kotlin/org/evomaster/core/output/compiler/CompilerForTestGeneratedTest.kt b/core/src/test/kotlin/org/evomaster/core/output/compiler/CompilerForTestGeneratedTest.kt index 512332cf25..914c50b0df 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/compiler/CompilerForTestGeneratedTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/compiler/CompilerForTestGeneratedTest.kt @@ -20,6 +20,43 @@ class CompilerForTestGeneratedTest{ } + @Test + fun testCompileJavaCode(){ + + /* + The Java half, which the AsyncAPI E2E is otherwise the only thing that exercises -- + and that one needs Docker and a broker to say anything at all. + */ + val name = "FooCompileJavaCodeClass" + val msg = "Hello World!" + val code = "public class $name { public String exe() { return \"$msg\"; } }" + + CompilerForTestGenerated.compile( + OutputFormat.JAVA_JUNIT_5, + code, + name + ) + + val klass = this.javaClass.classLoader.loadClass(name) + val obj = klass.newInstance() + val res = klass.getDeclaredMethod("exe").invoke(obj) as String + + assertEquals(msg, res) + } + + @Test + fun testJavaThatDoesNotCompileSaysWhy(){ + + val name = "FooBrokenJavaClass" + val code = "public class $name { this is not Java }" + + val error = assertThrows(RuntimeException::class.java) { + CompilerForTestGenerated.compile(OutputFormat.JAVA_JUNIT_5, code, name) + } + + assertTrue(error.message!!.contains("Failed to compile"), error.message) + } + @Test fun testCompileCode(){ diff --git a/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt b/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt index 56fe92c5a2..91a40077a1 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt @@ -18,7 +18,10 @@ import org.evomaster.core.search.Solution import org.evomaster.core.search.service.FitnessFunction import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.SearchGlobalState +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -71,7 +74,22 @@ class AsyncApiActionNamingStrategyTest { would change with a flag that is about reporting. */ val withoutOracles = namesOf({ FakeAsyncApiDriver.silence() }, "bessj") - val withOracles = namesOf({ FakeAsyncApiDriver.silence() }, "bessj", "--useExperimentalOracles=true") + + val reported = solutionOf({ FakeAsyncApiDriver.silence() }, "bessj", + arrayOf("--useExperimentalOracles=true")) + val withOracles = AsyncApiActionTestCaseNamingStrategy(reported, formatter, MAX_NAME_LENGTH) + .getTestCases() + .map { it.name } + + /* + The flag has to have done something, or both halves would be taking the same branch + and this would be comparing a name with itself. + */ + assertTrue( + DetectedFaultUtils.getDetectedFaultCategories(reported.individuals.first()) + .contains(ExperimentalFaultCategory.ASYNCAPI_NO_REPLY), + "the no-reply fault was not reported, so the fault branch is not what was named" + ) assertEquals(listOf("test_0_publishOnBessjGetsNoReply"), withoutOracles) assertEquals(withoutOracles, withOracles) diff --git a/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt b/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt new file mode 100644 index 0000000000..3be93c7ec5 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt @@ -0,0 +1,187 @@ +package org.evomaster.core.output.service + +import org.evomaster.core.EMConfig +import org.evomaster.core.output.Lines +import org.evomaster.core.output.OutputFormat +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +/** + * What a reply becomes, as assertions, in each language a suite can be written in. + * + * The three forms are spelled out rather than shared, because the point of this is to catch the + * one that was forgotten: a Python suite is only ever parsed, never run, by the tests here. + */ +class AsyncApiReplyAssertionsTest { + + private val variable = "res_0" + + private val body = "body_0" + + private fun emit(payload: String?, format: OutputFormat, config: EMConfig = EMConfig()): String { + val lines = Lines(format) + AsyncApiReplyAssertions.emit(lines, payload, variable, body, config, format) + return lines.toString() + } + + @Test + fun testNothingIsSaidWhenAssertionsAreOff() { + + val config = EMConfig().apply { enableBasicAssertions = false } + + OutputFormat.values().filter { it.isJavaOrKotlin() || it.isPython() }.forEach { + assertEquals("", emit("""{"a": 1}""", it, config).trim(), "for $it") + } + } + + @Test + fun testAnEmptyReplyIsAssertedInTheLanguageOfTheSuite() { + + /* + The case that was wrong: Python was handed Kotlin's isNullOrEmpty, which parses and + then fails at run time with an AttributeError. + */ + assertTrue(emit("", OutputFormat.KOTLIN_JUNIT_5).contains("assertTrue(res_0.isNullOrEmpty())")) + assertTrue(emit("", OutputFormat.JAVA_JUNIT_5).contains("assertTrue(res_0 == null || res_0.isEmpty());")) + + val python = emit("", OutputFormat.PYTHON_UNITTEST) + assertTrue(python.contains("assert res_0 is None or len(res_0) == 0"), python) + assertFalse(python.contains("isNullOrEmpty"), python) + } + + @Test + fun testAFieldIsReadTheWayItsLanguageReadsOne() { + + val payload = """{"name": "abc", "n": 7}""" + + val kotlin = emit(payload, OutputFormat.KOTLIN_JUNIT_5) + assertTrue(kotlin.contains("""assertTrue(body_0.has("name"))"""), kotlin) + assertTrue(kotlin.contains("""assertEquals("abc", body_0.get("name").asText())"""), kotlin) + assertTrue(kotlin.contains("""assertEquals(7, body_0.get("n").asLong())"""), kotlin) + + val python = emit(payload, OutputFormat.PYTHON_UNITTEST) + assertTrue(python.contains("""body_0 = json.loads(res_0)"""), python) + assertTrue(python.contains(""""name" in body_0"""), python) + assertTrue(python.contains("""body_0["name"] == "abc""""), python) + //a dict is not a tree of nodes, so nothing asks it for a typed accessor + assertFalse(python.contains("asText()"), python) + } + + @Test + fun testAFieldNamedLikeATemplateDoesNotBreakTheSuite() { + + /* + '$ref' is an ordinary JSON field name, and Kotlin would read it as the start of a + template unless it is escaped where it is written. + */ + val kotlin = emit("""{"${'$'}ref": "x"}""", OutputFormat.KOTLIN_JUNIT_5) + + assertTrue(kotlin.contains("""has("\${'$'}ref")"""), kotlin) + assertFalse(kotlin.contains("""has("${'$'}ref")"""), kotlin) + } + + @Test + fun testANumberOutsideADoubleIsNotAssertedOn() { + + //Jackson reads it as an infinity, which is not a literal in any of the three languages + val emitted = emit("""{"x": 1e400}""", OutputFormat.KOTLIN_JUNIT_5) + + assertFalse(emitted.contains("Infinity"), emitted) + assertTrue(emitted.contains("outside the range of a double"), emitted) + } + + @Test + fun testADoubleIsComparedWithinATolerance() { + + assertTrue(emit("""{"x": 1.5}""", OutputFormat.KOTLIN_JUNIT_5).contains("0.001")) + assertTrue(emit("""{"x": 1.5}""", OutputFormat.PYTHON_UNITTEST).contains("abs(body_0[\"x\"] - 1.5) < 0.001")) + } + + @Test + fun testOnlyTheFirstFewElementsOfACollectionAreAsserted() { + + val config = EMConfig().apply { maxAssertionForDataInCollection = 2 } + val emitted = emit("""{"xs": [1, 2, 3, 4]}""", OutputFormat.KOTLIN_JUNIT_5, config) + + assertTrue(emitted.contains("body_0.get(\"xs\").size()"), emitted) + assertTrue(emitted.contains("get(0)"), emitted) + assertTrue(emitted.contains("get(1)"), emitted) + assertFalse(emitted.contains("get(2)"), emitted) + } + + @Test + fun testAnEmptyCollectionIsAssertedEmptyInEachLanguage() { + + //an empty array is its size, which is how a full one is asserted too + assertTrue(emit("""{"xs": []}""", OutputFormat.KOTLIN_JUNIT_5).contains("""assertEquals(0, body_0.get("xs").size())""")) + assertTrue(emit("""{"xs": []}""", OutputFormat.PYTHON_UNITTEST).contains("""assert len(body_0["xs"]) == 0""")) + + //an empty object is asked whether it is empty, which each language spells its own way + assertTrue(emit("""{"o": {}}""", OutputFormat.KOTLIN_JUNIT_5).contains("""assertTrue(body_0.get("o").isEmpty)""")) + assertTrue(emit("""{"o": {}}""", OutputFormat.JAVA_JUNIT_5).contains("""assertTrue(body_0.get("o").isEmpty());""")) + assertTrue(emit("""{"o": {}}""", OutputFormat.PYTHON_UNITTEST).contains("""assert len(body_0["o"]) == 0""")) + } + + @Test + fun testTheFieldsThatChangeBetweenRunsAreSkipped() { + + val emitted = emit("""{"id": 7, "timestamp": 1, "self": "x", "kept": 2}""", OutputFormat.KOTLIN_JUNIT_5) + + /* + That the field is there is stable and is asserted; what it holds is not, so only the + value is left alone. This is what the REST writer does with the same three names. + */ + assertTrue(emitted.contains("""assertTrue(body_0.has("id"))"""), emitted) + listOf("id", "timestamp", "self").forEach { + assertFalse(emitted.contains("""body_0.get("$it")"""), emitted) + assertTrue(emitted.contains("$it is not asserted on"), emitted) + } + + assertTrue(emitted.contains("""assertEquals(2, body_0.get("kept").asLong())"""), emitted) + } + + @Test + fun testAValueThatIsNotStableBetweenRunsBecomesAComment() { + + //an address with a port is the REST writer's own example of what not to assert on + val emitted = emit("""{"where": "localhost:12345"}""", OutputFormat.KOTLIN_JUNIT_5) + + assertFalse(emitted.contains("localhost:12345"), emitted) + assertTrue(emitted.contains("not stable between runs"), emitted) + } + + @Test + fun testAReplyThatIsNotJsonIsAssertedOnAsText() { + + val emitted = emit("just text", OutputFormat.KOTLIN_JUNIT_5) + + assertTrue(emitted.contains("just text"), emitted) + assertFalse(emitted.contains("readTree"), emitted) + } + + @Test + fun testABooleanAndANullReadAsTheirOwnLanguageWritesThem() { + + val payload = """{"ok": true, "none": null}""" + + assertTrue(emit(payload, OutputFormat.KOTLIN_JUNIT_5).contains("assertTrue(body_0.get(\"none\").isNull)")) + assertTrue(emit(payload, OutputFormat.JAVA_JUNIT_5).contains("assertTrue(body_0.get(\"none\").isNull());")) + + val python = emit(payload, OutputFormat.PYTHON_UNITTEST) + assertTrue(python.contains("body_0[\"none\"] is None"), python) + //Python writes it True, not true + assertTrue(python.contains("== True"), python) + } + + @Test + fun testAReplyLargerThanTheThresholdIsNotAssertedOn() { + + val config = EMConfig().apply { maxResponseByteSize = 10 } + val emitted = emit("""{"x": "aaaaaaaaaaaaaaaaaaaaaaaaa"}""", OutputFormat.KOTLIN_JUNIT_5, config) + + assertTrue(emitted.contains("larger than the threshold"), emitted) + assertFalse(emitted.contains("assertTrue"), emitted) + assertFalse(emitted.contains("assertEquals"), emitted) + assertFalse(emitted.contains("readTree"), emitted) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt b/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt new file mode 100644 index 0000000000..12e23d643e --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt @@ -0,0 +1,100 @@ +package org.evomaster.core.output.sorting + +import com.google.inject.Key +import com.google.inject.TypeLiteral +import com.webfuzzing.asyncapi.access.AsyncApiAccess +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.problem.asyncapi.data.AsyncApiAction +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual +import org.evomaster.core.problem.asyncapi.service.AsyncApiSampler +import org.evomaster.core.problem.asyncapi.service.AsyncApiTestInjector +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver +import org.evomaster.core.problem.enterprise.SampleType +import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.Randomness +import org.evomaster.core.search.service.SearchGlobalState +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +/** + * The order the tests of an AsyncAPI suite end up in. + * + * Sorting happens only when a suite is written, which is why no other test reaches this: the + * comparator used to throw on an AsyncAPI individual, and nothing noticed until a whole run did. + */ +class AsyncApiSortingTest { + + companion object { + private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" + + private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" + + private const val ERROR = """{"code": 400, "message": "no"}""" + } + + @BeforeEach + fun reset() { + RestActionBuilderV3.cleanCache() + } + + @Test + fun testTestsAreGroupedByOperationThenByWhatPublishingDid() { + + val silent = evaluate("expint") { FakeAsyncApiDriver.silence() } + val replied = evaluate("bessj") { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } + val error = evaluate("expint") { FakeAsyncApiDriver.replied(ERROR) } + + val tests = mutableListOf(silent, error, replied) + + SortingHelper().sort(tests, SortingStrategy.TARGET_INCREMENTAL) + + /* + bessj before expint, as the operation groups first; then within expint the reply that + arrived before the silence, as REPLIED comes before NO_REPLY. + */ + assertEquals(listOf("bessj", "expint", "expint"), tests.map { operationOf(it) }) + assertEquals(replied, tests[0]) + assertEquals(error, tests[1]) + assertEquals(silent, tests[2]) + } + + @Test + fun testSortingASingleTestDoesNotThrow() { + + val tests = mutableListOf(evaluate("bessj") { FakeAsyncApiDriver.replied(DOUBLE_RESULT) }) + + SortingHelper().sort(tests, SortingStrategy.TARGET_INCREMENTAL) + + assertEquals(1, tests.size) + } + + private fun operationOf(evaluated: EvaluatedIndividual<*>) = + (evaluated.evaluatedMainActions().last().action as AsyncApiAction).operationId + + private fun evaluate( + operation: String, + answer: (AsyncApiActionDto) -> AsyncApiReplyDto? + ): EvaluatedIndividual { + + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS)), answer) + val injector = AsyncApiTestInjector.create(driver, "--blackBox=false") + + val sampler = injector.getInstance(AsyncApiSampler::class.java) + val fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + val randomness = injector.getInstance(Randomness::class.java) + + val template = sampler.seeAvailableActions().first { it.getName() == operation } + val action = (template.copy() as AsyncApiAction).apply { doInitialize(randomness) } + + val individual = AsyncApiIndividual(SampleType.RANDOM, mutableListOf(action)).apply { + doGlobalInitialize(injector.getInstance(SearchGlobalState::class.java)) + } + + return fitness.calculateCoverage(individual, modifiedSpec = null) + ?: throw IllegalStateException("the fitness gave up on the individual") + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index 4affe1cc00..8e6f6c10d3 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -25,6 +25,7 @@ import org.evomaster.core.search.service.FitnessFunction import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.SearchGlobalState import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Assumptions import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -148,7 +149,7 @@ class AsyncApiTestCaseWriterTest { private fun membersOf(evaluated: EvaluatedIndividual): String { val writer = injector.getInstance(TestCaseWriter::class.java) val solution = Solution(mutableListOf(evaluated), "Prefix", "Suffix", Termination.NONE, listOf(), listOf()) - val lines = Lines(OutputFormat.KOTLIN_JUNIT_5) + val lines = Lines(injector.getInstance(EMConfig::class.java).outputFormat) writer.addExtraClassMembers(lines, solution) return lines.toString() } @@ -174,7 +175,7 @@ class AsyncApiTestCaseWriterTest { //and what only the core knows about the reply assertTrue(body.contains("assertNotNull(res_0)"), body) - assertTrue(body.contains("bessj"), body) + assertTrue(body.contains("bessj: REPLIED"), body) } @Test @@ -212,23 +213,29 @@ class AsyncApiTestCaseWriterTest { start { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } val evaluated = evaluate("bessj") - val body = bodyOf(evaluated) - //one line per action: everything it needs came off the contract - assertTrue(body.contains("${KafkaTestClientEmitter.HELPER_NAME}("), body) - assertTrue(body.contains("\"localhost:9092\""), body) - assertTrue(body.contains("\"ncs.bessj.request\""), body) - assertTrue(body.contains("\"ncs.bessj.reply\""), body) - assertTrue(body.contains("\"correlationId\""), body) - assertTrue(body.contains("assertNotNull(res_0)"), body) - - //and what it calls is written once for the suite + /* + The members first, as the suite writer writes them: that is where the servers of a + suite are learnt, and a body written before them would not know they exist. + */ val members = membersOf(evaluated) assertTrue(members.contains("KafkaProducer"), members) assertTrue(members.contains("KafkaConsumer"), members) //a reply older than the test is not an answer to it, and each run stamps its own id assertTrue(members.contains("seekToEnd"), members) assertTrue(members.contains("UUID.randomUUID()"), members) + assertTrue(members.contains("asyncApiServer_kafka"), members) + + val body = bodyOf(evaluated) + + //one line per action: everything it needs came off the contract + assertTrue(body.contains("${KafkaTestClientEmitter.HELPER_NAME}(asyncApiServer_kafka,"), body) + assertTrue(body.contains("\"ncs.bessj.request\""), body) + assertTrue(body.contains("\"ncs.bessj.reply\""), body) + assertTrue(body.contains("\"correlationId\""), body) + assertTrue(body.contains("assertNotNull(res_0)"), body) + //the suite has a driver, so the document's address is not what a test publishes to + assertFalse(body.contains("\"localhost:9092\""), body) } @Test @@ -267,20 +274,36 @@ class AsyncApiTestCaseWriterTest { assertTrue(members.contains("kafka.KafkaProducer"), members) assertTrue(members.contains("seek_to_end"), members) + //there is no driver in a Python suite, so nothing is declared to ask one + assertFalse(members.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), members) + assertFalse(body.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), body) + + //and the driver was asked for no script, as its own enum cannot name Python + assertNull(driver.published.last().outputFormat) + //what comes out has to be parseable Python, indentation included val module = (members.trimEnd().lines() + body.trimEnd().lines()) .joinToString("\n") { " " + it } val file = java.io.File.createTempFile("asyncapi_suite", ".py") file.writeText("import json\nimport time\nimport uuid\nimport kafka\n\n\nclass Suite:\n$module\n") - val process = ProcessBuilder("python3", "-m", "py_compile", file.absolutePath) - .redirectErrorStream(true) - .start() - val output = process.inputStream.bufferedReader().readText() - val code = process.waitFor() + try { + val process = try { + ProcessBuilder("python3", "-m", "py_compile", file.absolutePath) + .redirectErrorStream(true) + .start() + } catch (e: java.io.IOException) { + //no Python on this machine: the rest of the suite has nothing to do with one + Assumptions.assumeTrue(false, "python3 is not on the PATH") + return + } + val output = process.inputStream.bufferedReader().readText() + val code = process.waitFor() - file.delete() - assertEquals(0, code, "the generated Python does not parse:\n$output\n\n$module") + assertEquals(0, code, "the generated Python does not parse:\n$output\n\n$module") + } finally { + file.delete() + } } @Test @@ -292,6 +315,7 @@ class AsyncApiTestCaseWriterTest { body is ours. Written to disk and read back, rather than asserted on fragments. */ val folder = java.nio.file.Files.createTempDirectory("asyncapi_suite").toFile() + folder.deleteOnExit() start { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } From 30fcca8bc0b917cc89fbdc9fae93b518457425f4 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Fri, 2 Oct 2026 18:31:53 -0300 Subject: [PATCH 14/20] AsyncAPI 3.x: one path to the document the tests all read Four test classes each declared the same resource path, and one of them imported the same name twice. The injector already holds it, so they take it from there. The module test said the writer pastes the driver's lines, which is the fallback rather than what it usually does. --- .../output/naming/AsyncApiActionNamingStrategyTest.kt | 4 +--- .../evomaster/core/output/sorting/AsyncApiSortingTest.kt | 4 +--- .../core/problem/asyncapi/service/AsyncApiModuleTest.kt | 3 +-- .../asyncapi/service/AsyncApiTestCaseWriterTest.kt | 8 +++----- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt b/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt index 91a40077a1..b9625d569a 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt @@ -33,8 +33,6 @@ import org.junit.jupiter.api.Test class AsyncApiActionNamingStrategyTest { companion object { - private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" - private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" private val outputFormat = OutputFormat.KOTLIN_JUNIT_5 @@ -127,7 +125,7 @@ class AsyncApiActionNamingStrategyTest { options: Array = arrayOf() ): Solution { - val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS)), answer) + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS)), answer) val injector = AsyncApiTestInjector.create(driver, "--blackBox=false", *options) val sampler = injector.getInstance(AsyncApiSampler::class.java) diff --git a/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt b/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt index 12e23d643e..79dae9dfab 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt @@ -29,8 +29,6 @@ import org.junit.jupiter.api.Test class AsyncApiSortingTest { companion object { - private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" - private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" private const val ERROR = """{"code": 400, "message": "no"}""" @@ -80,7 +78,7 @@ class AsyncApiSortingTest { answer: (AsyncApiActionDto) -> AsyncApiReplyDto? ): EvaluatedIndividual { - val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS)), answer) + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS)), answer) val injector = AsyncApiTestInjector.create(driver, "--blackBox=false") val sampler = injector.getInstance(AsyncApiSampler::class.java) diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt index 91b6e57cc6..dbf4dca86f 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt @@ -15,7 +15,6 @@ import org.evomaster.core.search.service.IdMapper import org.evomaster.core.output.service.AsyncApiTestCaseWriter import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.output.service.TestSuiteWriter -import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.FitnessFunction import org.evomaster.core.search.service.FlakinessDetector @@ -154,7 +153,7 @@ class AsyncApiModuleTest { injector.getInstance(TestSuiteWriter::class.java) injector.getInstance(RemoteController::class.java) - //the writer that pastes the driver's own publish-and-await lines + //the writer that writes the publish-and-await lines, from the contract or from the driver assertTrue(injector.getInstance(TestCaseWriter::class.java) is AsyncApiTestCaseWriter) } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index 8e6f6c10d3..ab3240ec83 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -36,8 +36,6 @@ import org.junit.jupiter.api.Test class AsyncApiTestCaseWriterTest { companion object { - private const val NCS = "/asyncapi/sut/ncs-kafka.yaml" - private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" /** @@ -118,7 +116,7 @@ class AsyncApiTestCaseWriterTest { } private fun start(answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) { - driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS)), answer) + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS)), answer) injector = AsyncApiTestInjector.create( driver, "--blackBox=false", @@ -247,7 +245,7 @@ class AsyncApiTestCaseWriterTest { that what comes out parses: indentation is part of the language, so a helper written as text can break it in a way no JVM format can. */ - driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS))) { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } injector = AsyncApiTestInjector.create( @@ -365,7 +363,7 @@ class AsyncApiTestCaseWriterTest { A driver that works the address out, rather than keeping one, would then do that work twice every time a suite starts. */ - driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS))) { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } injector = AsyncApiTestInjector.create( From 02efa0d41cd78567bd0cbadeaed39eea715f0e8c Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Sun, 4 Oct 2026 08:22:54 -0300 Subject: [PATCH 15/20] AsyncAPI 3.x: a black-box run can write its suite at all Writing a suite in black-box mode threw: the shared writer asks BlackBoxUtils for the base URL to put in it, and that answered "Black-box testing is currently not supported for ASYNCAPI". Which is the one mode this problem type is unusual in -- a driver publishes while the search runs, because nothing else can, but the suite that run leaves behind has none -- so it is the mode the option text and the black-box page both describe. Nothing had caught it because every writer test drove the test writer directly, and the suite writer is what calls this. A message-driven service has no one base URL to resolve: each server the document names has its own address, and the suite publishes to those, so there is nothing to answer with and nothing in the suite reads it. Both languages now have a test that writes a whole black-box suite and checks what it does not have -- no driver field, nothing started or stopped, nothing asked where a server is -- and that a message goes to the address the document declares. --- .../core/problem/rest/BlackBoxUtils.kt | 10 +++ .../service/AsyncApiTestCaseWriterTest.kt | 77 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/BlackBoxUtils.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/BlackBoxUtils.kt index 5d96e25007..30d4208e5e 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/BlackBoxUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/BlackBoxUtils.kt @@ -63,6 +63,16 @@ object BlackBoxUtils { } } } + EMConfig.ProblemType.ASYNCAPI -> { + /* + A message-driven service has no one base URL. Each server the document + names has its own address, and that is where a message is published, so + the suite takes them from there rather than from here. Nothing in it + reads this one, and --bbTargetUrl is still honoured above for a run + pointed somewhere else wholesale. + */ + return "" + } else -> throw IllegalStateException("Black-box testing is currently not supported for ${config.problemType}") } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index ab3240ec83..f19318d862 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -304,6 +304,83 @@ class AsyncApiTestCaseWriterTest { } } + /** + * A whole suite, written to disk by the shared suite writer and read back, which is the + * only way to reach what it puts around the tests: the class, its lifecycle, and whether + * there is a driver in it at all. + */ + private fun writeSuite(extension: String, vararg options: String): String { + + val folder = java.nio.file.Files.createTempDirectory("asyncapi_suite").toFile() + folder.deleteOnExit() + + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS))) { + FakeAsyncApiDriver.replied(DOUBLE_RESULT) + } + injector = AsyncApiTestInjector.create(driver, "--createTests=true", *options) + sampler = injector.getInstance(AsyncApiSampler::class.java) + fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + + val config = injector.getInstance(EMConfig::class.java) + config.outputFolder = folder.absolutePath + config.outputFilePrefix = "AsyncApiSuite" + config.outputFileSuffix = "" + + val evaluated = evaluate("bessj") + val solution = Solution(mutableListOf(evaluated), "AsyncApiSuite", "", Termination.NONE, listOf(), listOf()) + + injector.getInstance(TestSuiteWriter::class.java) + .writeTests(solution, FakeAsyncApiDriver::class.qualifiedName!!, null) + + return folder.walkTopDown().first { it.isFile && it.name.endsWith(extension) }.readText() + } + + @Test + fun testABlackBoxSuiteHasNoDriverAndPublishesWhereTheDocumentSays() { + + /* + Black-box is the mode this problem type is unusual in: a driver publishes while the + search runs, because nothing else can, but the suite that run leaves behind has no + driver in it. So there is nobody to ask where the broker is, and the only address + there is is the one the document declares. + */ + val written = writeSuite(".kt", "--blackBox=true", "--outputFormat=KOTLIN_JUNIT_5") + + //nothing of the driver: no field holding it, and nothing started or stopped + assertFalse(written.contains("SutHandler = "), written) + assertFalse(written.contains("controller.startSut()"), written) + assertFalse(written.contains("controller.stopSut()"), written) + assertFalse(written.contains("getAsyncApiServerAddress"), written) + assertFalse(written.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), written) + + //and so the document's own address, as a literal, where the variable would have been + assertTrue( + written.contains("${KafkaTestClientEmitter.HELPER_NAME}(\"localhost:9092\", \"ncs.bessj.request\""), + written + ) + + //the rest is what any other suite has + assertTrue(written.contains("This file has been generated by EvoMaster"), written) + assertTrue(written.contains("internal class AsyncApiSuite {"), written) + assertTrue(written.contains("fun test_0_publishOnBessj"), written) + assertTrue(written.contains("assertNotNull(res_0)"), written) + } + + @Test + fun testABlackBoxJavaSuiteFallsBackTheSameWay() { + + //the same, in the other language a JVM suite can be written in + val written = writeSuite(".java", "--blackBox=true", "--outputFormat=JAVA_JUNIT_5") + + assertFalse(written.contains("SutHandler "), written) + assertFalse(written.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), written) + assertTrue( + written.contains("${KafkaTestClientEmitter.HELPER_NAME}(\"localhost:9092\", \"ncs.bessj.request\""), + written + ) + assertTrue(written.contains("public class AsyncApiSuite {"), written) + } + @Test fun testTheWholeSuiteHasTheSameScaffoldingAsAnyOther() { From 70406ec3a11f20484303d76d96b8547d94ea6707 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Tue, 6 Oct 2026 20:26:42 -0300 Subject: [PATCH 16/20] AsyncAPI 3.x: the Python client missed every reply Seeking to the end of the reply destination is lazy in kafka-python as it is on the JVM: it takes effect on the first poll, by which time the reply has already been published and sits behind the position, so it is never seen. The JVM helpers were fixed for this; the Python one was written from the same shape and inherited the bug without the fix. Every Python Kafka test therefore waited out its whole deadline and then failed on the reply being absent. Found by running one against a real broker and a real service: the service answered -- the replies are on the topic -- and the test saw none of them. Asking for the position drops the same three tests from 16 seconds to 1.4, and the reply arrives. Nothing could have caught this before, because a Python suite was only ever checked by py_compile, and this parses. --- .../core/output/service/KafkaTestClientEmitter.kt | 3 +++ .../problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt index 28c037c0d0..2731d3f34c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt @@ -264,6 +264,9 @@ object KafkaTestClientEmitter { " else:", " consumer.assign(partitions)", " consumer.seek_to_end()", + " #seek_to_end is lazy: asking for the position is what makes it take effect", + " #now, rather than on the first poll, by when the reply would be behind it", + " consumer.position(partitions[0])", " producer = kafka.KafkaProducer(bootstrap_servers=broker)", " headers = []", " i = 0", diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index f19318d862..a8ee78bbfa 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -272,6 +272,13 @@ class AsyncApiTestCaseWriterTest { assertTrue(members.contains("kafka.KafkaProducer"), members) assertTrue(members.contains("seek_to_end"), members) + /* + Seeking to the end is lazy in every client written so far: without asking for the + position, it takes effect on the first poll, by which time the reply is behind it and + is never seen. A suite that only has to parse cannot catch that. + */ + assertTrue(members.contains("consumer.position(partitions[0])"), members) + //there is no driver in a Python suite, so nothing is declared to ask one assertFalse(members.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), members) assertFalse(body.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), body) From 424361e11722412fc3ddde938a658fbaef846eab Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Tue, 6 Oct 2026 21:17:31 -0300 Subject: [PATCH 17/20] AsyncAPI 3.x: make the tests check that the suite works, not that it reads right Of the assertions over emitted text, nearly all matched a substring and six matched a line. For a writer whose failures have all been code that parses and then does the wrong thing, that is the wrong ratio, and the measurements said so: line coverage sat at 95% while branches sat at 61 to 78. The JVM output is now compiled. core must not depend on a Kafka client -- the dependency belongs to the suite, not to the fuzzer -- so the helper is replaced by a declaration of the same shape and everything the writer decided is compiled around it: the call, its arguments, the escaping, the literals, the assertions. It earned its place immediately, by catching that a JUnit 4 suite needs a different import than the one assumed. The call every generated test runs on is pinned whole rather than by four substrings that matched in any order; swapping the topic published to with the one awaited would have left all four passing. Its deadline, which decides whether the test waits at all, was asserted nowhere. A second document reaches what the shared one cannot: a message with no reply at all, so the absent argument is seen as None and not null; a correlation header named something other than the default, which NCS's name is identical to; and headers of the message itself, which no document here declared, so the flattening had never run. The Python suite is now the one the run writes, parsed as written. It used to be a module assembled by the test, which supplied by hand the very imports the writer is responsible for emitting -- so the condition that decides whether they appear was checked by nothing. Four tests were not testing what they claimed. Sorting a one-element list never reaches a comparator, so that one passed against a comparator that always threw. Its sibling ordered by a payload the contract does not declare, so the key it meant to check was never the one that decided. The two black-box ones asserted the absence of a type that the shared header imports anyway. And the fix a real broker was needed to find was pinned in Python only, so deleting it from either JVM helper stayed green. Covered besides: what one suite found is no longer left for the next, two server names that collapse onto one identifier, a reply matching no declared message, a format named by the driver after every option has been checked, and which half of a split suite a test lands in -- none of which had a test. --- .../AsyncApiActionNamingStrategyTest.kt | 12 + .../service/AsyncApiReplyAssertionsTest.kt | 50 +- .../output/sorting/AsyncApiSortingTest.kt | 19 +- .../service/AsyncApiTestCaseWriterTest.kt | 547 +++++++++++++++--- .../asyncapi/service/AsyncApiTestInjector.kt | 7 +- 5 files changed, 548 insertions(+), 87 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt b/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt index b9625d569a..7b35568311 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/naming/AsyncApiActionNamingStrategyTest.kt @@ -63,6 +63,18 @@ class AsyncApiActionNamingStrategyTest { assertEquals(listOf("test_0_publishOnBessjReturnsNothing"), names) } + @Test + fun testAReplyMatchingNoDeclaredMessageIsNamedAfterThat() { + + /* + The one outcome whose name has to read like the fault it is: a reply arrived, it + correlated, and it still matches none of the messages the document declares. + */ + val names = namesOf({ FakeAsyncApiDriver.replied("""{"nothingTheContractDeclares": 1}""") }, "bessj") + + assertEquals(listOf("test_0_publishOnBessjRepliesWithUndeclaredMessage"), names) + } + @Test fun testSilenceIsNamedTheSameWhetherOrNotItIsReportedAsAFault() { diff --git a/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt b/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt index 3be93c7ec5..8adcf2b791 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt @@ -151,12 +151,54 @@ class AsyncApiReplyAssertionsTest { } @Test - fun testAReplyThatIsNotJsonIsAssertedOnAsText() { + fun testAReplyThatIsNotJsonIsAssertedOnAsTextInEveryLanguage() { - val emitted = emit("just text", OutputFormat.KOTLIN_JUNIT_5) + //a plain-text reply is ordinary for a transport that carries no JSON + assertTrue(emit("just text", OutputFormat.KOTLIN_JUNIT_5) + .contains("""assertTrue(res_0!!.contains("just text"))""")) + assertTrue(emit("just text", OutputFormat.JAVA_JUNIT_5) + .contains("""assertTrue(res_0.contains("just text"));""")) + assertTrue(emit("just text", OutputFormat.PYTHON_UNITTEST) + .contains("""assert "just text" in res_0""")) - assertTrue(emitted.contains("just text"), emitted) - assertFalse(emitted.contains("readTree"), emitted) + assertFalse(emit("just text", OutputFormat.KOTLIN_JUNIT_5).contains("readTree")) + } + + @Test + fun testAReplyThatCannotBePrintedSaysSoRatherThanAssertingNothing() { + + //an address with a port is the REST writer's own example of what is not stable + val emitted = emit("listening on localhost:12345", OutputFormat.KOTLIN_JUNIT_5) + + assertFalse(emitted.contains("localhost:12345"), emitted) + assertTrue(emitted.contains("not asserted on"), emitted) + } + + @Test + fun testANumberTooLargeForAnIntCarriesJavasSuffix() { + + /* + Java picks assertEquals(long, long) by widening within an int's range, but a literal + outside it is not an int at all and does not compile without the suffix. + */ + val big = """{"n": 5000000000}""" + + assertTrue(emit(big, OutputFormat.JAVA_JUNIT_5).contains("assertEquals(5000000000L,"), emit(big, OutputFormat.JAVA_JUNIT_5)) + //Kotlin types the literal itself, so a suffix there would be noise + assertTrue(emit(big, OutputFormat.KOTLIN_JUNIT_5).contains("assertEquals(5000000000,"), emit(big, OutputFormat.KOTLIN_JUNIT_5)) + assertTrue(emit(big, OutputFormat.PYTHON_UNITTEST).contains("== 5000000000"), emit(big, OutputFormat.PYTHON_UNITTEST)) + } + + @Test + fun testANegativeCollectionLimitAssertsTheWholeCollection() { + + //the option's way of saying "no limit", which must not truncate to nothing + val config = EMConfig().apply { maxAssertionForDataInCollection = -1 } + val emitted = emit("""{"xs": [1, 2, 3]}""", OutputFormat.KOTLIN_JUNIT_5, config) + + assertTrue(emitted.contains("get(0)"), emitted) + assertTrue(emitted.contains("get(1)"), emitted) + assertTrue(emitted.contains("get(2)"), emitted) } @Test diff --git a/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt b/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt index 79dae9dfab..6904cfbc4d 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/sorting/AsyncApiSortingTest.kt @@ -31,7 +31,8 @@ class AsyncApiSortingTest { companion object { private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" - private const val ERROR = """{"code": 400, "message": "no"}""" + //as the document declares it: the fields sit under "error", not at the top level + private const val ERROR = """{"error": {"code": 400, "message": "no"}}""" } @BeforeEach @@ -61,13 +62,23 @@ class AsyncApiSortingTest { } @Test - fun testSortingASingleTestDoesNotThrow() { + fun testRepliesToOneOperationAreOrderedByTheMessageTheyWereRecognisedAs() { - val tests = mutableListOf(evaluate("bessj") { FakeAsyncApiDriver.replied(DOUBLE_RESULT) }) + /* + The third key, which the operation and the outcome cannot separate: two tests of one + operation that both got a reply, telling them apart by which declared message it + was. A comparator of two keys leaves these in whatever order they arrived. + */ + val error = evaluate("expint") { FakeAsyncApiDriver.replied(ERROR) } + val result = evaluate("expint") { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } + + val tests = mutableListOf(result, error) SortingHelper().sort(tests, SortingStrategy.TARGET_INCREMENTAL) - assertEquals(1, tests.size) + //"doubleResult" before "error", as the comparator orders them by the message's own name + assertEquals(result, tests[0]) + assertEquals(error, tests[1]) } private fun operationOf(evaluated: EvaluatedIndividual<*>) = diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt index a8ee78bbfa..075118b7ad 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt @@ -8,9 +8,11 @@ import org.evomaster.client.java.controller.api.dto.SutInfoDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto import org.evomaster.core.output.TestCase +import org.evomaster.core.output.TestSuiteSplitter import org.evomaster.core.output.Lines import org.evomaster.core.output.OutputFormat import org.evomaster.core.output.Termination +import org.evomaster.core.output.compiler.CompilerForTestGenerated import org.evomaster.core.output.service.KafkaTestClientEmitter import org.evomaster.core.search.Solution import org.evomaster.core.EMConfig @@ -38,6 +40,84 @@ class AsyncApiTestCaseWriterTest { companion object { private const val DOUBLE_RESULT = """{"resultAsDouble": 1.5}""" + /** + * A document that reaches what the shared one cannot. + * + * NCS answers every operation and names its correlation header exactly what the default + * is, so three of the arguments a call carries are never seen as anything else: a + * message with no reply at all, a header named something of the service's own choosing, + * and headers of the message itself. + */ + private val VARIANTS = """ + asyncapi: 3.0.0 + info: + title: Variants + version: 1.0.0 + servers: + broker: + host: broker.local:19092 + protocol: kafka + channels: + askRequest: + address: ask.request + servers: + - ${'$'}ref: '#/servers/broker' + messages: + askRequest: + correlationId: + location: '${'$'}message.header#/x-corr-id' + headers: + type: object + required: [tenant] + properties: + tenant: + type: string + const: acme + payload: + type: object + required: [value] + properties: + value: + type: integer + askReply: + address: ask.reply + servers: + - ${'$'}ref: '#/servers/broker' + messages: + answer: + payload: + type: object + required: [answer] + properties: + answer: + type: integer + tellRequest: + address: tell.request + servers: + - ${'$'}ref: '#/servers/broker' + messages: + tellRequest: + payload: + type: object + required: [note] + properties: + note: + type: string + const: hello + operations: + ask: + action: receive + channel: + ${'$'}ref: '#/channels/askRequest' + reply: + channel: + ${'$'}ref: '#/channels/askReply' + tell: + action: receive + channel: + ${'$'}ref: '#/channels/tellRequest' + """.trimIndent() + /** * Every line any other suite opens with, taken verbatim from what the writer produces * for a suite with no tests in it (see TestSuiteWriterTest). The class name and the @@ -115,15 +195,29 @@ class AsyncApiTestCaseWriterTest { RestActionBuilderV3.cleanCache() } - private fun start(answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) { - driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS)), answer) - injector = AsyncApiTestInjector.create( - driver, + private fun start(vararg options: String, answer: (AsyncApiActionDto) -> AsyncApiReplyDto?) { + startWith(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS), *options, answer = answer) + } + + /** + * The same, over a document written for one test rather than the shared one. + */ + private fun startWith( + document: String, + vararg options: String, + answer: (AsyncApiActionDto) -> AsyncApiReplyDto? + ) { + driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(document), answer) + + //what a test did not ask for, so that asking for it does not pass the option twice + val defaults = listOf( "--blackBox=false", //the core only asks a driver to render lines when a test will be written "--createTests=true", "--outputFormat=KOTLIN_JUNIT_5" - ) + ).filterNot { d -> options.any { it.substringBefore('=') == d.substringBefore('=') } } + + injector = AsyncApiTestInjector.create(driver, *(defaults + options).toTypedArray()) sampler = injector.getInstance(AsyncApiSampler::class.java) fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) } @@ -152,6 +246,62 @@ class AsyncApiTestCaseWriterTest { return lines.toString() } + /** + * Both halves, in the order the suite writer produces them: the members first, as that is + * what tells the writer which servers the suite has. Asking for a body first would write one + * that publishes to the document's address even where a driver could have been asked. + */ + private fun membersAndBodyOf(evaluated: EvaluatedIndividual): Pair { + val members = membersOf(evaluated) + return members to bodyOf(evaluated) + } + + /** + * Whether what was emitted is a program at all. + * + * The suite as written calls a Kafka client, which core does not depend on and must not: the + * dependency belongs to the generated suite, not to the fuzzer. So the helper is replaced by + * a declaration of the same shape, and what is compiled is everything the writer decided -- + * the call, its arguments, the escaping of the payload, the literals and the assertions. + */ + private fun compiles(members: String, body: String, format: OutputFormat, name: String) { + + val servers = Regex("${KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX}\\w+") + .findAll(members + body) + .map { it.value } + .toSet() + + val indented = body.trimEnd().lines().joinToString("\n") { " $it" } + + val code = if (format.isJava()) { + buildString { + append("import org.junit.jupiter.api.Test;\n") + append("import org.junit.jupiter.api.Timeout;\n") + append("import static org.junit.jupiter.api.Assertions.*;\n\n") + append("public class $name {\n") + servers.forEach { append(" private String $it = \"localhost:9092\";\n") } + append(" private String ${KafkaTestClientEmitter.HELPER_NAME}(String broker, String topic,") + append(" String replyTopic, String payload, String correlationHeader, long timeoutMs,") + append(" String... headerPairs) throws Exception { return null; }\n") + append(indented).append("\n}\n") + } + } else { + buildString { + append("import org.junit.jupiter.api.Test\n") + append("import org.junit.jupiter.api.Timeout\n") + append("import org.junit.jupiter.api.Assertions.*\n\n") + append("internal class $name {\n") + servers.forEach { append(" private val $it: String = \"localhost:9092\"\n") } + append(" private fun ${KafkaTestClientEmitter.HELPER_NAME}(broker: String, topic: String,") + append(" replyTopic: String?, payload: String?, correlationHeader: String?, timeoutMs: Long,") + append(" vararg headerPairs: String): String? = null\n") + append(indented).append("\n}\n") + } + } + + CompilerForTestGenerated.compile(format, code, name) + } + /** * The generated body for one test, as the suite writer would ask for it. */ @@ -222,6 +372,18 @@ class AsyncApiTestCaseWriterTest { //a reply older than the test is not an answer to it, and each run stamps its own id assertTrue(members.contains("seekToEnd"), members) assertTrue(members.contains("UUID.randomUUID()"), members) + + /* + Seeking to the end is lazy in every client written so far: it takes effect on the + first poll, by which time the reply is behind it. Asking for the position is what + prevents that, and it is the one line a real broker was needed to find missing. + */ + assertTrue(members.contains("c.position(parts[0])"), members) + + //and the clients are closed however the call ends, not only when it succeeds + assertTrue(members.contains("} finally {"), members) + assertTrue(members.contains("producer?.close()"), members) + assertTrue(members.contains("consumer?.close()"), members) assertTrue(members.contains("asyncApiServer_kafka"), members) val body = bodyOf(evaluated) @@ -236,61 +398,87 @@ class AsyncApiTestCaseWriterTest { assertFalse(body.contains("\"localhost:9092\""), body) } + /** + * A whole suite, written to disk by the shared suite writer and read back, which is the + * only way to reach what it puts around the tests: the class, its lifecycle, and whether + * there is a driver in it at all. + */ + private fun writeSuite(extension: String, vararg options: String): String { + + val folder = java.nio.file.Files.createTempDirectory("asyncapi_suite").toFile() + folder.deleteOnExit() + + startWith(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS), *options) { + FakeAsyncApiDriver.replied(DOUBLE_RESULT) + } + + val config = injector.getInstance(EMConfig::class.java) + config.outputFolder = folder.absolutePath + config.outputFilePrefix = "AsyncApiSuite" + config.outputFileSuffix = "" + + val evaluated = evaluate("bessj") + val solution = Solution(mutableListOf(evaluated), "AsyncApiSuite", "", Termination.NONE, listOf(), listOf()) + + injector.getInstance(TestSuiteWriter::class.java) + .writeTests(solution, FakeAsyncApiDriver::class.qualifiedName!!, null) + + //by the prefix too: a Python suite is written beside the utilities it imports + return folder.walkTopDown() + .first { it.isFile && it.name == "AsyncApiSuite$extension" } + .readText() + } + @Test fun testThePythonSuiteIsValidPython() { /* A Python suite has no driver to ask where the broker is, since the controller is Java, so it publishes to the address the document declares. What matters here is - that what comes out parses: indentation is part of the language, so a helper written - as text can break it in a way no JVM format can. - */ - driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS))) { - FakeAsyncApiDriver.replied(DOUBLE_RESULT) - } - injector = AsyncApiTestInjector.create( - driver, - //Python output is for black-box only, which is also why its suite has no driver to ask - "--blackBox=true", - "--createTests=true", - "--outputFormat=PYTHON_UNITTEST" - ) - sampler = injector.getInstance(AsyncApiSampler::class.java) - fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) + that the file the run writes parses: indentation is part of the language, so a + helper written as text can break it in a way no JVM format can. - val evaluated = evaluate("bessj") - val body = bodyOf(evaluated) - val members = membersOf(evaluated) + The whole suite is written and checked, imports included, rather than a module + assembled here: whether those imports are emitted at all is itself a decision the + writer makes, and one this would otherwise be making for it. + */ + val written = writeSuite(".py", "--blackBox=true", "--outputFormat=PYTHON_UNITTEST") //the call, and the reply read as a dict rather than a tree of nodes - assertTrue(body.contains("${KafkaTestClientEmitter.HELPER_NAME}("), body) - assertTrue(body.contains("json.loads(res_0)"), body) - assertTrue(body.contains("\"resultAsDouble\" in body_0"), body) - assertTrue(body.contains("assert "), body) + assertTrue(written.contains("${KafkaTestClientEmitter.HELPER_NAME}("), written) + assertTrue(written.contains("json.loads(res_0)"), written) + assertTrue(written.contains("\"resultAsDouble\" in body_0"), written) - //and the helper it calls - assertTrue(members.contains("kafka.KafkaProducer"), members) - assertTrue(members.contains("seek_to_end"), members) + //the helper it calls, and the imports that helper needs + assertTrue(written.contains("kafka.KafkaProducer"), written) + assertTrue(written.contains("import kafka"), written) + assertTrue(written.contains("import uuid"), written) + assertTrue(written.contains("seek_to_end"), written) /* - Seeking to the end is lazy in every client written so far: without asking for the - position, it takes effect on the first poll, by which time the reply is behind it and - is never seen. A suite that only has to parse cannot catch that. + Seeking to the end is lazy in every client written so far: it takes effect on the + first poll, by which time the reply is behind it and is never seen. A suite that + only has to parse cannot catch that. */ - assertTrue(members.contains("consumer.position(partitions[0])"), members) + assertTrue(written.contains("consumer.position(partitions[0])"), written) //there is no driver in a Python suite, so nothing is declared to ask one - assertFalse(members.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), members) - assertFalse(body.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), body) + assertFalse(written.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), written) //and the driver was asked for no script, as its own enum cannot name Python assertNull(driver.published.last().outputFormat) - //what comes out has to be parseable Python, indentation included - val module = (members.trimEnd().lines() + body.trimEnd().lines()) - .joinToString("\n") { " " + it } + parsesAsPython(written) + } + + /** + * Whether python3 can read the file the run wrote. Skipped, visibly, where there is none. + */ + private fun parsesAsPython(suite: String) { + val file = java.io.File.createTempFile("asyncapi_suite", ".py") - file.writeText("import json\nimport time\nimport uuid\nimport kafka\n\n\nclass Suite:\n$module\n") + file.deleteOnExit() + file.writeText(suite) try { val process = try { @@ -305,43 +493,12 @@ class AsyncApiTestCaseWriterTest { val output = process.inputStream.bufferedReader().readText() val code = process.waitFor() - assertEquals(0, code, "the generated Python does not parse:\n$output\n\n$module") + assertEquals(0, code, "the generated Python does not parse:\n$output\n\n$suite") } finally { file.delete() } } - /** - * A whole suite, written to disk by the shared suite writer and read back, which is the - * only way to reach what it puts around the tests: the class, its lifecycle, and whether - * there is a driver in it at all. - */ - private fun writeSuite(extension: String, vararg options: String): String { - - val folder = java.nio.file.Files.createTempDirectory("asyncapi_suite").toFile() - folder.deleteOnExit() - - driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS))) { - FakeAsyncApiDriver.replied(DOUBLE_RESULT) - } - injector = AsyncApiTestInjector.create(driver, "--createTests=true", *options) - sampler = injector.getInstance(AsyncApiSampler::class.java) - fitness = injector.getInstance(Key.get(object : TypeLiteral>() {})) - - val config = injector.getInstance(EMConfig::class.java) - config.outputFolder = folder.absolutePath - config.outputFilePrefix = "AsyncApiSuite" - config.outputFileSuffix = "" - - val evaluated = evaluate("bessj") - val solution = Solution(mutableListOf(evaluated), "AsyncApiSuite", "", Termination.NONE, listOf(), listOf()) - - injector.getInstance(TestSuiteWriter::class.java) - .writeTests(solution, FakeAsyncApiDriver::class.qualifiedName!!, null) - - return folder.walkTopDown().first { it.isFile && it.name.endsWith(extension) }.readText() - } - @Test fun testABlackBoxSuiteHasNoDriverAndPublishesWhereTheDocumentSays() { @@ -353,8 +510,12 @@ class AsyncApiTestCaseWriterTest { */ val written = writeSuite(".kt", "--blackBox=true", "--outputFormat=KOTLIN_JUNIT_5") - //nothing of the driver: no field holding it, and nothing started or stopped - assertFalse(written.contains("SutHandler = "), written) + /* + Nothing of the driver: no field holding it, and nothing started or stopped. The type + is still imported, as it is in every suite the shared writer produces, so what says + there is no driver is that no line both names it and assigns one. + */ + assertFalse(written.lines().any { it.contains("SutHandler") && it.contains("=") }, written) assertFalse(written.contains("controller.startSut()"), written) assertFalse(written.contains("controller.stopSut()"), written) assertFalse(written.contains("getAsyncApiServerAddress"), written) @@ -379,7 +540,7 @@ class AsyncApiTestCaseWriterTest { //the same, in the other language a JVM suite can be written in val written = writeSuite(".java", "--blackBox=true", "--outputFormat=JAVA_JUNIT_5") - assertFalse(written.contains("SutHandler "), written) + assertFalse(written.lines().any { it.contains("SutHandler") && it.contains("=") }, written) assertFalse(written.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), written) assertTrue( written.contains("${KafkaTestClientEmitter.HELPER_NAME}(\"localhost:9092\", \"ncs.bessj.request\""), @@ -432,8 +593,15 @@ class AsyncApiTestCaseWriterTest { assertTrue(actual.contains("internal class AsyncApiSuite {"), written) assertTrue(actual.any { it.startsWith(" private val controller : SutHandler = ") }, written) - //and ours on top of it, not instead of it - assertTrue(written.contains("asyncApiServer_kafka = controller.getAsyncApiServerAddress"), written) + /* + And ours on top of it, not instead of it. Pinned whole: through a prefix, dropping + the fallback would still pass, and a lateinit var assigned null fails every test in + the suite at initClass. + */ + assertTrue( + actual.contains(" asyncApiServer_kafka = controller.getAsyncApiServerAddress(\"kafka\") ?: \"localhost:9092\""), + written + ) assertTrue(written.contains("publishOnBessj"), written) folder.deleteRecursively() @@ -481,6 +649,231 @@ class AsyncApiTestCaseWriterTest { ) } + @Test + fun testTheCallIsWrittenWholeAndCompiles() { + + /* + The one line every generated test runs on. Asserted whole rather than by substrings: + the arguments are all strings, so swapping the topic it publishes to with the one it + waits on would leave every substring assertion passing and every generated test + waiting on the wrong destination. + */ + //the JUnit 4 forms differ only in the annotation the shared writer puts above the method + OutputFormat.values().filter { it.isJavaOrKotlin() && it.isJUnit5() }.forEach { format -> + + startWith(VARIANTS, "--outputFormat=${format.name}") { FakeAsyncApiDriver.replied("""{"answer": 2}""") } + + val (members, body) = membersAndBodyOf(evaluate("ask")) + + val call = "${KafkaTestClientEmitter.HELPER_NAME}(asyncApiServer_broker, \"ask.request\"," + + " \"ask.reply\", \"{\\\"value\\\":" + + assertTrue(body.contains(call), "for $format:\n$body") + + //the header the id rides in is the document's, not whatever the default happens to be + assertTrue(body.contains("\"x-corr-id\""), "for $format:\n$body") + + //the deadline, without which a test polls once and finds nothing + assertTrue(body.contains("5000L"), "for $format:\n$body") + + //and the message's own headers, flattened onto the end of the call + assertTrue(body.contains("\"tenant\", \"acme\""), "for $format:\n$body") + + compiles(members, body, format, "AsyncApiCall${format.name}") + } + } + + @Test + fun testAMessageWithNoReplyPassesNothingWhereTheReplyWouldGo() { + + /* + A fire-and-forget operation: no reply destination, so no deadline and nothing to + correlate. Each language spells an absent argument its own way, and Python spelling + it "null" would parse and then fail with a NameError, which is how three of these + have gone wrong before. + */ + val expected = mapOf( + OutputFormat.KOTLIN_JUNIT_5 to "\"tell.request\", null, ", + OutputFormat.JAVA_JUNIT_5 to "\"tell.request\", null, ", + OutputFormat.PYTHON_UNITTEST to "\"tell.request\", None, " + ) + + expected.forEach { (format, fragment) -> + + val blackBox = if (format.isPython()) arrayOf("--blackBox=true") else arrayOf() + startWith(VARIANTS, "--outputFormat=${format.name}", *blackBox) { FakeAsyncApiDriver.fireAndForget() } + + val (members, body) = membersAndBodyOf(evaluate("tell")) + + assertTrue(body.contains(fragment), "for $format:\n$body") + //nothing came back, so nothing is asserted on + assertFalse(body.contains("assertNotNull"), "for $format:\n$body") + + if (format.isJavaOrKotlin()) { + compiles(members, body, format, "AsyncApiNoReply${format.name}") + } + } + } + + @Test + fun testWhatOneSuiteFoundIsNotLeftBehindForTheNext() { + + /* + One writer writes every suite of a run, and a run splits its tests into several. A + suite with nothing to publish over Kafka must come back with no members at all: if + the servers the last one found were still there, it would assign to a field it never + declared, and the suite would not compile. + */ + start { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } + + val kafka = membersOf(evaluate("bessj")) + assertTrue(kafka.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), kafka) + + //the second suite publishes through lines the driver rendered, so nothing is written + start { FakeAsyncApiDriver.replied(DOUBLE_RESULT).apply { testScript = listOf("publishSomehow()") } } + + val driverRendered = membersOf(evaluate("bessj")) + + assertFalse(driverRendered.contains(KafkaTestClientEmitter.SERVER_VARIABLE_PREFIX), driverRendered) + assertFalse(driverRendered.contains("KafkaProducer"), driverRendered) + + //and the init statements follow the members, so there is nothing to assign either + val writer = injector.getInstance(TestCaseWriter::class.java) + val init = Lines(OutputFormat.KOTLIN_JUNIT_5) + writer.addExtraInitStatement(init) + assertEquals("", init.toString().trim()) + } + + @Test + fun testTwoServersThatWouldShareOneVariableAreNotDeclaredTwice() { + + /* + A document may name its servers anything; an identifier may not. Two names that + differ only where a variable cannot would be declared twice under one name, and the + suite would not compile. The first keeps the variable, and the rest publish to the + address the document gave them. + */ + val collidingServers = """ + asyncapi: 3.0.0 + info: + title: Colliding + version: 1.0.0 + servers: + a-b: + host: one.local:9092 + protocol: kafka + a.b: + host: two.local:9092 + protocol: kafka + channels: + first: + address: first.request + servers: + - ${'$'}ref: '#/servers/a-b' + messages: + firstRequest: + payload: + type: object + properties: + value: + type: integer + second: + address: second.request + servers: + - ${'$'}ref: '#/servers/a.b' + messages: + secondRequest: + payload: + type: object + properties: + value: + type: integer + operations: + askFirst: + action: receive + channel: + ${'$'}ref: '#/channels/first' + askSecond: + action: receive + channel: + ${'$'}ref: '#/channels/second' + """.trimIndent() + + startWith(collidingServers) { FakeAsyncApiDriver.fireAndForget() } + + val members = membersOf(evaluate("askFirst", "askSecond")) + + //one declaration under that name, not two + assertEquals(1, members.split("asyncApiServer_a_b").size - 1, members) + + //and the one that lost it publishes to the address the document gave it + val body = bodyOf(evaluate("askFirst", "askSecond")) + assertTrue(body.contains("asyncApiServer_a_b"), body) + assertTrue(body.contains("\"two.local:9092\""), body) + } + + @Test + fun testAFormatNoTestCanBeWrittenInIsRefusedWhenTheDriverNamesIt() { + + /* + For this problem type the format is ordinarily left to the driver, in black-box mode + as well, so it arrives after every option has already been checked. A driver naming + a language no client is written for has to be refused there rather than reaching the + writer, which would otherwise emit a call with no client and assertions around it. + */ + driver = FakeAsyncApiDriver( + AsyncApiTestInjector.sutInfo( + AsyncApiAccess.readFromResource(AsyncApiTestInjector.NCS), + SutInfoDto.OutputFormat.JS_JEST + ) + ) { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } + + /* + The sampler is what asks the driver, and it is initialised as the injector is built, + so the refusal surfaces from there rather than from a later lookup. + */ + val error = assertThrows(Throwable::class.java) { + AsyncApiTestInjector.create(driver, "--blackBox=false", "--createTests=true") + .getInstance(AsyncApiSampler::class.java) + } + + val reasons = generateSequence(error as Throwable?) { it.cause }.mapNotNull { it.message }.toList() + + assertTrue( + reasons.any { it.contains("outputFormat") }, + "the format the driver named was not refused: $reasons" + ) + } + + @Test + fun testASuiteIsSplitByWhetherPublishingWentWell() { + + /* + Which file a test lands in, which is what a run with --testSuiteSplitType shows the + user. There is no status code to read here, so the outcome is what says whether the + message went out and was answered. + */ + start("--useExperimentalOracles=true") { FakeAsyncApiDriver.replied(DOUBLE_RESULT) } + val answered = evaluate("bessj") + + start("--useExperimentalOracles=true") { FakeAsyncApiDriver.silence() } + val unanswered = evaluate("bessj") + + val config = injector.getInstance(EMConfig::class.java) + config.testSuiteSplitType = EMConfig.TestSuiteSplitType.FAULTS + + val solution = Solution( + mutableListOf(answered, unanswered), "Prefix", "Suffix", Termination.NONE, listOf(), listOf() + ) + + val split = TestSuiteSplitter.split(solution, config) + val byIndividual = split.splitOutcome.flatMap { s -> s.individuals.map { it to s.termination } } + + //the one that was answered is a success; the one that was not is not + assertEquals(Termination.SUCCESSES, byIndividual.first { it.first == answered }.second) + assertNotEquals(Termination.SUCCESSES, byIndividual.first { it.first == unanswered }.second) + } + @Test fun testTheKafkaHelperIsLeftOutWhenNothingCallsIt() { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt index a0dab84872..17503421d1 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt @@ -27,9 +27,12 @@ object AsyncApiTestInjector { /** * What a driver declares for a service whose document it hands over as text. */ - fun sutInfo(schemaText: String): SutInfoDto = SutInfoDto().apply { + fun sutInfo( + schemaText: String, + declaredFormat: SutInfoDto.OutputFormat = SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 + ): SutInfoDto = SutInfoDto().apply { asyncApiProblem = AsyncApiProblemDto().apply { this.schemaText = schemaText } - defaultOutputFormat = SutInfoDto.OutputFormat.KOTLIN_JUNIT_5 + defaultOutputFormat = declaredFormat } fun create(driver: FakeAsyncApiDriver, vararg options: String): Injector { From 7932c9b2e45c166674789663c10169e00d2b2e41 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Tue, 6 Oct 2026 21:57:25 -0300 Subject: [PATCH 18/20] AsyncAPI 3.x: the two that are not services move out of the service package Every other file under output/service is injected; these two are stateless objects that take what they need as arguments. That is the same reason the reply classifier was moved out of the problem type's service package, so they go the same way, into a package of their own named after the protocol they write for -- which is how the REST naming helpers are already grouped. The writer itself stays: it is bound in the module and the suite writer asks the injector for it. Two tests follow what they test. The one for the writer had been sitting with the AsyncAPI fixtures rather than beside TestSuiteWriterTest, which is where a test for something under output/service belongs. --- .../{service => asyncapi}/AsyncApiReplyAssertions.kt | 2 +- .../{service => asyncapi}/KafkaTestClientEmitter.kt | 2 +- .../core/output/service/AsyncApiTestCaseWriter.kt | 2 ++ .../org/evomaster/core/output/service/TestSuiteWriter.kt | 1 + .../{service => asyncapi}/AsyncApiReplyAssertionsTest.kt | 2 +- .../service/AsyncApiTestCaseWriterTest.kt | 9 +++++---- 6 files changed, 11 insertions(+), 7 deletions(-) rename core/src/main/kotlin/org/evomaster/core/output/{service => asyncapi}/AsyncApiReplyAssertions.kt (99%) rename core/src/main/kotlin/org/evomaster/core/output/{service => asyncapi}/KafkaTestClientEmitter.kt (99%) rename core/src/test/kotlin/org/evomaster/core/output/{service => asyncapi}/AsyncApiReplyAssertionsTest.kt (99%) rename core/src/test/kotlin/org/evomaster/core/{problem/asyncapi => output}/service/AsyncApiTestCaseWriterTest.kt (99%) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt b/core/src/main/kotlin/org/evomaster/core/output/asyncapi/AsyncApiReplyAssertions.kt similarity index 99% rename from core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt rename to core/src/main/kotlin/org/evomaster/core/output/asyncapi/AsyncApiReplyAssertions.kt index 7063a8dd3a..87ceb5b4ae 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertions.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/asyncapi/AsyncApiReplyAssertions.kt @@ -1,4 +1,4 @@ -package org.evomaster.core.output.service +package org.evomaster.core.output.asyncapi import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt b/core/src/main/kotlin/org/evomaster/core/output/asyncapi/KafkaTestClientEmitter.kt similarity index 99% rename from core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt rename to core/src/main/kotlin/org/evomaster/core/output/asyncapi/KafkaTestClientEmitter.kt index 2731d3f34c..100d238fdc 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/KafkaTestClientEmitter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/asyncapi/KafkaTestClientEmitter.kt @@ -1,4 +1,4 @@ -package org.evomaster.core.output.service +package org.evomaster.core.output.asyncapi import com.fasterxml.jackson.databind.ObjectMapper import org.evomaster.core.logging.LoggingUtil diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt index cb3f321126..0e6c74958c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt @@ -1,5 +1,7 @@ package org.evomaster.core.output.service +import org.evomaster.core.output.asyncapi.AsyncApiReplyAssertions +import org.evomaster.core.output.asyncapi.KafkaTestClientEmitter import org.evomaster.core.output.Lines import org.evomaster.core.output.TestWriterUtils import org.evomaster.core.output.TestCase diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 5b9180fabc..9acd2729c1 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -7,6 +7,7 @@ import org.evomaster.core.EMConfig import org.evomaster.core.database.sql.schema.TableId import org.evomaster.core.llm.service.LlmService import org.evomaster.core.output.* +import org.evomaster.core.output.asyncapi.KafkaTestClientEmitter import org.evomaster.core.output.TestWriterUtils.getWireMockVariableName import org.evomaster.core.output.TestWriterUtils.handleDefaultStubForAsJavaOrKotlin import org.evomaster.core.output.dto.DtoWriter diff --git a/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt b/core/src/test/kotlin/org/evomaster/core/output/asyncapi/AsyncApiReplyAssertionsTest.kt similarity index 99% rename from core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt rename to core/src/test/kotlin/org/evomaster/core/output/asyncapi/AsyncApiReplyAssertionsTest.kt index 8adcf2b791..ae6e2c01f2 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiReplyAssertionsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/asyncapi/AsyncApiReplyAssertionsTest.kt @@ -1,4 +1,4 @@ -package org.evomaster.core.output.service +package org.evomaster.core.output.asyncapi import org.evomaster.core.EMConfig import org.evomaster.core.output.Lines diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriterTest.kt similarity index 99% rename from core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt rename to core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriterTest.kt index 075118b7ad..d4d939498f 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriterTest.kt @@ -1,4 +1,4 @@ -package org.evomaster.core.problem.asyncapi.service +package org.evomaster.core.output.service import com.google.inject.Injector import com.google.inject.Key @@ -7,17 +7,18 @@ import com.webfuzzing.asyncapi.access.AsyncApiAccess import org.evomaster.client.java.controller.api.dto.SutInfoDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.output.asyncapi.KafkaTestClientEmitter import org.evomaster.core.output.TestCase import org.evomaster.core.output.TestSuiteSplitter import org.evomaster.core.output.Lines import org.evomaster.core.output.OutputFormat import org.evomaster.core.output.Termination import org.evomaster.core.output.compiler.CompilerForTestGenerated -import org.evomaster.core.output.service.KafkaTestClientEmitter import org.evomaster.core.search.Solution import org.evomaster.core.EMConfig -import org.evomaster.core.output.service.TestSuiteWriter -import org.evomaster.core.output.service.TestCaseWriter +import org.evomaster.core.problem.asyncapi.service.AsyncApiSampler +import org.evomaster.core.problem.asyncapi.service.AsyncApiTestInjector +import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver import org.evomaster.core.problem.asyncapi.data.AsyncApiAction import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual import org.evomaster.core.problem.enterprise.SampleType From 33f7c08198fd261d76a23d57cdcc78e25e2a2aa1 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Tue, 6 Oct 2026 22:30:53 -0300 Subject: [PATCH 19/20] AsyncAPI 3.x: say what the injector does with an option, not why it once had to The note on the defaults called the tests here "suites", which is the one word in this area that already means something else -- what a run writes -- and said one test needed createTests on when four call sites do. Neither told a reader the rule: what a caller passes replaces the default of that name, because EMConfig refuses an option given twice. That is now said, on the function rather than inside it. It was also only half true. The match read the name before '=', so the spelling EMConfigTest uses two files away, --createTests true, would have slipped past and failed to parse. Both spellings now count as one option, and a test holds that, since nothing else would notice. --- .../asyncapi/service/AsyncApiModuleTest.kt | 21 +++++++++++++++++++ .../asyncapi/service/AsyncApiTestInjector.kt | 20 +++++++++++++----- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt index dbf4dca86f..50616db46a 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiModuleTest.kt @@ -6,6 +6,7 @@ import com.google.inject.TypeLiteral import com.webfuzzing.asyncapi.access.AsyncApiAccess import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.EMConfig import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual import org.evomaster.core.problem.asyncapi.service.FakeAsyncApiDriver.Companion.replied import org.evomaster.core.problem.rest.builder.RestActionBuilderV3 @@ -156,4 +157,24 @@ class AsyncApiModuleTest { //the writer that writes the publish-and-await lines, from the contract or from the driver assertTrue(injector.getInstance(TestCaseWriter::class.java) is AsyncApiTestCaseWriter) } + + @Test + fun testAnOptionACallerGivesReplacesTheDefaultOfThatName() { + + /* + The parser refuses an option given twice, so a default has to step aside rather than + be appended beside what a test asked for. Both spellings name the same option. + */ + val driver = FakeAsyncApiDriver(AsyncApiTestInjector.sutInfo(AsyncApiAccess.readFromResource(NCS))) { + replied("""{"resultAsDouble": 1.5}""") + } + + listOf( + arrayOf("--createTests=true"), + arrayOf("--createTests", "true") + ).forEach { options -> + val config = AsyncApiTestInjector.create(driver, *options).getInstance(EMConfig::class.java) + assertTrue(config.createTests, "for ${options.toList()}") + } + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt index 17503421d1..eb6702232d 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt @@ -35,14 +35,24 @@ object AsyncApiTestInjector { defaultOutputFormat = declaredFormat } + /** + * An injector for a search driven by [driver], under the options every test here wants + * unless it asks for something else. + * + * An option a caller passes **replaces** the default of the same name rather than joining + * it: EMConfig refuses an option given twice, so appending both would fail to parse rather + * than let the later one win. Which is how a test about what gets written turns createTests + * on where the rest of them want it off. + * + * Either spelling counts as the same option, `--name=value` and `--name value`, because + * both are used in this module and the parser takes either. + */ fun create(driver: FakeAsyncApiDriver, vararg options: String): Injector { - /* - Most suites here search without writing tests, but one is about what gets written, - and the option cannot be given twice. - */ + val named = options.map { it.removePrefix("--").substringBefore('=') }.toSet() + val defaults = listOf("--seed=42", "--problemType=ASYNCAPI", "--createTests=false") - .filterNot { d -> options.any { it.substringBefore('=') == d.substringBefore('=') } } + .filterNot { it.removePrefix("--").substringBefore('=') in named } val args = (defaults + options).toTypedArray() From 330d3f650460d96e1dfe76689318837d39c7618a Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Tue, 6 Oct 2026 22:35:03 -0300 Subject: [PATCH 20/20] AsyncAPI 3.x: one list of the options these tests run under The sampler test built the same three defaults itself, so a fourth added to one list would not have reached the other. They move to argsWith, which the injector already had to compute and which carries the replace-the-default rule with them. Only the list is shared. The sampler test keeps its own module graph, which binds the sampler alone rather than the whole problem type, and its own controller, whose start can be made to fail -- which is what those tests are about and what the driver the others use cannot express. --- .../asyncapi/service/AsyncApiSamplerTest.kt | 3 ++- .../asyncapi/service/AsyncApiTestInjector.kt | 20 +++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt index a3b8569f58..529dd8b8c6 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiSamplerTest.kt @@ -63,7 +63,8 @@ class AsyncApiSamplerTest { private fun injector(info: SutInfoDto, starts: Boolean = true, vararg options: String): Injector { - val args = arrayOf("--seed=42", "--problemType=ASYNCAPI", "--createTests=false") + options + //the same options every other test over this problem type runs under + val args = AsyncApiTestInjector.argsWith(*options) val modules = listOf(BaseModule(args), object : AbstractModule() { override fun configure() { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt index eb6702232d..592e01e9a2 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiTestInjector.kt @@ -36,8 +36,8 @@ object AsyncApiTestInjector { } /** - * An injector for a search driven by [driver], under the options every test here wants - * unless it asks for something else. + * The options every test over an AsyncAPI service wants, with what one of them asked for + * instead. * * An option a caller passes **replaces** the default of the same name rather than joining * it: EMConfig refuses an option given twice, so appending both would fail to parse rather @@ -46,15 +46,27 @@ object AsyncApiTestInjector { * * Either spelling counts as the same option, `--name=value` and `--name value`, because * both are used in this module and the parser takes either. + * + * Here rather than in each test class so that a default added later reaches all of them; + * the module graph and the fake controller stay each test's own, as they differ on purpose. */ - fun create(driver: FakeAsyncApiDriver, vararg options: String): Injector { + fun argsWith(vararg options: String): Array { val named = options.map { it.removePrefix("--").substringBefore('=') }.toSet() val defaults = listOf("--seed=42", "--problemType=ASYNCAPI", "--createTests=false") .filterNot { it.removePrefix("--").substringBefore('=') in named } - val args = (defaults + options).toTypedArray() + return (defaults + options).toTypedArray() + } + + /** + * An injector for a search driven by [driver], over the real module with the driver + * swapped, under [argsWith]'s options. + */ + fun create(driver: FakeAsyncApiDriver, vararg options: String): Injector { + + val args = argsWith(*options) val fake = object : AbstractModule() { override fun configure() {