AsyncAPI 3.x: write the test suites

client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java

@@ -1,5 +1,7 @@
 package org.evomaster.client.java.controller.api.dto.problem.asyncapi;
 
+import org.evomaster.client.java.controller.api.dto.SutInfoDto;
+
 import java.util.LinkedHashMap;
 import java.util.Map;
 
@@ -96,4 +98,25 @@ public enum CorrelationLocation {
      * Null when no reply is expected, as there is then nothing to wait for.
      */
     public Long replyTimeoutMs;
+
+    /**
+     * The language to render {@link AsyncApiReplyDto#testScript} in, or null when no script is
+     * wanted: either the core is not generating tests, or it is generating a format this enum
+     * cannot name, which is what a Python run is.
+     *
+     * A generated test talks to the broker with an ordinary client of the transport rather than
+     * through the driver, so the lines that publish and await have to come from the one side
+     * that knows the transport. Only Java and Kotlin are ever asked for.
+     */
+    public SutInfoDto.OutputFormat outputFormat;
+
+    /**
+     * The name the rendered lines must leave the reply payload in, as text, when a reply is
+     * expected. Named by the core so that two actions in one test cannot collide.
+     *
+     * Set whenever tests are being generated, whether or not a script was asked for: the core
+     * owns the name, whoever ends up writing the lines that assign to it. Null when no tests
+     * are being generated.
+     */
+    public String replyVariable;
 }

client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java

@@ -1,6 +1,7 @@
 package org.evomaster.client.java.controller.api.dto.problem.asyncapi;
 
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -86,4 +87,20 @@ public class AsyncApiReplyDto {
      * Why publishing failed, when it did.
      */
     public String errorMessage;
+
+    /**
+     * The publish and await this action just did, rendered as source lines for a generated test,
+     * in the language {@link AsyncApiActionDto#outputFormat} asked for. Null when none was asked
+     * for, or when the driver does not render them.
+     *
+     * A generated test talks to the broker directly rather than through the driver, so these
+     * lines stand up a client of the transport, publish the same message and read the reply.
+     * Only the driver can write them: it is the one side that knows the transport, and the core
+     * pastes them without reading them.
+     *
+     * Where a reply is expected, the lines must leave it in the variable the core named in
+     * {@link AsyncApiActionDto#replyVariable}, which is what the assertions the core appends are
+     * written against.
+     */
+    public List<String> testScript;
 }

client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java

@@ -147,6 +147,25 @@ default void setupForGeneratedTest(){}
      */
     default Object executeRPCEndpoint(String json) throws Exception {return null;}
 
+    /**
+     * <p>
+     * Where the AsyncAPI server called {@code serverName} can actually be reached, now.
+     * </p>
+     *
+     * The document gives an address, but it is written for the deployment the author had in
+     * mind. A system started for testing is often somewhere else: a broker in a container binds
+     * a port chosen when it starts, and a different one on the next run.
+     *
+     * A generated suite that carries the driver asks this when it starts, rather than holding an
+     * address that was only ever true once, in the same way it takes the base URL of a REST
+     * system from {@link #startSut()}. A plain black-box suite has no driver in it, so it
+     * publishes to the address the document declares, as it does when this returns null.
+     *
+     * @param serverName the key of the server in the document
+     * @return where to reach it, or null to fall back to what the document says
+     */
+    default String getAsyncApiServerAddress(String serverName) {return null;}
+
     /**
      * <p>
      * execute an RPC endpoint with evomaster driver

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -673,7 +673,15 @@ class EMConfig {
                         " 'problemType'. The system will default to RESTful API testing.")
                 problemType = ProblemType.REST
             }
-            if (outputFormat == OutputFormat.DEFAULT) {
+            /*
+                AsyncAPI is the exception: a driver publishes its messages even in black-box, since
+                there is no universal client for a broker, so the format is resolved from the driver
+                as it is in white-box rather than from the black-box default.
+
+                A driver can only name a format SutInfoDto.OutputFormat has, which Python is not, so
+                a Python suite has to be asked for with this option.
+             */
+            if (outputFormat == OutputFormat.DEFAULT && problemType != ProblemType.ASYNCAPI) {
                 LoggingUtil.uniqueUserWarn("You are doing Black-Box testing, but you did not specify the" +
                         " 'outputFormat'. The system will default to $defaultOutputFormatForBlackBox.")
                 outputFormat = defaultOutputFormatForBlackBox
@@ -816,10 +824,7 @@ class EMConfig {
             throw ConfigProblemException("When using the seedTestCases option, you must specify the file path of the test cases with the seedTestCasesPath option")
         }
 
-        if (problemType == ProblemType.ASYNCAPI && createTests) {
-            throw ConfigProblemException("Test generation for AsyncAPI services is not available yet." +
-                    " For the time being, run with '--createTests false' to only search for faults.")
-        }
+        checkOutputFormatIsWritable()
 
         if (problemType == ProblemType.ASYNCAPI && seedTestCases) {
             throw ConfigProblemException("Seeding test cases is not supported for AsyncAPI services yet")
@@ -1050,6 +1055,25 @@ class EMConfig {
      */
     fun usesDriver() = !blackBox || bbExperiments || problemType == ProblemType.ASYNCAPI
 
+    /**
+     * Whether a test can be written at all in the chosen format.
+     *
+     * Checked here with the rest of the options, and again once a DEFAULT has been resolved from
+     * the driver, which happens after every check here has run. An AsyncAPI run in black-box
+     * mode takes that road as well, so this is the only place that sees what it ended up with.
+     */
+    fun checkOutputFormatIsWritable() {
+
+        if (problemType == ProblemType.ASYNCAPI
+                && createTests
+                && outputFormat != OutputFormat.DEFAULT
+                && !outputFormat.isJavaOrKotlin() && !outputFormat.isPython()) {
+            throw ConfigProblemException("Tests for an AsyncAPI service publish with a client of the transport," +
+                    " and one is written only for Java, Kotlin and Python so far, so outputFormat supports" +
+                    " only those. Was: $outputFormat")
+        }
+    }
+
     fun activatedExperimentalFeatures(): List<String> {
 
         val properties = getConfigurationProperties()
@@ -1343,7 +1367,12 @@ class EMConfig {
     @Important(2.0)
     @Cfg("Specify in which format the tests should be outputted." +
             " If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used." +
-            " On the other hand, for black-box testing it will default to a predefined type (e.g., Python).")
+            " On the other hand, for black-box testing it will default to a predefined type (e.g., Python)." +
+            " AsyncAPI is the exception on both counts: a driver is what publishes during the search, in either" +
+            " mode, so a `DEFAULT` is resolved from the driver there as well rather than from the black-box" +
+            " default. Only Java, Kotlin and Python can be generated, as a generated test publishes with a client" +
+            " of the transport rather than through the driver, and only those three have one written for them so" +
+            " far. Python has to be asked for here explicitly, as a driver has no way of naming it.")
     var outputFormat = OutputFormat.DEFAULT
 
     @Important(2.1)
@@ -1547,7 +1576,8 @@ class EMConfig {
             " If left to DEFAULT, the type will be inferred from the EM Driver." +
             " However, in case of ambiguities (e.g., the driver specifies more than one type)," +
             " then this field must be set with a specific type." +
-            " This is also the case for Black-Box testing where there is no EM Driver." +
+            " This is also the case for Black-Box testing, where there is usually no EM Driver" +
+            " (AsyncAPI is the exception: see outputFormat)." +
             " In this latter case, the system defaults to handle REST APIs.")
     var problemType = ProblemType.DEFAULT
 

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -572,22 +572,6 @@ class Main {
                     config.problemType = EMConfig.ProblemType.WEBFRONTEND
                 } else if (info.asyncApiProblem != null) {
                     config.problemType = EMConfig.ProblemType.ASYNCAPI
-                    if (config.createTests) {
-                        /*
-                            There is no test writer for AsyncAPI yet, and the constraints checked
-                            below refuse the combination. As the problem type was inferred rather
-                            than asked for, turning test generation off is better than failing.
-
-                            TODO remove this block once there is a test writer for AsyncAPI: the
-                            combination will be valid then, and silently switching test generation
-                            off would hide it.
-                         */
-                        LoggingUtil.uniqueUserWarn(
-                            "The driver describes an AsyncAPI service, for which test generation is not" +
-                                    " available yet. Continuing with 'createTests' off."
-                        )
-                        config.createTests = false
-                    }
                 } else {
                     throw IllegalStateException("Can connect to the EM Driver, but cannot infer the 'problemType'")
                 }

core/src/main/kotlin/org/evomaster/core/output/TestSuiteSplitter.kt

@@ -6,6 +6,8 @@ import org.evomaster.core.output.service.PartialOracles
 import org.evomaster.core.problem.graphql.GraphQlCallResult
 import org.evomaster.core.problem.httpws.HttpWsCallResult
 import org.evomaster.core.problem.rpc.RPCCallResult
+import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult
+import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome
 import org.evomaster.core.problem.rpc.RPCIndividual
 import org.evomaster.core.search.*
 import com.google.gson.*
@@ -122,8 +124,17 @@ object TestSuiteSplitter {
                         &&
                         ind.evaluatedMainActions().all { ac ->
                             //TODO generic per type
-                            val code = (ac.result as HttpWsCallResult).getStatusCode()
-                            (code != null && code < 400)
+                            when (val r = ac.result) {
+                                is HttpWsCallResult -> r.getStatusCode().let { it != null && it < 400 }
+                                /*
+                                    A message that went out, and was answered when an answer was
+                                    promised. There is no status code to read here.
+                                 */
+                                is AsyncApiCallResult -> r.getOutcome().let {
+                                    it == AsyncApiOutcome.PUBLISHED || it == AsyncApiOutcome.REPLIED
+                                }
+                                else -> true
+                            }
                         }
             }.toMutableList()