AsyncAPI 3.x: an E2E test, NCS driven through Kafka

.github/workflows/ci.yml

@@ -460,7 +460,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: [ BaseTests, CoreTests, CoreITTests, OAV3Tests, OAV2Tests, GraphQLTests, MongoTests, H2V1Tests, MiscDB ]
+        profile: [ BaseTests, CoreTests, CoreITTests, OAV3Tests, OAV2Tests, GraphQLTests, MongoTests, H2V1Tests, MiscDB, AsyncApiTests ]
     steps:
       - uses: actions/checkout@v6
       - name: Setup JDK ${{env.build-jdk}}

.gitignore

@@ -171,6 +171,7 @@ Migrations/
 /core-tests/e2e-tests/spring/spring-rest-h2-v2/target/
 /core-tests/e2e-tests/spring/spring-rest-rsa/target/
 /core-tests/e2e-tests/spring/spring-rest-dynamodb/target/
+/core-tests/e2e-tests/spring/spring-asyncapi-kafka/target/
 /core-tests/e2e-tests/spring/spring-rest-h2-v1/em.yaml
 /core-tests/integration-tests/core-it/target/
 /core-tests/integration-tests/core-it/em.yaml

core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/AsyncApiTestBase.java

@@ -0,0 +1,91 @@
+package org.evomaster.e2etests.utils;
+
+import com.webfuzzing.commons.faults.FaultCategory;
+import org.evomaster.core.Main;
+import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult;
+import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual;
+import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome;
+import org.evomaster.core.problem.enterprise.DetectedFault;
+import org.evomaster.core.search.Solution;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * What an E2E test over an AsyncAPI service needs: to run the search, and to read what
+ * publishing to each operation was seen to do.
+ */
+public class AsyncApiTestBase extends EnterpriseTestBase {
+
+    protected Solution<AsyncApiIndividual> initAndRun(List<String> args) {
+        return (Solution<AsyncApiIndividual>) Main.initAndRun(args.toArray(new String[0]));
+    }
+
+    /**
+     * The result of every message published to {@code operation}, across the whole solution.
+     */
+    protected List<AsyncApiCallResult> resultsOf(Solution<AsyncApiIndividual> solution, String operation) {
+        return solution.getIndividuals().stream()
+                .flatMap(ind -> ind.evaluatedMainActions().stream())
+                .filter(e -> e.getAction().getName().equals(operation))
+                .map(e -> (AsyncApiCallResult) e.getResult())
+                .collect(Collectors.toList());
+    }
+
+    /**
+     * The declared messages the replies to {@code operation} were recognised as.
+     */
+    protected Set<String> repliesOf(Solution<AsyncApiIndividual> solution, String operation) {
+        return resultsOf(solution, operation).stream()
+                .map(AsyncApiCallResult::getReplyMessage)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toSet());
+    }
+
+    /**
+     * The fault categories reported across the whole solution. Read off the action results,
+     * which is where the reports count faults from, rather than off the covered targets.
+     */
+    protected Set<FaultCategory> faultsOf(Solution<AsyncApiIndividual> solution) {
+        return solution.getIndividuals().stream()
+                .flatMap(ind -> ind.evaluatedMainActions().stream())
+                .map(e -> (AsyncApiCallResult) e.getResult())
+                .flatMap(r -> r.getFaults().stream())
+                .map(DetectedFault::getCategory)
+                .collect(Collectors.toSet());
+    }
+
+    protected long countOutcome(Solution<AsyncApiIndividual> solution, AsyncApiOutcome outcome) {
+        return solution.getIndividuals().stream()
+                .flatMap(ind -> ind.evaluatedMainActions().stream())
+                .map(e -> (AsyncApiCallResult) e.getResult())
+                .filter(r -> r.getOutcome() == outcome)
+                .count();
+    }
+
+    protected void assertReplied(Solution<AsyncApiIndividual> solution, String operation) {
+        boolean ok = resultsOf(solution, operation).stream().anyMatch(r -> r.getOutcome() == AsyncApiOutcome.REPLIED);
+        assertTrue(ok, "With seed " + defaultSeed + ": no reply to '" + operation + "' was ever received");
+    }
+
+    /**
+     * That some reply to {@code operation} arrived and matched none of the messages the contract
+     * declares for it, which is the one fault only a reply classifier can find.
+     */
+    protected void assertRepliedWithUndeclaredMessage(Solution<AsyncApiIndividual> solution, String operation) {
+        boolean found = resultsOf(solution, operation).stream()
+                .anyMatch(r -> r.getOutcome() == AsyncApiOutcome.REPLIED && r.getReplyMessage() == null);
+        assertTrue(found, "With seed " + defaultSeed + ": every reply to '" + operation
+                + "' matched a declared message, so the undeclared-reply path was never reached");
+    }
+
+    protected void assertReplyReached(Solution<AsyncApiIndividual> solution, String operation, String messageId) {
+        Set<String> replies = repliesOf(solution, operation);
+        assertTrue(replies.contains(messageId),
+                "With seed " + defaultSeed + ": '" + operation + "' never replied with '" + messageId + "', only with " + replies);
+    }
+}

core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java

@@ -430,15 +430,37 @@ protected Class<?> loadClass(ClassName className){
         }
     }
 
+    private static boolean containsSourceWithExtension(File folder, String extension){
+        File[] children = folder.listFiles();
+        if (children == null) {
+            return false;
+        }
+        for (File f : children) {
+            if (f.isDirectory() && containsSourceWithExtension(f, extension)) {
+                return true;
+            }
+            if (f.isFile() && f.getName().endsWith(extension)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     protected void compile(String outputFolderName){
 
         long start = System.currentTimeMillis();
 
-        CompilerForTestGenerated.INSTANCE.compile(
-                OutputFormat.KOTLIN_JUNIT_5,
-                new File(outputFolderPath(outputFolderName)),
-                new File("target/test-classes")
-        );
+        File folder = new File(outputFolderPath(outputFolderName));
+
+        /*
+            Which language was generated is not passed down here, but the folder says: a suite is
+            written in one of them, so whichever is on disk is what has to be compiled.
+         */
+        OutputFormat format = containsSourceWithExtension(folder, ".java")
+                ? OutputFormat.JAVA_JUNIT_5
+                : OutputFormat.KOTLIN_JUNIT_5;
+
+        CompilerForTestGenerated.INSTANCE.compile(format, folder, new File("target/test-classes"));
 
         int passed = (int)(System.currentTimeMillis() - start) / 1000;
         System.out.println("Folder compiled in " + passed + " seconds: " + outputFolderName);

core-tests/e2e-tests/spring/pom.xml

@@ -35,6 +35,7 @@
         <module>spring-rpc-grpc</module>
         <module>spring-rpc-thrift</module>
         <module>spring-mcp-bb</module>
+        <module>spring-asyncapi-kafka</module>
     </modules>
 
     <dependencyManagement>
@@ -45,6 +46,11 @@
                 <artifactId>spring-boot</artifactId>
                 <version>${springboot.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-starter</artifactId>
+                <version>${springboot.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-starter-web</artifactId>

core-tests/e2e-tests/spring/spring-asyncapi-kafka/pom.xml

@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>evomaster-e2e-tests-spring</artifactId>
+        <groupId>org.evomaster</groupId>
+        <version>6.2.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <!--
+        NCS, the numerical case study, re-expressed as a service driven by Kafka messages:
+        a request topic and a reply topic per operation, described by an AsyncAPI document.
+        The SUT speaks only Kafka; there is no HTTP endpoint at all.
+    -->
+    <artifactId>evomaster-e2e-tests-spring-asyncapi-kafka</artifactId>
+    <packaging>jar</packaging>
+
+    <properties>
+        <skipTests>${skipAsyncApi}</skipTests>
+    </properties>
+
+    <dependencies>
+        <!--
+            Issue with Jersey (coming from client-java) using old version
+        -->
+        <dependency>
+            <groupId>javax.validation</groupId>
+            <artifactId>validation-api</artifactId>
+            <version>2.0.1.Final</version>
+        </dependency>
+        <dependency>
+            <groupId>org.evomaster</groupId>
+            <artifactId>evomaster-e2e-tests-utils</artifactId>
+            <type>test-jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.evomaster</groupId>
+            <artifactId>evomaster-client-java-controller</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.evomaster</groupId>
+            <artifactId>evomaster-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.evomaster</groupId>
+            <artifactId>evomaster-client-java-instrumentation</artifactId>
+            <type>test-jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>kafka</artifactId>
+        </dependency>
+        <dependency>
+            <!--
+                Not used by any source in this module, but the test suites EvoMaster generates
+                import org.hamcrest.Matchers, and they are compiled against this classpath.
+            -->
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-all</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <!-- a broker in a container per test class, as the other container-backed E2Es do -->
+                    <reuseForks>false</reuseForks>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

core-tests/e2e-tests/spring/spring-asyncapi-kafka/src/main/java/com/foo/asyncapi/ncs/NcsKafkaApplication.java

@@ -0,0 +1,29 @@
+package com.foo.asyncapi.ncs;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.WebApplicationType;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.validation.ValidationAutoConfiguration;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+
+/**
+ * NCS over Kafka: the six numerical operations of the NCS case study, each consuming a request
+ * topic and answering on a reply topic, as described by {@code asyncapi/ncs-kafka.yaml}.
+ *
+ * The service speaks only Kafka. What listens on it is {@link NcsRequestConsumer}, which hands
+ * every request to {@link NcsService}.
+ */
+/*
+    Bean validation is excluded: this service validates nothing, and the validation API that the
+    EvoMaster client puts on the classpath would otherwise make Spring look for an EL
+    implementation that is not there.
+ */
+@SpringBootApplication(exclude = ValidationAutoConfiguration.class)
+public class NcsKafkaApplication {
+
+    public static void main(String[] args) {
+        new SpringApplicationBuilder(NcsKafkaApplication.class)
+                .web(WebApplicationType.NONE)
+                .run(args);
+    }
+}