diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ActionDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ActionDto.java index 9f8f1c202d..7b8f00168e 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ActionDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ActionDto.java @@ -1,6 +1,7 @@ package org.evomaster.client.java.controller.api.dto; import org.evomaster.client.java.controller.api.dto.problem.ExternalServiceDto; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto; import org.evomaster.client.java.controller.api.dto.problem.rpc.RPCActionDto; import org.evomaster.client.java.controller.api.dto.problem.rpc.ScheduleTaskInvocationDto; @@ -39,6 +40,12 @@ public class ActionDto { * note that this is only used when handling RPC problem */ public RPCActionDto rpcCall; + + /** + * info to publish a message. + * note that this is only used when handling an AsyncAPI problem + */ + public AsyncApiActionDto asyncApiCall; /** diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/SutInfoDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/SutInfoDto.java index ea50070eea..b305be4331 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/SutInfoDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/SutInfoDto.java @@ -6,6 +6,7 @@ import org.evomaster.client.java.controller.api.dto.problem.RestProblemDto; import org.evomaster.client.java.controller.api.dto.problem.GraphQLProblemDto; import org.evomaster.client.java.controller.api.dto.problem.WebProblemDto; +import org.evomaster.client.java.controller.api.dto.problem.AsyncApiProblemDto; import java.util.List; @@ -35,6 +36,12 @@ public class SutInfoDto { */ public WebProblemDto webProblem; + /** + * If the SUT is a service driven by messages and described by an AsyncAPI document, + * here there will be the info on how to interact with it + */ + public AsyncApiProblemDto asyncApiProblem; + /** * Whether the SUT is running or not */ diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/AsyncApiProblemDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/AsyncApiProblemDto.java new file mode 100644 index 0000000000..ee34b8b079 --- /dev/null +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/AsyncApiProblemDto.java @@ -0,0 +1,26 @@ +package org.evomaster.client.java.controller.api.dto.problem; + +/** + * Info the driver gives about an AsyncAPI service, so that the core can learn what the service + * consumes and what shape those messages have. + * + * Note the transport client is not here, and never crosses: it is an open connection to a + * broker, held by the driver. Only the document travels. + */ +public class AsyncApiProblemDto extends ProblemInfoDto { + + /** + * Where the AsyncAPI document can be fetched from: a URL, or a path on the machine running + * the driver. Null when the document is given inline instead. + */ + public String schemaLocation; + + /** + * The AsyncAPI document itself. Null when a location is given instead. + * + * Useful when the document is packaged with the service rather than served by it, which is + * the common case: unlike OpenAPI, an AsyncAPI service rarely exposes its own contract over + * HTTP, since it may not speak HTTP at all. + */ + public String schemaText; +} diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java new file mode 100644 index 0000000000..5b1e5b441a --- /dev/null +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java @@ -0,0 +1,99 @@ +package org.evomaster.client.java.controller.api.dto.problem.asyncapi; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * One message for the driver to publish, and what to do about a reply. + * + * Everything here is decided by the core: which operation, where it goes, what it says. The + * driver's job is to put it on the wire and, when a reply is expected, wait for the one that + * answers it. + */ +public class AsyncApiActionDto { + + /** + * The two places a correlation id can travel, as {@link #correlationLocation} names them. + */ + public static final String CORRELATION_IN_HEADER = "HEADER"; + + public static final String CORRELATION_IN_PAYLOAD = "PAYLOAD"; + + /** + * Key of the operation in the AsyncAPI document. Sent along so the driver can report and + * log in terms the user will recognise from their own contract. + */ + public String operationId; + + /** + * Key of the channel the message is published on. + */ + public String channelName; + + /** + * Where the message actually goes on the wire: a topic, a queue, a routing key. Already + * resolved by the core, including any binding that overrides the channel's address. + */ + public String address; + + /** + * Id of the message being published, as the document names it. + */ + public String messageId; + + /** + * The message body, serialised. Its content type is in {@link #contentType}. + */ + public String payload; + + /** + * What the document declares the payload is encoded as, eg "application/json". + */ + public String contentType; + + /** + * Headers to publish alongside the body, for a transport that has them. + * Key is the header name as the document declares it, value is what to send under it, as text. + */ + public Map headers = new LinkedHashMap<>(); + + /** + * The value stamped into this message so that a reply can be recognised as answering it. + * + * It is minted fresh by the core for every execution rather than being part of the message + * the search varies: pairing needs a value unique to the execution, and the service only + * echoes it back. + */ + public String correlationId; + + /** + * Where the correlation id has to be written, as the document declares it. One of + * {@link #CORRELATION_IN_HEADER} or {@link #CORRELATION_IN_PAYLOAD}, or null when the + * document says nothing, in which case it is up to the driver to decide -- a transport with + * native correlation should use it. + */ + public String correlationLocation; + + /** + * JSON Pointer to the field the correlation id goes in, within whatever + * {@link #correlationLocation} names. Null when there is no declared location. + */ + public String correlationPointer; + + /** + * Where a reply is expected to arrive, when the operation declares one. Null for a + * fire-and-forget operation, in which case the driver publishes and returns. + */ + public String replyAddress; + + /** + * How long to wait for a reply before giving up, in milliseconds. + * + * There is no right answer here: a slow service and a stuck one look the same from outside, + * so this is a tuning parameter with no equivalent in a synchronous protocol. It is set + * generously and reported with the result. + * + * Null when no reply is expected, as there is then nothing to wait for. + */ + public Long replyTimeoutMs; +} diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java new file mode 100644 index 0000000000..c36e0e0703 --- /dev/null +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java @@ -0,0 +1,89 @@ +package org.evomaster.client.java.controller.api.dto.problem.asyncapi; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * What came of publishing one message. + * + * The four outcomes are deliberately distinguished, because they mean different things and only + * one of them is a fault: + * + * + */ +public class AsyncApiReplyDto { + + /** + * The index of the action this answers, echoing what was asked. + */ + public Integer index; + + /** + * Whether the message reached the broker. False means the driver could not publish, and + * {@link #errorMessage} says why. + * + * Null means the driver did not say, which is read the same way as false: without an answer + * here there is no knowing whether the message went out, and the rest of the test would + * mean nothing. + */ + public Boolean published; + + /** + * Whether a reply arrived and was recognised as answering this message. + * + * Null means the driver did not say, and is read as no reply having arrived. + */ + public Boolean replyReceived; + + /** + * Whether the driver waited for a reply at all. False for a fire-and-forget operation, so + * that the absence of a reply is not mistaken for silence in answer to a promise. + * + * Null means the driver did not say, and is read as not having waited. + */ + public Boolean replyExpected; + + /** + * The reply body, as it arrived. + */ + public String replyPayload; + + /** + * The reply's headers, for a transport that has them. + * Key is the header name, value is what arrived under it, as text. + */ + public Map replyHeaders = new LinkedHashMap<>(); + + /** + * Whether the reply carried back the correlation id that was stamped on the request. + * + * This is the honest answer to "did correlation work", which cannot be read off a contract: + * echoing the id is the service's own behaviour. A reply that arrives without it is + * recorded rather than treated as a fault, since from outside there is no telling a defect + * from a service that correlates by some business key instead. + * + * Null means the driver does not track correlation at all, which is not the same as having + * checked and found the id missing, and is recorded as neither. + */ + public Boolean correlationMatched; + + /** + * How long the driver waited, in milliseconds, whether or not anything arrived. Reported + * because the verdict on silence is only meaningful alongside how long it was waited for. + * + * Null when the driver did not wait at all: no reply was expected, or nothing was published. + */ + public Long waitedMs; + + /** + * Why publishing failed, when it did. + */ + public String errorMessage; +} diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java index 5951d369d9..97c789e324 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java @@ -5,6 +5,7 @@ import org.evomaster.client.java.controller.api.dto.*; import org.evomaster.client.java.controller.api.dto.database.operations.*; import org.evomaster.client.java.controller.api.dto.problem.*; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto; import org.evomaster.client.java.controller.api.dto.problem.param.DeriveParamResponseDto; import org.evomaster.client.java.controller.api.dto.problem.param.DerivedParamChangeReqDto; import org.evomaster.client.java.controller.api.dto.problem.param.RestDerivedParamDto; @@ -235,6 +236,13 @@ public Response getSutInfo(@Context HttpServletRequest httpServletRequest) { SimpleLogger.error(msg, e); return Response.status(500).entity(WrappedResponseDto.withError(msg)).build(); } + } else if (info instanceof AsyncApiProblem) { + AsyncApiProblem p = (AsyncApiProblem) info; + dto.asyncApiProblem = new AsyncApiProblemDto(); + dto.asyncApiProblem.schemaLocation = p.getSchemaLocation(); + dto.asyncApiProblem.schemaText = p.getSchemaText(); + dto.asyncApiProblem.servicesToNotMock = servicesToNotMock; + } else if(info instanceof WebProblem){ WebProblem p = (WebProblem) info; dto.webProblem = new WebProblemDto(); @@ -830,6 +838,28 @@ public Response newAction( } } + + if (dto.asyncApiCall != null) { + + AsyncApiReplyDto replyDto = new AsyncApiReplyDto(); + replyDto.index = index; + + try { + sutController.executeAsyncApiAction(dto.asyncApiCall, replyDto); + return Response.status(200).entity(WrappedResponseDto.withData(replyDto)).build(); + } catch (Exception e) { + /* + Failing to publish is not a finding about the service, it is a broken + setup, so it is reported as such rather than as silence in answer to a + promised reply. + */ + String msg = "Thrown exception when publishing a message: " + e.getMessage(); + SimpleLogger.error(msg, e); + replyDto.published = false; + replyDto.errorMessage = msg; + return Response.status(500).entity(WrappedResponseDto.withData(replyDto)).build(); + } + } } return Response.status(204).entity(WrappedResponseDto.withNoData()).build(); diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 053b1a8e8b..684dd9f878 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -23,6 +23,8 @@ import org.evomaster.client.java.controller.api.dto.MockDatabaseDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.controller.api.dto.problem.RPCProblemDto; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto; import org.evomaster.client.java.controller.api.dto.problem.rpc.*; import org.evomaster.client.java.controller.api.dto.problem.rpc.RPCTestDto; import org.evomaster.client.java.controller.internal.db.OpenSearchHandler; @@ -1929,6 +1931,41 @@ public boolean isScheduleTaskCompleted(ScheduleTaskInvocationResultDto invocatio return false; } + /** + * Publish one message, and wait for the reply that answers it when one is expected. + * + * This is where a driver for an AsyncAPI service does its work, and it is the counterpart + * of {@link #executeAction(RPCActionDto, ActionResponseDto)} for RPC: the core decides + * what to send and reads what comes back, while everything that knows about a broker lives + * on this side. Only publish and await are protocol-specific, and they never leave here. + * + * A driver that does not test an AsyncAPI service has no reason to override this. + * + * Sketch of what an implementation does, for a transport whose correlation rides in + * metadata: + * + * 
+     * publish(dto.address, dto.payload, dto.headers + {correlationId: dto.correlationId});
+     * reply.published = true;
+     * if (dto.replyAddress != null) {
+     *     reply.replyExpected = true;
+     *     awaitOn(dto.replyAddress, matching dto.correlationId, within dto.replyTimeoutMs);
+     * }
+     *
+ * + * Note what is not asked of the driver: it does not judge the reply, only reports it. + * Deciding what an outcome means is the core's job, so that it means the same thing + * whatever the transport. + * + * @param dto what to publish, and where a reply is expected + * @param reply to be filled in with what happened + */ + public void executeAsyncApiAction(AsyncApiActionDto dto, AsyncApiReplyDto reply) { + throw new IllegalStateException( + "Trying to publish a message, but this driver does not implement" + + " executeAsyncApiAction. It must be overridden to test an AsyncAPI service."); + } + @Override public void resetDatabase(List tablesToClean) { diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/problem/AsyncApiProblem.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/problem/AsyncApiProblem.java new file mode 100644 index 0000000000..4d3582dede --- /dev/null +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/problem/AsyncApiProblem.java @@ -0,0 +1,71 @@ +package org.evomaster.client.java.controller.problem; + +import java.util.List; +import java.util.Objects; + +/** + * Declares that the SUT is a service driven by messages, described by an AsyncAPI document. + * + * AsyncAPI is the event-driven counterpart of OpenAPI: it describes services that talk over a + * broker or a socket rather than over HTTP. That difference is why a driver is needed even for + * black-box testing, which is not so for REST or GraphQL. There is no universal wire to point + * at -- Kafka, AMQP, MQTT and WebSocket share nothing at the API level -- so something has to + * hold a client and move the bytes, and that something is the driver. + * + * The document is the only thing that crosses to the core. The connection to the broker stays + * here, in the driver, which is what keeps the core from depending on any broker library. + */ +public class AsyncApiProblem extends ProblemInfo { + + private final String schemaLocation; + + private final String schemaText; + + /** + * @param schemaLocation where the AsyncAPI document can be fetched from: a URL, or a path + * on the machine running the driver + */ + public AsyncApiProblem(String schemaLocation) { + this(schemaLocation, null); + } + + private AsyncApiProblem(String schemaLocation, String schemaText) { + + if ((schemaLocation == null && schemaText == null) + || (schemaLocation != null && schemaText != null)) { + throw new IllegalArgumentException( + "An AsyncAPI problem needs exactly one of a schema location and a schema text"); + } + + this.schemaLocation = schemaLocation; + this.schemaText = schemaText; + } + + /** + * Declare the problem with the document itself rather than somewhere to fetch it from. + * + * This is the common case, and the difference from OpenAPI is worth stating: a REST service + * usually serves its own contract over HTTP, whereas a service that speaks only Kafka has + * no endpoint to serve anything from. Its document is far more likely to be a file shipped + * beside it. + */ + public static AsyncApiProblem fromSchemaText(String schemaText) { + return new AsyncApiProblem(null, Objects.requireNonNull(schemaText)); + } + + public String getSchemaLocation() { + return schemaLocation; + } + + public String getSchemaText() { + return schemaText; + } + + @Override + public AsyncApiProblem withServicesToNotMock(List servicesToNotMock) { + Objects.requireNonNull(servicesToNotMock, "servicesToNotMock"); + AsyncApiProblem p = new AsyncApiProblem(this.schemaLocation, this.schemaText); + p.servicesToNotMock.addAll(servicesToNotMock); + return p; + } +} diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/problem/AsyncApiProblemTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/problem/AsyncApiProblemTest.java new file mode 100644 index 0000000000..4b48e53b6b --- /dev/null +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/problem/AsyncApiProblemTest.java @@ -0,0 +1,152 @@ +package org.evomaster.client.java.controller.problem; + +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto; +import org.evomaster.client.java.controller.EmbeddedSutController; +import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto; +import org.evomaster.client.java.controller.api.dto.SutInfoDto; +import org.evomaster.client.java.controller.internal.SutController; +import org.evomaster.client.java.sql.DbSpecification; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class AsyncApiProblemTest { + + @Test + public void testDeclaredWithSomewhereToFetchTheDocumentFrom() { + + AsyncApiProblem p = new AsyncApiProblem("http://localhost:8080/asyncapi.yaml"); + + assertEquals("http://localhost:8080/asyncapi.yaml", p.getSchemaLocation()); + assertNull(p.getSchemaText()); + } + + @Test + public void testDeclaredWithTheDocumentItself() { + + /* + The common case, and the difference from OpenAPI: a service that speaks only Kafka + has no HTTP endpoint to serve its own contract from, so the document is far more + likely to be a file shipped beside it than a URL. + */ + AsyncApiProblem p = AsyncApiProblem.fromSchemaText("asyncapi: 3.0.0"); + + assertEquals("asyncapi: 3.0.0", p.getSchemaText()); + assertNull(p.getSchemaLocation()); + } + + @Test + public void testTheDocumentHasToComeFromExactlyOnePlace() { + + //neither + assertThrows(IllegalArgumentException.class, () -> new AsyncApiProblem(null)); + + //and null is not a document + assertThrows(NullPointerException.class, () -> AsyncApiProblem.fromSchemaText(null)); + } + + @Test + public void testServicesToNotMockAreCarriedOver() { + + List services = + Arrays.asList(new ExternalService("foo.com", 80), new ExternalService("bar.com", 443)); + + AsyncApiProblem p = AsyncApiProblem.fromSchemaText("asyncapi: 3.0.0") + .withServicesToNotMock(services); + + assertEquals(2, p.getServicesToNotMock().size()); + assertEquals("asyncapi: 3.0.0", p.getSchemaText()); + } + + @Test + public void testServicesToNotMockCannotBeNull() { + + AsyncApiProblem p = AsyncApiProblem.fromSchemaText("asyncapi: 3.0.0"); + + assertThrows(NullPointerException.class, () -> p.withServicesToNotMock(null)); + } + + @Test + public void testServicesToNotMockCannotBeChangedFromOutside() { + + AsyncApiProblem p = AsyncApiProblem.fromSchemaText("asyncapi: 3.0.0") + .withServicesToNotMock(Collections.singletonList(new ExternalService("foo.com", 80))); + + assertThrows( + UnsupportedOperationException.class, + () -> p.getServicesToNotMock().add(new ExternalService("bar.com", 80)) + ); + } + + @Test + public void testADriverThatDoesNotPublishSaysSoClearly() { + + /* + The hook has a default rather than being abstract, so that adding it does not break + every existing driver. A driver that is pointed at an AsyncAPI service without + implementing it should fail with something that explains what to do. + */ + SutController controller = new AsyncApiControllerWithoutPublishing(); + + IllegalStateException e = assertThrows( + IllegalStateException.class, + () -> controller.executeAsyncApiAction(new AsyncApiActionDto(), new AsyncApiReplyDto()) + ); + + assertTrue(e.getMessage().contains("executeAsyncApiAction"), e.getMessage()); + } + + /** + * A driver that says its SUT is an AsyncAPI service, but never implements the publishing. + */ + private static class AsyncApiControllerWithoutPublishing extends EmbeddedSutController { + + @Override + public String startSut() { + return null; + } + + @Override + public boolean isSutRunning() { + return false; + } + + @Override + public void stopSut() { + } + + @Override + public String getPackagePrefixesToCover() { + return null; + } + + @Override + public void resetStateOfSUT() { + } + + @Override + public List getInfoForAuthentication() { + return null; + } + + @Override + public List getDbSpecifications() { + return null; + } + + @Override + public ProblemInfo getProblemInfo() { + return AsyncApiProblem.fromSchemaText("asyncapi: 3.0.0"); + } + + @Override + public SutInfoDto.OutputFormat getPreferredOutputFormat() { + return SutInfoDto.OutputFormat.JAVA_JUNIT_5; + } + } +}