diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java index 440ab53619..b9c3ea5fcd 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java @@ -93,6 +93,17 @@ public enum CorrelationLocation { */ public Long replyTimeoutMs; + /** + * How long to keep the action open once the driver has returned, in milliseconds. The SUT + * handles a message on a thread of its own, possibly after the driver saw the publish + * acknowledged or the reply arrive, and what it covers meanwhile must be credited to this + * action, not to the next one or to none. + * + * The wait is done by the controller, so drivers need not implement it. Null, zero or + * negative mean no wait. Only set when the SUT is instrumented. + */ + public Long settleMs; + /** * The language to render {@link AsyncApiReplyDto#testScript} in. Only Java and Kotlin are * ever asked for. diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java index 015024eee7..a575243c86 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java @@ -66,6 +66,16 @@ public class AsyncApiReplyDto { */ public Long waitedMs; + /** + * How long the controller kept the action open after the driver returned, in milliseconds, + * as asked by {@link AsyncApiActionDto#settleMs}. It can be shorter than what was asked when + * the wait was interrupted. + * + * Filled in by the controller rather than by the driver, which never sees this field set. + * Null when no wait was asked for, or when the message could not be published. + */ + public Long settledMs; + /** * Why publishing failed, when it did. */ diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/AsyncApiSettle.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/AsyncApiSettle.java new file mode 100644 index 0000000000..167843df9f --- /dev/null +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/AsyncApiSettle.java @@ -0,0 +1,80 @@ +package org.evomaster.client.java.controller.internal; + +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto; + +import java.util.Objects; + +/** + * Keeps an AsyncAPI action open for a while after the driver has returned from publishing it, + * so that what the service does with the message, on a thread of its own, is still attributed + * to that action. Done in the controller so that drivers need not implement it, and the core + * pays no extra round trip. + */ +public class AsyncApiSettle { + + private static final long NANOS_PER_MILLI = 1_000_000L; + + /** + * Where the time comes from, in milliseconds. Only the difference between two readings is + * ever used, so it does not have to be wall-clock time. + */ + interface Clock { + long nowMs(); + } + + /** + * How the waiting thread is put to sleep. + */ + interface Sleeper { + void sleep(long ms) throws InterruptedException; + } + + private final Clock clock; + + private final Sleeper sleeper; + + /** + * Sleeps for real, and measures with {@link System#nanoTime()}, which does not jump when the + * system clock changes. + */ + public AsyncApiSettle() { + this(() -> System.nanoTime() / NANOS_PER_MILLI, Thread::sleep); + } + + AsyncApiSettle(Clock clock, Sleeper sleeper) { + this.clock = Objects.requireNonNull(clock); + this.sleeper = Objects.requireNonNull(sleeper); + } + + /** + * Wait as long as {@link AsyncApiActionDto#settleMs} asks, and record in + * {@link AsyncApiReplyDto#settledMs} how long that actually was. Nothing is waited, and + * nothing recorded, when no wait is asked for. + * + * The thread calling this is a request thread of the controller, whose code is never + * instrumented, so the kill switch does not interrupt it. If it is interrupted all the same, + * the wait ends there, the interrupt is left set for whoever owns the thread, and the time + * actually waited is what gets recorded. + * + * @param dto the action that was just published + * @param reply what the driver reported about it, to which the wait is added + */ + public void settle(AsyncApiActionDto dto, AsyncApiReplyDto reply) { + Objects.requireNonNull(dto); + Objects.requireNonNull(reply); + + Long settleMs = dto.settleMs; + if (settleMs == null || settleMs <= 0) { + return; + } + + long start = clock.nowMs(); + try { + sleeper.sleep(settleMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + reply.settledMs = clock.nowMs() - start; + } +} diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java index 0bfe0f6f8c..b44679894c 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java @@ -65,6 +65,11 @@ public class EMController { private String baseUrlOfSUT; + /** + * Keeps an AsyncAPI action open after the driver has published it, as the core asks. + */ + private final AsyncApiSettle asyncApiSettle; + /** * Keep track of all host:port clients connect so far. * This is the mainly done for debugging, to check that we are using @@ -79,7 +84,15 @@ public class EMController { new Scanner(EMController.class.getResourceAsStream("/warning.html"), "UTF-8").useDelimiter("\\A").next(); public EMController(SutController sutController) { + this(sutController, new AsyncApiSettle()); + } + + /** + * Only for tests, to wait on a clock they control. + */ + EMController(SutController sutController, AsyncApiSettle asyncApiSettle) { this.sutController = Objects.requireNonNull(sutController); + this.asyncApiSettle = Objects.requireNonNull(asyncApiSettle); } private boolean trackRequestSource(HttpServletRequest request) { @@ -853,7 +866,6 @@ public Response newAction( try { sutController.executeAsyncApiAction(dto.asyncApiCall, replyDto); - return Response.status(200).entity(WrappedResponseDto.withData(replyDto)).build(); } catch (Exception e) { /* Failing to publish is not a finding about the service, it is a broken @@ -866,6 +878,26 @@ public Response newAction( replyDto.errorMessage = msg; return Response.status(500).entity(WrappedResponseDto.withData(replyDto)).build(); } + + if (!Boolean.TRUE.equals(replyDto.published)) { + /* + The driver says the message was not published, or does not say: nothing + to wait for. The core stops the test here, as after a failure. + */ + return Response.status(200).entity(WrappedResponseDto.withData(replyDto)).build(); + } + + /* + The driver is done, but the service may not be: it handles the message on a + thread of its own. The action stays open while it does, so that what it + covers is still credited to this action rather than to the next one, or + lost once the results are collected. Not inside a noKillSwitch: the switch + was already turned off for this action by newAction above, and must stay + off for the service's threads, not only this one. + */ + asyncApiSettle.settle(dto.asyncApiCall, replyDto); + + return Response.status(200).entity(WrappedResponseDto.withData(replyDto)).build(); } } diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/AsyncApiSettleTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/AsyncApiSettleTest.java new file mode 100644 index 0000000000..567938d5a1 --- /dev/null +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/AsyncApiSettleTest.java @@ -0,0 +1,130 @@ +package org.evomaster.client.java.controller.internal; + +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class AsyncApiSettleTest { + + /** + * A clock that only moves when the fake sleeper sleeps, so that nothing here depends on how + * long anything really takes. + */ + private long now = 1_000; + + /** + * Every sleep asked for, in order. + */ + private final List sleeps = new ArrayList<>(); + + private AsyncApiSettle settleThatSleeps() { + return new AsyncApiSettle(() -> now, ms -> { + sleeps.add(ms); + now += ms; + }); + } + + private static AsyncApiActionDto asking(Long settleMs) { + AsyncApiActionDto dto = new AsyncApiActionDto(); + dto.settleMs = settleMs; + return dto; + } + + @AfterEach + public void clearInterrupt() { + //a test that leaves the flag set would interrupt whatever runs next on this thread + Thread.interrupted(); + } + + @Test + public void testTheActionIsKeptOpenForAsLongAsAsked() { + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + + settleThatSleeps().settle(asking(250L), reply); + + assertEquals(1, sleeps.size()); + assertEquals(250L, sleeps.get(0)); + assertEquals(250L, reply.settledMs); + } + + @Test + public void testNothingIsWaitedNorRecordedWhenNoWaitIsAskedFor() { + + AsyncApiSettle settle = settleThatSleeps(); + + for (Long none : new Long[]{null, 0L, -5L}) { + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settle.settle(asking(none), reply); + assertNull(reply.settledMs, "a wait was recorded for settleMs=" + none); + } + + assertTrue(sleeps.isEmpty(), "slept although no wait was asked for: " + sleeps); + } + + @Test + public void testAnInterruptedWaitEndsThereAndKeepsTheInterrupt() { + + /* + The thread belongs to the HTTP server, so the interrupt is not this class's to + swallow: it ends the wait, and stays set for whoever owns the thread. + */ + AsyncApiSettle settle = new AsyncApiSettle(() -> now, ms -> { + now += 40; + throw new InterruptedException(); + }); + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settle.settle(asking(250L), reply); + + assertTrue(Thread.currentThread().isInterrupted(), "the interrupt was swallowed"); + assertEquals(40L, reply.settledMs, "what was recorded is not the time actually waited"); + } + + @Test + public void testWhatTheDriverReportedIsLeftAlone() { + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + reply.published = true; + reply.replyExpected = true; + reply.replyReceived = true; + reply.waitedMs = 12L; + + settleThatSleeps().settle(asking(100L), reply); + + assertTrue(reply.published); + assertTrue(reply.replyReceived); + assertEquals(12L, reply.waitedMs, "the driver's wait for the reply is not the controller's settle"); + assertEquals(100L, reply.settledMs); + } + + @Test + public void testTheActionAndTheReplyCannotBeNull() { + + AsyncApiSettle settle = settleThatSleeps(); + + assertThrows(NullPointerException.class, () -> settle.settle(null, new AsyncApiReplyDto())); + assertThrows(NullPointerException.class, () -> settle.settle(asking(1L), null)); + } + + @Test + public void testTheRealClockAndSleeperWait() { + + /* + Only that a wait is recorded, never how long it took: a loaded machine can always + take longer, and an assertion on that would be flaky. + */ + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + + new AsyncApiSettle().settle(asking(1L), reply); + + assertNotNull(reply.settledMs); + assertTrue(reply.settledMs >= 0, "negative wait: " + reply.settledMs); + } +} diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/EMControllerTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/EMControllerTest.java new file mode 100644 index 0000000000..ddbc95e1e0 --- /dev/null +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/EMControllerTest.java @@ -0,0 +1,230 @@ +package org.evomaster.client.java.controller.internal; + +import org.evomaster.client.java.controller.EmbeddedSutController; +import org.evomaster.client.java.controller.api.dto.ActionDto; +import org.evomaster.client.java.controller.api.dto.SutInfoDto; +import org.evomaster.client.java.controller.api.dto.WrappedResponseDto; +import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto; +import org.evomaster.client.java.controller.problem.AsyncApiProblem; +import org.evomaster.client.java.controller.problem.ProblemInfo; +import org.evomaster.client.java.instrumentation.staticstate.ExecutionTracer; +import org.evomaster.client.java.sql.DbSpecification; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +public class EMControllerTest { + + private static final int OK = 200; + + private static final int SERVER_ERROR = 500; + + /** + * What happened, in order, across the driver and the controller's wait. + */ + private final List events = new ArrayList<>(); + + private long now; + + private AsyncApiDriver driver; + + private EMController controller; + + @BeforeEach + public void init() { + events.clear(); + now = 0; + driver = new AsyncApiDriver(); + driver.newTest(); + controller = new EMController(driver, new AsyncApiSettle(() -> now, ms -> { + events.add("settle " + ms); + now += ms; + })); + } + + @AfterEach + public void tearDown() { + //static state, shared with whatever runs next in this JVM + ExecutionTracer.setKillSwitch(false); + } + + private Response publish(Long settleMs) { + ActionDto dto = new ActionDto(); + dto.index = 0; + dto.asyncApiCall = new AsyncApiActionDto(); + dto.asyncApiCall.settleMs = settleMs; + return controller.newAction(dto, false, mock(HttpServletRequest.class)); + } + + private static AsyncApiReplyDto replyIn(Response response) { + return (AsyncApiReplyDto) ((WrappedResponseDto) response.getEntity()).data; + } + + @Test + public void testTheWaitStartsOnceTheDriverHasReturned() { + + Response response = publish(300L); + + assertEquals(OK, response.getStatus()); + assertEquals(2, events.size(), events.toString()); + assertEquals("publish", events.get(0)); + assertEquals("settle 300", events.get(1)); + + AsyncApiReplyDto reply = replyIn(response); + assertTrue(reply.published); + assertEquals(300L, reply.settledMs); + } + + @Test + public void testNoWaitIsDoneWhenNoneIsAskedFor() { + + Response response = publish(null); + + assertEquals(OK, response.getStatus()); + assertEquals(1, events.size(), events.toString()); + assertNull(replyIn(response).settledMs); + } + + @Test + public void testAMessageThatWasNotPublishedIsNotWaitedOn() { + + /* + A broken setup, not a finding about the service: there is nothing it could be + handling, so the test stops straight away rather than after a wait. + */ + driver.fails = true; + + Response response = publish(300L); + + assertEquals(SERVER_ERROR, response.getStatus()); + assertEquals(1, events.size(), events.toString()); + + AsyncApiReplyDto reply = replyIn(response); + assertFalse(reply.published); + assertNull(reply.settledMs); + } + + @Test + public void testAMessageTheDriverSaysWasNotPublishedIsNotWaitedOn() { + + /* + A driver can also report a failed publish without throwing, or leave published unset. + Either way there is nothing the service could be handling. + */ + for (Boolean published : new Boolean[]{false, null}) { + events.clear(); + driver = new AsyncApiDriver() { + @Override + public void executeAsyncApiAction(AsyncApiActionDto dto, AsyncApiReplyDto reply) { + events.add("publish"); + reply.published = published; + } + }; + driver.newTest(); + controller = new EMController(driver, new AsyncApiSettle(() -> now, ms -> { + events.add("settle " + ms); + now += ms; + })); + + Response response = publish(300L); + + assertEquals(OK, response.getStatus(), "published=" + published); + assertEquals(1, events.size(), "published=" + published + ": " + events); + + AsyncApiReplyDto reply = replyIn(response); + assertEquals(published, reply.published); + assertNull(reply.settledMs, "published=" + published); + } + } + + @Test + public void testTheServiceStillRecordsCoverageWhileTheActionIsKeptOpen() { + + /* + The kill switch stops the SUT's threads from recording anything. It is turned off + when the action starts, and has to stay off for the whole wait, or the wait would be + for nothing. + */ + ExecutionTracer.setKillSwitch(true); + + controller = new EMController(driver, new AsyncApiSettle(() -> now, ms -> { + events.add("kill switch " + ExecutionTracer.isKillSwitch()); + now += ms; + })); + + publish(300L); + + assertEquals("kill switch false", events.get(1)); + } + + /** + * A driver that publishes nothing, and says it did unless told to fail. + */ + private class AsyncApiDriver extends EmbeddedSutController { + + private boolean fails = false; + + @Override + public void executeAsyncApiAction(AsyncApiActionDto dto, AsyncApiReplyDto reply) { + events.add("publish"); + if (fails) { + throw new IllegalStateException("no broker"); + } + reply.published = true; + reply.replyExpected = false; + } + + @Override + public String startSut() { + return null; + } + + @Override + public boolean isSutRunning() { + return true; + } + + @Override + public void stopSut() { + } + + @Override + public String getPackagePrefixesToCover() { + return null; + } + + @Override + public void resetStateOfSUT() { + } + + @Override + public List getInfoForAuthentication() { + return null; + } + + @Override + public List getDbSpecifications() { + return null; + } + + @Override + public ProblemInfo getProblemInfo() { + return AsyncApiProblem.fromSchemaText("asyncapi: 3.0.0"); + } + + @Override + public SutInfoDto.OutputFormat getPreferredOutputFormat() { + return SutInfoDto.OutputFormat.JAVA_JUNIT_5; + } + } +} diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/AsyncApiTestBase.java b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/AsyncApiTestBase.java index 3e9713ca96..5eb3f02034 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/AsyncApiTestBase.java +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/AsyncApiTestBase.java @@ -25,6 +25,16 @@ protected Solution initAndRun(List args) { return (Solution) Main.initAndRun(args.toArray(new String[0])); } + /** + * The result of every message published, across the whole solution. + */ + protected List allResultsOf(Solution solution) { + return solution.getIndividuals().stream() + .flatMap(ind -> ind.evaluatedMainActions().stream()) + .map(e -> (AsyncApiCallResult) e.getResult()) + .collect(Collectors.toList()); + } + /** * The result of every message published to {@code operation}, across the whole solution. */ @@ -67,6 +77,18 @@ protected long countOutcome(Solution solution, AsyncApiOutco .count(); } + /** + * That the controller kept every published message's action open after the driver returned, + * as a settle window asks it to. + */ + protected void assertEverySettled(Solution solution) { + List results = allResultsOf(solution); + assertTrue(!results.isEmpty(), "With seed " + defaultSeed + ": nothing was published"); + long unsettled = results.stream().filter(r -> r.getSettledMs() == null).count(); + assertTrue(unsettled == 0, "With seed " + defaultSeed + ": " + unsettled + " of " + results.size() + + " published messages were not kept open after the driver returned"); + } + protected void assertReplied(Solution solution, String operation) { boolean ok = resultsOf(solution, operation).stream().anyMatch(r -> r.getOutcome() == AsyncApiOutcome.REPLIED); assertTrue(ok, "With seed " + defaultSeed + ": no reply to '" + operation + "' was ever received"); diff --git a/core-tests/e2e-tests/spring/spring-asyncapi-kafka/src/test/java/org/evomaster/e2etests/spring/asyncapi/kafka/NcsKafkaEMTest.java b/core-tests/e2e-tests/spring/spring-asyncapi-kafka/src/test/java/org/evomaster/e2etests/spring/asyncapi/kafka/NcsKafkaEMTest.java index fb05312521..b43ce35287 100644 --- a/core-tests/e2e-tests/spring/spring-asyncapi-kafka/src/test/java/org/evomaster/e2etests/spring/asyncapi/kafka/NcsKafkaEMTest.java +++ b/core-tests/e2e-tests/spring/spring-asyncapi-kafka/src/test/java/org/evomaster/e2etests/spring/asyncapi/kafka/NcsKafkaEMTest.java @@ -143,6 +143,35 @@ public void testRunEMInJava() throws Throwable { 5); } + /** + * A search that keeps every action open for a while after the driver returns. The service + * goes on answering, and every message is recorded as having been waited on. + */ + @Test + public void testRunEMWithASettleWindow() throws Throwable { + + runTestHandlingFlaky( + "NcsKafkaSettleEM", + "org.foo.asyncapi.NcsKafkaSettleEM", + 40, + false, + (args) -> { + + setOption(args, "asyncApiReplyTimeoutMs", "3000"); + + //long enough to be noticed, short enough for the budget: every action takes at least this long + setOption(args, "asyncApiSettleMs", "200"); + + Solution solution = initAndRun(args); + + assertTrue(countOutcome(solution, AsyncApiOutcome.REPLIED) > 0, "nothing was answered"); + assertEquals(0, countOutcome(solution, AsyncApiOutcome.NO_REPLY), "a promised reply never came"); + assertEquals(0, countOutcome(solution, AsyncApiOutcome.PUBLISH_FAILED), "a message never left"); + assertEverySettled(solution); + }, + 3); + } + /** * The no-reply oracle, end to end. Given a deadline no round trip through a broker can meet, * every message goes unanswered, and that is reported as a fault once experimental oracles diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 0f37490b48..8e2fad7f04 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -3050,6 +3050,15 @@ class EMConfig { @Min(1.0) var asyncApiReplyTimeoutMs = 5000 + @Experimental + @Cfg("When testing an AsyncAPI service in white-box mode, how long to keep each published message's action" + + " open after the driver has returned, in milliseconds, so that the code the service runs to handle it" + + " is still credited to that action. A message-driven service handles a message on a thread of its own," + + " which may still be running once the driver has seen it published, or its reply arrive." + + " The wait happens in the driver's controller. 0 means no wait.") + @Min(0.0) + var asyncApiSettleMs = 0 + @Experimental @Cfg("When testing an AsyncAPI service, the probability of publishing a message's declared example as it is," + " rather than a value sampled from its schema. A service that silently drops what it does not recognise" + diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt index 640e2f74d1..2ee97f67a3 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriter.kt @@ -69,6 +69,7 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { testSuitePath, baseUrlOfSut ) + addSettleWait(lines, evaluated.result) } } @@ -306,4 +307,47 @@ class AsyncApiTestCaseWriter : ApiTestCaseWriter() { addFaultsCommentLine(lines, test) } + + /** + * After a published message, wait for the window the search kept it open, when it did, so + * that what the service does handling it does not land on the next message or test. + * + * The whole window is waited, not what the search ended up waiting: where the window ended + * early, it ended on what the instrumentation saw, which a generated test has not got. A + * message with a reply is waited on too, as the handler may still be running after it + * answered. + * + * Only for Java and Kotlin: only a white-box run keeps a message open, and those are the + * formats it writes. + */ + private fun addSettleWait(lines: Lines, result: ActionResult) { + + val res = result as? AsyncApiCallResult ?: return + + if (!isPublished(res) + || res.getSettledMs() == null + || config.asyncApiSettleMs <= 0 + || !format.isJavaOrKotlin() + ) { + return + } + + lines.addSingleCommentLine("the time the search gave the service to finish with this message") + lines.add("Thread.sleep(${config.asyncApiSettleMs})") + lines.appendSemicolon() + } + + /** + * Whether the test has lines that publish the message of [res]: the driver's, or those + * written from the contract. The same conditions [addActionLinesPerType] writes them on. + */ + private fun publishes(res: AsyncApiCallResult): Boolean = + res.getTestScript().any { it.isNotBlank() } + || (res.getReplyVariableName() != null && KafkaTestClientEmitter.canEmit(res)) + + /** + * Whether the message of [res] went out, whatever came back, and the test publishes it too. + */ + private fun isPublished(res: AsyncApiCallResult): Boolean = + res.getOutcome().let { it != null && it != AsyncApiOutcome.PUBLISH_FAILED } && publishes(res) } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt index faa0755e87..571d25e7dd 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt @@ -14,6 +14,7 @@ class AsyncApiCallResult : EnterpriseActionResult { const val REPLY_MESSAGE = "REPLY_MESSAGE" const val CORRELATION_MATCHED = "CORRELATION_MATCHED" const val WAITED_MS = "WAITED_MS" + const val SETTLED_MS = "SETTLED_MS" const val TEST_SCRIPT = "TEST_SCRIPT" /* @@ -83,6 +84,16 @@ class AsyncApiCallResult : EnterpriseActionResult { fun getWaitedMs(): Long? = getResultValue(WAITED_MS)?.toLong() + /** + * How long the action was kept open after the driver returned, so that what the service did + * with the message was still credited to it. Absent when no such wait was asked for. + */ + fun setSettledMs(ms: Long) { + addResultValue(SETTLED_MS, ms.toString()) + } + + fun getSettledMs(): Long? = getResultValue(SETTLED_MS)?.toLong() + /** * The lines the driver rendered for a generated test, or empty when it rendered none. */ diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index 673417a6b8..2759132a6d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -249,6 +249,7 @@ class AsyncApiFitness : ApiWsFitness() { result.setOutcome(outcome) reply.waitedMs?.let { result.setWaitedMs(it) } + reply.settledMs?.let { result.setSettledMs(it) } //what the generated test will publish with, when the driver rendered it //lines that are all blank are no script: the writer would publish nothing and assert on it @@ -440,6 +441,14 @@ class AsyncApiFitness : ApiWsFitness() { } } + if (!config.blackBox && config.asyncApiSettleMs > 0) { + /* + Only in white-box mode: the window exists so that the code a message reaches is + credited to it, and a black-box run collects no code coverage to credit. + */ + dto.settleMs = config.asyncApiSettleMs.toLong() + } + return dto } diff --git a/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriterTest.kt index d4d939498f..14c67718dc 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/service/AsyncApiTestCaseWriterTest.kt @@ -716,6 +716,70 @@ class AsyncApiTestCaseWriterTest { } } + @Test + fun testAMessageNothingAnswersIsGivenTheSettleWindowAndCompiles() { + + OutputFormat.values().filter { it.isJavaOrKotlin() && it.isJUnit5() }.forEach { format -> + + //the search ended the window early, and the test still waits all of it + startWith(VARIANTS, "--outputFormat=${format.name}", "--asyncApiSettleMs=1500") { + FakeAsyncApiDriver.fireAndForget().apply { settledMs = 300L } + } + + val (members, body) = membersAndBodyOf(evaluate("tell", "tell")) + + val publish = body.indexOf(KafkaTestClientEmitter.HELPER_NAME) + val wait = body.indexOf("Thread.sleep(1500)") + assertTrue(publish >= 0, "for $format:\n$body") + assertTrue(wait > publish, "for $format:\n$body") + //once per message, before the next one is published + assertTrue(body.indexOf(KafkaTestClientEmitter.HELPER_NAME, publish + 1) > wait, "for $format:\n$body") + assertEquals(2, Regex("Thread\\.sleep\\(1500\\)").findAll(body).count(), "for $format:\n$body") + + compiles(members, body, format, "AsyncApiSettleWait${format.name}") + } + } + + @Test + fun testAMessageWithAReplyIsGivenTheSettleWindowTooAndCompiles() { + + //the handler may still be running after it answered, or after the search stopped waiting + val answers = listOf( + FakeAsyncApiDriver.replied("""{"answer": 2}"""), + FakeAsyncApiDriver.silence() + ) + + OutputFormat.values().filter { it.isJavaOrKotlin() && it.isJUnit5() }.forEach { format -> + answers.forEach { answer -> + + startWith(VARIANTS, "--outputFormat=${format.name}", "--asyncApiSettleMs=1500") { + answer.apply { settledMs = 1500L } + } + + val (members, body) = membersAndBodyOf(evaluate("ask")) + + val publish = body.indexOf(KafkaTestClientEmitter.HELPER_NAME) + val wait = body.indexOf("Thread.sleep(1500)") + assertTrue(publish >= 0, "for $format:\n$body") + assertTrue(wait > publish, "for $format:\n$body") + + compiles(members, body, format, "AsyncApiSettleAfterReply${format.name}") + } + } + } + + @Test + fun testNoSettleWaitIsWrittenWhereTheSearchKeptNoMessageOpen() { + + //no window asked for, so the controller reported no wait + startWith(VARIANTS) { FakeAsyncApiDriver.fireAndForget() } + assertFalse(bodyOf(evaluate("tell")).contains("Thread.sleep")) + + //a message that was never published is not waited on either + startWith(VARIANTS, "--asyncApiSettleMs=1500") { FakeAsyncApiDriver.fireAndForget().apply { published = false } } + assertFalse(bodyOf(evaluate("tell")).contains("Thread.sleep")) + } + @Test fun testWhatOneSuiteFoundIsNotLeftBehindForTheNext() { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt index 3a0a32fd04..d15373a9d2 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt @@ -740,4 +740,58 @@ class AsyncApiFitnessTest { assertEquals(1234L, driver.published.single().replyTimeoutMs) } + + @Test + fun testNoSettleWindowIsAskedForByDefault() { + + startNcs { replied(DOUBLE_RESULT) } + + evaluate("bessj") + + assertNull(driver.published.single().settleMs) + } + + @Test + fun testTheSettleWindowIsAskedForInWhiteBox() { + + startNcs("--asyncApiSettleMs=250") { fireAndForget() } + + evaluate("bessj", "expint") + + assertEquals(listOf(250L, 250L), driver.published.map { it.settleMs }) + } + + @Test + fun testABlackBoxRunAsksForNoSettleWindow() { + + /* + The window is there so that the code a message reaches is credited to it. A + black-box run collects no code coverage, so it would only slow the search down. + */ + startNcs("$BLACK_BOX=true", "--asyncApiSettleMs=250") { fireAndForget() } + + evaluate("bessj") + + assertNull(driver.published.single().settleMs) + } + + @Test + fun testHowLongTheControllerWaitedIsKeptOnTheResult() { + + startNcs("--asyncApiSettleMs=250") { fireAndForget().apply { settledMs = 251 } } + + val evaluated = evaluate("bessj") + + assertEquals(251L, results(evaluated).single().getSettledMs()) + } + + @Test + fun testNoWaitIsRecordedWhenTheControllerReportsNone() { + + startNcs { fireAndForget() } + + val evaluated = evaluate("bessj") + + assertNull(results(evaluated).single().getSettledMs()) + } } diff --git a/docs/options.md b/docs/options.md index faab8e133f..bc0b50c465 100644 --- a/docs/options.md +++ b/docs/options.md @@ -284,6 +284,7 @@ There are 3 types of options: |`appendToTargetHeuristicsFile`| __Boolean__. Whether should add to an existing target heuristics file, instead of replacing it. It is only used when processFormat is TARGET_HEURISTIC. *Default value*: `false`.| |`arazzoLocation`| __String__. arazzo location on disk. *Default value*: `""`.| |`asyncApiReplyTimeoutMs`| __Int__. When testing an AsyncAPI service, how long to wait for the reply to a published message before treating it as unanswered, in milliseconds. *Constraints*: `min=1.0`. *Default value*: `5000`.| +|`asyncApiSettleMs`| __Int__. When testing an AsyncAPI service in white-box mode, how long to keep each published message's action open after the driver has returned, in milliseconds, so that the code the service runs to handle it is still credited to that action. A message-driven service handles a message on a thread of its own, which may still be running once the driver has seen it published, or its reply arrive. The wait happens in the driver's controller. 0 means no wait. *Constraints*: `min=0.0`. *Default value*: `0`.| |`breederParentsMin`| __Int__. Breeder GA: minimum number of individuals in parents pool after truncation. *Constraints*: `min=2.0`. *Default value*: `2`.| |`breederTruncationFraction`| __Double__. Breeder GA: fraction of top individuals to keep in parents pool (truncation). *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`callbackURLHostname`| __String__. HTTP callback verifier hostname. Default is set to 'localhost'. If the SUT is running inside a container (i.e., Docker), 'localhost' will refer to the container. This can be used to change the hostname. *Default value*: `localhost`.|