diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java index b9c3ea5fcd..db0ff0762d 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java @@ -22,6 +22,25 @@ public enum CorrelationLocation { PAYLOAD } + /** + * How the controller decides that the service is done with a message, and the action can + * be closed, within the {@link #settleMs} window. + */ + public enum Completion { + + /** + * Wait for the whole window. + */ + FIXED, + + /** + * Stop early once the SUT has gone {@link #stableMs} without reaching any coverage + * objective for the first time, or improving one. A heuristic: a handler that only runs + * code already covered earlier in the same test leaves coverage stable too. + */ + COVERAGE_STABLE + } + /** * Key of the operation in the AsyncAPI document, so a driver can log in terms the user will * recognise from their own contract. @@ -104,6 +123,19 @@ public enum CorrelationLocation { */ public Long settleMs; + /** + * How the controller decides, within {@link #settleMs}, that the action can be closed. Null + * is read as {@link Completion#FIXED}. Meaningless without a {@link #settleMs}. + */ + public Completion completion; + + /** + * For {@link Completion#COVERAGE_STABLE}: how long, in milliseconds, the SUT has to go + * without improving any coverage objective to end the wait early. Null, zero or negative + * make it a {@link Completion#FIXED} wait. Null for any other completion. + */ + public Long stableMs; + /** * The language to render {@link AsyncApiReplyDto#testScript} in. Only Java and Kotlin are * ever asked for. diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java index a575243c86..95b9fa3e19 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java @@ -14,6 +14,30 @@ */ public class AsyncApiReplyDto { + /** + * Why the controller stopped keeping the action open. + */ + public enum SettleEnd { + + /** + * The whole {@link AsyncApiActionDto#settleMs} window went by. Also what a + * {@link AsyncApiActionDto.Completion#COVERAGE_STABLE} wait ends with when the SUT's + * coverage cannot be read. + */ + DEADLINE, + + /** + * The SUT went {@link AsyncApiActionDto#stableMs} without improving any coverage + * objective. + */ + COVERAGE_STABLE, + + /** + * The waiting thread was interrupted. + */ + INTERRUPTED + } + /** * The index of the action this answers, echoing what was asked. */ @@ -68,14 +92,18 @@ public class AsyncApiReplyDto { /** * How long the controller kept the action open after the driver returned, in milliseconds, - * as asked by {@link AsyncApiActionDto#settleMs}. It can be shorter than what was asked when - * the wait was interrupted. - * - * Filled in by the controller rather than by the driver, which never sees this field set. - * Null when no wait was asked for, or when the message could not be published. + * as asked by {@link AsyncApiActionDto#settleMs}; shorter when it ended early, see + * {@link #settleEndedBy}. Set by the controller, not the driver. Null when no wait was asked + * for, or the message could not be published. */ public Long settledMs; + /** + * Why the controller stopped waiting, set together with {@link #settledMs}. Null exactly + * when that is. + */ + public SettleEnd settleEndedBy; + /** * Why publishing failed, when it did. */ diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/EmbeddedSutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/EmbeddedSutController.java index 06cf810835..224a4cb066 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/EmbeddedSutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/EmbeddedSutController.java @@ -144,6 +144,18 @@ public BootTimeInfoDto getBootTimeInfoDto() { return getBootTimeInfoDto(InstrumentationController.getBootTimeObjectiveInfo()); } + @Override + public final Long getObjectiveUpdateCount() { + /* + Without the agent nothing is instrumented, and a count stuck at zero would read as a + SUT doing nothing new rather than as unknown coverage. + */ + if (!isInstrumentationActivated()) { + return null; + } + return InstrumentationController.getObjectiveUpdateCount(); + } + @Override public final String getExecutableFullPath(){ return null; //not needed for embedded diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java index f3a6a8c83a..6dba4632dc 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java @@ -447,6 +447,14 @@ public BootTimeInfoDto getBootTimeInfoDto() { return getBootTimeInfoDto(serverController.handleBootTimeObjectiveInfo()); } + @Override + public final Long getObjectiveUpdateCount() { + if (!isInstrumentationActivated()) { + return null; + } + return serverController.getObjectiveUpdateCount(); + } + @Override public final void newActionSpecificHandler(ActionDto dto) { if (isInstrumentationActivated()) { diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/AsyncApiSettle.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/AsyncApiSettle.java index 167843df9f..06962b1aad 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/AsyncApiSettle.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/AsyncApiSettle.java @@ -4,6 +4,7 @@ import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto; import java.util.Objects; +import java.util.function.Supplier; /** * Keeps an AsyncAPI action open for a while after the driver has returned from publishing it, @@ -15,6 +16,16 @@ public class AsyncApiSettle { private static final long NANOS_PER_MILLI = 1_000_000L; + private static final String OBJECTIVE_UPDATES_FAILURE_WARNING = "Failed to read how many coverage objectives" + + " the SUT has reached or improved, so the whole settle window is waited"; + + /** + * How often the SUT's coverage is read while waiting for it to become stable, in milliseconds. + * Short next to any useful stable window, and long enough that reading it, a socket round + * trip for an external SUT, costs little. + */ + static final long POLL_INTERVAL_MS = 25; + /** * Where the time comes from, in milliseconds. Only the difference between two readings is * ever used, so it does not have to be wall-clock time. @@ -48,33 +59,112 @@ public AsyncApiSettle() { } /** - * Wait as long as {@link AsyncApiActionDto#settleMs} asks, and record in - * {@link AsyncApiReplyDto#settledMs} how long that actually was. Nothing is waited, and - * nothing recorded, when no wait is asked for. + * Wait as {@link AsyncApiActionDto#settleMs} and {@link AsyncApiActionDto#completion} ask, + * and record in {@link AsyncApiReplyDto#settledMs} how long that actually was and in + * {@link AsyncApiReplyDto#settleEndedBy} why it ended. Nothing is waited or recorded when no + * wait is asked for. * - * The thread calling this is a request thread of the controller, whose code is never - * instrumented, so the kill switch does not interrupt it. If it is interrupted all the same, - * the wait ends there, the interrupt is left set for whoever owns the thread, and the time - * actually waited is what gets recorded. + * If the thread is interrupted, the wait ends there and the interrupt flag is left set. * - * @param dto the action that was just published - * @param reply what the driver reported about it, to which the wait is added + * @param dto the action that was just published + * @param reply what the driver reported about it, to which the wait is added + * @param objectiveUpdates reads how many times the SUT has reached or improved a coverage + * objective in the current test, or null when that cannot be known. + * Only read for {@link AsyncApiActionDto.Completion#COVERAGE_STABLE} */ - public void settle(AsyncApiActionDto dto, AsyncApiReplyDto reply) { + public void settle(AsyncApiActionDto dto, AsyncApiReplyDto reply, Supplier objectiveUpdates) { Objects.requireNonNull(dto); Objects.requireNonNull(reply); + Objects.requireNonNull(objectiveUpdates); Long settleMs = dto.settleMs; if (settleMs == null || settleMs <= 0) { return; } + boolean coverageStable = dto.completion == AsyncApiActionDto.Completion.COVERAGE_STABLE + && dto.stableMs != null && dto.stableMs > 0; + long start = clock.nowMs(); + AsyncApiReplyDto.SettleEnd end; try { - sleeper.sleep(settleMs); + end = coverageStable + ? waitForStableCoverage(start, settleMs, dto.stableMs, objectiveUpdates) + : waitUntil(start + settleMs); } catch (InterruptedException e) { Thread.currentThread().interrupt(); + end = AsyncApiReplyDto.SettleEnd.INTERRUPTED; } reply.settledMs = clock.nowMs() - start; + reply.settleEndedBy = end; + } + + /** + * What {@code reader} reads, where a failure to read counts as not knowing. Reading is a + * socket round trip for an external SUT, and the action was published all the same: a + * failure here would otherwise turn a dropped connection into a broken setup. + * + * @param warning what to warn about, once, as reads come every few milliseconds; the reason + * goes to the debug log + */ + private static T readOrNull(Supplier reader, String warning) { + try { + return reader.get(); + } catch (RuntimeException e) { + org.evomaster.client.java.utils.SimpleLogger.uniqueWarn(warning); + org.evomaster.client.java.utils.SimpleLogger.debug(warning + ": " + e.getMessage()); + return null; + } + } + + private AsyncApiReplyDto.SettleEnd waitUntil(long deadline) throws InterruptedException { + long left = deadline - clock.nowMs(); + if (left > 0) { + sleeper.sleep(left); + } + return AsyncApiReplyDto.SettleEnd.DEADLINE; + } + + /** + * Read the SUT's coverage every {@link #POLL_INTERVAL_MS}, and stop once it has gone + * {@code stableMs} without moving, or at the deadline, whichever comes first. The stable + * period is counted from the start of the wait, so a service that does nothing new after + * the driver returned is let go after {@code stableMs}. + */ + private AsyncApiReplyDto.SettleEnd waitForStableCoverage( + long start, + long settleMs, + long stableMs, + Supplier objectiveUpdates + ) throws InterruptedException { + + long deadline = start + settleMs; + + Long last = readOrNull(objectiveUpdates, OBJECTIVE_UPDATES_FAILURE_WARNING); + long lastChange = start; + + while (true) { + if (last == null) { + //coverage unknown: wait out the window + return waitUntil(deadline); + } + + long now = clock.nowMs(); + if (now >= deadline) { + return AsyncApiReplyDto.SettleEnd.DEADLINE; + } + if (now - lastChange >= stableMs) { + return AsyncApiReplyDto.SettleEnd.COVERAGE_STABLE; + } + + long nap = Math.min(POLL_INTERVAL_MS, Math.min(deadline - now, lastChange + stableMs - now)); + sleeper.sleep(nap); + + Long current = readOrNull(objectiveUpdates, OBJECTIVE_UPDATES_FAILURE_WARNING); + if (current != null && !current.equals(last)) { + lastChange = clock.nowMs(); + } + last = current; + } } } diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java index b44679894c..656612d6fb 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java @@ -895,7 +895,7 @@ public Response newAction( was already turned off for this action by newAction above, and must stay off for the service's threads, not only this one. */ - asyncApiSettle.settle(dto.asyncApiCall, replyDto); + asyncApiSettle.settle(dto.asyncApiCall, replyDto, sutController::getObjectiveUpdateCount); return Response.status(200).entity(WrappedResponseDto.withData(replyDto)).build(); } diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index c753cf32f7..7ac96a8b74 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -1834,6 +1834,18 @@ public abstract List getTargetInfos(Collection ids, public abstract BootTimeInfoDto getBootTimeInfoDto(); + /** + * How many times, in the current test, a coverage objective of the SUT was reached for the + * first time or improved on its best value. Read while an action is kept open, to tell + * whether the SUT is still reaching anything new. + * + * @return the count, which only grows within a test, or null when unknown, eg because the + * SUT is not instrumented + */ + public Long getObjectiveUpdateCount() { + return null; + } + protected BootTimeInfoDto getBootTimeInfoDto(BootTimeObjectiveInfo info){ if (info == null) return null; diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/AsyncApiSettleTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/AsyncApiSettleTest.java index 567938d5a1..d7f4ad3944 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/AsyncApiSettleTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/AsyncApiSettleTest.java @@ -7,16 +7,32 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Supplier; import static org.junit.jupiter.api.Assertions.*; public class AsyncApiSettleTest { + /** + * A SUT whose coverage cannot be read. + */ + private static final Supplier NO_COVERAGE = () -> null; + + private static final long SETTLE_MS = 1_000; + + private static final long STABLE_MS = 100; + + /** + * Where the fake clock starts. Anything but zero, so that a wait measured from zero rather + * than from its start would show. + */ + private static final long START = 1_000; + /** * A clock that only moves when the fake sleeper sleeps, so that nothing here depends on how * long anything really takes. */ - private long now = 1_000; + private long now = START; /** * Every sleep asked for, in order. @@ -47,11 +63,12 @@ public void testTheActionIsKeptOpenForAsLongAsAsked() { AsyncApiReplyDto reply = new AsyncApiReplyDto(); - settleThatSleeps().settle(asking(250L), reply); + settleThatSleeps().settle(asking(250L), reply, NO_COVERAGE); assertEquals(1, sleeps.size()); assertEquals(250L, sleeps.get(0)); assertEquals(250L, reply.settledMs); + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); } @Test @@ -61,8 +78,9 @@ public void testNothingIsWaitedNorRecordedWhenNoWaitIsAskedFor() { for (Long none : new Long[]{null, 0L, -5L}) { AsyncApiReplyDto reply = new AsyncApiReplyDto(); - settle.settle(asking(none), reply); + settle.settle(asking(none), reply, NO_COVERAGE); assertNull(reply.settledMs, "a wait was recorded for settleMs=" + none); + assertNull(reply.settleEndedBy, "a wait was recorded for settleMs=" + none); } assertTrue(sleeps.isEmpty(), "slept although no wait was asked for: " + sleeps); @@ -81,10 +99,11 @@ public void testAnInterruptedWaitEndsThereAndKeepsTheInterrupt() { }); AsyncApiReplyDto reply = new AsyncApiReplyDto(); - settle.settle(asking(250L), reply); + settle.settle(asking(250L), reply, NO_COVERAGE); assertTrue(Thread.currentThread().isInterrupted(), "the interrupt was swallowed"); assertEquals(40L, reply.settledMs, "what was recorded is not the time actually waited"); + assertEquals(AsyncApiReplyDto.SettleEnd.INTERRUPTED, reply.settleEndedBy); } @Test @@ -96,7 +115,7 @@ public void testWhatTheDriverReportedIsLeftAlone() { reply.replyReceived = true; reply.waitedMs = 12L; - settleThatSleeps().settle(asking(100L), reply); + settleThatSleeps().settle(asking(100L), reply, NO_COVERAGE); assertTrue(reply.published); assertTrue(reply.replyReceived); @@ -105,12 +124,13 @@ public void testWhatTheDriverReportedIsLeftAlone() { } @Test - public void testTheActionAndTheReplyCannotBeNull() { + public void testNoneOfTheArgumentsCanBeNull() { AsyncApiSettle settle = settleThatSleeps(); - assertThrows(NullPointerException.class, () -> settle.settle(null, new AsyncApiReplyDto())); - assertThrows(NullPointerException.class, () -> settle.settle(asking(1L), null)); + assertThrows(NullPointerException.class, () -> settle.settle(null, new AsyncApiReplyDto(), NO_COVERAGE)); + assertThrows(NullPointerException.class, () -> settle.settle(asking(1L), null, NO_COVERAGE)); + assertThrows(NullPointerException.class, () -> settle.settle(asking(1L), new AsyncApiReplyDto(), null)); } @Test @@ -122,9 +142,163 @@ public void testTheRealClockAndSleeperWait() { */ AsyncApiReplyDto reply = new AsyncApiReplyDto(); - new AsyncApiSettle().settle(asking(1L), reply); + new AsyncApiSettle().settle(asking(1L), reply, NO_COVERAGE); assertNotNull(reply.settledMs); assertTrue(reply.settledMs >= 0, "negative wait: " + reply.settledMs); } + + private static AsyncApiActionDto coverageStable(Long stableMs) { + AsyncApiActionDto dto = asking(SETTLE_MS); + dto.completion = AsyncApiActionDto.Completion.COVERAGE_STABLE; + dto.stableMs = stableMs; + return dto; + } + + @Test + public void testASutThatReachesNothingNewIsLetGoOnceCoverageIsStable() { + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + + settleThatSleeps().settle(coverageStable(STABLE_MS), reply, () -> 7L); + + assertEquals(AsyncApiReplyDto.SettleEnd.COVERAGE_STABLE, reply.settleEndedBy); + assertEquals(STABLE_MS, reply.settledMs); + assertTrue(sleeps.stream().allMatch(ms -> ms <= AsyncApiSettle.POLL_INTERVAL_MS), + "the coverage was not read often enough: " + sleeps); + } + + @Test + public void testCoverageStillImprovingKeepsTheActionOpen() { + + /* + The count moves on each of the first reads, until 300 ms in, then stops: the wait + ends a stable window after the last move, not after the first. + */ + long[] count = {0}; + Supplier busyUntil300 = () -> { + if (now - START <= 300) { + count[0]++; + } + return count[0]; + }; + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settleThatSleeps().settle(coverageStable(STABLE_MS), reply, busyUntil300); + + assertEquals(AsyncApiReplyDto.SettleEnd.COVERAGE_STABLE, reply.settleEndedBy); + assertEquals(300L + STABLE_MS, reply.settledMs); + } + + @Test + public void testTheDeadlineCapsASutWhoseCoverageNeverSettles() { + + long[] count = {0}; + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settleThatSleeps().settle(coverageStable(STABLE_MS), reply, () -> ++count[0]); + + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); + assertEquals(SETTLE_MS, reply.settledMs); + } + + @Test + public void testCoverageThatCannotBeReadMeansTheWholeWindow() { + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settleThatSleeps().settle(coverageStable(STABLE_MS), reply, NO_COVERAGE); + + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); + assertEquals(SETTLE_MS, reply.settledMs); + } + + @Test + public void testCoverageLostHalfWayMeansTheRestOfTheWindow() { + + //eg the connection to an external SUT's agent dropping + Supplier lostAfter50 = () -> now - START < 50 ? Long.valueOf(now) : null; + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settleThatSleeps().settle(coverageStable(STABLE_MS), reply, lostAfter50); + + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); + assertEquals(SETTLE_MS, reply.settledMs); + } + + @Test + public void testCoverageThatFailsToBeReadMeansTheWholeWindow() { + + /* + Reading it for an external SUT is a round trip to its agent, which can fail. The + action was published all the same, so it still succeeds, only with the whole window + waited. + */ + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settleThatSleeps().settle(coverageStable(STABLE_MS), reply, () -> { + throw new IllegalStateException("agent gone"); + }); + + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); + assertEquals(SETTLE_MS, reply.settledMs); + } + + @Test + public void testCoverageThatFailsToBeReadHalfWayMeansTheRestOfTheWindow() { + + Supplier failingAfter50 = () -> { + if (now - START >= 50) { + throw new IllegalStateException("agent gone"); + } + return now; + }; + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settleThatSleeps().settle(coverageStable(STABLE_MS), reply, failingAfter50); + + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); + assertEquals(SETTLE_MS, reply.settledMs); + } + + @Test + public void testWithoutAStableWindowCoverageStableIsAFixedWait() { + + for (Long none : new Long[]{null, 0L, -1L}) { + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settleThatSleeps().settle(coverageStable(none), reply, () -> 7L); + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy, "stableMs=" + none); + } + } + + @Test + public void testAFixedWaitNeverReadsTheCoverage() { + + AsyncApiActionDto dto = asking(SETTLE_MS); + dto.completion = AsyncApiActionDto.Completion.FIXED; + dto.stableMs = STABLE_MS; + + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settleThatSleeps().settle(dto, reply, () -> fail("the coverage was read for a fixed wait")); + + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); + assertEquals(SETTLE_MS, reply.settledMs); + } + + @Test + public void testAnInterruptWhileWaitingForStableCoverageEndsTheWait() { + + AsyncApiSettle settle = new AsyncApiSettle(() -> now, ms -> { + now += ms; + if (now - START >= 50) { + throw new InterruptedException(); + } + }); + + long[] count = {0}; + AsyncApiReplyDto reply = new AsyncApiReplyDto(); + settle.settle(coverageStable(STABLE_MS), reply, () -> ++count[0]); + + assertTrue(Thread.currentThread().isInterrupted(), "the interrupt was swallowed"); + assertEquals(AsyncApiReplyDto.SettleEnd.INTERRUPTED, reply.settleEndedBy); + assertEquals(50L, reply.settledMs); + } } diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/EMControllerTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/EMControllerTest.java index ddbc95e1e0..1f7ee7ae62 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/EMControllerTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/EMControllerTest.java @@ -10,6 +10,7 @@ import org.evomaster.client.java.controller.problem.AsyncApiProblem; import org.evomaster.client.java.controller.problem.ProblemInfo; import org.evomaster.client.java.instrumentation.staticstate.ExecutionTracer; +import org.evomaster.client.java.instrumentation.InstrumentingAgent; import org.evomaster.client.java.sql.DbSpecification; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -17,6 +18,7 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Response; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; @@ -29,6 +31,15 @@ public class EMControllerTest { private static final int SERVER_ERROR = 500; + private static final long SETTLE_MS = 1_000; + + private static final long STABLE_MS = 100; + + /** + * What the coverage objectives a fake SUT thread reaches are named after. + */ + private static final String OBJECTIVE_PREFIX = "EMControllerTest_"; + /** * What happened, in order, across the driver and the controller's wait. */ @@ -40,6 +51,30 @@ public class EMControllerTest { private EMController controller; + /** + * An embedded controller only reads the instrumentation once the agent is active. Unit tests + * run without it, and only need the shared ExecutionTracer, so the flag is set by hand. + */ + private static void setAgentActive(boolean active) { + try { + Field flag = InstrumentingAgent.class.getDeclaredField("active"); + flag.setAccessible(true); + flag.setBoolean(null, active); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + } + + @BeforeEach + public void activateTheAgent() { + setAgentActive(true); + } + + @AfterEach + public void deactivateTheAgent() { + setAgentActive(false); + } + @BeforeEach public void init() { events.clear(); @@ -55,17 +90,30 @@ public void init() { @AfterEach public void tearDown() { //static state, shared with whatever runs next in this JVM - ExecutionTracer.setKillSwitch(false); + ExecutionTracer.reset(); } private Response publish(Long settleMs) { + AsyncApiActionDto call = new AsyncApiActionDto(); + call.settleMs = settleMs; + return publishCall(call); + } + + private Response publishCall(AsyncApiActionDto call) { ActionDto dto = new ActionDto(); dto.index = 0; - dto.asyncApiCall = new AsyncApiActionDto(); - dto.asyncApiCall.settleMs = settleMs; + dto.asyncApiCall = call; return controller.newAction(dto, false, mock(HttpServletRequest.class)); } + private static AsyncApiActionDto untilCoverageStable() { + AsyncApiActionDto call = new AsyncApiActionDto(); + call.settleMs = SETTLE_MS; + call.completion = AsyncApiActionDto.Completion.COVERAGE_STABLE; + call.stableMs = STABLE_MS; + return call; + } + private static AsyncApiReplyDto replyIn(Response response) { return (AsyncApiReplyDto) ((WrappedResponseDto) response.getEntity()).data; } @@ -83,6 +131,7 @@ public void testTheWaitStartsOnceTheDriverHasReturned() { AsyncApiReplyDto reply = replyIn(response); assertTrue(reply.published); assertEquals(300L, reply.settledMs); + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); } @Test @@ -167,6 +216,51 @@ public void testTheServiceStillRecordsCoverageWhileTheActionIsKeptOpen() { assertEquals("kill switch false", events.get(1)); } + @Test + public void testASutThatReachesNothingNewIsLetGoOnceCoverageIsStable() { + + Response response = publishCall(untilCoverageStable()); + + AsyncApiReplyDto reply = replyIn(response); + assertEquals(AsyncApiReplyDto.SettleEnd.COVERAGE_STABLE, reply.settleEndedBy); + assertEquals(STABLE_MS, reply.settledMs); + } + + @Test + public void testWithoutTheAgentCoverageStableWaitsTheWholeWindow() { + + /* + Nothing is instrumented, so the coverage never moves: that says nothing about whether + the SUT is done, and must not end the wait early. + */ + setAgentActive(false); + + Response response = publishCall(untilCoverageStable()); + + AsyncApiReplyDto reply = replyIn(response); + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); + assertEquals(SETTLE_MS, reply.settledMs); + } + + @Test + public void testASutStillReachingNewCodeIsWaitedOnUntilTheDeadline() { + + /* + Stands in for the service's consumer thread, which keeps reaching new objectives + while the controller waits. + */ + controller = new EMController(driver, new AsyncApiSettle(() -> now, ms -> { + now += ms; + ExecutionTracer.executedNumericComparison(OBJECTIVE_PREFIX + now, 0.5, 1, 0.5); + })); + + Response response = publishCall(untilCoverageStable()); + + AsyncApiReplyDto reply = replyIn(response); + assertEquals(AsyncApiReplyDto.SettleEnd.DEADLINE, reply.settleEndedBy); + assertEquals(SETTLE_MS, reply.settledMs); + } + /** * A driver that publishes nothing, and says it did unless told to fail. */ diff --git a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/InstrumentationController.java b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/InstrumentationController.java index b6b6a84db8..2809ffb8ac 100644 --- a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/InstrumentationController.java +++ b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/InstrumentationController.java @@ -39,6 +39,14 @@ public static void resetForNewTest(){ ObjectiveRecorder.clearFirstTimeEncountered(); } + /** + * @return how many times an objective was reached or improved in the current test, see + * {@link ExecutionTracer#getObjectiveUpdateCount()} + */ + public static long getObjectiveUpdateCount(){ + return ExecutionTracer.getObjectiveUpdateCount(); + } + public static void newAction(Action action){ ExecutionTracer.setAction(action); diff --git a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/AgentController.java b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/AgentController.java index 923754d3f8..a2bfe9aa87 100644 --- a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/AgentController.java +++ b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/AgentController.java @@ -124,6 +124,9 @@ public static void start(int port){ case EXTRACT_JVM_DTO: handleExtractingSpecifiedDto(); break; + case OBJECTIVE_UPDATE_COUNT: + handleObjectiveUpdateCount(); + break; default: SimpleLogger.error("Unrecognized command: "+command); return; @@ -271,6 +274,27 @@ private static void handleBootTimeObjectiveInfo(){ } } + /** + * Answer null, read by the controller as unknown, when handling a command failed. Otherwise + * the controller would stall until its read times out. + */ + private static void sendUnknown(){ + try { + sendObject(null); + } catch (Exception e) { + SimpleLogger.error("Failure in sending an unknown answer: "+e.getMessage()); + } + } + + private static void handleObjectiveUpdateCount(){ + try { + sendObject(InstrumentationController.getObjectiveUpdateCount()); + } catch (Exception e) { + SimpleLogger.error("Failure in handling objective update count: "+e.getMessage()); + sendUnknown(); + } + } + private static void handleExtractingSpecifiedDto(){ try { Object msg = in.readObject(); diff --git a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/Command.java b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/Command.java index 0048b3e6db..e6a92ca039 100644 --- a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/Command.java +++ b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/Command.java @@ -26,5 +26,6 @@ public enum Command implements Serializable { EXECUTING_ACTION, BOOT_TIME_INFO, EXTRACT_JVM_DTO, - BOOTING_SUT + BOOTING_SUT, + OBJECTIVE_UPDATE_COUNT } diff --git a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/ServerController.java b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/ServerController.java index 74ab775623..9ad7048ec4 100644 --- a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/ServerController.java +++ b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/external/ServerController.java @@ -327,6 +327,33 @@ public synchronized BootTimeObjectiveInfo handleBootTimeObjectiveInfo() { return (BootTimeObjectiveInfo) response; } + /** + * Cheap enough to be polled every few milliseconds. + * + * @return how many times an objective was reached or improved in the SUT's current test, + * or null if the agent could not be asked, or failed to answer + */ + public synchronized Long getObjectiveUpdateCount() { + + boolean sent = sendCommand(Command.OBJECTIVE_UPDATE_COUNT); + if (!sent) { + SimpleLogger.error("Failed to send message"); + return null; + } + + Object response = waitAndGetResponse(); + if (response == null) { + SimpleLogger.error("Failed to read response about objective update count"); + return null; + } + + if (!(response instanceof Long)) { + throw new IllegalStateException(errorMsgExpectingResponse(response, "a Long")); + } + + return (Long) response; + } + public synchronized UnitsInfoRecorder getUnitsInfoRecorder(){ boolean sent = sendCommand(Command.UNITS_INFO); diff --git a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/staticstate/ExecutionTracer.java b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/staticstate/ExecutionTracer.java index 259ef9fa9e..5ca110ae9e 100644 --- a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/staticstate/ExecutionTracer.java +++ b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/staticstate/ExecutionTracer.java @@ -9,6 +9,7 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import static org.evomaster.client.java.instrumentation.shared.ExternalServiceSharedUtils.RESERVED_RESOLVED_LOCAL_IP; @@ -64,6 +65,18 @@ public class ExecutionTracer { private static final Map objectiveCoverage = new ConcurrentHashMap<>(65536); + /** + * How many times, in the current test, an objective was reached for the first time or got a + * better value than it had. Reset with {@link #objectiveCoverage}. + * + * Improvements rather than executions, as it is read to tell whether the SUT is still doing + * something new: a service polling a broker runs the same lines forever. As a consequence, a + * handler re-running only paths already covered in the same test does not move it either. + * + * Declared before the static block that calls {@link #reset()}. + */ + private static final AtomicLong objectiveUpdates = new AtomicLong(0); + /** * A test case can be composed by 1 or more actions, eg HTTP calls. * When we get the best distance for a testing target, we might @@ -134,6 +147,7 @@ public class ExecutionTracer { public static void reset() { synchronized (lock) { objectiveCoverage.clear(); + objectiveUpdates.set(0); actionIndex = 0; actionName = null; additionalInfoList.clear(); @@ -550,6 +564,14 @@ public static Set getNonCoveredObjectives(String prefix) { .collect(Collectors.toSet()); } + /** + * @return how many times, since the last {@link #reset()}, an objective was reached for the + * first time or improved on its best value. Only grows between resets. + */ + public static long getObjectiveUpdateCount() { + return objectiveUpdates.get(); + } + public static Double getValue(String id) { return objectiveCoverage.get(id).value; } @@ -568,9 +590,11 @@ private static void updateObjective(String id, double value) { double previous = objectiveCoverage.get(id).value; if (value > previous) { objectiveCoverage.put(id, new TargetInfo(null, id, value, actionIndex)); + objectiveUpdates.incrementAndGet(); } } else { objectiveCoverage.put(id, new TargetInfo(null, id, value, actionIndex)); + objectiveUpdates.incrementAndGet(); } } diff --git a/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/external/ServerControllerTest.java b/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/external/ServerControllerTest.java new file mode 100644 index 0000000000..fd45e2619b --- /dev/null +++ b/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/external/ServerControllerTest.java @@ -0,0 +1,102 @@ +package org.evomaster.client.java.instrumentation.external; + +import org.evomaster.client.java.instrumentation.staticstate.ExecutionTracer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The controller's side of the connection to the agent of an external SUT, talking to a real + * {@link AgentController} over a local socket. Both run in this JVM, so the agent answers from + * the same {@link ExecutionTracer} the test drives. + */ +public class ServerControllerTest { + + /** + * Generous, as it only bounds how long a broken connection takes to fail the test. Despite + * the parameter's name, waitForIncomingConnection reads it as milliseconds. + */ + private static final int TIMEOUT_MS = 10_000; + + private static final String COMPARISON = "ServerControllerTest_comparison"; + + /** + * A numeric comparison reports three objectives at once: less than, equal and greater than. + */ + private static final long OBJECTIVES_PER_COMPARISON = 3; + + /** + * Can close its end of the connection, which is what ends the agent's thread. + */ + private static class ClosableServerController extends ServerController { + void closeConnection() throws IOException { + out.close(); + } + } + + private ClosableServerController server; + + @BeforeEach + public void connect() throws InterruptedException { + ExecutionTracer.reset(); + + server = new ClosableServerController(); + int port = server.startServer(); + + //the agent blocks until the server accepts, so it connects from a thread of its own + Thread agent = new Thread(() -> AgentController.start(port)); + agent.start(); + + assertTrue(server.waitForIncomingConnection(TIMEOUT_MS), "the agent did not connect"); + agent.join(TIMEOUT_MS); + } + + @AfterEach + public void disconnect() throws IOException { + server.closeConnection(); + server.closeServer(); + ExecutionTracer.reset(); + } + + @Test + public void testTheObjectiveUpdateCountComesBackFromTheAgent() { + + assertEquals(0L, server.getObjectiveUpdateCount()); + + ExecutionTracer.executedNumericComparison(COMPARISON, 0.2, 0.2, 0.2); + + assertEquals(OBJECTIVES_PER_COMPARISON, server.getObjectiveUpdateCount()); + //asking does not change it + assertEquals(OBJECTIVES_PER_COMPARISON, server.getObjectiveUpdateCount()); + } + + @Test + public void testANewTestResetsTheCountInTheAgent() { + + ExecutionTracer.executedNumericComparison(COMPARISON, 0.2, 0.2, 0.2); + + assertTrue(server.resetForNewTest()); + + assertEquals(0L, server.getObjectiveUpdateCount()); + } + + @Test + public void testTheCountCanBeAskedForBetweenOtherCommands() { + + /* + Commands share one connection and are answered in order, so a reply read out of turn + would show up as the wrong type here. + */ + ExecutionTracer.executedNumericComparison(COMPARISON, 0.2, 0.2, 0.2); + + assertTrue(server.setKillSwitch(false)); + assertEquals(OBJECTIVES_PER_COMPARISON, server.getObjectiveUpdateCount()); + assertTrue(server.setExecutingAction(true)); + assertEquals(OBJECTIVES_PER_COMPARISON, server.getObjectiveUpdateCount()); + } +} diff --git a/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/staticstate/ExecutionTracerTest.java b/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/staticstate/ExecutionTracerTest.java new file mode 100644 index 0000000000..5aa54eef95 --- /dev/null +++ b/client-java/instrumentation/src/test/java/org/evomaster/client/java/instrumentation/staticstate/ExecutionTracerTest.java @@ -0,0 +1,87 @@ +package org.evomaster.client.java.instrumentation.staticstate; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ExecutionTracerTest { + + /** + * A numeric comparison reports three objectives at once: less than, equal and greater than. + */ + private static final int OBJECTIVES_PER_COMPARISON = 3; + + private static final String COMPARISON = "ExecutionTracerTest_comparison"; + + private static final String OTHER_COMPARISON = "ExecutionTracerTest_other"; + + @BeforeEach + @AfterEach + public void reset() { + ExecutionTracer.reset(); + } + + @Test + public void testNothingReachedMeansNoUpdates() { + assertEquals(0, ExecutionTracer.getObjectiveUpdateCount()); + } + + @Test + public void testAnObjectiveReachedForTheFirstTimeCounts() { + + ExecutionTracer.executedNumericComparison(COMPARISON, 0.2, 0.2, 0.2); + + assertEquals(OBJECTIVES_PER_COMPARISON, ExecutionTracer.getObjectiveUpdateCount()); + + ExecutionTracer.executedNumericComparison(OTHER_COMPARISON, 0.2, 0.2, 0.2); + + assertEquals(2 * OBJECTIVES_PER_COMPARISON, ExecutionTracer.getObjectiveUpdateCount()); + } + + @Test + public void testRunningTheSameCodeAgainDoesNotCount() { + + /* + The point of counting improvements rather than executions: a consumer polling a + broker runs the same lines forever, and must still be seen to settle. + */ + ExecutionTracer.executedNumericComparison(COMPARISON, 0.2, 0.2, 0.2); + long first = ExecutionTracer.getObjectiveUpdateCount(); + + for (int i = 0; i < 10; i++) { + ExecutionTracer.executedNumericComparison(COMPARISON, 0.2, 0.2, 0.2); + } + + assertEquals(first, ExecutionTracer.getObjectiveUpdateCount()); + } + + @Test + public void testOnlyABetterValueCounts() { + + ExecutionTracer.executedNumericComparison(COMPARISON, 0.2, 0.2, 0.2); + long first = ExecutionTracer.getObjectiveUpdateCount(); + + //worse on all three + ExecutionTracer.executedNumericComparison(COMPARISON, 0.1, 0.1, 0.1); + assertEquals(first, ExecutionTracer.getObjectiveUpdateCount()); + + //better on one of the three + ExecutionTracer.executedNumericComparison(COMPARISON, 0.1, 0.9, 0.1); + assertEquals(first + 1, ExecutionTracer.getObjectiveUpdateCount()); + } + + @Test + public void testEachTestStartsFromZero() { + + ExecutionTracer.executedNumericComparison(COMPARISON, 0.2, 0.2, 0.2); + + ExecutionTracer.reset(); + assertEquals(0, ExecutionTracer.getObjectiveUpdateCount()); + + //and what the previous test reached counts again, since coverage is per test + ExecutionTracer.executedNumericComparison(COMPARISON, 0.2, 0.2, 0.2); + assertEquals(OBJECTIVES_PER_COMPARISON, ExecutionTracer.getObjectiveUpdateCount()); + } +} diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/AsyncApiTestBase.java b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/AsyncApiTestBase.java index 5eb3f02034..1c44d27849 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/AsyncApiTestBase.java +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/AsyncApiTestBase.java @@ -1,6 +1,7 @@ package org.evomaster.e2etests.utils; import com.webfuzzing.commons.faults.FaultCategory; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto; import org.evomaster.core.Main; import org.evomaster.core.problem.asyncapi.data.AsyncApiCallResult; import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual; @@ -89,6 +90,15 @@ protected void assertEverySettled(Solution solution) { + " published messages were not kept open after the driver returned"); } + /** + * How many published messages the controller stopped waiting on for {@code end}. + */ + protected long countSettleEnd(Solution solution, AsyncApiReplyDto.SettleEnd end) { + return allResultsOf(solution).stream() + .filter(r -> r.getSettleEndedBy() == end) + .count(); + } + protected void assertReplied(Solution solution, String operation) { boolean ok = resultsOf(solution, operation).stream().anyMatch(r -> r.getOutcome() == AsyncApiOutcome.REPLIED); assertTrue(ok, "With seed " + defaultSeed + ": no reply to '" + operation + "' was ever received"); diff --git a/core-tests/e2e-tests/spring/spring-asyncapi-kafka/src/test/java/org/evomaster/e2etests/spring/asyncapi/kafka/NcsKafkaEMTest.java b/core-tests/e2e-tests/spring/spring-asyncapi-kafka/src/test/java/org/evomaster/e2etests/spring/asyncapi/kafka/NcsKafkaEMTest.java index b43ce35287..3950aff9f7 100644 --- a/core-tests/e2e-tests/spring/spring-asyncapi-kafka/src/test/java/org/evomaster/e2etests/spring/asyncapi/kafka/NcsKafkaEMTest.java +++ b/core-tests/e2e-tests/spring/spring-asyncapi-kafka/src/test/java/org/evomaster/e2etests/spring/asyncapi/kafka/NcsKafkaEMTest.java @@ -1,6 +1,7 @@ package org.evomaster.e2etests.spring.asyncapi.kafka; import com.foo.asyncapi.ncs.NcsKafkaController; +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto; import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual; import org.evomaster.core.problem.asyncapi.data.AsyncApiOutcome; import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory; @@ -172,6 +173,47 @@ public void testRunEMWithASettleWindow() throws Throwable { 3); } + /** + * The same, but each action ends once the service stops reaching new code. NCS handles a + * message well within the window and then only polls lines already covered, so most waits + * end early. + */ + @Test + public void testRunEMWithCoverageStable() throws Throwable { + + runTestHandlingFlaky( + "NcsKafkaCoverageStableEM", + "org.foo.asyncapi.NcsKafkaCoverageStableEM", + 40, + false, + (args) -> { + + setOption(args, "asyncApiReplyTimeoutMs", "3000"); + + /* + A window far longer than NCS ever needs, and a stable period far shorter: + a wait that went to the deadline every time would mean stable coverage was + never detected. + */ + setOption(args, "asyncApiSettleMs", "5000"); + setOption(args, "asyncApiCompletion", "COVERAGE_STABLE"); + setOption(args, "asyncApiStableMs", "100"); + + Solution solution = initAndRun(args); + + assertTrue(countOutcome(solution, AsyncApiOutcome.REPLIED) > 0, "nothing was answered"); + assertEquals(0, countOutcome(solution, AsyncApiOutcome.NO_REPLY), "a promised reply never came"); + assertEquals(0, countOutcome(solution, AsyncApiOutcome.PUBLISH_FAILED), "a message never left"); + assertEverySettled(solution); + + assertTrue(countSettleEnd(solution, AsyncApiReplyDto.SettleEnd.COVERAGE_STABLE) > 0, + "no wait ended on stable coverage"); + assertEquals(0, countSettleEnd(solution, AsyncApiReplyDto.SettleEnd.INTERRUPTED), + "a wait was interrupted"); + }, + 3); + } + /** * The no-reply oracle, end to end. Given a deadline no round trip through a broker can meet, * every message goes unanswered, and that is reported as a fault once experimental oracles diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 8e2fad7f04..d771f7335e 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -896,10 +896,32 @@ class EMConfig { checkOutputFormatIsWritable() + if (blackBox && (asyncApiSettleMs > 0 || asyncApiCompletion != AsyncApiCompletion.FIXED)) { + /* + A black-box run never asks the controller to keep an action open, so these options + do nothing. Reset to their defaults rather than failing on how they combine. + */ + LoggingUtil.uniqueUserWarn("'asyncApiSettleMs' and 'asyncApiCompletion' only apply in white-box mode," + + " so they are ignored in this black-box run") + asyncApiSettleMs = 0 + asyncApiCompletion = AsyncApiCompletion.FIXED + } + + if (asyncApiCompletion == AsyncApiCompletion.COVERAGE_STABLE + && asyncApiSettleMs > 0 && asyncApiStableMs >= asyncApiSettleMs) { + LoggingUtil.uniqueUserWarn("'asyncApiStableMs' ($asyncApiStableMs) is not shorter than 'asyncApiSettleMs'" + + " ($asyncApiSettleMs), so ${AsyncApiCompletion.COVERAGE_STABLE} never ends a settle window early") + } + if (problemType == ProblemType.ASYNCAPI && seedTestCases) { throw ConfigProblemException("Seeding test cases is not supported for AsyncAPI services yet") } + if (asyncApiCompletion == AsyncApiCompletion.COVERAGE_STABLE && asyncApiSettleMs <= 0) { + throw ConfigProblemException("'asyncApiCompletion' ${AsyncApiCompletion.COVERAGE_STABLE} ends a" + + " settle window early, so it needs one: set 'asyncApiSettleMs' to the longest wait to allow") + } + if (problemType == ProblemType.RPC && createTests && (enablePureRPCTestGeneration || enableRPCAssertionWithInstance) @@ -3059,6 +3081,38 @@ class EMConfig { @Min(0.0) var asyncApiSettleMs = 0 + /** + * How the driver's controller decides, within the `asyncApiSettleMs` window, that a service + * is done with a message. + */ + enum class AsyncApiCompletion { + + /** + * Wait for the whole window. + */ + FIXED, + + /** + * Stop early once the service has gone `asyncApiStableMs` without reaching any coverage + * objective for the first time, or improving one. + */ + COVERAGE_STABLE + } + + @Experimental + @Cfg("When testing an AsyncAPI service in white-box mode with a settle window (asyncApiSettleMs), how the" + + " driver's controller decides the service is done with a message. FIXED waits for the whole window." + + " COVERAGE_STABLE ends it early once the service has gone asyncApiStableMs without reaching any new" + + " code, or improving on what it reached. That is a heuristic: a handler that only runs code already" + + " covered earlier in the same test looks done too.") + var asyncApiCompletion = AsyncApiCompletion.FIXED + + @Experimental + @Cfg("When testing an AsyncAPI service with asyncApiCompletion COVERAGE_STABLE, how long the service has" + + " to go without improving its coverage, in milliseconds, for the settle window to end early.") + @Min(1.0) + var asyncApiStableMs = 200 + @Experimental @Cfg("When testing an AsyncAPI service, the probability of publishing a message's declared example as it is," + " rather than a value sampled from its schema. A service that silently drops what it does not recognise" + diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt index 571d25e7dd..9b4e0f61b4 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/data/AsyncApiCallResult.kt @@ -1,5 +1,6 @@ package org.evomaster.core.problem.asyncapi.data +import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto import org.evomaster.core.problem.enterprise.EnterpriseActionResult import org.evomaster.core.search.action.Action @@ -15,6 +16,7 @@ class AsyncApiCallResult : EnterpriseActionResult { const val CORRELATION_MATCHED = "CORRELATION_MATCHED" const val WAITED_MS = "WAITED_MS" const val SETTLED_MS = "SETTLED_MS" + const val SETTLE_ENDED_BY = "SETTLE_ENDED_BY" const val TEST_SCRIPT = "TEST_SCRIPT" /* @@ -94,6 +96,16 @@ class AsyncApiCallResult : EnterpriseActionResult { fun getSettledMs(): Long? = getResultValue(SETTLED_MS)?.toLong() + /** + * Why the controller stopped keeping the action open. Absent when no wait was asked for. + */ + fun setSettleEndedBy(end: AsyncApiReplyDto.SettleEnd) { + addResultValue(SETTLE_ENDED_BY, end.name) + } + + fun getSettleEndedBy(): AsyncApiReplyDto.SettleEnd? = + getResultValue(SETTLE_ENDED_BY)?.let { AsyncApiReplyDto.SettleEnd.valueOf(it) } + /** * The lines the driver rendered for a generated test, or empty when it rendered none. */ diff --git a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt index 2759132a6d..8bde5a6bd1 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitness.kt @@ -9,6 +9,7 @@ import com.webfuzzing.asyncapi.models.AsyncApiReply import org.evomaster.client.java.controller.api.dto.SutInfoDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiActionDto import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto +import org.evomaster.core.EMConfig import org.evomaster.core.Lazy import org.evomaster.core.database.sql.SqlAction import org.evomaster.core.logging.LoggingUtil @@ -250,6 +251,7 @@ class AsyncApiFitness : ApiWsFitness() { result.setOutcome(outcome) reply.waitedMs?.let { result.setWaitedMs(it) } reply.settledMs?.let { result.setSettledMs(it) } + reply.settleEndedBy?.let { result.setSettleEndedBy(it) } //what the generated test will publish with, when the driver rendered it //lines that are all blank are no script: the writer would publish nothing and assert on it @@ -447,6 +449,13 @@ class AsyncApiFitness : ApiWsFitness() { credited to it, and a black-box run collects no code coverage to credit. */ dto.settleMs = config.asyncApiSettleMs.toLong() + dto.completion = when (config.asyncApiCompletion) { + EMConfig.AsyncApiCompletion.FIXED -> AsyncApiActionDto.Completion.FIXED + EMConfig.AsyncApiCompletion.COVERAGE_STABLE -> { + dto.stableMs = config.asyncApiStableMs.toLong() + AsyncApiActionDto.Completion.COVERAGE_STABLE + } + } } return dto diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 33423903d2..48d60e1272 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -927,6 +927,62 @@ internal class EMConfigTest{ assertTrue(rest.isKillSwitchOn()) } + @Test + fun testSettleOptionsAreIgnoredInBlackBox(){ + + fun parse(vararg args: String) = + EMConfig().apply { updateProperties(EMConfig.getOptionParser().parse(*args)) } + + //no settle window to end early, which would be an error in white-box + val noWindow = parse("--$blackBox", "true", "--problemType", "ASYNCAPI", "--createTests", "false", + "--asyncApiCompletion", "COVERAGE_STABLE") + assertEquals(EMConfig.AsyncApiCompletion.FIXED, noWindow.asyncApiCompletion) + assertEquals(0, noWindow.asyncApiSettleMs) + + val window = parse("--$blackBox", "true", "--problemType", "ASYNCAPI", "--createTests", "false", + "--asyncApiSettleMs", "2000") + assertEquals(0, window.asyncApiSettleMs, "a black-box run never keeps an action open") + + //while in white-box they are kept as given + val whiteBox = parse("--$blackBox", "false", "--problemType", "ASYNCAPI", + "--asyncApiCompletion", "COVERAGE_STABLE", "--asyncApiSettleMs", "2000") + assertEquals(EMConfig.AsyncApiCompletion.COVERAGE_STABLE, whiteBox.asyncApiCompletion) + assertEquals(2000, whiteBox.asyncApiSettleMs) + } + + @Test + fun testAStablePeriodNoShorterThanTheWindowIsAllowed(){ + + //it can never end the window early, which is worth a warning, not a failure + val config = EMConfig().apply { updateProperties(EMConfig.getOptionParser().parse( + "--blackBox", "false", "--problemType", "ASYNCAPI", "--asyncApiCompletion", "COVERAGE_STABLE", + "--asyncApiSettleMs", "200", "--asyncApiStableMs", "500")) } + assertEquals(EMConfig.AsyncApiCompletion.COVERAGE_STABLE, config.asyncApiCompletion) + assertEquals(500, config.asyncApiStableMs) + } + + @Test + fun testCoverageStableNeedsASettleWindowToEndEarly(){ + + //in white-box, as the settle options are ignored in black-box + fun parse(vararg args: String) = + EMConfig().apply { updateProperties(EMConfig.getOptionParser().parse("--blackBox", "false", *args)) } + + val e = assertThrows { + parse("--problemType", "ASYNCAPI", "--asyncApiCompletion", "COVERAGE_STABLE") + } + assertTrue(e.message!!.contains("asyncApiSettleMs"), e.message) + + val config = parse("--problemType", "ASYNCAPI", + "--asyncApiCompletion", "COVERAGE_STABLE", "--asyncApiSettleMs", "2000") + assertEquals(EMConfig.AsyncApiCompletion.COVERAGE_STABLE, config.asyncApiCompletion) + + //and the defaults ask for no wait at all + val defaults = parse("--problemType", "ASYNCAPI") + assertEquals(0, defaults.asyncApiSettleMs) + assertEquals(EMConfig.AsyncApiCompletion.FIXED, defaults.asyncApiCompletion) + } + @Test fun testSetFalseOnDependsOn(){ diff --git a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt index d15373a9d2..9276474f4d 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/asyncapi/service/AsyncApiFitnessTest.kt @@ -759,6 +759,58 @@ class AsyncApiFitnessTest { evaluate("bessj", "expint") assertEquals(listOf(250L, 250L), driver.published.map { it.settleMs }) + //the whole window unless asked otherwise, and no stable window to go with it + assertEquals(AsyncApiActionDto.Completion.FIXED, driver.published.first().completion) + assertNull(driver.published.first().stableMs) + } + + @Test + fun testCoverageStableIsAskedForWithItsStableWindow() { + + startNcs("--asyncApiSettleMs=2000", "--asyncApiCompletion=COVERAGE_STABLE", "--asyncApiStableMs=150") { + fireAndForget() + } + + evaluate("bessj") + + val dto = driver.published.single() + assertEquals(2000L, dto.settleMs) + assertEquals(AsyncApiActionDto.Completion.COVERAGE_STABLE, dto.completion) + assertEquals(150L, dto.stableMs) + } + + @Test + fun testABlackBoxRunAsksForNoStableCoverageEither() { + + startNcs("$BLACK_BOX=true") { fireAndForget() } + //set after start, as EMConfig puts both back to their defaults in black-box + injector.getInstance(EMConfig::class.java).apply { + asyncApiSettleMs = 2000 + asyncApiCompletion = EMConfig.AsyncApiCompletion.COVERAGE_STABLE + } + + evaluate("bessj") + + val dto = driver.published.single() + assertNull(dto.settleMs) + assertNull(dto.completion) + assertNull(dto.stableMs) + } + + @Test + fun testWhyTheControllerStoppedWaitingIsKeptOnTheResult() { + + startNcs("--asyncApiSettleMs=2000", "--asyncApiCompletion=COVERAGE_STABLE") { + fireAndForget().apply { + settledMs = 230 + settleEndedBy = AsyncApiReplyDto.SettleEnd.COVERAGE_STABLE + } + } + + val result = results(evaluate("bessj")).single() + + assertEquals(230L, result.getSettledMs()) + assertEquals(AsyncApiReplyDto.SettleEnd.COVERAGE_STABLE, result.getSettleEndedBy()) } @Test @@ -768,7 +820,9 @@ class AsyncApiFitnessTest { The window is there so that the code a message reaches is credited to it. A black-box run collects no code coverage, so it would only slow the search down. */ - startNcs("$BLACK_BOX=true", "--asyncApiSettleMs=250") { fireAndForget() } + startNcs("$BLACK_BOX=true") { fireAndForget() } + //set after start, as EMConfig puts it back to 0 in black-box + injector.getInstance(EMConfig::class.java).asyncApiSettleMs = 250 evaluate("bessj") @@ -793,5 +847,6 @@ class AsyncApiFitnessTest { val evaluated = evaluate("bessj") assertNull(results(evaluated).single().getSettledMs()) + assertNull(results(evaluated).single().getSettleEndedBy()) } } diff --git a/docs/options.md b/docs/options.md index bc0b50c465..fa48dff8c6 100644 --- a/docs/options.md +++ b/docs/options.md @@ -283,8 +283,10 @@ There are 3 types of options: |`aiResponseClassifierWarmup`| __Int__. Number of training iterations required to update classifier parameters. For example, in the Gaussian model this affects mean and variance updates. For neural network (NN) models, the warm-up should typically be larger than 1000. *Default value*: `100`.| |`appendToTargetHeuristicsFile`| __Boolean__. Whether should add to an existing target heuristics file, instead of replacing it. It is only used when processFormat is TARGET_HEURISTIC. *Default value*: `false`.| |`arazzoLocation`| __String__. arazzo location on disk. *Default value*: `""`.| +|`asyncApiCompletion`| __Enum__. When testing an AsyncAPI service in white-box mode with a settle window (asyncApiSettleMs), how the driver's controller decides the service is done with a message. FIXED waits for the whole window. COVERAGE_STABLE ends it early once the service has gone asyncApiStableMs without reaching any new code, or improving on what it reached. That is a heuristic: a handler that only runs code already covered earlier in the same test looks done too. *Valid values*: `FIXED, COVERAGE_STABLE`. *Default value*: `FIXED`.| |`asyncApiReplyTimeoutMs`| __Int__. When testing an AsyncAPI service, how long to wait for the reply to a published message before treating it as unanswered, in milliseconds. *Constraints*: `min=1.0`. *Default value*: `5000`.| |`asyncApiSettleMs`| __Int__. When testing an AsyncAPI service in white-box mode, how long to keep each published message's action open after the driver has returned, in milliseconds, so that the code the service runs to handle it is still credited to that action. A message-driven service handles a message on a thread of its own, which may still be running once the driver has seen it published, or its reply arrive. The wait happens in the driver's controller. 0 means no wait. *Constraints*: `min=0.0`. *Default value*: `0`.| +|`asyncApiStableMs`| __Int__. When testing an AsyncAPI service with asyncApiCompletion COVERAGE_STABLE, how long the service has to go without improving its coverage, in milliseconds, for the settle window to end early. *Constraints*: `min=1.0`. *Default value*: `200`.| |`breederParentsMin`| __Int__. Breeder GA: minimum number of individuals in parents pool after truncation. *Constraints*: `min=2.0`. *Default value*: `2`.| |`breederTruncationFraction`| __Double__. Breeder GA: fraction of top individuals to keep in parents pool (truncation). *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`callbackURLHostname`| __String__. HTTP callback verifier hostname. Default is set to 'localhost'. If the SUT is running inside a container (i.e., Docker), 'localhost' will refer to the container. This can be used to change the hostname. *Default value*: `localhost`.|