AsyncAPI 3.x: stable coverage as a completion strategy

client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiActionDto.java

@@ -22,6 +22,25 @@ public enum CorrelationLocation {
         PAYLOAD
     }
 
+    /**
+     * How the controller decides that the service is done with a message, and the action can
+     * be closed, within the {@link #settleMs} window.
+     */
+    public enum Completion {
+
+        /**
+         * Wait for the whole window.
+         */
+        FIXED,
+
+        /**
+         * Stop early once the SUT has gone {@link #stableMs} without reaching any coverage
+         * objective for the first time, or improving one. A heuristic: a handler that only runs
+         * code already covered earlier in the same test leaves coverage stable too.
+         */
+        COVERAGE_STABLE
+    }
+
     /**
      * Key of the operation in the AsyncAPI document, so a driver can log in terms the user will
      * recognise from their own contract.
@@ -104,6 +123,19 @@ public enum CorrelationLocation {
      */
     public Long settleMs;
 
+    /**
+     * How the controller decides, within {@link #settleMs}, that the action can be closed. Null
+     * is read as {@link Completion#FIXED}. Meaningless without a {@link #settleMs}.
+     */
+    public Completion completion;
+
+    /**
+     * For {@link Completion#COVERAGE_STABLE}: how long, in milliseconds, the SUT has to go
+     * without improving any coverage objective to end the wait early. Null, zero or negative
+     * make it a {@link Completion#FIXED} wait. Null for any other completion.
+     */
+    public Long stableMs;
+
     /**
      * The language to render {@link AsyncApiReplyDto#testScript} in. Only Java and Kotlin are
      * ever asked for.

client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/problem/asyncapi/AsyncApiReplyDto.java

@@ -14,6 +14,30 @@
  */
 public class AsyncApiReplyDto {
 
+    /**
+     * Why the controller stopped keeping the action open.
+     */
+    public enum SettleEnd {
+
+        /**
+         * The whole {@link AsyncApiActionDto#settleMs} window went by. Also what a
+         * {@link AsyncApiActionDto.Completion#COVERAGE_STABLE} wait ends with when the SUT's
+         * coverage cannot be read.
+         */
+        DEADLINE,
+
+        /**
+         * The SUT went {@link AsyncApiActionDto#stableMs} without improving any coverage
+         * objective.
+         */
+        COVERAGE_STABLE,
+
+        /**
+         * The waiting thread was interrupted.
+         */
+        INTERRUPTED
+    }
+
     /**
      * The index of the action this answers, echoing what was asked.
      */
@@ -68,14 +92,18 @@ public class AsyncApiReplyDto {
 
     /**
      * How long the controller kept the action open after the driver returned, in milliseconds,
-     * as asked by {@link AsyncApiActionDto#settleMs}. It can be shorter than what was asked when
-     * the wait was interrupted.
-     *
-     * Filled in by the controller rather than by the driver, which never sees this field set.
-     * Null when no wait was asked for, or when the message could not be published.
+     * as asked by {@link AsyncApiActionDto#settleMs}; shorter when it ended early, see
+     * {@link #settleEndedBy}. Set by the controller, not the driver. Null when no wait was asked
+     * for, or the message could not be published.
      */
     public Long settledMs;
 
+    /**
+     * Why the controller stopped waiting, set together with {@link #settledMs}. Null exactly
+     * when that is.
+     */
+    public SettleEnd settleEndedBy;
+
     /**
      * Why publishing failed, when it did.
      */

client-java/controller/src/main/java/org/evomaster/client/java/controller/EmbeddedSutController.java

@@ -144,6 +144,18 @@ public BootTimeInfoDto getBootTimeInfoDto() {
         return getBootTimeInfoDto(InstrumentationController.getBootTimeObjectiveInfo());
     }
 
+    @Override
+    public final Long getObjectiveUpdateCount() {
+        /*
+            Without the agent nothing is instrumented, and a count stuck at zero would read as a
+            SUT doing nothing new rather than as unknown coverage.
+         */
+        if (!isInstrumentationActivated()) {
+            return null;
+        }
+        return InstrumentationController.getObjectiveUpdateCount();
+    }
+
     @Override
     public final String getExecutableFullPath(){
         return null; //not needed for embedded

client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java

@@ -447,6 +447,14 @@ public BootTimeInfoDto getBootTimeInfoDto() {
         return getBootTimeInfoDto(serverController.handleBootTimeObjectiveInfo());
     }
 
+    @Override
+    public final Long getObjectiveUpdateCount() {
+        if (!isInstrumentationActivated()) {
+            return null;
+        }
+        return serverController.getObjectiveUpdateCount();
+    }
+
     @Override
     public final void newActionSpecificHandler(ActionDto dto) {
         if (isInstrumentationActivated()) {

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/AsyncApiSettle.java

@@ -4,6 +4,7 @@
 import org.evomaster.client.java.controller.api.dto.problem.asyncapi.AsyncApiReplyDto;
 
 import java.util.Objects;
+import java.util.function.Supplier;
 
 /**
  * Keeps an AsyncAPI action open for a while after the driver has returned from publishing it,
@@ -15,6 +16,16 @@ public class AsyncApiSettle {
 
     private static final long NANOS_PER_MILLI = 1_000_000L;
 
+    private static final String OBJECTIVE_UPDATES_FAILURE_WARNING = "Failed to read how many coverage objectives" +
+            " the SUT has reached or improved, so the whole settle window is waited";
+
+    /**
+     * How often the SUT's coverage is read while waiting for it to become stable, in milliseconds.
+     * Short next to any useful stable window, and long enough that reading it, a socket round
+     * trip for an external SUT, costs little.
+     */
+    static final long POLL_INTERVAL_MS = 25;
+
     /**
      * Where the time comes from, in milliseconds. Only the difference between two readings is
      * ever used, so it does not have to be wall-clock time.
@@ -48,33 +59,112 @@ public AsyncApiSettle() {
     }
 
     /**
-     * Wait as long as {@link AsyncApiActionDto#settleMs} asks, and record in
-     * {@link AsyncApiReplyDto#settledMs} how long that actually was. Nothing is waited, and
-     * nothing recorded, when no wait is asked for.
+     * Wait as {@link AsyncApiActionDto#settleMs} and {@link AsyncApiActionDto#completion} ask,
+     * and record in {@link AsyncApiReplyDto#settledMs} how long that actually was and in
+     * {@link AsyncApiReplyDto#settleEndedBy} why it ended. Nothing is waited or recorded when no
+     * wait is asked for.
      *
-     * The thread calling this is a request thread of the controller, whose code is never
-     * instrumented, so the kill switch does not interrupt it. If it is interrupted all the same,
-     * the wait ends there, the interrupt is left set for whoever owns the thread, and the time
-     * actually waited is what gets recorded.
+     * If the thread is interrupted, the wait ends there and the interrupt flag is left set.
      *
-     * @param dto   the action that was just published
-     * @param reply what the driver reported about it, to which the wait is added
+     * @param dto               the action that was just published
+     * @param reply             what the driver reported about it, to which the wait is added
+     * @param objectiveUpdates  reads how many times the SUT has reached or improved a coverage
+     *                          objective in the current test, or null when that cannot be known.
+     *                          Only read for {@link AsyncApiActionDto.Completion#COVERAGE_STABLE}
      */
-    public void settle(AsyncApiActionDto dto, AsyncApiReplyDto reply) {
+    public void settle(AsyncApiActionDto dto, AsyncApiReplyDto reply, Supplier<Long> objectiveUpdates) {
         Objects.requireNonNull(dto);
         Objects.requireNonNull(reply);
+        Objects.requireNonNull(objectiveUpdates);
 
         Long settleMs = dto.settleMs;
         if (settleMs == null || settleMs <= 0) {
             return;
         }
 
+        boolean coverageStable = dto.completion == AsyncApiActionDto.Completion.COVERAGE_STABLE
+                && dto.stableMs != null && dto.stableMs > 0;
+
         long start = clock.nowMs();
+        AsyncApiReplyDto.SettleEnd end;
         try {
-            sleeper.sleep(settleMs);
+            end = coverageStable
+                    ? waitForStableCoverage(start, settleMs, dto.stableMs, objectiveUpdates)
+                    : waitUntil(start + settleMs);
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
+            end = AsyncApiReplyDto.SettleEnd.INTERRUPTED;
         }
         reply.settledMs = clock.nowMs() - start;
+        reply.settleEndedBy = end;
+    }
+
+    /**
+     * What {@code reader} reads, where a failure to read counts as not knowing. Reading is a
+     * socket round trip for an external SUT, and the action was published all the same: a
+     * failure here would otherwise turn a dropped connection into a broken setup.
+     *
+     * @param warning what to warn about, once, as reads come every few milliseconds; the reason
+     *                goes to the debug log
+     */
+    private static <T> T readOrNull(Supplier<T> reader, String warning) {
+        try {
+            return reader.get();
+        } catch (RuntimeException e) {
+            org.evomaster.client.java.utils.SimpleLogger.uniqueWarn(warning);
+            org.evomaster.client.java.utils.SimpleLogger.debug(warning + ": " + e.getMessage());
+            return null;
+        }
+    }
+
+    private AsyncApiReplyDto.SettleEnd waitUntil(long deadline) throws InterruptedException {
+        long left = deadline - clock.nowMs();
+        if (left > 0) {
+            sleeper.sleep(left);
+        }
+        return AsyncApiReplyDto.SettleEnd.DEADLINE;
+    }
+
+    /**
+     * Read the SUT's coverage every {@link #POLL_INTERVAL_MS}, and stop once it has gone
+     * {@code stableMs} without moving, or at the deadline, whichever comes first. The stable
+     * period is counted from the start of the wait, so a service that does nothing new after
+     * the driver returned is let go after {@code stableMs}.
+     */
+    private AsyncApiReplyDto.SettleEnd waitForStableCoverage(
+            long start,
+            long settleMs,
+            long stableMs,
+            Supplier<Long> objectiveUpdates
+    ) throws InterruptedException {
+
+        long deadline = start + settleMs;
+
+        Long last = readOrNull(objectiveUpdates, OBJECTIVE_UPDATES_FAILURE_WARNING);
+        long lastChange = start;
+
+        while (true) {
+            if (last == null) {
+                //coverage unknown: wait out the window
+                return waitUntil(deadline);
+            }
+
+            long now = clock.nowMs();
+            if (now >= deadline) {
+                return AsyncApiReplyDto.SettleEnd.DEADLINE;
+            }
+            if (now - lastChange >= stableMs) {
+                return AsyncApiReplyDto.SettleEnd.COVERAGE_STABLE;
+            }
+
+            long nap = Math.min(POLL_INTERVAL_MS, Math.min(deadline - now, lastChange + stableMs - now));
+            sleeper.sleep(nap);
+
+            Long current = readOrNull(objectiveUpdates, OBJECTIVE_UPDATES_FAILURE_WARNING);
+            if (current != null && !current.equals(last)) {
+                lastChange = clock.nowMs();
+            }
+            last = current;
+        }
     }
 }

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java

@@ -895,7 +895,7 @@ public Response newAction(
                     was already turned off for this action by newAction above, and must stay
                     off for the service's threads, not only this one.
                  */
-                asyncApiSettle.settle(dto.asyncApiCall, replyDto);
+                asyncApiSettle.settle(dto.asyncApiCall, replyDto, sutController::getObjectiveUpdateCount);
 
                 return Response.status(200).entity(WrappedResponseDto.withData(replyDto)).build();
             }

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java

@@ -1834,6 +1834,18 @@ public abstract List<TargetInfo> getTargetInfos(Collection<Integer> ids,
 
     public abstract BootTimeInfoDto getBootTimeInfoDto();
 
+    /**
+     * How many times, in the current test, a coverage objective of the SUT was reached for the
+     * first time or improved on its best value. Read while an action is kept open, to tell
+     * whether the SUT is still reaching anything new.
+     *
+     * @return the count, which only grows within a test, or null when unknown, eg because the
+     * SUT is not instrumented
+     */
+    public Long getObjectiveUpdateCount() {
+        return null;
+    }
+
     protected BootTimeInfoDto getBootTimeInfoDto(BootTimeObjectiveInfo info){
         if (info == null)
             return null;