diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index b6c6524c4c..d7447270d5 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -1020,6 +1020,15 @@ class EMConfig { fun couldSupportDtoForPayload() = problemType == ProblemType.REST && outputFormat.isJavaOrKotlin() + /** + * Whether an EvoMaster Driver takes part in this run. + * + * Always in white-box mode, and in black-box mode only for experiments. AsyncAPI is the + * exception: there is no universal wire to speak to a message-driven service, so the driver + * holds the connection to the broker even when the SUT itself is treated as a black box. + */ + fun usesDriver() = !blackBox || bbExperiments || problemType == ProblemType.ASYNCAPI + fun activatedExperimentalFeatures(): List { val properties = getConfigurationProperties() @@ -1506,7 +1515,8 @@ class EMConfig { GRAPHQL(experimental = false), RPC(experimental = true), WEBFRONTEND(experimental = true), - MCP(experimental = true); + MCP(experimental = true), + ASYNCAPI(experimental = true); override fun isExperimental() = experimental } diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 96e093ac14..435b6fc8ad 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -19,6 +19,7 @@ import org.evomaster.core.output.TestSuiteCode import org.evomaster.core.output.TestSuiteSplitter import org.evomaster.core.output.clustering.SplitResult import org.evomaster.core.output.service.TestSuiteWriter +import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual import org.evomaster.core.problem.enterprise.service.WFCReportWriter import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHttpWsResponseHandler import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler @@ -364,7 +365,7 @@ class Main { solution: Solution<*>, idMapper: IdMapper ) { - if (!config.blackBox || config.bbExperiments) { + if (config.usesDriver()) { val rc = injector.getInstance(RemoteController::class.java) val unitsInfo = rc.getSutInfo()?.unitsInfoDto val bootTimeInfo = rc.getSutInfo()?.bootTimeInfoDto @@ -539,7 +540,7 @@ class Main { /* Note that, in case ob BB-testing, this would had been already modified */ - assert(!config.blackBox || config.bbExperiments) + assert(config.usesDriver()) val rc = RemoteControllerImplementation(base.getEMConfig()) @@ -568,6 +569,8 @@ class Main { config.problemType = EMConfig.ProblemType.RPC } else if (info.webProblem != null) { config.problemType = EMConfig.ProblemType.WEBFRONTEND + } else if (info.asyncApiProblem != null) { + config.problemType = EMConfig.ProblemType.ASYNCAPI } else { throw IllegalStateException("Can connect to the EM Driver, but cannot infer the 'problemType'") } @@ -623,6 +626,11 @@ class Main { McpBlackBoxModule(false) } + EMConfig.ProblemType.ASYNCAPI -> { + //the sampler, fitness and module for it are being added one at a time + throw IllegalStateException("AsyncAPI is not wired into the search yet") + } + //this should never happen, unless we add new type and forget to add it here else -> throw IllegalStateException("Unrecognized problem type: ${config.problemType}") } @@ -830,6 +838,58 @@ class Main { } } + private fun getAlgorithmKeyAsyncApi(config: EMConfig): Key> { + + return when (config.algorithm) { + EMConfig.Algorithm.SMARTS -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.RANDOM -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.MIO -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.WTS -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.MOSA -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.StandardGA -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.MonotonicGA -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.SteadyStateGA -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.RW -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.LIPS -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.MuPlusLambdaEA -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.MuLambdaEA -> + Key.get(object : TypeLiteral>(){}) + + EMConfig.Algorithm.BreederGA -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.CellularGA -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.OnePlusLambdaLambdaGA -> + Key.get(object : TypeLiteral>() {}) + + else -> throw IllegalStateException("Unrecognized algorithm ${config.algorithm}") + } + } + private fun getAlgorithmKeyMcp(config: EMConfig): Key> { return when (config.algorithm) { @@ -913,7 +973,7 @@ class Main { val epc = injector.getInstance(ExecutionPhaseController::class.java) epc.markStartingSearch() - if (!config.blackBox || config.bbExperiments) { + if (config.usesDriver()) { val rc = injector.getInstance(RemoteController::class.java) rc.startANewSearch() } @@ -924,6 +984,7 @@ class Main { EMConfig.ProblemType.RPC -> getAlgorithmKeyRPC(config) EMConfig.ProblemType.WEBFRONTEND -> getAlgorithmKeyWeb(config) EMConfig.ProblemType.MCP -> getAlgorithmKeyMcp(config) + EMConfig.ProblemType.ASYNCAPI -> getAlgorithmKeyAsyncApi(config) else -> throw IllegalStateException("Unrecognized problem type ${config.problemType}") } @@ -977,7 +1038,7 @@ class Main { val config = injector.getInstance(EMConfig::class.java) - if (config.blackBox && !config.bbExperiments) { + if (!config.usesDriver()) { return null } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt index fc034570de..acf9737f6e 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt @@ -469,7 +469,7 @@ class Statistics : SearchListener { fun getData(solution: Solution<*>): List { - val sutInfo : SutInfoDto? = if(!config.blackBox || config.bbExperiments) { + val sutInfo : SutInfoDto? = if(config.usesDriver()) { remoteController?.getSutInfo() } else { null @@ -852,7 +852,7 @@ class Statistics : SearchListener { // append boot-time targets - if(!config.blackBox || config.bbExperiments) { + if(config.usesDriver()) { remoteController?.getSutInfo()?.bootTimeInfoDto?.targets?.map { it.descriptiveId }?.sorted()?.apply { if (isNotEmpty()){ content.add(System.lineSeparator()) diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index a31343e4a9..4a67d5de98 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -762,4 +762,26 @@ internal class EMConfigTest{ assertThrows (Exception::class.java,{config.updateProperties(optionAllSpecifiedNotRest)}) } + + @Test + fun testAsyncApiNeedsTheDriverEvenAsABlackBox(){ + + val parser = EMConfig.getOptionParser() + + val rest = EMConfig() + rest.updateProperties(parser.parse("--$blackBox", "true", "--problemType", "REST")) + assertFalse(rest.usesDriver()) + + /* + There is no universal wire to a message-driven service, so the driver holds the + connection to the broker even when the service itself is a black box. + */ + val asyncApi = EMConfig() + asyncApi.updateProperties(parser.parse("--$blackBox", "true", "--problemType", "ASYNCAPI")) + assertTrue(asyncApi.usesDriver()) + + val whiteBox = EMConfig() + whiteBox.updateProperties(parser.parse("--$blackBox", "false")) + assertTrue(whiteBox.usesDriver()) + } } diff --git a/docs/options.md b/docs/options.md index 23b9589dc1..ba38a4c4e5 100644 --- a/docs/options.md +++ b/docs/options.md @@ -200,7 +200,7 @@ There are 3 types of options: |`probRestDefault`| __Double__. In REST, specify probability of using 'default' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.05`.| |`probRestExamples`| __Double__. In REST, specify probability of using 'example(s)' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.| |`probUseRestLinks`| __Double__. In REST, enable the supports of 'links' between resources defined in the OpenAPI schema, if any. When sampling a test case, if the last call has links, given this probability new calls are added for the link. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| -|`problemType`| __Enum__. The type of SUT we want to generate tests for, e.g., a RESTful API. If left to DEFAULT, the type will be inferred from the EM Driver. However, in case of ambiguities (e.g., the driver specifies more than one type), then this field must be set with a specific type. This is also the case for Black-Box testing where there is no EM Driver. In this latter case, the system defaults to handle REST APIs. *Valid values*: `DEFAULT, REST, GRAPHQL`. *Experimental values*: `RPC, WEBFRONTEND, MCP`. *Default value*: `DEFAULT`.| +|`problemType`| __Enum__. The type of SUT we want to generate tests for, e.g., a RESTful API. If left to DEFAULT, the type will be inferred from the EM Driver. However, in case of ambiguities (e.g., the driver specifies more than one type), then this field must be set with a specific type. This is also the case for Black-Box testing where there is no EM Driver. In this latter case, the system defaults to handle REST APIs. *Valid values*: `DEFAULT, REST, GRAPHQL`. *Experimental values*: `RPC, WEBFRONTEND, MCP, ASYNCAPI`. *Default value*: `DEFAULT`.| |`processFiles`| __String__. Specify a folder to save results when a search monitor is enabled. *DEBUG option*. *Default value*: `process_data`.| |`processFormat`| __Enum__. Specify a format to save the process data. *DEBUG option*. *Valid values*: `JSON_ALL, TEST_IND, TARGET_TEST_IND, TARGET_HEURISTIC`. *Default value*: `JSON_ALL`.| |`processInterval`| __Double__. Specify how often to save results when a search monitor is enabled, and 0.0 presents to record all evaluated individual. *DEBUG option*. *Constraints*: `min=0.0, max=50.0`. *Default value*: `0.0`.|