Skip to content

AsyncAPI 3.x: the problem type #1751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LautaroPetaccio wants to merge 1 commit into from
Open

AsyncAPI 3.x: the problem type #1751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion core/src/main/kotlin/org/evomaster/core/EMConfig.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1020,6 +1020,15 @@ class EMConfig {

fun couldSupportDtoForPayload() = problemType == ProblemType.REST && outputFormat.isJavaOrKotlin()

/**
* Whether an EvoMaster Driver takes part in this run.
*
* Always in white-box mode, and in black-box mode only for experiments. AsyncAPI is the
* exception: there is no universal wire to speak to a message-driven service, so the driver
* holds the connection to the broker even when the SUT itself is treated as a black box.
*/
fun usesDriver() = !blackBox || bbExperiments || problemType == ProblemType.ASYNCAPI

fun activatedExperimentalFeatures(): List<String> {

val properties = getConfigurationProperties()
Expand Down Expand Up @@ -1506,7 +1515,8 @@ class EMConfig {
GRAPHQL(experimental = false),
RPC(experimental = true),
WEBFRONTEND(experimental = true),
MCP(experimental = true);
MCP(experimental = true),
ASYNCAPI(experimental = true);

override fun isExperimental() = experimental
}
Expand Down
69 changes: 65 additions & 4 deletions core/src/main/kotlin/org/evomaster/core/Main.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import org.evomaster.core.output.TestSuiteCode
import org.evomaster.core.output.TestSuiteSplitter
import org.evomaster.core.output.clustering.SplitResult
import org.evomaster.core.output.service.TestSuiteWriter
import org.evomaster.core.problem.asyncapi.data.AsyncApiIndividual
import org.evomaster.core.problem.enterprise.service.WFCReportWriter
import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHttpWsResponseHandler
import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler
Expand Down Expand Up @@ -364,7 +365,7 @@ class Main {
solution: Solution<*>,
idMapper: IdMapper
) {
if (!config.blackBox || config.bbExperiments) {
if (config.usesDriver()) {
val rc = injector.getInstance(RemoteController::class.java)
val unitsInfo = rc.getSutInfo()?.unitsInfoDto
val bootTimeInfo = rc.getSutInfo()?.bootTimeInfoDto
Expand Down Expand Up @@ -539,7 +540,7 @@ class Main {
/*
Note that, in case ob BB-testing, this would had been already modified
*/
assert(!config.blackBox || config.bbExperiments)
assert(config.usesDriver())

val rc = RemoteControllerImplementation(base.getEMConfig())

Expand Down Expand Up @@ -568,6 +569,8 @@ class Main {
config.problemType = EMConfig.ProblemType.RPC
} else if (info.webProblem != null) {
config.problemType = EMConfig.ProblemType.WEBFRONTEND
} else if (info.asyncApiProblem != null) {
config.problemType = EMConfig.ProblemType.ASYNCAPI
} else {
throw IllegalStateException("Can connect to the EM Driver, but cannot infer the 'problemType'")
}
Expand Down Expand Up @@ -623,6 +626,11 @@ class Main {
McpBlackBoxModule(false)
}

EMConfig.ProblemType.ASYNCAPI -> {
//the sampler, fitness and module for it are being added one at a time
throw IllegalStateException("AsyncAPI is not wired into the search yet")
}

//this should never happen, unless we add new type and forget to add it here
else -> throw IllegalStateException("Unrecognized problem type: ${config.problemType}")
}
Expand Down Expand Up @@ -830,6 +838,58 @@ class Main {
}
}

private fun getAlgorithmKeyAsyncApi(config: EMConfig): Key<out SearchAlgorithm<AsyncApiIndividual>> {

return when (config.algorithm) {
EMConfig.Algorithm.SMARTS ->
Key.get(object : TypeLiteral<SmartsAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.RANDOM ->
Key.get(object : TypeLiteral<RandomAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.MIO ->
Key.get(object : TypeLiteral<MioAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.WTS ->
Key.get(object : TypeLiteral<WtsAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.MOSA ->
Key.get(object : TypeLiteral<MosaAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.StandardGA ->
Key.get(object : TypeLiteral<StandardGeneticAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.MonotonicGA ->
Key.get(object : TypeLiteral<MonotonicGeneticAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.SteadyStateGA ->
Key.get(object : TypeLiteral<SteadyStateGeneticAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.RW ->
Key.get(object : TypeLiteral<RandomWalkAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.LIPS ->
Key.get(object : TypeLiteral<LIPSAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.MuPlusLambdaEA ->
Key.get(object : TypeLiteral<MuPlusLambdaEvolutionaryAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.MuLambdaEA ->
Key.get(object : TypeLiteral<MuLambdaEvolutionaryAlgorithm<AsyncApiIndividual>>(){})

EMConfig.Algorithm.BreederGA ->
Key.get(object : TypeLiteral<BreederGeneticAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.CellularGA ->
Key.get(object : TypeLiteral<CellularGeneticAlgorithm<AsyncApiIndividual>>() {})

EMConfig.Algorithm.OnePlusLambdaLambdaGA ->
Key.get(object : TypeLiteral<OnePlusLambdaLambdaGeneticAlgorithm<AsyncApiIndividual>>() {})

else -> throw IllegalStateException("Unrecognized algorithm ${config.algorithm}")
}
}

private fun getAlgorithmKeyMcp(config: EMConfig): Key<out SearchAlgorithm<McpIndividual>> {

return when (config.algorithm) {
Expand Down Expand Up @@ -913,7 +973,7 @@ class Main {
val epc = injector.getInstance(ExecutionPhaseController::class.java)
epc.markStartingSearch()

if (!config.blackBox || config.bbExperiments) {
if (config.usesDriver()) {
val rc = injector.getInstance(RemoteController::class.java)
rc.startANewSearch()
}
Expand All @@ -924,6 +984,7 @@ class Main {
EMConfig.ProblemType.RPC -> getAlgorithmKeyRPC(config)
EMConfig.ProblemType.WEBFRONTEND -> getAlgorithmKeyWeb(config)
EMConfig.ProblemType.MCP -> getAlgorithmKeyMcp(config)
EMConfig.ProblemType.ASYNCAPI -> getAlgorithmKeyAsyncApi(config)
else -> throw IllegalStateException("Unrecognized problem type ${config.problemType}")
}

Expand Down Expand Up @@ -977,7 +1038,7 @@ class Main {

val config = injector.getInstance(EMConfig::class.java)

if (config.blackBox && !config.bbExperiments) {
if (!config.usesDriver()) {
return null
}

Expand Down
4 changes: 2 additions & 2 deletions core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ class Statistics : SearchListener {

fun getData(solution: Solution<*>): List<Pair> {

val sutInfo : SutInfoDto? = if(!config.blackBox || config.bbExperiments) {
val sutInfo : SutInfoDto? = if(config.usesDriver()) {
remoteController?.getSutInfo()
} else {
null
Expand Down Expand Up @@ -852,7 +852,7 @@ class Statistics : SearchListener {


// append boot-time targets
if(!config.blackBox || config.bbExperiments) {
if(config.usesDriver()) {
remoteController?.getSutInfo()?.bootTimeInfoDto?.targets?.map { it.descriptiveId }?.sorted()?.apply {
if (isNotEmpty()){
content.add(System.lineSeparator())
Expand Down
22 changes: 22 additions & 0 deletions core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -762,4 +762,26 @@ internal class EMConfigTest{
assertThrows (Exception::class.java,{config.updateProperties(optionAllSpecifiedNotRest)})

}

@Test
fun testAsyncApiNeedsTheDriverEvenAsABlackBox(){

val parser = EMConfig.getOptionParser()

val rest = EMConfig()
rest.updateProperties(parser.parse("--$blackBox", "true", "--problemType", "REST"))
assertFalse(rest.usesDriver())

/*
There is no universal wire to a message-driven service, so the driver holds the
connection to the broker even when the service itself is a black box.
*/
val asyncApi = EMConfig()
asyncApi.updateProperties(parser.parse("--$blackBox", "true", "--problemType", "ASYNCAPI"))
assertTrue(asyncApi.usesDriver())

val whiteBox = EMConfig()
whiteBox.updateProperties(parser.parse("--$blackBox", "false"))
assertTrue(whiteBox.usesDriver())
}
}
2 changes: 1 addition & 1 deletion docs/options.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ There are 3 types of options:
|`probRestDefault`| __Double__. In REST, specify probability of using 'default' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.05`.|
|`probRestExamples`| __Double__. In REST, specify probability of using 'example(s)' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.|
|`probUseRestLinks`| __Double__. In REST, enable the supports of 'links' between resources defined in the OpenAPI schema, if any. When sampling a test case, if the last call has links, given this probability new calls are added for the link. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
|`problemType`| __Enum__. The type of SUT we want to generate tests for, e.g., a RESTful API. If left to DEFAULT, the type will be inferred from the EM Driver. However, in case of ambiguities (e.g., the driver specifies more than one type), then this field must be set with a specific type. This is also the case for Black-Box testing where there is no EM Driver. In this latter case, the system defaults to handle REST APIs. *Valid values*: `DEFAULT, REST, GRAPHQL`. *Experimental values*: `RPC, WEBFRONTEND, MCP`. *Default value*: `DEFAULT`.|
|`problemType`| __Enum__. The type of SUT we want to generate tests for, e.g., a RESTful API. If left to DEFAULT, the type will be inferred from the EM Driver. However, in case of ambiguities (e.g., the driver specifies more than one type), then this field must be set with a specific type. This is also the case for Black-Box testing where there is no EM Driver. In this latter case, the system defaults to handle REST APIs. *Valid values*: `DEFAULT, REST, GRAPHQL`. *Experimental values*: `RPC, WEBFRONTEND, MCP, ASYNCAPI`. *Default value*: `DEFAULT`.|
|`processFiles`| __String__. Specify a folder to save results when a search monitor is enabled. *DEBUG option*. *Default value*: `process_data`.|
|`processFormat`| __Enum__. Specify a format to save the process data. *DEBUG option*. *Valid values*: `JSON_ALL, TEST_IND, TARGET_TEST_IND, TARGET_HEURISTIC`. *Default value*: `JSON_ALL`.|
|`processInterval`| __Double__. Specify how often to save results when a search monitor is enabled, and 0.0 presents to record all evaluated individual. *DEBUG option*. *Constraints*: `min=0.0, max=50.0`. *Default value*: `0.0`.|
Expand Down
Loading