diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 262efca03b..9e12eeae3d 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -839,6 +839,16 @@ class EMConfig { "'extractDynamoDbExecutionInfo'") } + if (shouldGenerateCassandraData() && !heuristicsForCassandra) { + throw ConfigProblemException("Cannot generate Cassandra data if you did not enable " + + "collecting heuristics with 'heuristicsForCassandra'") + } + + if (shouldGenerateCassandraData() && !extractCassandraExecutionInfo) { + throw ConfigProblemException("Cannot generate Cassandra data if you did not enable " + + "extracting Cassandra execution info with 'extractCassandraExecutionInfo'") + } + if (enableTrackEvaluatedIndividual && enableTrackIndividual) { throw ConfigProblemException("When tracking EvaluatedIndividual, it is not necessary to track individual") } @@ -1088,6 +1098,8 @@ class EMConfig { fun shouldGenerateNeo4jData() = isUsingAdvancedTechniques() && generateNeo4jData + fun shouldGenerateCassandraData() = isUsingAdvancedTechniques() && generateCassandraData + fun dtoSupportedForPayload() = dtoForRequestPayload && couldSupportDtoForPayload() fun couldSupportDtoForPayload() = problemType == ProblemType.REST && outputFormat.isJavaOrKotlin() @@ -2096,6 +2108,12 @@ class EMConfig { @DependsOnFalseFor("blackBox") var heuristicsForNeo4j = false + @Experimental + @Cfg("Tracking of Cassandra commands to improve test generation") + @DependsOnFalseFor("blackBox") + @DependsOnTrueFor("extractCassandraExecutionInfo") + var heuristicsForCassandra = false + @Cfg("Enable extracting SQL execution info") @DependsOnFalseFor("blackBox") var extractSqlExecutionInfo = true @@ -2119,6 +2137,11 @@ class EMConfig { @DependsOnFalseFor("blackBox") var extractNeo4jExecutionInfo = false + @Experimental + @Cfg("Enable extracting Cassandra execution info") + @DependsOnFalseFor("blackBox") + var extractCassandraExecutionInfo = false + @Experimental @Cfg("Enable EvoMaster to generate SQL data with direct accesses to the database. Use the Z3 SMT solver") @DependsOnFalseFor("blackBox") @@ -2190,6 +2213,12 @@ class EMConfig { @DependsOnTrueFor("extractNeo4jExecutionInfo") var generateNeo4jData = false + @Experimental + @Cfg("Enable EvoMaster to generate Cassandra data with direct accesses to the database") + @DependsOnFalseFor("blackBox") + @DependsOnTrueFor("extractCassandraExecutionInfo") + var generateCassandraData = false + @Cfg("When generating SQL data, how many new rows (max) to generate for each specific SQL Select") @Min(1.0) @DependsOnFalseFor("blackBox") diff --git a/core/src/main/kotlin/org/evomaster/core/database/cassandra/CassandraExecution.kt b/core/src/main/kotlin/org/evomaster/core/database/cassandra/CassandraExecution.kt new file mode 100644 index 0000000000..6050f2ee2e --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/database/cassandra/CassandraExecution.kt @@ -0,0 +1,23 @@ +package org.evomaster.core.database.cassandra + +import org.evomaster.client.java.controller.api.dto.database.execution.CassandraExecutionsDto +import org.evomaster.client.java.controller.api.dto.database.execution.CassandraFailedQuery + +/** + * Encapsulates every CQL query that matched no row during an execution, as a data-generation hint. + */ +class CassandraExecution( + /** + * One entry per CQL query that matched no row, + * each with the table it targeted and the shape of its rows. + */ + val failedQueries: MutableList? +) { + + companion object { + + fun fromDto(dto: CassandraExecutionsDto?): CassandraExecution { + return CassandraExecution(dto?.failedQueries?.toMutableList()) + } + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt index 85f8bb7341..9f25c7a543 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt @@ -1,13 +1,24 @@ package org.evomaster.core.problem.api.service import com.google.inject.Inject -import org.evomaster.client.java.controller.api.dto.database.execution.DynamoDbFailedQuery -import org.evomaster.client.java.controller.api.dto.database.execution.MongoFailedQuery -import org.evomaster.client.java.controller.api.dto.database.execution.RedisFailedCommand +import org.evomaster.client.java.controller.api.dto.database.execution.* import org.evomaster.client.java.instrumentation.shared.ExternalServiceSharedUtils import org.evomaster.core.EMConfig import org.evomaster.core.Lazy +import org.evomaster.core.database.cassandra.CassandraDbAction +import org.evomaster.core.database.cassandra.CassandraInsertBuilder +import org.evomaster.core.database.dynamodb.DynamoDbAction +import org.evomaster.core.database.dynamodb.DynamoDbInsertBuilder import org.evomaster.core.database.mongo.MongoDbAction +import org.evomaster.core.database.neo4j.Neo4jDbAction +import org.evomaster.core.database.neo4j.Neo4jInsertBuilder +import org.evomaster.core.database.redis.RedisDbAction +import org.evomaster.core.database.redis.RedisInsertBuilder +import org.evomaster.core.database.sql.SqlAction +import org.evomaster.core.database.sql.SqlActionUtils +import org.evomaster.core.database.sql.SqlInsertBuilder +import org.evomaster.core.database.sql.schema.TableId +import org.evomaster.core.database.sql.solver.service.SMTLibZ3DbConstraintSolver import org.evomaster.core.problem.api.ApiWsIndividual import org.evomaster.core.problem.enterprise.EnterpriseActionGroup import org.evomaster.core.problem.externalservice.HostnameResolutionAction @@ -15,13 +26,6 @@ import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceActi import org.evomaster.core.problem.externalservice.httpws.param.HttpWsResponseParam import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHttpWsResponseHandler import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler -import org.evomaster.core.database.redis.RedisDbAction -import org.evomaster.core.database.redis.RedisInsertBuilder -import org.evomaster.core.database.dynamodb.DynamoDbAction -import org.evomaster.core.database.dynamodb.DynamoDbInsertBuilder -import org.evomaster.core.database.neo4j.Neo4jDbAction -import org.evomaster.core.database.neo4j.Neo4jInsertBuilder -import org.evomaster.client.java.controller.api.dto.database.execution.Neo4jFailedQueryDto import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.GroupsOfChildren import org.evomaster.core.search.Individual @@ -31,11 +35,6 @@ import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.impact.impactinfocollection.ImpactsOfIndividual import org.evomaster.core.search.service.mutator.MutatedGeneSpecification import org.evomaster.core.search.service.mutator.StructureMutator -import org.evomaster.core.database.sql.solver.service.SMTLibZ3DbConstraintSolver -import org.evomaster.core.database.sql.SqlAction -import org.evomaster.core.database.sql.SqlActionUtils -import org.evomaster.core.database.sql.SqlInsertBuilder -import org.evomaster.core.database.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import kotlin.math.max @@ -203,6 +202,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { addInitializingRedisDbActions(individual, mutatedGenes, sampler) addInitializingDynamoDbActions(individual, mutatedGenes, sampler) addInitializingNeo4jDbActions(individual, mutatedGenes, sampler) + addInitializingCassandraDbActions(individual, mutatedGenes, sampler) addInitializingHostnameResolutionActions(individual, mutatedGenes, sampler) // TODO if we handle schedule actions with structure mutator } @@ -345,6 +345,41 @@ abstract class ApiWsStructureMutator : StructureMutator() { } } + private fun addInitializingCassandraDbActions( + individual: EvaluatedIndividual<*>, + mutatedGenes: MutatedGeneSpecification?, + sampler: ApiWsSampler + ) { + if (!config.shouldGenerateCassandraData()) { + return + } + + val ind = individual.individual as? T + ?: throw IllegalArgumentException("Invalid individual type") + + val failedQueries = individual.fitness.getViewOfAggregatedFailedCassandraQueries() + + if (failedQueries.isEmpty()) { + return + } + + val oldCassandraDbActions = mutableListOf().plus(ind.seeInitializingActions()) + + val addedCassandraDbInsertions = handleFailedCql(ind, failedQueries, mutatedGenes, sampler) + + ind.repairInitializationActions(randomness) + // update impact based on added genes + if (mutatedGenes != null && config.isEnabledArchiveGeneSelection()) { + individual.updateImpactGeneDueToAddedInitializationGenes( + mutatedGenes, + oldCassandraDbActions, + addedCassandraDbInsertions, + ImpactsOfIndividual.CASSANDRADB_ACTION_KEY, + config + ) + } + } + private fun addInitializingHostnameResolutionActions( individual: EvaluatedIndividual<*>, mutatedGenes: MutatedGeneSpecification?, @@ -660,6 +695,28 @@ abstract class ApiWsStructureMutator : StructureMutator() { return addedActions } + private fun handleFailedCql( + ind: T, + failedQueries: List, + mutatedGenes: MutatedGeneSpecification?, + sampler: ApiWsSampler + ): MutableList>? { + + val builder = CassandraInsertBuilder() + val addedCassandraDbInsertions = if (mutatedGenes != null) mutableListOf>() else null + + failedQueries + .mapNotNull { it.tableSchema } + .filter { builder.canBuildInsertionFor(it) } + .forEach { + val insertion = listOf(sampler.sampleCassandraInsertion(it)) + ind.addInitializingCassandraDbActions(actions = insertion) + addedCassandraDbInsertions?.add(insertion) + } + + return addedCassandraDbInsertions + } + private fun findMissing( fw: Map>, dbactions: List diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt index 0183d97873..1dfa0eb4c5 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt @@ -8,6 +8,7 @@ import org.evomaster.core.StaticCounter import org.evomaster.core.database.cassandra.CassandraDbAction import org.evomaster.core.database.cassandra.CassandraDbActionResult import org.evomaster.core.database.cassandra.CassandraDbActionTransformer +import org.evomaster.core.database.cassandra.CassandraExecution import org.evomaster.core.database.mongo.MongoDbAction import org.evomaster.core.database.mongo.MongoDbActionResult import org.evomaster.core.database.mongo.MongoDbActionTransformer @@ -537,6 +538,18 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual } fv.aggregateNeo4jDatabaseData() } + + if (configuration.heuristicsForCassandra) { + handleCassandraHeuristics(dto, fv) + } + + if (configuration.extractCassandraExecutionInfo) { + for (i in 0 until dto.extraHeuristics.size) { + val extra = dto.extraHeuristics[i] + fv.setCassandraExecution(i, CassandraExecution.fromDto(extra.cassandraExecutionsDto)) + } + fv.aggregateCassandraDatabaseData() + } } private fun handleSqlHeuristics( @@ -646,6 +659,40 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual } } + /** Applies CQL WHERE distances and records their evaluation metrics. */ + private fun handleCassandraHeuristics(dto: TestResultsDto, fv: FitnessValue) { + for (i in 0 until dto.extraHeuristics.size) { + + val extra = dto.extraHeuristics[i] + + extraHeuristicsLogger.writeHeuristics(extra.heuristics, i) + + val toMinimize = extra.heuristics + .filter { + it != null + && it.objective == ExtraHeuristicEntryDto.Objective.MINIMIZE_TO_ZERO + && it.type == ExtraHeuristicEntryDto.Type.CASSANDRA + }.map { it.value } + .toList() + + if (toMinimize.isNotEmpty()) { + fv.addExtraObjectivesToMinimize(i, toMinimize) + } + + extra.heuristics + .filterNotNull().forEach { + if (it.type == ExtraHeuristicEntryDto.Type.CASSANDRA) { + statistics.reportNumberOfEvaluatedRowsForCassandraHeuristic(it.numberOfEvaluatedRecords) + if (it.extraHeuristicEvaluationFailure) { + statistics.reportCassandraHeuristicEvaluationFailure() + } else { + statistics.reportCassandraHeuristicEvaluationSuccess() + } + } + } + } + } + /** Applies DynamoDB predicate distances and records their evaluation metrics. */ private fun handleDynamoDbHeuristics(dto: TestResultsDto, fv: FitnessValue) { for (i in 0 until dto.extraHeuristics.size) { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt index 61bd4adc81..f48744beb9 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt @@ -4,6 +4,9 @@ import com.google.inject.Inject import org.evomaster.client.java.controller.api.dto.SutInfoDto import org.evomaster.client.java.controller.api.dto.problem.param.DerivedParamChangeReqDto import org.evomaster.client.java.controller.api.dto.problem.param.RestDerivedParamDto +import org.evomaster.client.java.controller.api.dto.database.cassandra.CassandraTableSchemaDto +import org.evomaster.core.database.cassandra.CassandraDbAction +import org.evomaster.core.database.cassandra.CassandraInsertBuilder import org.evomaster.core.database.mongo.MongoDbAction import org.evomaster.core.database.mongo.MongoInsertBuilder import org.evomaster.core.output.OutputFormat @@ -133,6 +136,12 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { return action } + fun sampleCassandraInsertion(tableSchema: CassandraTableSchemaDto): CassandraDbAction { + val action = CassandraInsertBuilder().createCassandraInsertionAction(tableSchema) + action.seeTopGenes().forEach{it.doInitialize(randomness)} + return action + } + fun canInsertInto(tableName: TableId) : Boolean { //TODO might need to refactor/remove once we deal with VIEWs return sqlInsertBuilder?.isTable(tableName) ?: false diff --git a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt index 89ab6483d8..49f6347a4a 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt @@ -2,23 +2,21 @@ package org.evomaster.core.search import com.webfuzzing.commons.faults.DefinedFaultCategory import org.evomaster.client.java.controller.api.dto.BootTimeInfoDto -import org.evomaster.client.java.controller.api.dto.database.execution.DynamoDbFailedQuery -import org.evomaster.client.java.controller.api.dto.database.execution.MongoFailedQuery -import org.evomaster.client.java.controller.api.dto.database.execution.RedisFailedCommand +import org.evomaster.client.java.controller.api.dto.database.execution.* import org.evomaster.core.EMConfig -import org.evomaster.core.database.dynamodb.DynamoDbExecution -import org.evomaster.core.database.sql.DatabaseExecution import org.evomaster.core.EMConfig.SecondaryObjectiveStrategy.* +import org.evomaster.core.database.cassandra.CassandraExecution +import org.evomaster.core.database.dynamodb.DynamoDbExecution import org.evomaster.core.database.mongo.MongoExecution +import org.evomaster.core.database.neo4j.Neo4jExecution +import org.evomaster.core.database.redis.RedisExecution +import org.evomaster.core.database.sql.DatabaseExecution +import org.evomaster.core.database.sql.schema.TableId import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory -import org.evomaster.core.problem.externalservice.httpws.HttpWsExternalService import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceRequest -import org.evomaster.core.database.redis.RedisExecution -import org.evomaster.core.database.neo4j.Neo4jExecution -import org.evomaster.client.java.controller.api.dto.database.execution.Neo4jFailedQueryDto +import org.evomaster.core.problem.externalservice.httpws.HttpWsExternalService import org.evomaster.core.search.service.IdMapper import org.evomaster.core.search.service.mutator.EvaluatedMutation -import org.evomaster.core.database.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import kotlin.math.max @@ -106,6 +104,13 @@ class FitnessValue( */ val neo4jExecutions: MutableMap = mutableMapOf() + /** + * Key -> the index of a main action of the test, ie of the HTTP call that made the queries + * + * Value -> the CQL queries executed by the SUT while serving that call which matched no row + */ + val cassandraExecutions: MutableMap = mutableMapOf() + /** * When SUT does SQL commands using WHERE, keep track of when those "fails" (ie evaluate * to false), in particular, the tables and columns in them involved @@ -138,6 +143,12 @@ class FitnessValue( */ private val aggregatedFailedNeo4jQueries: MutableList = mutableListOf() + /** + * When SUT executes CQL commands with a WHERE, keep track of when those match no row, in + * particular, the table involved and the shape of its rows + */ + private val aggregatedFailedCassandraQueries: MutableList = mutableListOf() + /** * To keep track of accessed external services prevent from adding them again * TODO: This is not completed, not need to consider for review for now @@ -172,11 +183,13 @@ class FitnessValue( copy.redisExecutions.putAll(this.redisExecutions) copy.dynamoDbExecutions.putAll(this.dynamoDbExecutions) copy.neo4jExecutions.putAll(this.neo4jExecutions) + copy.cassandraExecutions.putAll(this.cassandraExecutions) copy.aggregateDatabaseData() copy.aggregateMongoDatabaseData() copy.aggregateRedisDatabaseData() copy.aggregateDynamoDbData() copy.aggregateNeo4jDatabaseData() + copy.aggregateCassandraDatabaseData() copy.executionTimeMs = executionTimeMs copy.accessedExternalServiceRequests.putAll(this.accessedExternalServiceRequests) copy.accessedDefaultWM.putAll(this.accessedDefaultWM.toMap()) @@ -234,6 +247,11 @@ class FitnessValue( neo4jExecutions.values.forEach { aggregatedFailedNeo4jQueries.addAll(it.failedQueries) } } + fun aggregateCassandraDatabaseData(){ + aggregatedFailedCassandraQueries.clear() + cassandraExecutions.values.map { it.failedQueries?.let { it1 -> aggregatedFailedCassandraQueries.addAll(it1) } } + } + fun addExtraObjectivesToMinimize(actionIndex: Int, list: List) { if (extraToMinimize[actionIndex] == null) { extraToMinimize[actionIndex] = list.sorted() @@ -263,6 +281,10 @@ class FitnessValue( neo4jExecutions[actionIndex] = neo4jExecution } + fun setCassandraExecution(actionIndex: Int, cassandraExecution: CassandraExecution){ + cassandraExecutions[actionIndex] = cassandraExecution + } + fun isAnyDatabaseExecutionInfo() = databaseExecutions.isNotEmpty() fun getViewOfData(): Map { @@ -281,6 +303,8 @@ class FitnessValue( fun getViewOfAggregatedFailedNeo4jQueries() = aggregatedFailedNeo4jQueries + fun getViewOfAggregatedFailedCassandraQueries() = aggregatedFailedCassandraQueries + fun doesCover(target: Int): Boolean { return targets[target]?.score == MAX_VALUE } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt index 52210cac54..d04ba725a8 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt @@ -156,6 +156,11 @@ class Statistics : SearchListener { private var neo4jHeuristicEvaluationFailureCount = 0 private val neo4jNodesAverageCalculator = IncrementalAverage() + // cassandra heuristic evaluation statistic + private var cassandraHeuristicEvaluationSuccessCount = 0 + private var cassandraHeuristicEvaluationFailureCount = 0 + private val cassandraRowsAverageCalculator = IncrementalAverage() + //how long time spent in choosing names for the generated test cases private var timeSpentChoosingTestNamesMs = 0L @@ -353,6 +358,10 @@ class Statistics : SearchListener { neo4jNodesAverageCalculator.addValue(numberOfEvaluatedNodes) } + fun reportNumberOfEvaluatedRowsForCassandraHeuristic(numberOfEvaluatedRows: Int) { + cassandraRowsAverageCalculator.addValue(numberOfEvaluatedRows) + } + fun reportSqlParsingFailures(numberOfParsingFailures: Int) { if (numberOfParsingFailures<0) { throw IllegalArgumentException("Invalid number of parsing failures: $numberOfParsingFailures") @@ -394,6 +403,14 @@ class Statistics : SearchListener { dynamoDbHeuristicEvaluationFailureCount++ } + fun reportCassandraHeuristicEvaluationSuccess() { + cassandraHeuristicEvaluationSuccessCount++ + } + + fun reportCassandraHeuristicEvaluationFailure() { + cassandraHeuristicEvaluationFailureCount++ + } + fun reportSqlZ3Sat(z3TimeMs: Long) { sqlZ3CacheMissCount++ sqlZ3SatCount++ @@ -530,6 +547,12 @@ class Statistics : SearchListener { /** Returns the average number of nodes inspected by Neo4j heuristics. */ fun averageNumberOfEvaluatedNodesForNeo4jHeuristics(): Double = neo4jNodesAverageCalculator.mean + /** Returns the total number of Cassandra heuristic evaluations. */ + fun getCassandraHeuristicsEvaluationCount(): Int = cassandraHeuristicEvaluationSuccessCount + cassandraHeuristicEvaluationFailureCount + + /** Returns the average number of rows inspected by DynamoDB heuristics. */ + fun averageNumberOfEvaluatedRowsForCassandraHeuristics(): Double = cassandraRowsAverageCalculator.mean + override fun newActionsEvaluated(n: Int) { if(!epc.isInSearch()){ @@ -712,6 +735,10 @@ class Statistics : SearchListener { add(Pair("averageNumberOfEvaluatedNodesForNeo4jHeuristics","${averageNumberOfEvaluatedNodesForNeo4jHeuristics()}")) add(Pair("neo4jHeuristicsEvaluationCount","${getNeo4jHeuristicsEvaluationCount()}")) + // statistics info for Cassandra Heuristics + add(Pair("averageNumberOfEvaluatedRowsForCassandraHeuristics","${averageNumberOfEvaluatedRowsForCassandraHeuristics()}")) + add(Pair("cassandraHeuristicsEvaluationCount","${getCassandraHeuristicsEvaluationCount()}")) + add(Pair("timeSpentChoosingTestNamesMs", "$timeSpentChoosingTestNamesMs")) for(phase in ExecutionPhaseController.Phase.entries){ diff --git a/core/src/test/kotlin/org/evomaster/core/database/cassandra/CassandraExecutionTest.kt b/core/src/test/kotlin/org/evomaster/core/database/cassandra/CassandraExecutionTest.kt new file mode 100644 index 0000000000..5e5ac3cae9 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/database/cassandra/CassandraExecutionTest.kt @@ -0,0 +1,55 @@ +package org.evomaster.core.database.cassandra + +import org.evomaster.client.java.controller.api.dto.database.cassandra.CassandraColumnDto +import org.evomaster.client.java.controller.api.dto.database.cassandra.CassandraTableSchemaDto +import org.evomaster.client.java.controller.api.dto.database.execution.CassandraExecutionsDto +import org.evomaster.client.java.controller.api.dto.database.execution.CassandraFailedQuery +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class CassandraExecutionTest { + + private fun tableSchema(table: String) = CassandraTableSchemaDto( + "ks", + table, + listOf(CassandraColumnDto("id", "int", true, false)) + ) + + private fun failedQuery(table: String, schema: CassandraTableSchemaDto?) = + CassandraFailedQuery("ks", table, schema) + + @Test + fun testNoDtoGivesNoFailedQueries() { + assertNull(CassandraExecution.fromDto(null).failedQueries) + } + + @Test + fun testFailedQueriesAreKeptWithTheirSchema() { + val dto = CassandraExecutionsDto() + dto.failedQueries = listOf( + failedQuery("person", tableSchema("person")), + failedQuery("event", tableSchema("event")) + ) + + val execution = CassandraExecution.fromDto(dto) + + assertEquals(2, execution.failedQueries!!.size) + assertEquals(listOf("person", "event"), execution.failedQueries!!.map { it.tableName }) + assertEquals("id", execution.failedQueries!![0].tableSchema!!.columns[0].name) + } + + /** + * The schema is null for a table whose shape was never captured, and the execution must still + * carry the query, as it is the aggregation that decides what to do with it. + */ + @Test + fun testFailedQueryWithNoSchemaIsKept() { + val dto = CassandraExecutionsDto() + dto.failedQueries = listOf(failedQuery("person", null)) + + val execution = CassandraExecution.fromDto(dto) + + assertEquals(1, execution.failedQueries!!.size) + assertNull(execution.failedQueries!![0].tableSchema) + } +} \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt b/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt index f670d3656f..bd9721654a 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/FitnessValueTest.kt @@ -5,6 +5,8 @@ import org.evomaster.client.java.controller.api.dto.BootTimeInfoDto import org.evomaster.client.java.controller.api.dto.TargetInfoDto import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming import org.evomaster.core.EMConfig +import org.evomaster.client.java.controller.api.dto.database.execution.CassandraFailedQuery +import org.evomaster.core.database.cassandra.CassandraExecution import org.evomaster.core.search.service.IdMapper import org.evomaster.core.database.sql.DatabaseExecution import org.evomaster.core.database.sql.SqlExecutionInfo @@ -231,6 +233,36 @@ class FitnessValueTest { assertTrue(queries.contains("SELECT * FROM foo WHERE id = 1")) } + private fun cassandraExecutionWith(vararg tables: String) = CassandraExecution( + tables.map { CassandraFailedQuery("ks", it, null) }.toMutableList() + ) + + @Test + fun testAggregatedFailedCassandraQueries() { + val fv = FitnessValue(1.0) + + fv.setCassandraExecution(0, cassandraExecutionWith("person")) + fv.setCassandraExecution(1, cassandraExecutionWith("event", "log")) + fv.aggregateCassandraDatabaseData() + + val queries = fv.getViewOfAggregatedFailedCassandraQueries() + assertEquals(3, queries.size) + assertEquals(setOf("person", "event", "log"), queries.map { it.tableName }.toSet()) + } + + @Test + fun testCopyKeepsCassandraExecutions() { + val fv = FitnessValue(1.0) + fv.setCassandraExecution(0, cassandraExecutionWith("person")) + fv.aggregateCassandraDatabaseData() + + val copy = fv.copy() + + assertEquals(1, copy.cassandraExecutions.size) + assertEquals(1, copy.getViewOfAggregatedFailedCassandraQueries().size) + assertEquals("person", copy.getViewOfAggregatedFailedCassandraQueries()[0].tableName) + } + @Test fun testAddExtraObjectivesToMinimize() { val fv = FitnessValue(1.0) diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt index 1d2f4c4209..c8a4c4ae0c 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt @@ -68,6 +68,20 @@ class StatisticsTest { assertEquals(20.0, statistics.averageNumberOfEvaluatedNodesForNeo4jHeuristics()) } + @Test + fun testCassandraHeuristicsAverage() { + val statistics = Statistics() + statistics.reportNumberOfEvaluatedRowsForCassandraHeuristic(2) + statistics.reportNumberOfEvaluatedRowsForCassandraHeuristic(4) + + statistics.reportCassandraHeuristicEvaluationSuccess() + statistics.reportCassandraHeuristicEvaluationSuccess() + statistics.reportCassandraHeuristicEvaluationFailure() + + assertEquals(3, statistics.getCassandraHeuristicsEvaluationCount()) + assertEquals((2 + 4).toDouble() / 2, statistics.averageNumberOfEvaluatedRowsForCassandraHeuristics()) + } + @Test fun testSqlZ3CacheAccountingInvariant() { val statistics = Statistics() diff --git a/docs/options.md b/docs/options.md index 0e7aea2ab2..6821c3df86 100644 --- a/docs/options.md +++ b/docs/options.md @@ -312,14 +312,17 @@ There are 3 types of options: |`externalRequestResponseSelectionStrategy`| __Enum__. Harvested external request response selection strategy. *Valid values*: `EXACT, CLOSEST_SAME_DOMAIN, CLOSEST_SAME_PATH, RANDOM`. *Default value*: `EXACT`.| |`externalServiceIP`| __String__. User provided external service IP. When EvoMaster mocks external services, mock server instances will run on local addresses starting from this provided address. Min value is 127.0.0.4. Lower values like 127.0.0.2 and 127.0.0.3 are reserved. *Constraints*: `regex (?!^0*127(\.0*0){2}\.0*[0123]$)^0*127(\.0*(25[0-5]\|2[0-4][0-9]\|1?[0-9]?[0-9])){3}$`. *Default value*: `127.0.0.4`.| |`externalServiceIPSelectionStrategy`| __Enum__. Specify a method to select the first external service spoof IP address. *Valid values*: `NONE, DEFAULT, USER, RANDOM`. *Default value*: `NONE`.| +|`extractCassandraExecutionInfo`| __Boolean__. Enable extracting Cassandra execution info. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`extractDynamoDbExecutionInfo`| __Boolean__. Enable extracting DynamoDB execution info. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`extractNeo4jExecutionInfo`| __Boolean__. Enable extracting Neo4j execution info. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`extractRedisExecutionInfo`| __Boolean__. Enable extracting Redis execution info. *Depends on*: `blackBox=false`. *Default value*: `false`.| +|`generateCassandraData`| __Boolean__. Enable EvoMaster to generate Cassandra data with direct accesses to the database. *Depends on*: `extractCassandraExecutionInfo=true,blackBox=false`. *Default value*: `false`.| |`generateDynamoDbData`| __Boolean__. Enable EvoMaster to generate DynamoDB data with direct database access. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`generateNeo4jData`| __Boolean__. Enable EvoMaster to generate Neo4j data with direct accesses to the database. *Depends on*: `extractNeo4jExecutionInfo=true,blackBox=false`. *Default value*: `false`.| |`generateRedisData`| __Boolean__. Enable EvoMaster to generate Redis data with direct accesses to the database. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`generateSqlDataWithZ3`| __Boolean__. Enable EvoMaster to generate SQL data with direct accesses to the database. Use the Z3 SMT solver. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`handleFlakiness`| __Boolean__. Specify whether to detect flakiness and handle the flakiness in assertions during post handling of fuzzing. Note that flakiness is now supported only for fuzzing REST APIs. *Default value*: `false`.| +|`heuristicsForCassandra`| __Boolean__. Tracking of Cassandra commands to improve test generation. *Depends on*: `extractCassandraExecutionInfo=true,blackBox=false`. *Default value*: `false`.| |`heuristicsForDynamoDb`| __Boolean__. Tracking of DynamoDB commands to improve test generation. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`heuristicsForNeo4j`| __Boolean__. Tracking of Neo4j commands to improve test generation. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`heuristicsForRedis`| __Boolean__. Tracking of Redis commands to improve test generation. *Depends on*: `blackBox=false`. *Default value*: `false`.|