diff --git a/java-agent/src/jvmMain/kotlin/com/epam/drill/agent/test/prioritization/RecommendedTestsReceiver.kt b/java-agent/src/jvmMain/kotlin/com/epam/drill/agent/test/prioritization/RecommendedTestsReceiver.kt index 3aaa84f0a..b6ab3a97f 100644 --- a/java-agent/src/jvmMain/kotlin/com/epam/drill/agent/test/prioritization/RecommendedTestsReceiver.kt +++ b/java-agent/src/jvmMain/kotlin/com/epam/drill/agent/test/prioritization/RecommendedTestsReceiver.kt @@ -98,10 +98,12 @@ class RecommendedTestsReceiverImpl( logger.debug { "Loading tests to skip from file: $filePath" } return runCatching { val content = File(filePath).readText() - val entries = fileJson.decodeFromString(ListSerializer(TestDefinitionResponse.serializer()), content) - entries.map { it.toTestMethodInfo() }.also { - logger.info { "Loaded ${it.size} tests to skip from file: $filePath" } - } + fileJson.decodeFromString(ListSerializer(TestDefinitionResponse.serializer()), content) + .filter { it.impactStatus == "NOT_IMPACTED" } + .map { it.toTestMethodInfo() } + .also { + logger.info { "Loaded ${it.size} tests to skip from file: $filePath" } + } }.onFailure { logger.warn { "Unable to load tests to skip from file '$filePath'. Error message: $it" } }.getOrElse { diff --git a/lib-jvm-shared/test2code-common/src/commonMain/kotlin/com/epam/drill/agent/test2code/common/transport/BuildFinalizePayload.kt b/lib-jvm-shared/test2code-common/src/commonMain/kotlin/com/epam/drill/agent/test2code/common/transport/BuildFinalizePayload.kt new file mode 100644 index 000000000..ab766c86f --- /dev/null +++ b/lib-jvm-shared/test2code-common/src/commonMain/kotlin/com/epam/drill/agent/test2code/common/transport/BuildFinalizePayload.kt @@ -0,0 +1,36 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.agent.test2code.common.transport + +import kotlinx.serialization.Serializable +import com.epam.drill.agent.common.transport.AgentMessage + +/** + * Finalizes a build after all its method metadata has been sent. + * + * @param methodsCount the total number of methods sent for the build + * @param methodsChecksum the build checksum combined from the checksums of all methods of the build + */ +@Serializable +data class BuildFinalizePayload( + val groupId: String, + val appId: String, + val commitSha: String? = null, + val buildVersion: String? = null, + val instanceId: String? = null, + val methodsCount: Int, + val methodsChecksum: String, +): AgentMessage() diff --git a/test2code/src/main/kotlin/com/epam/drill/agent/test2code/Test2Code.kt b/test2code/src/main/kotlin/com/epam/drill/agent/test2code/Test2Code.kt index 4a9a9f621..6d2f9e148 100644 --- a/test2code/src/main/kotlin/com/epam/drill/agent/test2code/Test2Code.kt +++ b/test2code/src/main/kotlin/com/epam/drill/agent/test2code/Test2Code.kt @@ -32,9 +32,11 @@ import com.epam.drill.agent.configuration.AgentParametersValidator import com.epam.drill.agent.configuration.CapabilityParameterDefinitions.CLASS_SCANNING_ENABLED import com.epam.drill.agent.configuration.CapabilityParameterDefinitions.COVERAGE_COLLECTION_ENABLED import com.epam.drill.agent.test2code.common.api.AstMethod +import com.epam.drill.agent.test2code.common.transport.BuildFinalizePayload import com.epam.drill.agent.test2code.common.transport.ClassMetadata import com.epam.drill.agent.test2code.classloading.ClassLoadersScanner import com.epam.drill.agent.test2code.classloading.ClassScanner +import com.epam.drill.agent.test2code.classparsing.CumulativeChecksumCalculator import com.epam.drill.agent.test2code.classparsing.parseAstClass import com.epam.drill.agent.test2code.configuration.Test2CodeParameterDefinitions import com.epam.drill.agent.common.lifecycle.AgentShutdownRegistry @@ -153,7 +155,7 @@ class Test2Code( } var classCount = 0 var totalMethodsCount = 0 - var filteredMethodsCount = 0 + val checksumCalculator = CumulativeChecksumCalculator() val excludeMethodsByAnnotationPackage = configuration.parameters[Test2CodeParameterDefinitions.EXCLUDE_METHODS_BY_ANNOTATION_PACKAGE] as List @@ -171,17 +173,20 @@ class Test2Code( excludeMethodsByAnnotationPackage.any { key.contains(it) } } ?: false } - .onEach { filteredMethodsCount++ } + .onEach(checksumCalculator::add) .chunkedLazy(configuration.parameters[Test2CodeParameterDefinitions.METHODS_SEND_PAGE_SIZE]) .forEach(::sendClassMetadata) } - logger.info { """Scanned $classCount classes with $filteredMethodsCount methods - | total methods: ${totalMethodsCount + filteredMethodsCount}; - | methods excluded by annotations: ${totalMethodsCount - filteredMethodsCount} + sendBuildFinalize(checksumCalculator) + logger.info { """Scanned $classCount classes with ${checksumCalculator.methodsCount} methods + | methods checksum: ${checksumCalculator.methodsChecksum} + | total methods: ${totalMethodsCount}; + | methods excluded by annotations: ${totalMethodsCount - checksumCalculator.methodsCount} | packages for annotations-based exclusion are specified in ${Test2CodeParameterDefinitions.EXCLUDE_METHODS_BY_ANNOTATION_PACKAGE.name} parameter """.trimMargin() } } private val classMetadataDestination = AgentMessageDestination("PUT", "methods") + private val buildFinalizeDestination = AgentMessageDestination("PUT", "builds/finalize") private fun sendClassMetadata(methods: List) { val message = ClassMetadata( @@ -195,6 +200,20 @@ class Test2Code( logger.debug { "sendClassMetadata: Sending methods: $message" } sender.send(classMetadataDestination, message, ClassMetadata.serializer()) } + + private fun sendBuildFinalize(buildChecksumCalculator: CumulativeChecksumCalculator) { + val message = BuildFinalizePayload( + groupId = configuration.agentMetadata.groupId, + appId = configuration.agentMetadata.appId, + commitSha = configuration.agentMetadata.commitSha, + buildVersion = configuration.agentMetadata.buildVersion, + instanceId = configuration.agentMetadata.instanceId, + methodsChecksum = buildChecksumCalculator.methodsChecksum, + methodsCount = buildChecksumCalculator.methodsCount + ) + logger.debug { "sendBuildFinalize: Finalizing build: $message" } + sender.send(buildFinalizeDestination, message, BuildFinalizePayload.serializer()) + } } private fun Sequence.chunkedLazy(size: Int): Sequence> = sequence { diff --git a/test2code/src/main/kotlin/com/epam/drill/agent/test2code/classparsing/Checksum.kt b/test2code/src/main/kotlin/com/epam/drill/agent/test2code/classparsing/Checksum.kt index e182fc9f8..0d8ddd79d 100644 --- a/test2code/src/main/kotlin/com/epam/drill/agent/test2code/classparsing/Checksum.kt +++ b/test2code/src/main/kotlin/com/epam/drill/agent/test2code/classparsing/Checksum.kt @@ -16,6 +16,7 @@ package com.epam.drill.agent.test2code.classparsing +import com.epam.drill.agent.test2code.common.api.AstMethod import mu.KotlinLogging import org.apache.bcel.classfile.ClassParser import org.apache.bcel.classfile.Method @@ -51,4 +52,40 @@ private fun calculateChecksum( logger.error { "Failed to calculate method checksum. Class: $className. Method: ${method.name}. Opcode: ${ex.opcode}. Error: ${ex.error}. Stacktrace: ${ex.stackTraceToString()}" } return "" } +} + +const val CHECKSUM_RADIX = 36 +class InvalidChecksumException(checksum: String) : Exception("Invalid checksum value: $checksum") + +/** + * Incrementally combines the CRC64 checksums of all methods of a build into a single build checksum + * by summing them modulo 2^64 (i.e. relying on natural `Long` overflow), then re-encoding the result the same way. + */ +class CumulativeChecksumCalculator { + + private var sum = 0L + private var count = 0 + + /** + * The combined build checksum, encoded as a signed base-36 string. + */ + val methodsChecksum: String + get() = sum.toString(CHECKSUM_RADIX) + + /** + * The total number of methods added to the build. + */ + val methodsCount: Int + get() = count + + /** + * Add a single method to the build checksum. + * + * @throws InvalidChecksumException if the checksum is not blank and cannot be parsed as base-36. + */ + fun add(method: AstMethod) { + count++ + if (method.bodyChecksum.isBlank()) return + sum += method.bodyChecksum.toLongOrNull(CHECKSUM_RADIX) ?: throw InvalidChecksumException(method.bodyChecksum) + } } \ No newline at end of file diff --git a/test2code/src/test/kotlin/com/epam/drill/agent/test2code/classparsing/BuildChecksumCalculatorTest.kt b/test2code/src/test/kotlin/com/epam/drill/agent/test2code/classparsing/BuildChecksumCalculatorTest.kt new file mode 100644 index 000000000..1391886b8 --- /dev/null +++ b/test2code/src/test/kotlin/com/epam/drill/agent/test2code/classparsing/BuildChecksumCalculatorTest.kt @@ -0,0 +1,101 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.agent.test2code.classparsing + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import com.epam.drill.agent.test2code.common.api.AstMethod + +class BuildChecksumCalculatorTest { + + private fun expectedChecksum(vararg checksums: String): String = + checksums.fold(0L) { acc, c -> acc + c.toLong(CHECKSUM_RADIX) }.toString(CHECKSUM_RADIX) + + private fun String.astMethod() = AstMethod( + classname = "Test", + name = "test", + params = "", + returnType = "void", + probesCount = 0, + probesStartPos = 0, + bodyChecksum = this + ) + + @Test + fun `should produce base-36 checksum equal to the sum of parsed values`() { + val a = 123456789L.toString(CHECKSUM_RADIX) + val b = (-987654321L).toString(CHECKSUM_RADIX) + val calculator = CumulativeChecksumCalculator() + + calculator.add(a.astMethod()) + calculator.add(b.astMethod()) + + assertEquals(expectedChecksum(a, b), calculator.methodsChecksum) + assertEquals(2, calculator.methodsCount) + } + + @Test + fun `should be order independent`() { + val checksums = listOf(1L, 42L, -7L, Long.MAX_VALUE, Long.MIN_VALUE) + .map { it.toString(CHECKSUM_RADIX) } + .map { it.astMethod() } + + val forward = CumulativeChecksumCalculator().apply { checksums.forEach(::add) } + val backward = CumulativeChecksumCalculator().apply { checksums.reversed().forEach(::add) } + + assertEquals(forward.methodsChecksum, backward.methodsChecksum) + } + + @Test + fun `should skip blank checksums from the sum but count them`() { + val a = 555L.toString(CHECKSUM_RADIX) + val calculator = CumulativeChecksumCalculator() + + calculator.add(a.astMethod()) + calculator.add("".astMethod()) + calculator.add(" ".astMethod()) + + assertEquals(expectedChecksum(a), calculator.methodsChecksum) + assertEquals(3, calculator.methodsCount) + } + + @Test + fun `should wrap on overflow like data-ingest fold`() { + val max = Long.MAX_VALUE.toString(CHECKSUM_RADIX) + val one = 1L.toString(CHECKSUM_RADIX) + val calculator = CumulativeChecksumCalculator() + + calculator.add(max.astMethod()) + calculator.add(one.astMethod()) + + assertEquals(Long.MIN_VALUE.toString(CHECKSUM_RADIX), calculator.methodsChecksum) + } + + @Test + fun `should throw InvalidChecksumException on non-base-36 checksum`() { + val calculator = CumulativeChecksumCalculator() + assertFailsWith { calculator.add("not-a-checksum!".astMethod()) } + } + + @Test + fun `empty calculator should have zero checksum and no methods`() { + val calculator = CumulativeChecksumCalculator() + assertEquals(0L.toString(CHECKSUM_RADIX), calculator.methodsChecksum) + assertEquals(0, calculator.methodsCount) + } + +}