Feature/build checksum epmdj 11199

java-agent/src/jvmMain/kotlin/com/epam/drill/agent/test/prioritization/RecommendedTestsReceiver.kt

@@ -98,10 +98,12 @@ class RecommendedTestsReceiverImpl(
         logger.debug { "Loading tests to skip from file: $filePath" }
         return runCatching {
             val content = File(filePath).readText()
-            val entries = fileJson.decodeFromString(ListSerializer(TestDefinitionResponse.serializer()), content)
-            entries.map { it.toTestMethodInfo() }.also {
-                logger.info { "Loaded ${it.size} tests to skip from file: $filePath" }
-            }
+            fileJson.decodeFromString(ListSerializer(TestDefinitionResponse.serializer()), content)
+                .filter { it.impactStatus == "NOT_IMPACTED" }
+                .map { it.toTestMethodInfo() }
+                .also {
+                    logger.info { "Loaded ${it.size} tests to skip from file: $filePath" }
+                }
         }.onFailure {
             logger.warn { "Unable to load tests to skip from file '$filePath'. Error message: $it" }
         }.getOrElse {

lib-jvm-shared/test2code-common/src/commonMain/kotlin/com/epam/drill/agent/test2code/common/transport/BuildFinalizePayload.kt

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 - 2022 EPAM Systems
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.epam.drill.agent.test2code.common.transport
+
+import kotlinx.serialization.Serializable
+import com.epam.drill.agent.common.transport.AgentMessage
+
+/**
+ * Finalizes a build after all its method metadata has been sent.
+ *
+ * @param methodsCount the total number of methods sent for the build
+ * @param methodsChecksum the build checksum combined from the checksums of all methods of the build
+ */
+@Serializable
+data class BuildFinalizePayload(
+    val groupId: String,
+    val appId: String,
+    val commitSha: String? = null,
+    val buildVersion: String? = null,
+    val instanceId: String? = null,
+    val methodsCount: Int,
+    val methodsChecksum: String,
+): AgentMessage()

test2code/src/main/kotlin/com/epam/drill/agent/test2code/Test2Code.kt

@@ -32,9 +32,11 @@ import com.epam.drill.agent.configuration.AgentParametersValidator
 import com.epam.drill.agent.configuration.CapabilityParameterDefinitions.CLASS_SCANNING_ENABLED
 import com.epam.drill.agent.configuration.CapabilityParameterDefinitions.COVERAGE_COLLECTION_ENABLED
 import com.epam.drill.agent.test2code.common.api.AstMethod
+import com.epam.drill.agent.test2code.common.transport.BuildFinalizePayload
 import com.epam.drill.agent.test2code.common.transport.ClassMetadata
 import com.epam.drill.agent.test2code.classloading.ClassLoadersScanner
 import com.epam.drill.agent.test2code.classloading.ClassScanner
+import com.epam.drill.agent.test2code.classparsing.CumulativeChecksumCalculator
 import com.epam.drill.agent.test2code.classparsing.parseAstClass
 import com.epam.drill.agent.test2code.configuration.Test2CodeParameterDefinitions
 import com.epam.drill.agent.common.lifecycle.AgentShutdownRegistry
@@ -153,7 +155,7 @@ class Test2Code(
         }
         var classCount = 0
         var totalMethodsCount = 0
-        var filteredMethodsCount = 0
+        val checksumCalculator = CumulativeChecksumCalculator()
 
         val excludeMethodsByAnnotationPackage =
             configuration.parameters[Test2CodeParameterDefinitions.EXCLUDE_METHODS_BY_ANNOTATION_PACKAGE]  as List<String>
@@ -171,17 +173,20 @@ class Test2Code(
                                 excludeMethodsByAnnotationPackage.any { key.contains(it) }
                             } ?: false
                 }
-                .onEach { filteredMethodsCount++ }
+                .onEach(checksumCalculator::add)
                 .chunkedLazy(configuration.parameters[Test2CodeParameterDefinitions.METHODS_SEND_PAGE_SIZE])
                 .forEach(::sendClassMetadata)
         }
-        logger.info { """Scanned $classCount classes with $filteredMethodsCount methods
-            | total methods: ${totalMethodsCount + filteredMethodsCount};
-            | methods excluded by annotations: ${totalMethodsCount - filteredMethodsCount}
+        sendBuildFinalize(checksumCalculator)
+        logger.info { """Scanned $classCount classes with ${checksumCalculator.methodsCount} methods
+            | methods checksum: ${checksumCalculator.methodsChecksum}
+            | total methods: ${totalMethodsCount};
+            | methods excluded by annotations: ${totalMethodsCount - checksumCalculator.methodsCount}
             | packages for annotations-based exclusion are specified in ${Test2CodeParameterDefinitions.EXCLUDE_METHODS_BY_ANNOTATION_PACKAGE.name} parameter """.trimMargin() }
     }
 
     private val classMetadataDestination = AgentMessageDestination("PUT", "methods")
+    private val buildFinalizeDestination = AgentMessageDestination("PUT", "builds/finalize")
 
     private fun sendClassMetadata(methods: List<AstMethod>) {
         val message = ClassMetadata(
@@ -195,6 +200,20 @@ class Test2Code(
         logger.debug { "sendClassMetadata: Sending methods: $message" }
         sender.send(classMetadataDestination, message, ClassMetadata.serializer())
     }
+
+    private fun sendBuildFinalize(buildChecksumCalculator: CumulativeChecksumCalculator) {
+        val message = BuildFinalizePayload(
+            groupId = configuration.agentMetadata.groupId,
+            appId = configuration.agentMetadata.appId,
+            commitSha = configuration.agentMetadata.commitSha,
+            buildVersion = configuration.agentMetadata.buildVersion,
+            instanceId = configuration.agentMetadata.instanceId,
+            methodsChecksum = buildChecksumCalculator.methodsChecksum,
+            methodsCount = buildChecksumCalculator.methodsCount
+        )
+        logger.debug { "sendBuildFinalize: Finalizing build: $message" }
+        sender.send(buildFinalizeDestination, message, BuildFinalizePayload.serializer())
+    }
 }
 
 private fun <T> Sequence<T>.chunkedLazy(size: Int): Sequence<List<T>> = sequence {

test2code/src/main/kotlin/com/epam/drill/agent/test2code/classparsing/Checksum.kt

@@ -16,6 +16,7 @@
 
 package com.epam.drill.agent.test2code.classparsing
 
+import com.epam.drill.agent.test2code.common.api.AstMethod
 import mu.KotlinLogging
 import org.apache.bcel.classfile.ClassParser
 import org.apache.bcel.classfile.Method
@@ -51,4 +52,40 @@ private fun calculateChecksum(
         logger.error { "Failed to calculate method checksum. Class: $className. Method: ${method.name}. Opcode: ${ex.opcode}. Error: ${ex.error}. Stacktrace: ${ex.stackTraceToString()}" }
         return ""
     }
+}
+
+const val CHECKSUM_RADIX = 36
+class InvalidChecksumException(checksum: String) : Exception("Invalid checksum value: $checksum")
+
+/**
+ * Incrementally combines the CRC64 checksums of all methods of a build into a single build checksum
+ * by summing them modulo 2^64 (i.e. relying on natural `Long` overflow), then re-encoding the result the same way.
+ */
+class CumulativeChecksumCalculator {
+
+    private var sum = 0L
+    private var count = 0
+
+    /**
+     * The combined build checksum, encoded as a signed base-36 string.
+     */
+    val methodsChecksum: String
+        get() = sum.toString(CHECKSUM_RADIX)
+
+    /**
+     * The total number of methods added to the build.
+     */
+    val methodsCount: Int
+        get() = count
+
+    /**
+     * Add a single method to the build checksum.
+     *
+     * @throws InvalidChecksumException if the checksum is not blank and cannot be parsed as base-36.
+     */
+    fun add(method: AstMethod) {
+        count++
+        if (method.bodyChecksum.isBlank()) return
+        sum += method.bodyChecksum.toLongOrNull(CHECKSUM_RADIX) ?: throw InvalidChecksumException(method.bodyChecksum)
+    }
 }

test2code/src/test/kotlin/com/epam/drill/agent/test2code/classparsing/BuildChecksumCalculatorTest.kt

@@ -0,0 +1,101 @@
+/**
+ * Copyright 2020 - 2022 EPAM Systems
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.epam.drill.agent.test2code.classparsing
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+import com.epam.drill.agent.test2code.common.api.AstMethod
+
+class BuildChecksumCalculatorTest {
+
+    private fun expectedChecksum(vararg checksums: String): String =
+        checksums.fold(0L) { acc, c -> acc + c.toLong(CHECKSUM_RADIX) }.toString(CHECKSUM_RADIX)
+
+    private fun String.astMethod() = AstMethod(
+        classname = "Test",
+        name = "test",
+        params = "",
+        returnType = "void",
+        probesCount = 0,
+        probesStartPos = 0,
+        bodyChecksum = this
+    )
+
+    @Test
+    fun `should produce base-36 checksum equal to the sum of parsed values`() {
+        val a = 123456789L.toString(CHECKSUM_RADIX)
+        val b = (-987654321L).toString(CHECKSUM_RADIX)
+        val calculator = CumulativeChecksumCalculator()
+
+        calculator.add(a.astMethod())
+        calculator.add(b.astMethod())
+
+        assertEquals(expectedChecksum(a, b), calculator.methodsChecksum)
+        assertEquals(2, calculator.methodsCount)
+    }
+
+    @Test
+    fun `should be order independent`() {
+        val checksums = listOf(1L, 42L, -7L, Long.MAX_VALUE, Long.MIN_VALUE)
+            .map { it.toString(CHECKSUM_RADIX) }
+            .map { it.astMethod() }
+
+        val forward = CumulativeChecksumCalculator().apply { checksums.forEach(::add) }
+        val backward = CumulativeChecksumCalculator().apply { checksums.reversed().forEach(::add) }
+
+        assertEquals(forward.methodsChecksum, backward.methodsChecksum)
+    }
+
+    @Test
+    fun `should skip blank checksums from the sum but count them`() {
+        val a = 555L.toString(CHECKSUM_RADIX)
+        val calculator = CumulativeChecksumCalculator()
+
+        calculator.add(a.astMethod())
+        calculator.add("".astMethod())
+        calculator.add("   ".astMethod())
+
+        assertEquals(expectedChecksum(a), calculator.methodsChecksum)
+        assertEquals(3, calculator.methodsCount)
+    }
+
+    @Test
+    fun `should wrap on overflow like data-ingest fold`() {
+        val max = Long.MAX_VALUE.toString(CHECKSUM_RADIX)
+        val one = 1L.toString(CHECKSUM_RADIX)
+        val calculator = CumulativeChecksumCalculator()
+
+        calculator.add(max.astMethod())
+        calculator.add(one.astMethod())
+
+        assertEquals(Long.MIN_VALUE.toString(CHECKSUM_RADIX), calculator.methodsChecksum)
+    }
+
+    @Test
+    fun `should throw InvalidChecksumException on non-base-36 checksum`() {
+        val calculator = CumulativeChecksumCalculator()
+        assertFailsWith<InvalidChecksumException> { calculator.add("not-a-checksum!".astMethod()) }
+    }
+
+    @Test
+    fun `empty calculator should have zero checksum and no methods`() {
+        val calculator = CumulativeChecksumCalculator()
+        assertEquals(0L.toString(CHECKSUM_RADIX), calculator.methodsChecksum)
+        assertEquals(0, calculator.methodsCount)
+    }
+
+}