diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index ea8a23d204c..8c7493c8485 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -240,6 +240,54 @@ jobs: /root/.m2/repository key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-lint + # Compiles main and test sources with -Pstrict-warnings, which promotes scalac warnings to + # errors, so a change that reintroduces a cleared warning (an Int widened into a Long metric, + # a discarded builder result) fails here rather than accumulating. Spark 3.5 only: the profile + # passes on the Scala 2.12 profiles, and the Scala 2.13 remainder is tracked in + # https://github.com/apache/datafusion-comet/issues/5893. + strict-scala-warnings: + needs: lint + name: Strict Scala warnings (Spark 3.5, JDK 17) + if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} + runs-on: ubuntu-24.04 + container: + image: amd64/rust + steps: + - uses: actions/checkout@v7 + + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ env.RUST_VERSION }} + jdk-version: 17 + + - name: Restore Maven dependencies + id: maven-cache + uses: actions/cache/restore@v6 + with: + path: | + ~/.m2/repository + /root/.m2/repository + key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-strict-warnings + restore-keys: | + ${{ runner.os }}-java-maven- + + - name: Compile with strict warnings + run: ./mvnw -B test-compile -Pspark-3.5 -Pstrict-warnings -DskipTests + + # Saved only on main: an entry written from a pull request or a + # `gh-readonly-queue/*` branch cannot be restored by any later run, and + # it evicts main's from the shared budget. See "Large caches are written + # on main only" in README.md. + - name: Save Maven dependencies + if: ${{ github.ref == 'refs/heads/main' && steps.maven-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v6 + with: + path: | + ~/.m2/repository + /root/.m2/repository + key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-strict-warnings + # Compile-only verification for Spark 4.1. Tests are intentionally skipped: the spark-4.1 # profile is currently a build target only, and several runtime/test failures are tracked # in follow-up PRs. Excluded from lint-java because semanticdb-scalac_2.13.17 is not yet diff --git a/docs/source/contributor-guide/ci.md b/docs/source/contributor-guide/ci.md index fa367de798a..06fc265ac93 100644 --- a/docs/source/contributor-guide/ci.md +++ b/docs/source/contributor-guide/ci.md @@ -410,7 +410,7 @@ actionlint --shellcheck=off ``` A new test suite has to be registered in the workflow files by hand; see -[Register New Test Suites in CI](development.md#5-register-new-test-suites-in-ci). A new job in +[Register New Test Suites in CI](development.md#6-register-new-test-suites-in-ci). A new job in `ci.yml` needs an entry in both `FILTERS` and `POLICY` in `dev/ci/compute-changes.py`, a case in `dev/ci/check-ci-config.py`, and a line in `required_checks.needs`. The check will tell you which of those is missing. diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index e817baddaa0..008af4132d9 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -613,7 +613,17 @@ cargo clippy --color=never --all-targets --workspace -- -D warnings Make sure to resolve any Clippy warnings before submitting your pull request, as the CI/CD pipeline will fail if warnings are present. -### 4. Run Tests +### 4. Compile With Strict Scala Warnings (Recommended) + +The `Strict Scala warnings` job runs on every pull request and in the merge queue. It compiles the main and test sources with scalac warnings promoted to errors, so anything it reports fails the build — an `Int` widened into a `Long` metric, or a discarded builder result, for example. Reproduce it locally with: + +```sh +./mvnw test-compile -Pspark-3.5 -Pstrict-warnings -DskipTests +``` + +Use the Spark 3.5 profile: it is the one the job runs, and the default build profile will not reproduce it. The default is Spark 4.1 on Scala 2.13, where `-Pstrict-warnings` still fails on warnings unrelated to your change (tracked in [#5893](https://github.com/apache/datafusion-comet/issues/5893)), and where the compiler reports a different set — an adapted argument list, for instance, is flagged under Scala 2.12 but not under 2.13. + +### 5. Run Tests Run the relevant tests for your changes: @@ -628,7 +638,7 @@ make test-rust make test-jvm ``` -### 5. Register New Test Suites in CI +### 6. Register New Test Suites in CI Comet's CI does not automatically discover test suites. Instead, test suites are explicitly listed in the GitHub Actions workflow files so they can be grouped by category and run as separate parallel diff --git a/pom.xml b/pom.xml index fd991f055fd..020bd898d73 100644 --- a/pom.xml +++ b/pom.xml @@ -856,29 +856,84 @@ under the License. + - strict-warnings - - - - net.alchim31.maven - scala-maven-plugin - - - -deprecation - -unchecked - -feature - -Xlint:_ - -Ywarn-dead-code - -Ywarn-numeric-widen - -Ywarn-value-discard - -Ywarn-unused:imports,patvars,privates,locals,params,-implicits - -Xfatal-warnings - - - - - + strict-warnings + + + + net.alchim31.maven + scala-maven-plugin + + + scala-compile-first + + + -deprecation + -unchecked + -feature + -Xlint:_ + -Ywarn-dead-code + -Ywarn-numeric-widen + -Ywarn-value-discard + -Ywarn-unused:imports,patvars,privates,locals,-implicits + -Xfatal-warnings + + + + + scala-test-compile-first + + + -deprecation + -unchecked + -feature + -Xlint:_ + -Ywarn-dead-code + -Ywarn-numeric-widen + -Ywarn-unused:imports,patvars,privates,locals,-implicits + -Xfatal-warnings + + + + + + + diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala index 5b70fbaaf24..4dc28957d22 100644 --- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala +++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala @@ -96,7 +96,7 @@ class CometExecIterator( private val nativeLib = new Native() private val nativeUtil = new NativeUtil() private val taskAttemptId = TaskContext.get().taskAttemptId() - private val taskCPUs = TaskContext.get().cpus() + private val taskCPUs = TaskContext.get().cpus().toLong private val cometTaskMemoryManager = new CometTaskMemoryManager(id, taskAttemptId) private val plan = { @@ -411,7 +411,7 @@ object CometExecIterator extends Logging { if (intervalMs > 0) { val nativeLib = new Native() val limitBytes = nativeMemoryLimit(conf) - Executors + val _ = Executors .newSingleThreadScheduledExecutor(new ThreadFactory { override def newThread(runnable: Runnable): Thread = { val thread = new Thread(runnable, "comet-memory-usage-log") diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index c1cb43b153b..d4498d71436 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -444,13 +444,13 @@ object IcebergReflection extends Logging { * `taskGroups()`, so for staged scans we flatten the groups instead. Both methods are protected * and require reflection. */ - def getTasks(scan: Any): Option[java.util.List[_]] = + def getTasks(scan: Any): Option[java.util.List[AnyRef]] = if (isStagedScan(scan)) tasksFromTaskGroups(scan) else tasksFromTasksAccessor(scan) - private def tasksFromTasksAccessor(scan: Any): Option[java.util.List[_]] = + private def tasksFromTasksAccessor(scan: Any): Option[java.util.List[AnyRef]] = findMethodInHierarchy(scan.getClass, "tasks") match { case Some(method) => - Some(method.invoke(scan).asInstanceOf[java.util.List[_]]) + Some(method.invoke(scan).asInstanceOf[java.util.List[AnyRef]]) case None => logError( "Iceberg reflection failure: Failed to get tasks from SparkScan: " + @@ -458,7 +458,7 @@ object IcebergReflection extends Logging { None } - private def tasksFromTaskGroups(scan: Any): Option[java.util.List[_]] = + private def tasksFromTaskGroups(scan: Any): Option[java.util.List[AnyRef]] = findMethodInHierarchy(scan.getClass, "taskGroups") match { case Some(method) => try { @@ -473,7 +473,7 @@ object IcebergReflection extends Logging { groups.forEach { group => val groupTasks = groupTasksMethod.invoke(group).asInstanceOf[java.util.Collection[_ <: AnyRef]] - flat.addAll(groupTasks) + val _ = flat.addAll(groupTasks) } Some(flat) } @@ -1705,13 +1705,19 @@ object IcebergReflection extends Logging { private def newDataManifestFile(inputFile: AnyRef, specId: Int): AnyRef = { val inputFileClass = loadClass(ClassNames.INPUT_FILE) val cls = loadClass(ClassNames.GENERIC_MANIFEST_FILE) - val (ctor, args): (java.lang.reflect.Constructor[_], Array[Object]) = + // `Constructor[AnyRef]` rather than `Constructor[_]`: the two `try`/`catch` branches + // would otherwise infer a top-level existential, which `-Xlint:existential` rejects. + val (ctor, args): (java.lang.reflect.Constructor[AnyRef], Array[Object]) = try { - val c = cls.getDeclaredConstructor(inputFileClass, classOf[Int], classOf[Long]) + val c = cls + .getDeclaredConstructor(inputFileClass, classOf[Int], classOf[Long]) + .asInstanceOf[java.lang.reflect.Constructor[AnyRef]] (c, Array[Object](inputFile, Integer.valueOf(specId), java.lang.Long.valueOf(0L))) } catch { case _: NoSuchMethodException => - val c = cls.getDeclaredConstructor(inputFileClass, classOf[Int]) + val c = cls + .getDeclaredConstructor(inputFileClass, classOf[Int]) + .asInstanceOf[java.lang.reflect.Constructor[AnyRef]] (c, Array[Object](inputFile, Integer.valueOf(specId))) } ctor.setAccessible(true) @@ -1747,10 +1753,11 @@ object IcebergReflection extends Logging { } result } finally { - try reader.getClass.getMethod("close").invoke(reader) - catch { - case e: Exception => logWarning(s"Failed to close ManifestReader: ${e.getMessage}") - } + val _ = + try reader.getClass.getMethod("close").invoke(reader) + catch { + case e: Exception => logWarning(s"Failed to close ManifestReader: ${e.getMessage}") + } } } diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index 19c88a6a9d8..f11c97029e1 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -992,7 +992,7 @@ case class CometExecRule(session: SparkSession) .flatten .toSet if (reasons.nonEmpty) { - withFallbackReasons(op, reasons) + val _ = withFallbackReasons(op, reasons) } } @@ -1024,7 +1024,7 @@ case class CometExecRule(session: SparkSession) "operator or any of its expressions. Add a withFallbackReason call stating why " + s"conversion failed. Operator:\n$op") } - withFallbackReason(op, s"${op.nodeName} is not supported") + val _ = withFallbackReason(op, s"${op.nodeName} is not supported") } } @@ -1053,7 +1053,8 @@ case class CometExecRule(session: SparkSession) CometExplainInfo.collectExprTagValues(allExprs, CometExplainInfo.CODEGEN_DISPATCH_EXPRS) appendTagValues(exec, CometExplainInfo.CODEGEN_DISPATCH_EXPRS, routedNames) if (routedNames.nonEmpty && CometConf.COMET_EXPLAIN_CODEGEN_ENABLED.get()) { - withInfo(exec, s"JVM codegen dispatcher: ${routedNames.toSeq.sorted.mkString(", ")}") + val _ = + withInfo(exec, s"JVM codegen dispatcher: ${routedNames.toSeq.sorted.mkString(", ")}") } } diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index bb83297e636..bc6f67218d0 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -31,7 +31,7 @@ import scala.jdk.CollectionConverters._ import org.apache.hadoop.conf.Configuration import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.expressions.{Attribute, DynamicPruningExpression, Expression, GenericInternalRow, InputFileBlockLength, InputFileBlockStart, InputFileName, PlanExpression} +import org.apache.spark.sql.catalyst.expressions.{Attribute, DynamicPruningExpression, Expression, GenericInternalRow, InputFileBlockLength, InputFileBlockStart, InputFileName} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.{sideBySide, ArrayBasedMapData, GenericArrayData, MetadataColumnHelper} import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues @@ -1001,9 +1001,6 @@ case class CometScanRule(session: SparkSession) } } - private def isDynamicPruningFilter(e: Expression): Boolean = - e.exists(_.isInstanceOf[PlanExpression[_]]) - /** * Detects AQE DPP (SubqueryAdaptiveBroadcastExec), as opposed to non-AQE DPP. * diff --git a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala index 9249ab280f0..57d609c158b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala @@ -181,7 +181,7 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] { target.foreach { case _: AttributeReference | _: Literal => case node if !(node eq target) => - withCodegenDispatchExpr(expr, CometExplainInfo.exprDisplayName(node)) + val _ = withCodegenDispatchExpr(expr, CometExplainInfo.exprDisplayName(node)) case _ => } Some( diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 1d87294c20b..493743a5067 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -979,7 +979,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { e.getTagValue(CometExplainInfo.FALLBACK_REASONS).foreach(reasons ++= _) } if (reasons.nonEmpty) { - withFallbackReasons(to, reasons.toSet) + val _ = withFallbackReasons(to, reasons.toSet) } } diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index b368f80320c..98a3757ffc8 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -86,8 +86,8 @@ object CometLiteral extends CometExpressionSerde[Literal] with CometTypeShim wit exprBuilder.setIsNull(false) dataType match { case _: BooleanType => exprBuilder.setBoolVal(value.asInstanceOf[Boolean]) - case _: ByteType => exprBuilder.setByteVal(value.asInstanceOf[Byte]) - case _: ShortType => exprBuilder.setShortVal(value.asInstanceOf[Short]) + case _: ByteType => exprBuilder.setByteVal(value.asInstanceOf[Byte].toInt) + case _: ShortType => exprBuilder.setShortVal(value.asInstanceOf[Short].toInt) case _: IntegerType | _: DateType => exprBuilder.setIntVal(value.asInstanceOf[Int]) case _: LongType | _: TimestampType | _: TimestampNTZType | _: DayTimeIntervalType => exprBuilder.setLongVal(value.asInstanceOf[Long]) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala index 86790d118b7..689ed68389e 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala @@ -357,7 +357,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit val equalityIds = equalityIdsMethod .invoke(deleteFile) .asInstanceOf[java.util.List[Integer]] - equalityIds.forEach(id => deleteBuilder.addEqualityIds(id)) + deleteBuilder.addAllEqualityIds(equalityIds) } catch { case _: Exception => } @@ -519,7 +519,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit commonBuilder.addPartitionTypePool(partitionTypeJson) idx }) - taskBuilder.setPartitionSpecIdx(specIdx) + val _ = taskBuilder.setPartitionSpecIdx(specIdx) } catch { case e: Exception => logWarning(s"Failed to serialize partition spec to JSON: ${e.getMessage}") @@ -579,7 +579,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit commonBuilder.addPartitionDataPool(partitionDataProto) idx }) - taskBuilder.setPartitionDataIdx(partitionDataIdx) + val _ = taskBuilder.setPartitionDataIdx(partitionDataIdx) } else { // Defensive: ContentScanTask.partition() returns an empty struct (never null) for // unpartitioned tables in practice. If it is ever null we cannot compute values, so diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala index de8652ad90c..571d2e43ffb 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala @@ -68,7 +68,9 @@ object CometNativeScan extends CometOperatorSerde[CometScanExec] with CometTypeS } else { Some(Literal.create(value, field.dataType)) } - expression.flatMap(exprToProto(_, output)).map(_ -> java.lang.Long.valueOf(index)) + expression + .flatMap(exprToProto(_, output)) + .map(_ -> java.lang.Long.valueOf(index.toLong)) } .toSeq // Never drop a value independently of its index: that would shift every later default. diff --git a/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala b/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala index 37d67f55ca4..27ecec8f559 100644 --- a/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala +++ b/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala @@ -246,7 +246,7 @@ object CelebornShufflePusherFactory { client: AnyRef, taskContext: TaskContext, handle: ShuffleHandle): Unit = { - sparkUtilsClass + val _ = sparkUtilsClass .getMethod( "addFailureListenerIfBarrierTask", shuffleClientClass, @@ -336,7 +336,7 @@ object CelebornShufflePusherFactory { /** Remove task-independent state for one unregistered Celeborn generation. */ def cleanupShuffle(client: AnyRef, celebornShuffleId: Int): Unit = { try { - client.getClass + val _ = client.getClass .getMethod("cleanupShuffle", java.lang.Integer.TYPE) .invoke(client, Int.box(celebornShuffleId)) } catch { @@ -354,7 +354,8 @@ object CelebornShufflePusherFactory { try { val shuffleClientClass = ClassLoaders.loadClass(CELEBORN_SHUFFLE_CLIENT) try { - shuffleClientClass.getMethod("removeInstance", shuffleClientClass).invoke(null, client) + val _ = + shuffleClientClass.getMethod("removeInstance", shuffleClientClass).invoke(null, client) } catch { case _: NoSuchMethodException => // Celeborn 0.6 owns one shared application client and cannot remove a single instance. diff --git a/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala b/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala index ef541fb3e2d..1cab882df93 100644 --- a/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala +++ b/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala @@ -296,7 +296,7 @@ object CometScalaUDFCodegen { private[codegen] def recordCompiledSignature( specs: IndexedSeq[ArrowColumnSpec], outputType: DataType): Unit = { - compiledSignatures.add((specs.map(_.vectorClass), outputType)) + val _ = compiledSignatures.add((specs.map(_.vectorClass), outputType)) } /** diff --git a/spark/src/main/scala/org/apache/spark/CometSource.scala b/spark/src/main/scala/org/apache/spark/CometSource.scala index 95d75236166..6581cb2018a 100644 --- a/spark/src/main/scala/org/apache/spark/CometSource.scala +++ b/spark/src/main/scala/org/apache/spark/CometSource.scala @@ -54,9 +54,9 @@ object CometSource extends Source { }) def recordStats(stats: CometCoverageStats): Unit = { - NATIVE_OPERATORS.inc(stats.cometOperators) - SPARK_OPERATORS.inc(stats.sparkOperators) - TRANSITIONS.inc(stats.transitions) + NATIVE_OPERATORS.inc(stats.cometOperators.toLong) + SPARK_OPERATORS.inc(stats.sparkOperators.toLong) + TRANSITIONS.inc(stats.transitions.toLong) QUERIES_PLANNED.inc() } } diff --git a/spark/src/main/scala/org/apache/spark/Plugins.scala b/spark/src/main/scala/org/apache/spark/Plugins.scala index 65fb606edc2..c542bf0099e 100644 --- a/spark/src/main/scala/org/apache/spark/Plugins.scala +++ b/spark/src/main/scala/org/apache/spark/Plugins.scala @@ -235,13 +235,13 @@ object CometDriverPlugin extends Logging { val listeners = sc.conf.get(listenerKey, "") if (listeners.isEmpty) { logInfo(s"Setting $listenerKey=$listenerClass") - sc.conf.set(listenerKey, listenerClass) + val _ = sc.conf.set(listenerKey, listenerClass) } else { val currentListeners = listeners.split(",").map(_.trim) if (!currentListeners.contains(listenerClass)) { val newValue = s"$listeners,$listenerClass" logInfo(s"Setting $listenerKey=$newValue") - sc.conf.set(listenerKey, newValue) + val _ = sc.conf.set(listenerKey, newValue) } } } else { @@ -256,13 +256,13 @@ object CometDriverPlugin extends Logging { val extensions = conf.get(extensionKey, "") if (extensions.isEmpty) { logInfo(s"Setting $extensionKey=$extensionClass") - conf.set(extensionKey, extensionClass) + val _ = conf.set(extensionKey, extensionClass) } else { val currentExtensions = extensions.split(",").map(_.trim) if (!currentExtensions.contains(extensionClass)) { val newValue = s"$extensions,$extensionClass" logInfo(s"Setting $extensionKey=$newValue") - conf.set(extensionKey, newValue) + val _ = conf.set(extensionKey, newValue) } } } diff --git a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala index d3536850f68..887dae05bca 100644 --- a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala +++ b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala @@ -27,7 +27,7 @@ class RowPartition(initialSize: Int) { def addRow(addr: Long, size: Int): Unit = { rowAddresses += addr - rowSizes += size + val _ = rowSizes += size } def getNumRows: Int = if (rowAddresses == null) { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala index d900889e176..4f807dc1c79 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala @@ -76,7 +76,7 @@ case class CometBatchScanExec( override def next(): ColumnarBatch = { val batch = batches.next() - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong batch } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala index 9ae92845c9c..615f7bada26 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala @@ -107,7 +107,7 @@ case class CometBroadcastExchangeExec( private val timeout: Long = conf.broadcastTimeout @transient - private lazy val maxBroadcastRows = 512000000 + private lazy val maxBroadcastRows = 512000000L private def getByteArrayRdd(plan: SparkPlan): RDD[(Long, ChunkedByteBuffer)] = { plan.executeColumnar().mapPartitionsInternal { iter => @@ -195,7 +195,7 @@ case class CometBroadcastExchangeExec( override protected def doPrepare(): Unit = { // Materialize the future. - relationFuture + val _ = relationFuture } override protected def doExecute(): RDD[InternalRow] = { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala index 6d43fe156b2..4d560ef8d3f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala @@ -112,7 +112,7 @@ case class CometCollectLimitExec( outputPartitioning, serializer, metrics) - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) new CometShuffledBatchRDD(dep, readMetrics) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala index 2fe870ed069..da1d9ba296f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala @@ -80,7 +80,7 @@ case class CometColumnarToRowExec(child: SparkPlan) val toUnsafe = UnsafeProjection.create(localOutput, localOutput) batches.flatMap { batch => numInputBatches += 1 - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong batch.rowIterator().asScala.map(toUnsafe) } } @@ -111,14 +111,14 @@ case class CometColumnarToRowExec(child: SparkPlan) val numOutputRows = longMetric("numOutputRows") val numInputBatches = longMetric("numInputBatches") val localOutput = this.output - val broadcastColumnar = child.executeBroadcast() + val broadcastColumnar = child.executeBroadcast[Any]() val serializedBatches = broadcastColumnar.value.asInstanceOf[Array[ChunkedByteBuffer]] val toUnsafe = UnsafeProjection.create(localOutput, localOutput) val rows = serializedBatches.iterator .flatMap(CometUtils.decodeBatches(_, this.getClass.getSimpleName)) .flatMap { batch => numInputBatches += 1 - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong batch.rowIterator().asScala.map(toUnsafe) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala index 00a5047625f..c95c0d41e46 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala @@ -373,7 +373,8 @@ object CometIcebergWriteExec { /** Take ownership of the locations the native writer reported. */ def own(written: Seq[String]): Unit = locations = written - override def onTaskFailure(context: TaskContext, error: Throwable): Unit = - IcebergReflection.deleteFilesQuietly(io, locations, describeTask) + override def onTaskFailure(context: TaskContext, error: Throwable): Unit = { + val _ = IcebergReflection.deleteFilesQuietly(io, locations, describeTask) + } } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala index 0f6f531fdc3..103f95f2e53 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala @@ -123,7 +123,7 @@ case class CometInMemoryTableScanExec( serializer .convertCachedBatchToColumnarBatch(filteredBuffers, relationOutput, scanOutput, conf) .map { cb => - numOutputRows += cb.numRows() + numOutputRows += cb.numRows().toLong cb } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala index 8ac9a70de3e..8e6ff6c2ad3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala @@ -91,7 +91,7 @@ case class CometMapInBatchExec( def processPartition(batches: Iterator[ColumnarBatch]): Iterator[ColumnarBatch] = { val context = TaskContext.get() - val counting = batches.map { b => numInputRows += b.numRows(); b } + val counting = batches.map { b => numInputRows += b.numRows().toLong; b } val columnarBatchIter = computeArrowPython( resolvedRunnerInputs, @@ -113,7 +113,7 @@ case class CometMapInBatchExec( outputAttrs.indices.map(i => structVector.getChild(i)).toArray val flattenedBatch = new ColumnarBatch(outputVectors) flattenedBatch.setNumRows(batch.numRows()) - numOutputRows += flattenedBatch.numRows() + numOutputRows += flattenedBatch.numRows().toLong numOutputBatches += 1 flattenedBatch } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala index 806c9d00eda..5b974f150b5 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala @@ -110,7 +110,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM * be called in a TaskCompletionListener after the iterator is fully consumed. */ def reportScanInputMetrics(ctx: TaskContext): Unit = { - ctx.addTaskCompletionListener[Unit] { _ => + val _ = ctx.addTaskCompletionListener[Unit] { _ => val scanLeaves = leafNodes.filter(_.metrics.contains("bytes_scanned")) if (scanLeaves.nonEmpty) { val totalBytes = scanLeaves.map(_.metrics("bytes_scanned").value).sum @@ -137,7 +137,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM * before this listener runs. */ def reportNativeWriteOutputMetrics(ctx: TaskContext): Unit = { - ctx.addTaskCompletionListener[Unit] { _ => + val _ = ctx.addTaskCompletionListener[Unit] { _ => metrics.get("bytes_written").foreach { m => ctx.taskMetrics().outputMetrics.setBytesWritten(m.value) } @@ -162,7 +162,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM */ def reportSpillMetrics(ctx: TaskContext): Unit = { val seenMetrics = CometMetricNode.taskSeenSpillMetrics(ctx) - ctx.addTaskCompletionListener[Unit] { _ => + val _ = ctx.addTaskCompletionListener[Unit] { _ => val diskBytesSpilled = sumMetricValues("spilled_bytes", seenMetrics.disk) if (diskBytesSpilled > 0L) { ctx.taskMetrics().incDiskBytesSpilled(diskBytesSpilled) @@ -244,7 +244,9 @@ object CometMetricNode { // The task thread is the only registrant for its attempt id, so there is no put race. val created = SeenSpillMetrics(new IdentityHashMap(), new IdentityHashMap()) seenSpillMetricsByTask.put(attemptId, created) - ctx.addTaskCompletionListener[Unit](_ => seenSpillMetricsByTask.remove(attemptId)) + ctx.addTaskCompletionListener[Unit](_ => { + val _ = seenSpillMetricsByTask.remove(attemptId) + }) created } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala index ce800ddf098..0c3eea71f8b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala @@ -97,7 +97,7 @@ case class CometNativeColumnarToRowExec(child: SparkPlan) val numInputBatches = longMetric("numInputBatches") val localSchema = this.schema val batchSize = CometConf.COMET_BATCH_SIZE.get() - val broadcastColumnar = child.executeBroadcast() + val broadcastColumnar = child.executeBroadcast[Any]() val serializedBatches = broadcastColumnar.value.asInstanceOf[Array[org.apache.spark.util.io.ChunkedByteBuffer]] @@ -108,7 +108,7 @@ case class CometNativeColumnarToRowExec(child: SparkPlan) .flatMap(CometUtils.decodeBatches(_, this.getClass.getSimpleName)) .flatMap { batch => numInputBatches += 1 - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong val result = converter.convert(batch) // Wrap iterator to close batch after consumption new Iterator[InternalRow] { @@ -202,7 +202,7 @@ case class CometNativeColumnarToRowExec(child: SparkPlan) batches.flatMap { batch => numInputBatches += 1 val numRows = batch.numRows() - numOutputRows += numRows + numOutputRows += numRows.toLong val startTime = System.nanoTime() val result = converter.convert(batch) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala index f0d10b17667..ced525c1869 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala @@ -188,7 +188,7 @@ case class CometNativeWriteExec( val taskAttemptId = org.apache.spark.TaskContext.get().taskAttemptId() // Setup task-level commit protocol if provided - val (workDir, taskContext, commitMsg) = capturedCommitter + val (workDir, taskContext, _) = capturedCommitter .map { committer => val taskContext = createTaskContext(capturedJobTrackerID, partitionId, taskAttemptId.toInt) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala index f88e3d380e9..3e1beec716d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala @@ -228,7 +228,7 @@ case class CometScanExec( driverMetrics("staticFilesSize") = filesSize } if (relation.partitionSchema.nonEmpty) { - driverMetrics("numPartitions") = partitions.length + driverMetrics("numPartitions") = partitions.length.toLong } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala index a96069b7dc3..6f91248e787 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala @@ -105,7 +105,7 @@ case class CometSparkToColumnarExec(child: SparkPlan) CometArrowStream .countingIterator( sparkBatches, - (b: ColumnarBatch) => numInputRows.add(b.numRows())), + (b: ColumnarBatch) => numInputRows.add(b.numRows().toLong)), maxBatchInt, onConversionNs)) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala index 61254794489..419b99aef70 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala @@ -128,7 +128,7 @@ case class CometSubqueryBroadcastExec( val beforeBuild = System.nanoTime() longMetric("collectTime") += (beforeBuild - beforeCollect) / 1000000 - longMetric("numOutputRows") += rows.length + longMetric("numOutputRows") += rows.length.toLong val dataSize = rows.map(_.asInstanceOf[UnsafeRow].getSizeInBytes.toLong).sum longMetric("dataSize") += dataSize SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq) @@ -139,7 +139,7 @@ case class CometSubqueryBroadcastExec( } protected override def doPrepare(): Unit = { - relationFuture + val _ = relationFuture } protected override def doExecute(): RDD[InternalRow] = { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala index 09dd944d93e..4cb5047f7d7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala @@ -160,7 +160,7 @@ case class CometTakeOrderedAndProjectExec( outputPartitioning, serializer, metrics) - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) new CometShuffledBatchRDD(dep, readMetrics) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala index 04fdfd7155d..77005852e30 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala @@ -99,7 +99,7 @@ case class IcebergCommitExec( } throw failure } - longMetric("numCommittedMessages").add(messages.length) + longMetric("numCommittedMessages").add(messages.length.toLong) try { messages.foreach(batchWrite.onDataWriterCommit) @@ -141,7 +141,7 @@ case class IcebergCommitExec( .flatMap(IcebergReflection.getTableIO) io match { case Some(fileIO) => - IcebergReflection.deleteFilesQuietly( + val _ = IcebergReflection.deleteFilesQuietly( fileIO, locations, s"job abort, ${completed.length} completed task(s)") diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala index 3048456ea78..3592095a37c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala @@ -121,7 +121,7 @@ class CometBlockStoreShuffleReader[K, C]( // Update the context task metrics for each record read. val metricIter = CompletionIterator[(Any, Any), Iterator[(Any, Any)]]( recordIter.map { record => - readMetrics.incRecordsRead(record._2.numRows()) + readMetrics.incRecordsRead(record._2.numRows().toLong) record }, context.taskMetrics().mergeShuffleReadMetrics()) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala index c067bc6e49b..b4c2a258cf9 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala @@ -23,6 +23,7 @@ import java.lang.reflect.InvocationTargetException import java.util.Locale import java.util.concurrent.ConcurrentHashMap +import scala.annotation.nowarn import scala.collection.mutable import scala.concurrent.ExecutionContext import scala.jdk.CollectionConverters._ @@ -125,6 +126,7 @@ class CometCelebornShuffleManager private[shuffle] ( } } + @nowarn("msg=references private") override def getWriter[K, V]( handle: ShuffleHandle, mapId: Long, @@ -213,6 +215,7 @@ class CometCelebornShuffleManager private[shuffle] ( } // Spark's final all-mapper reader overload delegates to this mapper-range overload. + @nowarn("msg=references private") override def getReader[K, C]( handle: ShuffleHandle, startMapIndex: Int, @@ -257,9 +260,9 @@ class CometCelebornShuffleManager private[shuffle] ( endPartition), context, metrics, - client => ownedNativeClients.put(client, java.lang.Boolean.TRUE), + client => { val _ = ownedNativeClients.put(client, java.lang.Boolean.TRUE) }, (client, celebornShuffleId) => { - nativeShuffleClients + val _ = nativeShuffleClients .computeIfAbsent(handle.shuffleId, _ => new ConcurrentHashMap[Int, AnyRef]()) .put(celebornShuffleId, client) }, @@ -397,7 +400,7 @@ class CometCelebornShuffleManager private[shuffle] ( conf, handle, context, - client => ownedNativeClients.put(client, java.lang.Boolean.TRUE), + client => { val _ = ownedNativeClients.put(client, java.lang.Boolean.TRUE) }, onGenerationResolved, onGenerationInvalidated, onInvalidationUnsafe) @@ -416,7 +419,7 @@ class CometCelebornShuffleManager private[shuffle] ( } private[shuffle] def removeSizeLimitFallback(shuffleId: Int): Unit = { - sizeLimitFallbacks.remove(shuffleId) + val _ = sizeLimitFallbacks.remove(shuffleId) } private def isLocalNativeHandle(handle: ShuffleHandle): Boolean = handle match { @@ -1075,7 +1078,9 @@ private[shuffle] final class CelebornShuffleGenerationCoordinator( invalidatedGenerations.remove(shuffleId) generationEpochs.remove(shuffleId) claimOwners.retain { case ((ownerShuffleId, _, _, _), _) => ownerShuffleId != shuffleId } - deniedAttempts.retain { case ((ownerShuffleId, _, _, _), _) => ownerShuffleId != shuffleId } + val _ = deniedAttempts.retain { case ((ownerShuffleId, _, _, _), _) => + ownerShuffleId != shuffleId + } } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala index 891290a441e..b7deac55b49 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala @@ -143,7 +143,7 @@ private[shuffle] final class CometCelebornShuffleReader[K, C]( } val rows = decoder.map { batch: ColumnarBatch => - readMetrics.incRecordsRead(batch.numRows()) + readMetrics.incRecordsRead(batch.numRows().toLong) (0, batch) } val completed = CompletionIterator[(Int, ColumnarBatch), Iterator[(Int, ColumnarBatch)]]( @@ -511,7 +511,7 @@ private[shuffle] object CelebornRawPartitionReader { java.util.Objects.checkFromIndexSize(offset, length, buffer.length) if (length == 0) return 0 val readingHeader = bodyRemaining == 0 - val remaining = if (readingHeader) header.length - headerBytes else bodyRemaining + val remaining = if (readingHeader) (header.length - headerBytes).toLong else bodyRemaining val count = in.read(buffer, offset, math.min(length.toLong, remaining).toInt) if (count < 0) { if (headerBytes != 0 || bodyRemaining != 0) { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala index c97dc62744b..fcd85b6f6a1 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala @@ -69,7 +69,10 @@ class CometNativeShuffleWriter[K, V]( with Logging { var partitionLengths: Array[Long] = _ - var mapStatus: MapStatus = _ + // `private[shuffle]` rather than public: `MapStatus` is `private[spark]`, and a public + // var exposing it trips `-Xlint:inaccessible`. The shuffle package, which includes the + // tests that read it, is the only thing that ever touches this. + private[shuffle] var mapStatus: MapStatus = _ private var stopped = false private lazy val effectivePartitionCount = remoteDestination.map(_.numPartitions).getOrElse(outputPartitioning.numPartitions) @@ -113,7 +116,9 @@ class CometNativeShuffleWriter[K, V]( val resolver = SparkEnv.get.shuffleManager.shuffleBlockResolver.asInstanceOf[IndexShuffleBlockResolver] val dataFile = resolver.getDataFile(shuffleId, mapId) - Some(LocalShuffleOutput(resolver, dataFile.getPath.replace(".data", ".data.tmp"))) + Some( + CometNativeShuffleWriter + .LocalShuffleOutput(resolver, dataFile.getPath.replace(".data", ".data.tmp"))) } else { None } @@ -491,13 +496,18 @@ class CometNativeShuffleWriter[K, V]( } override def getPartitionLengths(): Array[Long] = partitionLengths +} + +private[shuffle] object CometNativeShuffleWriter { - private final case class LocalShuffleOutput( + /** + * Declared here rather than inside the class: as an inner case class every type test against it + * carries an outer reference that cannot be checked at run time, which `-Xlint` reports. + */ + private[shuffle] final case class LocalShuffleOutput( resolver: IndexShuffleBlockResolver, dataFile: String) -} -private[shuffle] object CometNativeShuffleWriter { private[shuffle] def isSizeLimitFailure(failure: Throwable): Boolean = { var cause = failure val visited = new java.util.IdentityHashMap[Throwable, java.lang.Boolean]() diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala index 3ad5527d8cd..7a0d231a141 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala @@ -234,7 +234,7 @@ case class CometShuffleExchangeExec( serializer, metrics) } - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) SQLMetrics.postDriverMetricUpdates( sparkContext, @@ -248,7 +248,7 @@ case class CometShuffleExchangeExec( outputPartitioning, serializer, metrics) - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) SQLMetrics.postDriverMetricUpdates( sparkContext, @@ -387,7 +387,7 @@ object CometShuffleExchangeExec * `native_shuffle.md` for why the starts must be decorrelated rather than merely distinct. */ def positionalStartPartition(mapPartitionId: Int, numPartitions: Int): Int = - new XORShiftRandom(mapPartitionId).nextInt(math.max(numPartitions, 1)) + 1 + new XORShiftRandom(mapPartitionId.toLong).nextInt(math.max(numPartitions, 1)) + 1 /** * Whether re-executing this subtree yields the same rows in the same order. @@ -1117,7 +1117,7 @@ object CometShuffleExchangeExec // end up being almost the same regardless of the index. substantially scrambling the // seed by hashing will help. Refer to SPARK-21782 for more details. val partitionId = TaskContext.get().partitionId() - var position = new XORShiftRandom(partitionId).nextInt(numPartitions) + var position = new XORShiftRandom(partitionId.toLong).nextInt(numPartitions) (_: InternalRow) => { // The HashPartitioner will handle the `mod` by the number of partitions position += 1 @@ -1164,7 +1164,7 @@ object CometShuffleExchangeExec row: InternalRow): UnsafeExternalRowSorter.PrefixComputer.Prefix = { // The hashcode generated from the binary form of a [[UnsafeRow]] should not be null. result.isNull = false - result.value = row.hashCode() + result.value = row.hashCode().toLong result } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala index 82a59707777..4cf42c5d312 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala @@ -21,6 +21,7 @@ package org.apache.spark.sql.comet.execution.shuffle import java.util.concurrent.ConcurrentHashMap +import scala.annotation.nowarn import scala.jdk.CollectionConverters._ import org.apache.spark.ShuffleDependency @@ -165,6 +166,7 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { } } + @nowarn("msg=references private") override def getReader[K, C]( handle: ShuffleHandle, startMapIndex: Int, @@ -217,6 +219,7 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { } /** Get a writer for a given partition. Called on executors by map tasks. */ + @nowarn("msg=references private") override def getWriter[K, V]( handle: ShuffleHandle, mapId: Long, diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index ed2d62a1a3a..580dd7a5023 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -137,7 +137,7 @@ private[comet] object PlanDataInjector extends Logging { * xxhash64 and runs at memory speed over the byte array. */ def planFingerprint(planBytes: Array[Byte]): Long = - XXH64.hashUnsafeBytes(planBytes, Platform.BYTE_ARRAY_OFFSET, planBytes.length, 42L) + XXH64.hashUnsafeBytes(planBytes, Platform.BYTE_ARRAY_OFFSET.toLong, planBytes.length, 42L) /** * A prepared common message together with the exact finalized bytes it was prepared from. @@ -290,8 +290,9 @@ private[comet] object PlanDataInjector extends Logging { // SparkContext in the JVM, so a recreated context would otherwise keep stacking new scan keys // under ids the last context already used. The shuffle managers call this from // unregisterShuffle. - private[comet] def releasePreparedShuffle(shuffleId: Int): Unit = - shufflePreparedCommons.remove(Integer.valueOf(shuffleId)) + private[comet] def releasePreparedShuffle(shuffleId: Int): Unit = { + val _ = shufflePreparedCommons.remove(Integer.valueOf(shuffleId)) + } // Both stores are JVM-wide statics that assume one active SparkContext per JVM, so the shuffle // managers drop them together from stop, before the next context can fill them. diff --git a/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala b/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala index 15c480b057c..57db09d5f38 100644 --- a/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala +++ b/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala @@ -37,7 +37,7 @@ object ShimCometUnionExec { def unionRDDs[T: ClassTag]( sc: SparkContext, rdds: Seq[RDD[T]], - @annotation.nowarn("cat=unused") outputPartitioning: Partitioning): RDD[T] = { + outputPartitioning: Partitioning): RDD[T] = { sc.union(rdds) } } diff --git a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala index 15c480b057c..57db09d5f38 100644 --- a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala +++ b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala @@ -37,7 +37,7 @@ object ShimCometUnionExec { def unionRDDs[T: ClassTag]( sc: SparkContext, rdds: Seq[RDD[T]], - @annotation.nowarn("cat=unused") outputPartitioning: Partitioning): RDD[T] = { + outputPartitioning: Partitioning): RDD[T] = { sc.union(rdds) } } diff --git a/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala index 1e5d1686f96..567999fc1c6 100644 --- a/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala +++ b/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala @@ -22,40 +22,37 @@ package org.apache.comet.shims import java.nio.ByteBuffer import java.nio.charset.{CharacterCodingException, CodingErrorAction, StandardCharsets} -import scala.annotation.nowarn - import org.apache.spark.sql.catalyst.expressions.aggregate.Mode import org.apache.spark.sql.types.{DataType, StructType} import org.apache.spark.unsafe.types.UTF8String trait CometTypeShim { - @nowarn // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. + // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. def isStringCollationType(dt: DataType): Boolean = false // `mode() WITHIN GROUP (ORDER BY ...)` and the deterministic-flag form (which set `reverseOpt`) // are Spark 4.0 features; Spark 3.x `Mode` is always the plain `mode(col)` form. - @nowarn def modeHasUnsupportedOrdering(expr: Mode): Boolean = false - @nowarn // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. + // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. def hasNonDefaultStringCollation(dt: DataType): Boolean = false - @nowarn // Spark 4 feature; collation does not exist in Spark 3.x. + // Spark 4 feature; collation does not exist in Spark 3.x. def hasCollationSupport: Boolean = false - @nowarn // Spark 4 feature; Variant shredding doesn't exist in Spark 3.x. + // Spark 4 feature; Variant shredding doesn't exist in Spark 3.x. def isVariantStruct(s: StructType): Boolean = false - @nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x. + // Spark 4 feature; VariantType doesn't exist in Spark 3.x. def isVariantType(dt: DataType): Boolean = false - @nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x. + // Spark 4 feature; VariantType doesn't exist in Spark 3.x. def containsVariantType(dt: DataType): Boolean = false - @nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x. + // Spark 4 feature; VariantType doesn't exist in Spark 3.x. def variantType: Option[DataType] = None - @nowarn // Spark 4.1 feature; TimeType doesn't exist in Spark 3.x. + // Spark 4.1 feature; TimeType doesn't exist in Spark 3.x. def isTimeType(dt: DataType): Boolean = false /** diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index f26ef595a64..078572feb42 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -198,7 +198,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { (3, 0.0f, 0.0d), (4, 1.0f, 1.0d)) val expected = rows.map { case (id, f, d) => - id -> (java.lang.Float.floatToRawIntBits(f), java.lang.Double.doubleToRawLongBits(d)) + id -> ((java.lang.Float.floatToRawIntBits(f), java.lang.Double.doubleToRawLongBits(d))) }.toMap rows.toDF("id", "f", "d").createOrReplaceTempView("strict_fp_bits") @@ -1569,7 +1569,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { def makeDecimalRDD(num: Int, decimal: DecimalType, useDictionary: Boolean): DataFrame = { val div = if (useDictionary) 5 else num // narrow the space to make it dictionary encoded spark - .range(num) + .range(num.toLong) .map(_ % div) // Parquet doesn't allow column names with spaces, have to add an alias here. // Minus 500 here so that negative decimals are also tested. @@ -2102,7 +2102,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { // must stay native; interval types are unsupported and fall back to Spark. withParquetTable( (1 to 5).map(i => - (i.toByte, i.toShort, i, i.toLong, i.toFloat, i.toDouble, BigDecimal(i * 3, 2))), + (i.toByte, i.toShort, i, i.toLong, i.toFloat, i.toDouble, BigDecimal((i * 3).toLong, 2))), "umt") { checkSparkAnswerAndOperator("SELECT -_1, -_2, -_3, -_4, -_5, -_6, -_7 FROM umt") } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index 256ed221355..cd0778134c4 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -6355,7 +6355,8 @@ class CometIcebergNativeSuite conflictingField.put("name", "id_as_region") conflictingField.put("transform", "identity") conflictingFields.add(conflictingField) - conflictingSpec.set("fields", conflictingFields) + conflictingSpec + .set[com.fasterxml.jackson.databind.node.ObjectNode]("fields", conflictingFields) specs.add(conflictingSpec) // Round-trip through Iceberg's own parser before writing, so a malformed hand-edit diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala index e7dd961150a..118b2371d53 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala @@ -483,7 +483,7 @@ class CometIcebergSystemFunctionSuite maybeNull(randomDecimal38()), maybeNull(randomString()), maybeNull(randomBinary()), - maybeNull(LocalDate.ofEpochDay(random.nextInt(40000) - 20000)), + maybeNull(LocalDate.ofEpochDay((random.nextInt(40000) - 20000).toLong)), maybeNull(instant(randomMicros())), maybeNull(localDateTime(randomMicros()))) } diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 3c9dbacd156..9909aeb6120 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -2706,28 +2706,7 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { values.map(v => Some(v)) ++ Seq(None) } - private def castFallbackTest( - input: DataFrame, - toType: DataType, - expectedMessage: String): Unit = { - withTempPath { dir => - val data = roundtripParquet(input, dir).coalesce(1) - data.createOrReplaceTempView("t") - - withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) { - val df = data.withColumn("converted", col("a").cast(toType)) - df.collect() - val str = - new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan) - assert(str.contains(expectedMessage)) - } - } - } - - private def castTimestampTest( - input: DataFrame, - toType: DataType, - assertNative: Boolean = false) = { + private def castTimestampTest(input: DataFrame, toType: DataType, assertNative: Boolean) = { withTempPath { dir => val data = roundtripParquet(input, dir).coalesce(1) data.createOrReplaceTempView("t") diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index cbe86fadf34..aededc434b4 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -3450,19 +3450,11 @@ class CometExecSuite extends CometTestBase { val df2 = (0 until 50).map(i => (i % 7, i % 11, i.toString)).toDF("i", "j", "k").as("df2") - val BucketedTableTestSpec( - bucketSpecLeft, - numPartitionsLeft, - shuffleLeft, - sortLeft, - numOutputPartitionsLeft) = bucketedTableTestSpecLeft - - val BucketedTableTestSpec( - bucketSpecRight, - numPartitionsRight, - shuffleRight, - sortRight, - numOutputPartitionsRight) = bucketedTableTestSpecRight + val BucketedTableTestSpec(bucketSpecLeft, numPartitionsLeft, _, _, _) = + bucketedTableTestSpecLeft + + val BucketedTableTestSpec(bucketSpecRight, numPartitionsRight, _, _, _) = + bucketedTableTestSpecRight withTable("bucketed_table1", "bucketed_table2") { withBucket(df1.repartition(numPartitionsLeft).write.format("parquet"), bucketSpecLeft) @@ -3650,7 +3642,7 @@ class CometExecSuite extends CometTestBase { withTable("t1") { val numRows = 10 spark - .range(numRows) + .range(numRows.toLong) .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b") .repartition(3) // Move data across multiple partitions .write @@ -3687,7 +3679,7 @@ class CometExecSuite extends CometTestBase { withTable("t1") { val numRows = 10 spark - .range(numRows) + .range(numRows.toLong) .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b") .repartition(3) // Force repartition to test data will come to single partition .write @@ -3718,7 +3710,7 @@ class CometExecSuite extends CometTestBase { withTable("t1") { val numRows = 10 spark - .range(numRows) + .range(numRows.toLong) .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b") .repartition(3) // Force repartition to test data will come to single partition .write diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala index b12ef02c5f7..05777a5db90 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala @@ -1472,7 +1472,7 @@ class CometInMemoryCacheSuite extends CometTestBase { spark.catalog.clearCache() spark - .range(0, projectionCacheRows, 1, 2) + .range(0, projectionCacheRows.toLong, 1, 2) .selectExpr(columns: _*) .createOrReplaceTempView(view) spark.catalog.cacheTable(view) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index 2163ab05e26..0083ab9222d 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -663,7 +663,7 @@ class CometJoinSuite extends CometTestBase { SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { withParquetTable( - (0 until 100).map(i => (Some(i), i.toLong)) :+ (None, -1L), + (0 until 100).map(i => (Some(i), i.toLong)) :+ ((None, -1L)), "dynamic_probe") { for (build <- Seq( Seq((Some(10), 1L), (None, 2L), (Some(10), 3L), (Some(90), 4L)), diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala index 800264bb141..818be0b1ecd 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala @@ -357,7 +357,7 @@ class CometNativeColumnarToRowSuite extends CometTestBase with AdaptiveSparkPlan if (isNull) null else s"string_$i", if (isNull) null else new Date(baseDate.getTime + i * 24 * 60 * 60 * 1000L), if (isNull) null else new Timestamp(baseTs.getTime + i * 1000L), - if (isNull) null else BigDecimal(i * 100 + i, 2).bigDecimal) + if (isNull) null else BigDecimal((i * 100 + i).toLong, 2).bigDecimal) } val schema = StructType( diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index 353ee66d4d1..66e5ccbce94 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -424,7 +424,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") makeParquetFileAllPrimitiveTypes(path, dictionaryEnabled = dictionaryEnabled, 1000) - var allTypes: Seq[Int] = (1 to 20) + val allTypes: Seq[Int] = (1 to 20) allTypes.map(i => s"_$i").foreach { c => withSQLConf("parquet.enable.dictionary" -> dictionaryEnabled.toString) { readParquetFile(path.toString) { df => diff --git a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala index 8e5a60d63b0..1cfd1d36b92 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala @@ -30,6 +30,14 @@ import org.apache.comet.CometConf.COMET_S3_COMPLIANT_SCHEMES_KEY class NativeConfigSuite extends AnyFunSuite with Matchers { + /** + * A Hadoop `Configuration` variable reference to `key`, as `Configuration#get` expands it. + * Built by concatenation: written as a literal, it reads to scalac as a string missing its `s` + * interpolator, and adding the `s` (escaping the dollar) is then flagged by scalafix as + * redundant. + */ + private def varRef(key: String): String = "${" + key + "}" + test("extractObjectStoreOptions - multiple cloud provider configurations") { val hadoopConf = new Configuration() // S3A configs @@ -108,13 +116,15 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { } } - test("extractObjectStoreOptions - forwards the substituted value of a ${...} reference") { + test( + "extractObjectStoreOptions - forwards the substituted value of a " + + s"${varRef("...")} reference") { // Hadoop's own consumers read values through Configuration#get, which expands a ${...} // reference against another conf entry. Forwarding the raw, unexpanded literal here would // give native a different credential than every Hadoop-side consumer sees. val hadoopConf = new Configuration() hadoopConf.set("my.custom.access.key", "expanded-access-key") - hadoopConf.set("fs.s3a.access.key", "${my.custom.access.key}") + hadoopConf.set("fs.s3a.access.key", varRef("my.custom.access.key")) val options = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/test-object")) @@ -122,19 +132,19 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { } test( - "extractObjectStoreOptions - a cyclic ${...} reference falls back to the raw value " + - "instead of throwing") { + s"extractObjectStoreOptions - a cyclic ${varRef("...")} reference falls back to the raw " + + "value instead of throwing") { // Configuration#get raises IllegalStateException once ${...} expansion recurses past // Hadoop's MAX_SUBST bound; a two-key mutual cycle triggers this on every call. Extraction // must still return a full options map rather than aborting for the whole object store. val hadoopConf = new Configuration() - hadoopConf.set("fs.s3a.access.key", "${fs.s3a.secret.key}") - hadoopConf.set("fs.s3a.secret.key", "${fs.s3a.access.key}") + hadoopConf.set("fs.s3a.access.key", varRef("fs.s3a.secret.key")) + hadoopConf.set("fs.s3a.secret.key", varRef("fs.s3a.access.key")) val options = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/test-object")) - assert(options("fs.s3a.access.key") == "${fs.s3a.secret.key}") - assert(options("fs.s3a.secret.key") == "${fs.s3a.access.key}") + assert(options("fs.s3a.access.key") == varRef("fs.s3a.secret.key")) + assert(options("fs.s3a.secret.key") == varRef("fs.s3a.access.key")) } test( diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala index c46524e2917..babe86b3c72 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala @@ -1752,5 +1752,5 @@ class RecordingStatsTracker extends WriteTaskStatsTracker { override def closeFile(filePath: String): Unit = {} override def newRow(filePath: String, row: InternalRow): Unit = rows += ((filePath, row)) override def getFinalStats(taskCommitTime: Long): WriteTaskStats = - BasicWriteTaskStats(Seq.empty, 0, 0, rows.size) + BasicWriteTaskStats(Seq.empty, 0, 0, rows.size.toLong) } diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 287d4ecb14a..979a976786b 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -736,8 +736,8 @@ abstract class ParquetReadSuite extends CometTestBase { opt match { case Some(i) => record.add(0, i % 2 == 0) - record.add(1, i.toByte) - record.add(2, i.toShort) + record.add(1, i.toByte.toInt) + record.add(2, i.toShort.toInt) record.add(3, i) record.add(4, i.toLong) record.add(5, i.toFloat) @@ -1153,7 +1153,7 @@ abstract class ParquetReadSuite extends CometTestBase { var b = record.addGroup("b") b.add("b1", 1) b.add("b2", 1) - var c = record.addGroup("c") + val c = record.addGroup("c") c.add("c1", 1) c.add("c2", 1) writer.write(record) @@ -1223,7 +1223,7 @@ abstract class ParquetReadSuite extends CometTestBase { var b = record.addGroup("b") b.add("b1", 1) b.add("b2", 1) - var c = record.addGroup("c") + val c = record.addGroup("c") c.add("c1", 1) c.add("c2", 1) writer.write(record) @@ -1900,7 +1900,7 @@ abstract class ParquetReadSuite extends CometTestBase { } private def withId(id: Int) = - new MetadataBuilder().putLong(ParquetUtils.FIELD_ID_METADATA_KEY, id).build() + new MetadataBuilder().putLong(ParquetUtils.FIELD_ID_METADATA_KEY, id.toLong).build() // Based on Spark ParquetIOSuite.test("vectorized reader: array of nested struct") test("array of nested struct with and without field id") { diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index 9a475ae2bfd..b9b4b09657f 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -1577,7 +1577,7 @@ class CometExecRuleSuite extends CometTestBase { s"expected one report containing '$marker', got:\n${reports.mkString("\n\n")}") assert( coverageOf(matching.head) == - (executed.cometOperators, executed.cometOperators + executed.sparkOperators), + ((executed.cometOperators, executed.cometOperators + executed.sparkOperators)), s"report disagrees with the executed plan ($executed):\n${matching.head}") } plan diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala index c0f3c0a6ce4..c848501a71a 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.ServiceLoader +import scala.annotation.nowarn import scala.jdk.CollectionConverters._ import org.scalatest.funsuite.AnyFunSuite @@ -331,6 +332,7 @@ class RecordingClaimingScanContrib extends ClaimingScanContrib { class NotAContribAtAll /** Implements the service but cannot be constructed; `next` throws. */ +@nowarn("msg=dead code") class ThrowingCtorScanContrib extends CometScanContrib { throw new IllegalStateException("contrib constructor blew up") } diff --git a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala index 6b82b2fcfbb..300d19081f1 100644 --- a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala @@ -30,10 +30,10 @@ class SerdeRegistrationSuite extends AnyFunSuite { test("version shims register only classes the shared serde maps do not") { import QueryPlanSerde._ val overlaps = Seq( - "math" -> (baseMathExpressions, sparkVersionSpecificMathExpressions), - "map" -> (baseMapExpressions, sparkVersionSpecificMapExpressions), - "string" -> (baseStringExpressions, sparkVersionSpecificStringExpressions), - "misc" -> (baseMiscExpressions, sparkVersionSpecificMiscExpressions)) + "math" -> ((baseMathExpressions, sparkVersionSpecificMathExpressions)), + "map" -> ((baseMapExpressions, sparkVersionSpecificMapExpressions)), + "string" -> ((baseStringExpressions, sparkVersionSpecificStringExpressions)), + "misc" -> ((baseMiscExpressions, sparkVersionSpecificMiscExpressions))) .flatMap { case (group, (base, shim)) => base.keySet.intersect(shim.keySet).map(cls => s"$group: ${cls.getSimpleName}") } diff --git a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala index d16e7fe504f..47a9b9a706b 100644 --- a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala @@ -93,10 +93,10 @@ class CometExecIteratorLifecycleSuite extends CometTestBase { // Unique synthetic task attempt ids keep each iteration's pool entry independent. val taskAttemptId = 4200000L + i withTaskContext(taskAttemptId) { - val manager = new CometTaskMemoryManager(i, taskAttemptId) + val manager = new CometTaskMemoryManager(i.toLong, taskAttemptId) val thrown = intercept[Throwable] { nativeLib.createPlan( - i, + i.toLong, Array.empty[Object], emptyPlan, badConfigs, diff --git a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala index 42ecc7611e8..42020a2af66 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala @@ -58,7 +58,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { taskMemoryManager = new TaskMemoryManager(memoryManager, 0) // One allocator per test: pages are addressed by a page number in the allocator's own table, // so everything a sorter touches has to come from the same instance. - allocator = CometShuffleMemoryAllocator.getInstance(taskMemoryManager, PAGE_SIZE) + allocator = CometShuffleMemoryAllocator.getInstance(taskMemoryManager, PAGE_SIZE.toLong) } override def afterEach(): Unit = { @@ -119,7 +119,11 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { val partitionId = 0 sorter.initialCurrentPage(recordData.length + UAO_SIZE) - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, partitionId) + sorter.insertRecord( + recordData, + Platform.BYTE_ARRAY_OFFSET.toLong, + recordData.length, + partitionId) assert(sorter.numRecords() === 1) } finally { @@ -140,7 +144,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { val partitionId = i % 10 sorter.insertRecord( recordData, - Platform.BYTE_ARRAY_OFFSET, + Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, partitionId) } @@ -157,7 +161,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { try { val recordData = Array[Byte](1, 2, 3, 4) sorter.initialCurrentPage(recordData.length + UAO_SIZE) - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, 0) + sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, 0) assert(sorter.numRecords() === 1) @@ -194,7 +198,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { try { sorter.initialCurrentPage(1024) val recordData = Array[Byte](1, 2, 3, 4) - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, 0) + sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, 0) assert(spillCount.get() === 0, "Spill callback should not be triggered during normal ops") } finally { @@ -229,7 +233,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { val sorter = createSpillSorter() try { val initialMemory = sorter.getMemoryUsage() - val newArray = allocator.allocateArray(INITIAL_SIZE * 2) + val newArray = allocator.allocateArray((INITIAL_SIZE * 2).toLong) sorter.expandPointerArray(newArray) assert(sorter.getMemoryUsage() >= initialMemory) @@ -251,7 +255,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { for (p <- 0 until numPartitions) { for (_ <- 0 until recordsPerPartition) { - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, p) + sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, p) } } @@ -272,7 +276,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { offHeapMemoryManager.limit(64L * 1024 * 1024) val offHeapTaskMemoryManager = new TaskMemoryManager(offHeapMemoryManager, 0) val allocator = - CometShuffleMemoryAllocator.getInstance(offHeapTaskMemoryManager, PAGE_SIZE) + CometShuffleMemoryAllocator.getInstance(offHeapTaskMemoryManager, PAGE_SIZE.toLong) // The block manager is only touched when spilling, which this test never does. val sorter = new CometShuffleExternalSorter( allocator, @@ -287,7 +291,11 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { try { val recordData = new Array[Byte](16) def insert(i: Int): Unit = - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, i % 2) + sorter.insertRecord( + recordData, + Platform.BYTE_ARRAY_OFFSET.toLong, + recordData.length, + i % 2) val initialArrayBytes = INITIAL_SIZE * 8L assert(allocator.getUsed === initialArrayBytes) diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala index 1cf39c3a248..2b67f6ffaa2 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala @@ -102,7 +102,7 @@ trait CometTPCQueryBase extends Logging { // Recover partitions but don't fail if a table is not partitioned. Try { cometSpark.sql(s"ALTER TABLE $tableName RECOVER PARTITIONS") - }.getOrElse { + }.failed.foreach { _ => logInfo(s"Recovering partitions of table $tableName failed") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala index cdd466f03d3..8903f43769b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala @@ -819,9 +819,7 @@ abstract class CometTestBase .builder(path) .withDictionaryEncoding(dictionaryEnabled) .withType(schema) - // TODO we need to shim this and use withRowGroupSize(Long) with later parquet-hadoop versions to remove - // the deprecated warning here - .withRowGroupSize(rowGroupSize.toInt) + .withRowGroupSize(rowGroupSize) .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) .withPageRowCountLimit(pageRowCountLimit) @@ -937,15 +935,15 @@ abstract class CometTestBase opt match { case Some(i) => record.add(0, i % 2 == 0) - record.add(1, i.toByte) - record.add(2, i.toShort) + record.add(1, i.toByte.toInt) + record.add(2, i.toShort.toInt) record.add(3, i) record.add(4, i.toLong) record.add(5, i.toFloat) record.add(6, i.toDouble) record.add(7, i.toString * 48) - record.add(8, (-i).toByte) - record.add(9, (-i).toShort) + record.add(8, (-i).toByte.toInt) + record.add(9, (-i).toShort.toInt) record.add(10, -i) record.add(11, (-i).toLong) record.add(12, i.toString) @@ -966,15 +964,15 @@ abstract class CometTestBase val i = rand.nextLong() val record = new SimpleGroup(schema) record.add(0, i % 2 == 0) - record.add(1, i.toByte) - record.add(2, i.toShort) + record.add(1, i.toByte.toInt) + record.add(2, i.toShort.toInt) record.add(3, i.toInt) record.add(4, i) record.add(5, java.lang.Float.intBitsToFloat(i.toInt)) record.add(6, java.lang.Double.longBitsToDouble(i)) record.add(7, i.toString * 24) - record.add(8, (-i).toByte) - record.add(9, (-i).toShort) + record.add(8, (-i).toByte.toInt) + record.add(9, (-i).toShort.toInt) record.add(10, (-i).toInt) record.add(11, -i) record.add(12, i.toString) @@ -1023,7 +1021,7 @@ abstract class CometTestBase if (rand.nextBoolean()) { None } else { - Some(getValue(i, div)) + Some(getValue(i.toLong, div.toLong)) } } expected.foreach { opt => @@ -1077,7 +1075,7 @@ abstract class CometTestBase if (rand.nextBoolean()) { None } else { - Some(getValue(i, div)) + Some(getValue(i.toLong, div.toLong)) } } expected.foreach { opt => @@ -1255,7 +1253,7 @@ abstract class CometTestBase val div = if (dictionaryEnabled) 10 else n // maps value to a small range for dict to kick in val expected = (0 until n).map { i => - Some(getValue(i, div)) + Some(getValue(i.toLong, div.toLong)) } expected.foreach { opt => val timestampFormats = List( @@ -1303,7 +1301,7 @@ abstract class CometTestBase def makeDecimalRDD(num: Int, decimal: DecimalType, useDictionary: Boolean): DataFrame = { val div = if (useDictionary) 5 else num // narrow the space to make it dictionary encoded spark - .range(num) + .range(num.toLong) .map(_ % div) // Parquet doesn't allow column names with spaces, have to add an alias here. // Minus 500 here so that negative decimals are also tested. @@ -1483,8 +1481,8 @@ abstract class CometTestBase val record = new SimpleGroup(schema) opt match { case Some(i) => - record.add(0, i.toByte) - record.add(1, i.toShort) + record.add(0, i.toByte.toInt) + record.add(1, i.toShort.toInt) record.add(2, i) record.add(3, i.toLong) record.add(4, rand.nextFloat()) diff --git a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala index e25d4e51e46..6e8ce177d62 100644 --- a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala +++ b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala @@ -65,7 +65,7 @@ object GenTPCHData { // Install the data generators in all nodes // TODO: think a better way to install on each worker node // such as https://stackoverflow.com/a/40876671 - spark.range(0, workers, 1, workers).foreach(worker => installDBGEN(baseDir)(worker)) + spark.range(0, workers.toLong, 1, workers).foreach(worker => installDBGEN(baseDir)(worker)) s"${baseDir}/dbgen" } else { config.dbgenDir @@ -91,7 +91,7 @@ object GenTPCHData { // Clean up if (defaultDbgenDir != null) { - spark.range(0, workers, 1, workers).foreach { _ => + spark.range(0, workers.toLong, 1, workers).foreach { _ => val _ = FileUtils.deleteQuietly(defaultDbgenDir) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala index 2772d37ffbf..f577ed102d5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala @@ -217,7 +217,7 @@ object CometAggregateExpressionBenchmark extends CometBenchmarkBase { approxCountDistinctAggregates ++ maxMinByAggregates allAggregates.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala index a513aa1a77b..7594ab62fb5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala @@ -44,7 +44,7 @@ object CometArithmeticBenchmark extends CometBenchmarkBase { val name = s"Binary op ${dataType.sql}, dictionary = $useDictionary" val query = s"SELECT c1 ${op.sig} c2 FROM $table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -64,7 +64,7 @@ object CometArithmeticBenchmark extends CometBenchmarkBase { val name = s"Binary op ${dataType.sql}, dictionary = $useDictionary" val query = s"SELECT c1 ${op.sig} c2 FROM $table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala index 624abdebe88..90c247a9996 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala @@ -65,7 +65,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { prepareSortArrayTable(width) { runExpressionBenchmark( s"sort_array int ascending (width=$width)", - values, + values.toLong, "SELECT sort_array(int_arr) FROM parquetV1Table") } } @@ -74,7 +74,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { prepareSortArrayTable(width) { runExpressionBenchmark( s"sort_array int descending (width=$width)", - values, + values.toLong, "SELECT sort_array(int_arr, false) FROM parquetV1Table") } } @@ -83,7 +83,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { prepareSortArrayTable(width) { runExpressionBenchmark( s"element_at(sort_array(int_arr), 1) (width=$width)", - values, + values.toLong, "SELECT element_at(sort_array(int_arr), 1) FROM parquetV1Table") } } @@ -112,7 +112,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { runExpressionBenchmark( "array_position - int array", - values, + values.toLong, "SELECT array_position(int_arr, search_val) FROM parquetV1Table") } } @@ -140,7 +140,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { runExpressionBenchmark( "array_position - string array", - values, + values.toLong, "SELECT array_position(str_arr, search_val) FROM parquetV1Table") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala index 11033703afc..c8073e04c66 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala @@ -97,7 +97,10 @@ object CometArrowWriterBenchmark extends BenchmarkBase { val root = VectorSchemaRoot.create(arrowSchema, allocator) try { val benchmark = - new Benchmark(s"Spark columnar to Arrow ($numRows rows)", numRows, output = output) + new Benchmark( + s"Spark columnar to Arrow ($numRows rows)", + numRows.toLong, + output = output) benchmark.addCase("on-heap optimized path") { _ => writeBatch(onHeap, bulkCopy = true, root) } @@ -133,7 +136,7 @@ object CometArrowWriterBenchmark extends BenchmarkBase { arrowSchema, Iterator.from(0).map(i => if ((i & 1) == 0) nullRow else row), numRows) - val rowBenchmark = new Benchmark("Spark rows to Arrow", numRows, output = output) + val rowBenchmark = new Benchmark("Spark rows to Arrow", numRows.toLong, output = output) try { rowBenchmark.addCase("fixed-width, no nulls") { _ => noNullRowReader.loadNextBatch() } rowBenchmark.addCase("fixed-width, 50% nulls") { _ => nullableRowReader.loadNextBatch() } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala index 385c1eb1f93..7edbae22c82 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala @@ -106,7 +106,7 @@ trait CometBenchmarkBase // generator, so that results are comparable across runs. Seeding a driver-side `Random` // would not work here: the closure runs per row on the executor. spark - .range(values) + .range(values.toLong) .map(i => if (useDictionary) CometBenchmarkBase.mix64(i) % 5 else CometBenchmarkBase.mix64(i)) .createOrReplaceTempView(tbl) @@ -394,7 +394,7 @@ trait CometBenchmarkBase val div = if (useDictionary) 5 else values spark - .range(values) + .range(values.toLong) .map(_ % div) .select((($"value" - 500) / 100.0) cast decimal as Symbol("dec")) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBinaryLengthBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBinaryLengthBenchmark.scala index 04e65ab354b..23699ccdd3e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBinaryLengthBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBinaryLengthBenchmark.scala @@ -57,7 +57,7 @@ object CometBinaryLengthBenchmark extends CometBenchmarkBase { cases.foreach { case (name, query) => runBenchmark(name) { - runExpressionBenchmark(name, v, query) + runExpressionBenchmark(name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala index da1a857f588..c485bc524bd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala @@ -66,12 +66,12 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id AS k", "id * 10 AS w") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -82,7 +82,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - inner count") { runExpressionBenchmark( "inner count", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -90,7 +90,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - inner projected") { runExpressionBenchmark( "inner projected", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ p.k, p.v, b.w FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -98,7 +98,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - left outer") { runExpressionBenchmark( "left outer", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p LEFT JOIN build b ON p.k = b.k", cometConfigs) } @@ -106,7 +106,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - left semi") { runExpressionBenchmark( "left semi", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p LEFT SEMI JOIN build b ON p.k = b.k", cometConfigs) } @@ -114,7 +114,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - right outer") { runExpressionBenchmark( "right outer", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(p) */ count(*) FROM probe p RIGHT JOIN build b ON p.k = b.k", cometConfigs) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala index c1cfab60fcb..a6e2c43be15 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala @@ -71,12 +71,12 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id * 1000 AS lo", "id * 1000 + 500 AS hi") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -87,7 +87,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - range") { runExpressionBenchmark( "range join (BETWEEN)", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p " + "JOIN build b ON p.k BETWEEN b.lo AND b.hi", cometConfigs) @@ -96,7 +96,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - inequality") { runExpressionBenchmark( "inequality join (>)", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p " + "JOIN build b ON p.k > b.lo", cometConfigs) @@ -105,7 +105,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - left outer with non-equi") { runExpressionBenchmark( "left outer non-equi", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p " + "LEFT OUTER JOIN build b ON p.k BETWEEN b.lo AND b.hi", cometConfigs) @@ -114,7 +114,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - range, materialized rows") { runExpressionBenchmark( "range join (BETWEEN, projected)", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ p.k, p.v, b.lo, b.hi FROM probe p " + "JOIN build b ON p.k BETWEEN b.lo AND b.hi", cometConfigs) diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala index 57b8e88a7b5..8419ade9e63 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala @@ -89,7 +89,7 @@ object CometCastBooleanBenchmark extends CometBenchmarkBase { """)) (boolToStringConfigs ++ boolToNumericConfigs).foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -115,7 +115,7 @@ object CometCastBooleanBenchmark extends CometBenchmarkBase { """)) numericToBoolConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala index 2141b1bea08..12ae85374d0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala @@ -148,7 +148,7 @@ object CometCastNumericToNumericBenchmark extends CometBenchmarkBase { generateConfigs(floatToIntPairs) ++ generateConfigs(decimalPairs) ++ generateConfigs(floatToDecimalPairs)).foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala index 1fd2138c581..cdf719b0406 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala @@ -95,7 +95,7 @@ object CometCastNumericToStringBenchmark extends CometBenchmarkBase { """)) castConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala index ec2d9ab12f1..e54127c9dc6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala @@ -70,7 +70,7 @@ object CometCastNumericToTemporalBenchmark extends CometBenchmarkBase { """)) intToDateConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -92,7 +92,7 @@ object CometCastNumericToTemporalBenchmark extends CometBenchmarkBase { """)) longToTimestampConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala index c71eadad8c4..ac9c172e5b8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala @@ -89,7 +89,7 @@ object CometCastStringToNumericBenchmark extends CometBenchmarkBase { """)) castConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala index 77cc009ae10..31520824b73 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala @@ -71,7 +71,7 @@ object CometCastStringToTemporalBenchmark extends CometBenchmarkBase { // Run date cast benchmarks with the same data dateCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -94,7 +94,7 @@ object CometCastStringToTemporalBenchmark extends CometBenchmarkBase { // Run timestamp cast benchmarks with the same data timestampCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala index 1468cbe086f..6a4b9f81b06 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala @@ -74,7 +74,7 @@ object CometCastTemporalToNumericBenchmark extends CometBenchmarkBase { """)) dateToNumericConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -96,7 +96,7 @@ object CometCastTemporalToNumericBenchmark extends CometBenchmarkBase { """)) timestampToNumericConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala index 1ef3e7711d2..5e77742a345 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala @@ -67,7 +67,7 @@ object CometCastTemporalToStringBenchmark extends CometBenchmarkBase { """)) dateCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -89,7 +89,7 @@ object CometCastTemporalToStringBenchmark extends CometBenchmarkBase { """)) timestampCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala index f2e25724873..c030846861a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala @@ -69,7 +69,7 @@ object CometCastTemporalToTemporalBenchmark extends CometBenchmarkBase { """)) dateToTimestampConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -91,7 +91,7 @@ object CometCastTemporalToTemporalBenchmark extends CometBenchmarkBase { """)) timestampToDateConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala index ab2c15affd9..294b5ec7bc4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala @@ -294,7 +294,7 @@ object CometCodegenDispatchBenchmark extends CometBenchmarkBase { private def runSteadyState(c: DispatchCase, rows: Int): Unit = { runBenchmark(s"${c.name} -- $rows rows") { - val benchmark = new Benchmark(s"${c.name} -- $rows rows", rows, output = output) + val benchmark = new Benchmark(s"${c.name} -- $rows rows", rows.toLong, output = output) checkPlans(benchmark, c) // The dispatch-off arm goes first so the `Relative` column reads as the speedup this // change buys over the behaviour that shipped before it. @@ -364,7 +364,7 @@ object CometCodegenDispatchBenchmark extends CometBenchmarkBase { private def withCorpus(rows: Int)(f: => Unit): Unit = { withTempPath { dir => withTempTable(tbl, "parquetV1Table") { - spark.range(rows).createOrReplaceTempView(tbl) + spark.range(rows.toLong).createOrReplaceTempView(tbl) prepareTable(dir, spark.sql(corpusQuery)) f } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala index 22e7dbb54dd..1a05bdbcd73 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala @@ -150,13 +150,13 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def primitiveTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Primitive Types", values, output = output) + new Benchmark("Columnar to Row - Primitive Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { // Create a table with various primitive types (includes strings) val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id as long_col", "cast(id as int) as int_col", @@ -182,13 +182,16 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def fixedWidthOnlyBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Fixed Width Only (no strings)", values, output = output) + new Benchmark( + "Columnar to Row - Fixed Width Only (no strings)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { // Create a table with ONLY fixed-width primitive types (no strings!) val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id as long_col", "cast(id as int) as int_col", @@ -214,12 +217,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def stringTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - String Types", values, output = output) + new Benchmark("Columnar to Row - String Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", "concat('short_', cast(id % 100 as string)) as short_str", @@ -239,12 +242,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def structTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Struct Types", values, output = output) + new Benchmark("Columnar to Row - Struct Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Simple struct @@ -279,12 +282,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def arrayTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Array Types", values, output = output) + new Benchmark("Columnar to Row - Array Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Array of primitives @@ -310,12 +313,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def mapTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Map Types", values, output = output) + new Benchmark("Columnar to Row - Map Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Map with string keys and int values @@ -343,12 +346,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def complexNestedTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Complex Nested Types", values, output = output) + new Benchmark("Columnar to Row - Complex Nested Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Array of structs @@ -380,7 +383,7 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def wideRowsBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Wide Rows (50 columns)", values, output = output) + new Benchmark("Columnar to Row - Wide Rows (50 columns)", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -395,7 +398,7 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { } } - val df = spark.range(values).selectExpr(columns: _*) + val df = spark.range(values.toLong).selectExpr(columns: _*) prepareTable(dir, df) val query = "SELECT * FROM parquetV1Table" @@ -411,12 +414,15 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def dictionaryEncodedBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Dictionary-encoded strings", values, output = output) + new Benchmark( + "Columnar to Row - Dictionary-encoded strings", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", "concat('val_', cast(id % 1000 as string)) as low_card_str", @@ -446,14 +452,14 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def jvmConsumerBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - JVM UDF consumer", values, output = output) + new Benchmark("Columnar to Row - JVM UDF consumer", values.toLong, output = output) spark.udf.register("plus_one", (x: Long) => x + 1) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id as long_col", "cast(id as int) as int_col", diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala index 56ce2ac2640..060b96b8ea4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala @@ -85,7 +85,7 @@ object CometComparisonExpressionBenchmark extends CometBenchmarkBase { """)) comparisonExpressions.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala index 6db28b3072b..54391073bbd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala @@ -98,7 +98,7 @@ object CometConcatWsBenchmark extends CometBenchmarkBase { // scalastyle:on println } runBenchmark(name) { - runExpressionBenchmark(name, rows, query, cometConfigs) + runExpressionBenchmark(name, rows.toLong, query, cometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala index 5b3532d5f23..877318e96b4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala @@ -57,7 +57,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { prepareTestTable(values) { val query = "SELECT CASE WHEN c1 < 0 THEN '<0' WHEN c1 = 0 THEN '=0' ELSE '>0' END FROM parquetV1Table" - runExpressionBenchmark("Case When Literal (3 branches)", values, query) + runExpressionBenchmark("Case When Literal (3 branches)", values.toLong, query) } } @@ -78,7 +78,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { ELSE 'j' END FROM parquetV1Table """ - runExpressionBenchmark("Case When Literal (10 branches)", values, query) + runExpressionBenchmark("Case When Literal (10 branches)", values.toLong, query) } } @@ -87,7 +87,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { // Result expressions are column references, not literals val query = "SELECT CASE WHEN c1 < 0 THEN c3 WHEN c1 = 0 THEN c1 ELSE c3 + c1 END FROM parquetV1Table" - runExpressionBenchmark("Case When Column Result (3 branches)", values, query) + runExpressionBenchmark("Case When Column Result (3 branches)", values.toLong, query) } } @@ -108,14 +108,14 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { ELSE c1 + c2 + c3 END FROM parquetV1Table """ - runExpressionBenchmark("Case When Column Result (10 branches)", values, query) + runExpressionBenchmark("Case When Column Result (10 branches)", values.toLong, query) } } def ifLiteralBenchmark(values: Int): Unit = { prepareTestTable(values) { val query = "SELECT IF(c1 < 0, '<0', '>=0') FROM parquetV1Table" - runExpressionBenchmark("If Literal", values, query) + runExpressionBenchmark("If Literal", values.toLong, query) } } @@ -123,7 +123,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { prepareTestTable(values) { // Result expressions are column references val query = "SELECT IF(c1 < 0, c3, c1 + c3) FROM parquetV1Table" - runExpressionBenchmark("If Column Result", values, query) + runExpressionBenchmark("If Column Result", values.toLong, query) } } @@ -136,7 +136,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { IF(c2 < 75, 'c', 'd'))) FROM parquetV1Table """ - runExpressionBenchmark("Nested If Literal (4 outcomes)", values, query) + runExpressionBenchmark("Nested If Literal (4 outcomes)", values.toLong, query) } } @@ -148,7 +148,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { IF(c2 < 75, c1 + c3, c3 * 2))) FROM parquetV1Table """ - runExpressionBenchmark("Nested If Column Result (4 outcomes)", values, query) + runExpressionBenchmark("Nested If Column Result (4 outcomes)", values.toLong, query) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala index 1495b0320e2..3e0e02a4cc8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala @@ -63,7 +63,7 @@ object CometCsvExpressionBenchmark extends CometBenchmarkBase { CometConf.getExprAllowIncompatConfigKey( classOf[CsvToStructs]) -> "true") ++ config.extraCometConfigs - runExpressionBenchmark(config.name, values, config.query, extraConfigs) + runExpressionBenchmark(config.name, values.toLong, config.query, extraConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala index 8e2978714aa..1ff82a6d450 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala @@ -45,7 +45,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { Seq("YEAR", "MONTH").foreach { level => val name = s"Date Truncate - $level" val query = s"select trunc(dt, '$level') from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -70,7 +70,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { "MICROSECOND").foreach { level => val name = s"Timestamp Truncate - $level" val query = s"select date_trunc('$level', ts) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -85,7 +85,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { val name = s"Unix Timestamp from Timestamp ($timeZone)" val query = "select unix_timestamp(ts) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -101,7 +101,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { val name = s"Unix Timestamp from Date ($timeZone)" val query = "select unix_timestamp(dt) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -116,7 +116,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { s"select concat(cast(abs(value) % 24 as string), ':', lpad(cast(abs(value) % 60 as string), 2, '0'), ':', lpad(cast(abs(value) % 60 as string), 2, '0')) as s FROM $tbl")) val name = "to_time" val query = "select to_time(s) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -130,7 +130,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { s"select cast(abs(value) % 24 as int) as h, cast(abs(value) % 60 as int) as m, cast(abs(value) % 60 as decimal(16,6)) as s FROM $tbl")) val name = "make_time" val query = "select make_time(h, m, s) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -154,7 +154,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { def consumeIntervals(): Unit = { spark.sql(query).queryExecution.toRdd.foreachPartition(_.foreach(_.getInterval(0))) } - val benchmark = new Benchmark("MakeInterval", values, output = output) + val benchmark = new Benchmark("MakeInterval", values.toLong, output = output) val cometConfigs = Map( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", @@ -209,11 +209,14 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { END AS dow FROM $tbl """)) - runExpressionBenchmark("NextDay", values, "select next_day(dt, dow) from parquetV1Table") + runExpressionBenchmark( + "NextDay", + values.toLong, + "select next_day(dt, dow) from parquetV1Table") if (isSpark40Plus) { runExpressionBenchmark( "NextDay - collated dayOfWeek", - values, + values.toLong, "select next_day(dt, dow collate utf8_lcase) from parquetV1Table") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala index 8fc866ff4aa..11425bbfe8c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala @@ -70,7 +70,10 @@ object CometExecBenchmark extends CometBenchmarkBase { def numericFilterExecBenchmark(values: Int, fractionOfZeros: Double): Unit = { val percentageOfZeros = fractionOfZeros * 100 val benchmark = - new Benchmark(s"Project + Filter Exec ($percentageOfZeros% zeros)", values, output = output) + new Benchmark( + s"Project + Filter Exec ($percentageOfZeros% zeros)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -108,7 +111,7 @@ object CometExecBenchmark extends CometBenchmarkBase { } def subqueryExecBenchmark(values: Int): Unit = { - val benchmark = new Benchmark("Subquery", values, output = output) + val benchmark = new Benchmark("Subquery", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -140,7 +143,7 @@ object CometExecBenchmark extends CometBenchmarkBase { } def sortExecBenchmark(values: Int): Unit = { - val benchmark = new Benchmark("Sort Exec", values, output = output) + val benchmark = new Benchmark("Sort Exec", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -164,7 +167,8 @@ object CometExecBenchmark extends CometBenchmarkBase { } def sampleExecBenchmark(values: Int, fraction: Double): Unit = { - val benchmark = new Benchmark(s"Sample Exec (fraction $fraction)", values, output = output) + val benchmark = + new Benchmark(s"Sample Exec (fraction $fraction)", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -194,7 +198,7 @@ object CometExecBenchmark extends CometBenchmarkBase { } def expandExecBenchmark(values: Int): Unit = { - val benchmark = new Benchmark("Expand Exec", values, output = output) + val benchmark = new Benchmark("Expand Exec", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -232,7 +236,7 @@ object CometExecBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"BloomFilterAggregate Exec (cardinality $cardinality)", - values, + values.toLong, output = output) val funcId_bloom_filter_agg = new FunctionIdentifier("bloom_filter_agg") diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala index aa16417ca2c..86909f839c5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala @@ -79,7 +79,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { * Rows of an `rows`-row dataset whose [[arrayColumn]] is neither NULL nor empty, which is how * many rows reach a non-outer generator with something to emit. */ - private def nonEmptyRows(rows: Int): Long = rows - (rows + 9) / 10 - (rows + 8) / 10 + private def nonEmptyRows(rows: Int): Long = (rows - (rows + 9) / 10 - (rows + 8) / 10).toLong /** * A SQL expression for an array of `len` elements of `elementExpr`, where `elementExpr` may @@ -88,7 +88,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { * Elements are wrapped in a never-taken null branch so the array types as `containsNull`; see * [[nullableExpr]]. */ - private def fullArray(elementExpr: String, len: Int, v: String = "x"): String = + private def fullArray(elementExpr: String, len: Int, v: String): String = s"transform(sequence(1, $len), $v -> ${nullableExpr(elementExpr, s"$v = 0")})" /** @@ -214,7 +214,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { /** Writes a view's rows to Parquet and registers it. */ private def createView(dir: File, view: TempView): Unit = { val path = s"${dir.getAbsolutePath}/${view.name}" - spark.range(view.rows).selectExpr(view.columns: _*).write.parquet(path) + spark.range(view.rows.toLong).selectExpr(view.columns: _*).write.parquet(path) spark.read.parquet(path).createOrReplaceTempView(view.name) } @@ -306,7 +306,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { /** Verifies a case's aggregate under both engines, then times it. */ private def runCase(name: String, benchmarkCase: Case, rows: Int = numRows): Unit = { verifySink(name, benchmarkCase.query, benchmarkCase.expected) - runExpressionBenchmark(name, rows, benchmarkCase.query) + runExpressionBenchmark(name, rows.toLong, benchmarkCase.query) } override def runCometBenchmark(mainArgs: Array[String]): Unit = { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala index fa639453475..6b18aed5dee 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala @@ -38,7 +38,7 @@ object CometGetJsonObjectBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(numRows) + .range(numRows.toLong) .map { i => val name = s"user_$i" val age = (i % 80 + 18).toInt @@ -76,7 +76,11 @@ object CometGetJsonObjectBenchmark extends CometBenchmarkBase { benchmarks.foreach { config => runBenchmark(config.name) { - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark( + config.name, + v.toLong, + config.query, + config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala index 68ee3c1d1e0..238d8b662b1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala @@ -65,7 +65,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { """)) hashExpressions.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -124,7 +124,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { HashExprConfig("hash_decimal", "SELECT hash(c_decimal) FROM parquetV1Table")) primitiveHashBenchmarks.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -175,7 +175,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { "SELECT hash(c_struct_multi) FROM parquetV1Table")) complexHashBenchmarks.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -232,7 +232,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { "SELECT hash(c_array_of_struct) FROM parquetV1Table")) nestedHashBenchmarks.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala index f339ef7aa72..159c91faebc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala @@ -73,12 +73,12 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id AS k", "id * 10 AS w") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -89,7 +89,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - inner count") { runExpressionBenchmark( "inner count", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ count(*) FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -97,7 +97,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - inner projected") { runExpressionBenchmark( "inner projected", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ p.k, p.v, b.w FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -105,7 +105,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - left outer") { runExpressionBenchmark( "left outer", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ count(*) FROM probe p LEFT JOIN build b ON p.k = b.k", cometConfigs) } @@ -113,7 +113,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - left semi") { runExpressionBenchmark( "left semi", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ count(*) FROM probe p LEFT SEMI JOIN build b ON p.k = b.k", cometConfigs) } @@ -121,7 +121,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - right outer") { runExpressionBenchmark( "right outer", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(p) */ count(*) FROM probe p RIGHT JOIN build b ON p.k = b.k", cometConfigs) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala index b90b893712b..1a171a5adac 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala @@ -34,7 +34,10 @@ object CometIcebergReadBenchmark extends CometBenchmarkBase { def icebergScanBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"SQL Single ${dataType.sql} Iceberg Column Scan", values, output = output) + new Benchmark( + s"SQL Single ${dataType.sql} Iceberg Column Scan", + values.toLong, + output = output) withTempPath { dir => withTempTable("icebergTable") { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala index 7369a532304..97b11a54f3c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala @@ -154,7 +154,7 @@ object CometIcebergSystemFunctionBenchmark extends CometBenchmarkBase { cases.foreach { case (name, query) => verifyOutputsMatch(name, query) runBenchmark(name) { - runExpressionBenchmark(name, v, query) + runExpressionBenchmark(name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala index 55f14d9613a..509bcbf8796 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala @@ -328,7 +328,7 @@ object CometIcebergWriteBenchmark extends CometBenchmarkBase { // better-jitted measurement. val benchmark = new Benchmark( workload.title, - values, + values.toLong, // The iceberg-java baseline is the least repeatable of the three arms - it carries a fifth of // its own runtime as spread between iterations, against a few percent for the two Comet arms // - and it is also the divisor of every `Relative` figure. Five iterations rather than the diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala index f33a53af299..1b0d6eff78a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala @@ -76,7 +76,10 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { // rejects it. private val codecs = Seq("zstd", "none") - @volatile private var statsResult: (Array[Any], Array[Any], Array[Int]) = _ + // A sink for the benchmarked call's result: written but never read, so that neither the + // compiler nor the JIT can treat `gatherColumnStats` as dead code. Not `private`, because + // a private field that is only ever written is what `-Ywarn-unused:privates` reports. + @volatile var statsResult: (Array[Any], Array[Any], Array[Int]) = _ override def getSparkSession: SparkSession = { val conf = new SparkConf() @@ -122,7 +125,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { // Remainder can divide by zero, while `id + 1` is not, so relying on that is what let the // mislabelling through in the first place. spark - .range(0, numRows, 1, 16) + .range(0, numRows.toLong, 1, 16) .selectExpr( "if(id % 8 = 0, null, id) AS id", "if(id % 8 = 1, null, id % 1000) AS k", @@ -148,7 +151,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { // nothing to do with the cache. Counting a nullable field reads the whole column regardless, // since the cache scan selects whole top-level columns. spark - .range(0, nestedNumRows, 1, 16) + .range(0, nestedNumRows.toLong, 1, 16) .selectExpr( "if(id % 8 = 0, null, id) AS id", "named_struct(" + @@ -277,7 +280,10 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { } val materialize = - new Benchmark("in-memory cache materialize by codec", relation.rows, output = output) + new Benchmark( + "in-memory cache materialize by codec", + relation.rows.toLong, + output = output) codecs.foreach { codec => // Timed around the caching alone: dropping the previous copy is setup, and a plain // addCase would charge it to whichever codec happens to be running. @@ -316,7 +322,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { verifyPlan(query, nativeCacheEnabled = true, scanned) } val benchmark = - new Benchmark(s"in-memory cache $label by codec", relation.rows, output = output) + new Benchmark(s"in-memory cache $label by codec", relation.rows.toLong, output = output) codecs.foreach { codec => // Re-caching under this case's codec is setup, so it is outside the timer, and it only // happens on the case's first call, which is a warmup iteration. @@ -379,7 +385,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { 6)).foreach { case (label, query, scanned) => val benchmark = new Benchmark( s"in-memory cache read by Spark operators, $label", - relation.rows, + relation.rows.toLong, output = output) formats.foreach { case (name, serializer) => var verified = false @@ -462,8 +468,8 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { var r = 0 while (r < batchSize) { columns(0).putLong(r, r.toLong) - columns(1).putLong(r, r % 1000) - columns(2).putLong(r, r + 1) + columns(1).putLong(r, (r % 1000).toLong) + columns(2).putLong(r, (r + 1).toLong) columns(3).putByteArray(r, s"str_a_${r % 100000}".getBytes(StandardCharsets.UTF_8)) columns(4).putByteArray(r, s"str_b_${r % 7919}".getBytes(StandardCharsets.UTF_8)) columns(5).putByteArray(r, s"str_c_$r".getBytes(StandardCharsets.UTF_8)) @@ -473,7 +479,8 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { // Resolved outside the timed loop because that is where the serializer resolves it: once // per partition, not once per batch. val orderings = serializer.boundsOrderings(attrs) - val benchmark = new Benchmark("in-memory cache statistics", numRows, output = output) + val benchmark = + new Benchmark("in-memory cache statistics", numRows.toLong, output = output) // One case measures this collector across commits; Spark's default cache has its own collector. benchmark.addCase("Comet statistics collector") { _ => var i = 0 @@ -499,7 +506,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { verifyPlan(query, nativeCacheEnabled = true, scanned) } - val benchmark = new Benchmark(name, relation.rows, output = output) + val benchmark = new Benchmark(name, relation.rows.toLong, output = output) benchmark.addCase("Spark cache scan + CometSparkColumnarToColumnar") { _ => withSQLConf(cacheConf(nativeCacheEnabled = false): _*) { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala index 82aae3d7b93..8a422bb27ba 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala @@ -159,7 +159,7 @@ object CometJsonExpressionBenchmark extends CometBenchmarkBase { CometConf.getExprAllowIncompatConfigKey( classOf[StructsToJson]) -> "true") ++ config.extraCometConfigs - runExpressionBenchmark(config.name, values, config.query, extraConfigs) + runExpressionBenchmark(config.name, values.toLong, config.query, extraConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala index 8c09ce01cf1..53acadbc729 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala @@ -41,7 +41,7 @@ object CometLengthOfJsonArrayBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(numRows) + .range(numRows.toLong) .map { i => val arrayLength = (i % 100).toInt (0 until arrayLength) @@ -61,7 +61,11 @@ object CometLengthOfJsonArrayBenchmark extends CometBenchmarkBase { benchmarks.foreach { config => runBenchmark(config.name) { - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark( + config.name, + v.toLong, + config.query, + config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala index 2f3904c5df6..fa54e0dfe3b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala @@ -219,7 +219,7 @@ object CometOperatorSerdeBenchmark extends CometBenchmarkBase { val iterations = 100 val benchmark = new Benchmark( s"IcebergScan serde ($numPartitions partitions, ${tasks.size()} tasks)", - iterations, + iterations.toLong, output = output) // Benchmark: Convert FileScanTasks to protobuf (the convert() method) @@ -342,7 +342,7 @@ object CometOperatorSerdeBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"CometScanRule apply ($numPartitions partitions)", - iterations, + iterations.toLong, output = output) benchmark.addCase("CometScanRule.apply(sparkPlan)") { _ => diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala index 6fab1ea7a76..58750ee9f4c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala @@ -34,7 +34,7 @@ object CometPaddingExpressionBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(rows) + .range(rows.toLong) .selectExpr( "CAST(id AS STRING) AS s", "CAST(id % 32 + 8 AS INT) AS len", @@ -48,7 +48,7 @@ object CometPaddingExpressionBenchmark extends CometBenchmarkBase { runBenchmark(name) { runExpressionBenchmark( name, - rows, + rows.toLong, s"SELECT $function($arguments) FROM parquetV1Table") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala index a7d170057f4..cdc93a44ff1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala @@ -39,7 +39,7 @@ object CometPartitionColumnBenchmark extends CometBenchmarkBase { def partitionColumnScanBenchmark(values: Int, numPartitionCols: Int): Unit = { val sqlBenchmark = new Benchmark( s"Partitioned Scan with $numPartitionCols partition column(s)", - values, + values.toLong, output = output) withTempPath { dir => diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala index 6506c5665d9..a7e22d8e90d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala @@ -38,7 +38,7 @@ object CometPredicateExpressionBenchmark extends CometBenchmarkBase { val query = "select * from parquetV1Table where c1 in ('positive', 'zero')" - runExpressionBenchmark("in Expr", values, query) + runExpressionBenchmark("in Expr", values.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala index 1055240cd73..6b809d381b1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala @@ -77,7 +77,11 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val operators = 1024 val creation = - new Benchmark("Scan I/O SQL metrics: creation", operators, minNumIters = 5, output = output) + new Benchmark( + "Scan I/O SQL metrics: creation", + operators.toLong, + minNumIters = 5, + output = output) creation.addCase("nine metrics per operator") { _ => var count = 0 while (count < operators) { @@ -91,7 +95,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val driverMetrics = createMetrics() val updates = new Benchmark( "Scan I/O SQL metrics: task snapshots", - tasks, + tasks.toLong, minNumIters = 5, output = output) updates.addCase("copy, update and merge nine metrics") { _ => @@ -112,7 +116,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val partitions = spark.sparkContext.parallelize(0 until tasks, tasks) val jobs = new Benchmark( "Scan I/O SQL metrics: 10000-task Spark job", - tasks, + tasks.toLong, minNumIters = 3, output = output) val cases = if (reverseCases) Seq(9, 0) else Seq(0, 9) @@ -131,7 +135,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def numericScanBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"SQL Single ${dataType.sql} Column Scan", values, output = output) + new Benchmark(s"SQL Single ${dataType.sql} Column Scan", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -150,7 +154,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def encryptedScanBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"SQL Single ${dataType.sql} Encrypted Column Scan", values, output = output) + new Benchmark( + s"SQL Single ${dataType.sql} Encrypted Column Scan", + values.toLong, + output = output) val encoder = Base64.getEncoder val footerKey = @@ -191,7 +198,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def decimalScanBenchmark(values: Int, precision: Int, scale: Int): Unit = { val sqlBenchmark = new Benchmark( s"SQL Single Decimal(precision: $precision, scale: $scale) Column Scan", - values, + values.toLong, output = output) withTempPath { dir => @@ -210,7 +217,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def readerBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"Parquet reader benchmark for $dataType", values, output = output) + new Benchmark(s"Parquet reader benchmark for $dataType", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -268,7 +275,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def numericFilterScanBenchmark(values: Int, fractionOfZeros: Double): Unit = { val percentageOfZeros = fractionOfZeros * 100 val benchmark = - new Benchmark(s"Numeric Filter Scan ($percentageOfZeros% zeros)", values, output = output) + new Benchmark( + s"Numeric Filter Scan ($percentageOfZeros% zeros)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table", "parquetV2Table") { @@ -286,7 +296,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def stringWithDictionaryScanBenchmark(values: Int): Unit = { val sqlBenchmark = - new Benchmark("String Scan with Dictionary Encoding", values, output = output) + new Benchmark("String Scan with Dictionary Encoding", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table", "parquetV2Table") { @@ -316,7 +326,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def stringWithNullsScanBenchmark(values: Int, fractionOfNulls: Double): Unit = { val percentageOfNulls = fractionOfNulls * 100 val benchmark = - new Benchmark(s"String with Nulls Scan ($percentageOfNulls%)", values, output = output) + new Benchmark( + s"String with Nulls Scan ($percentageOfNulls%)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -337,7 +350,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def columnsBenchmark(values: Int, width: Int): Unit = { val benchmark = - new Benchmark(s"Single Column Scan from $width columns", values, output = output) + new Benchmark(s"Single Column Scan from $width columns", values.toLong, output = output) withTempPath { dir => withTempTable("t1", "parquetV1Table") { @@ -358,7 +371,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"Large String Filter Scan ($percentageOfZeros% zeros)", - values, + values.toLong, output = output) withTempPath { dir => @@ -380,7 +393,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"Sorted Lg Str Filter Scan ($percentageOfZeros% zeros)", - values, + values.toLong, output = output) withTempPath { dir => diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala index 21a4b42024d..3766b7748e6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala @@ -82,13 +82,13 @@ object CometRegExpBenchmark extends CometBenchmarkBase { inSubsetPatterns.foreach { p => val query = rlikeQuery(p.pattern) runBenchmark(p.name) { - runInSubsetModes(p.name, v, query) + runInSubsetModes(p.name, v.toLong, query) } } outOfSubsetPatterns.foreach { p => val query = rlikeQuery(p.pattern) runBenchmark(p.name) { - runOutOfSubsetModes(p.name, v, query) + runOutOfSubsetModes(p.name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala index 26cc7f73f14..1d8d423f060 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala @@ -112,14 +112,14 @@ object CometRegExpExtractBenchmark extends CometBenchmarkBase { val extractQuery = s"select regexp_extract(c1, '${p.pattern}', ${p.idx}) from parquetV1Table" runBenchmark(extractName) { - runModes("RegExpExtract", extractName, v, extractQuery) + runModes("RegExpExtract", extractName, v.toLong, extractQuery) } val extractAllName = s"regexp_extract_all / ${p.name}" val extractAllQuery = s"select regexp_extract_all(c1, '${p.pattern}', ${p.idx}) from parquetV1Table" runBenchmark(extractAllName) { - runModes("RegExpExtractAll", extractAllName, v, extractAllQuery) + runModes("RegExpExtractAll", extractAllName, v.toLong, extractAllQuery) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala index 93f41ccf6b2..1b559ee6ddc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala @@ -70,7 +70,7 @@ object CometSequenceBenchmark extends CometBenchmarkBase { sequenceQueries.foreach { case (name, query) => runBenchmark(name) { - runExpressionBenchmark(name, v, query) + runExpressionBenchmark(name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala index 17d96571c14..0bb78d93f78 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala @@ -111,7 +111,9 @@ object CometShuffleBenchmark extends CometBenchmarkBase { def shuffleArrayBenchmark(values: Int, dataType: DataType, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"SQL ${dataType.sql} shuffle on array ($partitionNum Partition)", values) + microBenchmark( + s"SQL ${dataType.sql} shuffle on array ($partitionNum Partition)", + values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { @@ -159,7 +161,9 @@ object CometShuffleBenchmark extends CometBenchmarkBase { def shuffleStructBenchmark(values: Int, dataType: DataType, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"SQL ${dataType.sql} shuffle on struct ($partitionNum Partition)", values) + microBenchmark( + s"SQL ${dataType.sql} shuffle on struct ($partitionNum Partition)", + values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { @@ -214,7 +218,9 @@ object CometShuffleBenchmark extends CometBenchmarkBase { def shuffleDictionaryBenchmark(values: Int, dataType: DataType, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"SQL ${dataType.sql} Dictionary Shuffle($partitionNum Partition)", values) + microBenchmark( + s"SQL ${dataType.sql} Dictionary Shuffle($partitionNum Partition)", + values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { @@ -299,7 +305,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { val benchmark = microBenchmark( s"SQL Wide ($width cols) ${dataType.sql} Shuffle($partitionNum Partition)", - values) + values.toLong) val projection = (1 to width) .map(i => s"CAST(CAST(RAND(1) * 100 AS INTEGER) AS ${dataType.sql}) AS c$i") @@ -368,7 +374,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { val benchmark = microBenchmark( s"SQL Wide ($width cols) ${dataType.sql} Range Partition Shuffle($partitionNum Partition)", - values) + values.toLong) val projection = (1 to width) .map(i => s"CAST(CAST(RAND(1) * 100 AS INTEGER) AS ${dataType.sql}) AS c$i") @@ -435,7 +441,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { numRows: Int, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"Shuffle with nested schema ($name)", numRows) + microBenchmark(s"Shuffle with nested schema ($name)", numRows.toLong) val df = spark.read.parquet(filename) withTempTable("deeplyNestedTable") { df.createOrReplaceTempView("deeplyNestedTable") @@ -497,7 +503,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { values: Int, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"Nested hash key: $name ($partitionNum Partition)", values) + microBenchmark(s"Nested hash key: $name ($partitionNum Partition)", values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala index c0f976563a7..832fc28e67f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala @@ -72,12 +72,12 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id AS k", "id * 10 AS w") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -88,7 +88,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - inner count") { runExpressionBenchmark( "inner count", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ count(*) FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -96,7 +96,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - inner projected") { runExpressionBenchmark( "inner projected", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ p.k, p.v, b.w FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -104,7 +104,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - left outer") { runExpressionBenchmark( "left outer", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ count(*) FROM probe p LEFT JOIN build b ON p.k = b.k", cometConfigs) } @@ -112,7 +112,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - left semi") { runExpressionBenchmark( "left semi", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ count(*) FROM probe p LEFT SEMI JOIN build b ON p.k = b.k", cometConfigs) } @@ -120,7 +120,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - right outer") { runExpressionBenchmark( "right outer", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(p) */ count(*) FROM probe p RIGHT JOIN build b ON p.k = b.k", cometConfigs) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala index fe3c78dac93..aede98a129d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala @@ -127,7 +127,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase { (stringExpressions ++ collatedStringExpressions).foreach { config => val allConfigs = extraConfigs ++ config.extraCometConfigs runBenchmark(config.name) { - runExpressionBenchmark(config.name, v, config.query, allConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, allConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringFfiImportBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringFfiImportBenchmark.scala index 284438b40be..49c3cfde804 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringFfiImportBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringFfiImportBenchmark.scala @@ -137,7 +137,7 @@ object CometStringFfiImportBenchmark extends CometBenchmarkBase { /** A filter over a wide string column, with the column reaching native code from a scan. */ def scanImportBenchmark(values: Int, kind: String): Unit = { val benchmark = - new Benchmark(s"Filter over wide $kind strings", values, output = output) + new Benchmark(s"Filter over wide $kind strings", values.toLong, output = output) val query = "SELECT count(*) FROM parquetV1Table WHERE s LIKE '%999%'" benchmark.addCase("Spark") { _ => @@ -168,7 +168,7 @@ object CometStringFfiImportBenchmark extends CometBenchmarkBase { /** Wide string columns returned to Spark as rows. */ def columnarToRowImportBenchmark(values: Int, kind: String): Unit = { val benchmark = - new Benchmark(s"Columnar to row of wide $kind strings", values, output = output) + new Benchmark(s"Columnar to row of wide $kind strings", values.toLong, output = output) val query = "SELECT s FROM parquetV1Table" val nativeScan = cometConf :+ (CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "true") @@ -210,7 +210,7 @@ object CometStringFfiImportBenchmark extends CometBenchmarkBase { strings.foreach { case (kind, expr) => withTempPath { dir => withTempTable("parquetV1Table") { - prepareTable(dir, spark.range(values).selectExpr(s"$expr AS s")) + prepareTable(dir, spark.range(values.toLong).selectExpr(s"$expr AS s")) runBenchmark(s"FFI String Import - Scan ($kind)") { scanImportBenchmark(values, kind) diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala index 20ebfb9df8f..e70005fa063 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala @@ -88,7 +88,7 @@ object CometUnBase64Benchmark extends CometBenchmarkBase { shapes.foreach { s => val query = s"select unbase64(${s.column}) from parquetV1Table" runBenchmark(s.name) { - runUnBase64Modes(s.name, v, query) + runUnBase64Modes(s.name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnixTimestampBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnixTimestampBenchmark.scala index 85f2a30b14a..20d2aec2aa7 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnixTimestampBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnixTimestampBenchmark.scala @@ -34,7 +34,7 @@ object CometUnixTimestampBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(rows) + .range(rows.toLong) .selectExpr( "timestamp_seconds(id) AS ts", "date_format(timestamp_seconds(id), 'yyyy-MM-dd HH:mm:ss') AS s", @@ -48,7 +48,7 @@ object CometUnixTimestampBenchmark extends CometBenchmarkBase { runBenchmark(name) { runExpressionBenchmark( name, - rows, + rows.toLong, s"SELECT unix_timestamp($arguments) FROM parquetV1Table") } } @@ -57,7 +57,7 @@ object CometUnixTimestampBenchmark extends CometBenchmarkBase { runBenchmark(name) { runExpressionBenchmark( name, - rows, + rows.toLong, s"SELECT unix_timestamp($arguments) AS u, count(*) FROM parquetV1Table GROUP BY u") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index 4d648574494..b63db32128b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -28,11 +28,9 @@ import org.apache.commons.io.FileUtils import org.apache.spark.SparkContext import org.apache.spark.internal.config.{MEMORY_OFFHEAP_ENABLED, MEMORY_OFFHEAP_SIZE} import org.apache.spark.sql.TPCDSBase -import org.apache.spark.sql.catalyst.expressions.AttributeSet import org.apache.spark.sql.catalyst.util.resourceToString -import org.apache.spark.sql.execution.{ReusedSubqueryExec, SparkPlan, SubqueryBroadcastExec, SubqueryExec} import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite -import org.apache.spark.sql.execution.exchange.{Exchange, ReusedExchangeExec, ValidateRequirements} +import org.apache.spark.sql.execution.exchange.ValidateRequirements import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.TestSparkSession @@ -66,7 +64,6 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa getWorkspaceFilePath("spark", "src", "test", "resources", "tpcds-plan-stability").toFile } - private val referenceRegex = "#\\d+".r private val normalizeRegex = "#\\d+L?".r private val planIdRegex = "plan_id=\\d+".r @@ -182,64 +179,6 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa } } - /** - * Get the simplified plan for a specific SparkPlan. In the simplified plan, the node only has - * its name and all the sorted reference and produced attributes names(without ExprId) and its - * simplified children as well. And we'll only identify the performance sensitive nodes, e.g., - * Exchange, Subquery, in the simplified plan. Given such a identical but simplified plan, we'd - * expect to avoid frequent plan changing and catch the possible meaningful regression. - */ - private def getSimplifiedPlan(plan: SparkPlan): String = { - val exchangeIdMap = new mutable.HashMap[Int, Int]() - val subqueriesMap = new mutable.HashMap[Int, Int]() - - def getId(plan: SparkPlan): Int = plan match { - case exchange: Exchange => - exchangeIdMap.getOrElseUpdate(exchange.id, exchangeIdMap.size + 1) - case ReusedExchangeExec(_, exchange) => - exchangeIdMap.getOrElseUpdate(exchange.id, exchangeIdMap.size + 1) - case subquery: SubqueryExec => - subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1) - case subquery: SubqueryBroadcastExec => - subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1) - case ReusedSubqueryExec(subquery) => - subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1) - case _ => -1 - } - - /** - * Some expression names have ExprId in them due to using things such as - * "sum(sr_return_amt#14)", so we remove all of these using regex - */ - def cleanUpReferences(references: AttributeSet): String = { - referenceRegex.replaceAllIn(references.map(_.name).mkString(","), "") - } - - /** - * Generate a simplified plan as a string Example output: TakeOrderedAndProject - * [c_customer_id] WholeStageCodegen Project [c_customer_id] - */ - def simplifyNode(node: SparkPlan, depth: Int): String = { - val padding = " " * depth - var thisNode = node.nodeName - if (node.references.nonEmpty) { - thisNode += s" [${cleanUpReferences(node.references)}]" - } - if (node.producedAttributes.nonEmpty) { - thisNode += s" [${cleanUpReferences(node.producedAttributes)}]" - } - val id = getId(node) - if (id > 0) { - thisNode += s" #$id" - } - val childrenSimplified = node.children.map(simplifyNode(_, depth + 1)) - val subqueriesSimplified = node.subqueries.map(simplifyNode(_, depth + 1)) - s"$padding$thisNode\n${subqueriesSimplified.mkString("")}${childrenSimplified.mkString("")}" - } - - simplifyNode(plan, 0) - } - private def normalizeIds(plan: String): String = { val map = new mutable.HashMap[String, String]() normalizeRegex diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala index 64a12c0b7e6..319114e08b2 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala @@ -307,11 +307,11 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper { def insert(value: Int): Unit = { val bytes = new Array[Byte](4 + 16) - Platform.putInt(bytes, Platform.BYTE_ARRAY_OFFSET, value) + Platform.putInt(bytes, Platform.BYTE_ARRAY_OFFSET.toLong, value) val row = new UnsafeRow(1) - row.pointTo(bytes, Platform.BYTE_ARRAY_OFFSET + 4, 16) + row.pointTo(bytes, (Platform.BYTE_ARRAY_OFFSET + 4).toLong, 16) row.setInt(0, value) - sorter.insertRecord(bytes, Platform.BYTE_ARRAY_OFFSET, bytes.length, value % 2) + sorter.insertRecord(bytes, Platform.BYTE_ARRAY_OFFSET.toLong, bytes.length, value % 2) } try { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala index 3ad59e96ef2..f2ff9e97903 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala @@ -130,7 +130,7 @@ class CometArrowStreamSuite extends AnyFunSuite with Matchers { } requiredInput.putInt(i, -i) booleanInput.putBoolean(i, (i & 1) == 0) - val decimal = Decimal(i % 10000, decimalType.precision, decimalType.scale) + val decimal = Decimal((i % 10000).toLong, decimalType.precision, decimalType.scale) decimalInput.putDecimal(i, decimal, decimalType.precision) i += 1 } @@ -155,7 +155,7 @@ class CometArrowStreamSuite extends AnyFunSuite with Matchers { requiredArrow.get(i) shouldBe -i booleanArrow.get(i) shouldBe (if ((i & 1) == 0) 1 else 0) decimalArrow.getObject(i) shouldBe - Decimal(i % 10000, decimalType.precision, decimalType.scale).toJavaBigDecimal + Decimal((i % 10000).toLong, decimalType.precision, decimalType.scale).toJavaBigDecimal i += 1 } // A realloc frees the old buffers, so cumulative allocations would exceed live memory. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala index 60132c984fd..7603c22d784 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala @@ -315,7 +315,9 @@ private[shuffle] class ConcurrentRemoteMapStarts { } } -class CometCelebornConcurrentMaterializationTestManager(conf: SparkConf, isDriver: Boolean) +private[shuffle] class CometCelebornConcurrentMaterializationTestManager( + conf: SparkConf, + isDriver: Boolean) extends CometCelebornFallbackTestShuffleManager(conf, isDriver) { @volatile private[shuffle] var remoteMapStarts: Option[ConcurrentRemoteMapStarts] = None diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala index 85829401d4d..a2f8f1e41e5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala @@ -88,7 +88,9 @@ class CometCelebornLocalFetchFailureSuite extends CometTestBase { } /** Injects one local block loss only after Spark accepts another result partition. */ -class CometCelebornLocalFetchFailureTestManager(conf: SparkConf, isDriver: Boolean) +private[shuffle] class CometCelebornLocalFetchFailureTestManager( + conf: SparkConf, + isDriver: Boolean) extends CometCelebornFallbackTestShuffleManager(conf, isDriver) { // This local-mode fixture keeps state in its SparkContext-owned shuffle manager. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala index 826ac7e86df..de3381febfe 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala @@ -542,7 +542,7 @@ private[shuffle] class CompletedMaterializationJobs extends SparkListener { } } -class CometCelebornFallbackTestShuffleManager(conf: SparkConf, isDriver: Boolean) +private[shuffle] class CometCelebornFallbackTestShuffleManager(conf: SparkConf, isDriver: Boolean) extends CometCelebornShuffleManager( conf, isDriver, diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 7a9a4cb0637..8ce68239999 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -29,7 +29,7 @@ import org.apache.spark.executor.{ShuffleWriteMetrics, TaskMetrics} import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager} import org.apache.spark.shuffle.api.{ShuffleExecutorComponents, ShuffleMapOutputWriter, ShufflePartitionWriter} import org.apache.spark.shuffle.api.metadata.MapOutputCommitMessage -import org.apache.spark.shuffle.comet.{CometShuffleMemoryAllocator, CometShuffleMemoryAllocatorTrait} +import org.apache.spark.shuffle.comet.CometShuffleMemoryAllocator import org.apache.spark.shuffle.sort.SpillSorter import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{UnsafeProjection, UnsafeRow} @@ -195,23 +195,6 @@ class CometDiskBlockWriterSuite extends AnyFunSuite { } } - private def newWriter( - file: File, - allocator: CometShuffleMemoryAllocatorTrait, - taskContext: TaskContextImpl, - conf: SparkConf): CometDiskBlockWriter = { - new CometDiskBlockWriter( - file, - allocator, - taskContext, - new UnsafeRowSerializer(1).newInstance(), - schema, - new ShuffleWriteMetrics, - conf, - false, - new JLinkedList[CometDiskBlockWriter]()) - } - test("a fatal error during write() frees the task's buffered pages") { // Spark's ShuffleWriteProcessor only calls stop(false) when write() throws an Exception, so a // fatal error such as SparkOutOfMemoryError skips it. write() itself must therefore free the diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala index 20a2e1592a9..bfab3727cab 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala @@ -75,7 +75,7 @@ class CometNativePositionalRoundRobinSuite extends CometTestBase with AdaptiveSp withTempPath { dir => val path = dir.getAbsolutePath spark - .range(rows) + .range(rows.toLong) .selectExpr("id", "cast(id % 7 as string) as s", "id % 3 as g") .write .parquet(path)