From 0e23925889514f3f1fe3725b855dcb0c59fb0d02 Mon Sep 17 00:00:00 2001 From: Sudarshan Date: Sat, 12 Sep 2026 13:29:03 +0530 Subject: [PATCH 1/5] fix scala warnings --- pom.xml | 100 ++++++++++++++---- .../org/apache/comet/CometExecIterator.scala | 2 +- .../comet/iceberg/IcebergReflection.scala | 23 ++-- .../apache/comet/rules/CometExecRule.scala | 7 +- .../apache/comet/rules/CometScanRule.scala | 13 ++- .../apache/comet/serde/CometScalaUDF.scala | 2 +- .../apache/comet/serde/QueryPlanSerde.scala | 2 +- .../org/apache/comet/serde/literals.scala | 4 +- .../operator/CometIcebergNativeScan.scala | 6 +- .../CelebornShufflePusherFactory.scala | 7 +- .../udf/codegen/CometScalaUDFCodegen.scala | 2 +- .../scala/org/apache/spark/CometSource.scala | 6 +- .../main/scala/org/apache/spark/Plugins.scala | 8 +- .../spark/shuffle/sort/RowPartition.scala | 2 +- .../spark/sql/comet/CometBatchScanExec.scala | 2 +- .../comet/CometBroadcastExchangeExec.scala | 4 +- .../sql/comet/CometCollectLimitExec.scala | 2 +- .../sql/comet/CometColumnarToRowExec.scala | 6 +- .../sql/comet/CometIcebergWriteExec.scala | 5 +- .../comet/CometInMemoryTableScanExec.scala | 2 +- .../spark/sql/comet/CometMapInBatchExec.scala | 4 +- .../spark/sql/comet/CometMetricNode.scala | 10 +- .../comet/CometNativeColumnarToRowExec.scala | 6 +- .../sql/comet/CometNativeWriteExec.scala | 2 +- .../spark/sql/comet/CometScanExec.scala | 2 +- .../sql/comet/CometSparkToColumnarExec.scala | 2 +- .../comet/CometSubqueryBroadcastExec.scala | 4 +- .../CometTakeOrderedAndProjectExec.scala | 2 +- .../spark/sql/comet/IcebergCommitExec.scala | 4 +- .../CometBlockStoreShuffleReader.scala | 2 +- .../shuffle/CometCelebornShuffleManager.scala | 15 ++- .../shuffle/CometCelebornShuffleReader.scala | 4 +- .../shuffle/CometNativeShuffleWriter.scala | 18 +++- .../shuffle/CometShuffleExchangeExec.scala | 8 +- .../shuffle/CometShuffleManager.scala | 3 + .../sql/comet/shims/ShimCometUnionExec.scala | 2 +- .../sql/comet/shims/ShimCometUnionExec.scala | 2 +- .../apache/comet/shims/CometTypeShim.scala | 19 ++-- .../apache/comet/CometExpressionSuite.scala | 4 +- .../comet/CometIcebergNativeSuite.scala | 3 +- .../CometIcebergSystemFunctionSuite.scala | 2 +- .../apache/comet/CometNativeCastSuite.scala | 23 +--- .../apache/comet/exec/CometExecSuite.scala | 24 ++--- .../apache/comet/exec/CometJoinSuite.scala | 2 +- .../exec/CometNativeColumnarToRowSuite.scala | 2 +- .../comet/exec/CometNativeShuffleSuite.scala | 2 +- .../comet/objectstore/NativeConfigSuite.scala | 14 +-- .../parquet/CometParquetWriterSuite.scala | 2 +- .../comet/parquet/ParquetReadSuite.scala | 10 +- .../comet/rules/CometScanContribSuite.scala | 2 + .../CometExecIteratorLifecycleSuite.scala | 4 +- .../spark/shuffle/sort/SpillSorterSuite.scala | 22 ++-- .../apache/spark/sql/CometTPCQueryBase.scala | 2 +- .../org/apache/spark/sql/CometTestBase.scala | 30 +++--- .../org/apache/spark/sql/GenTPCHData.scala | 4 +- .../CometAggregateExpressionBenchmark.scala | 2 +- .../benchmark/CometArithmeticBenchmark.scala | 4 +- .../CometArrayExpressionBenchmark.scala | 10 +- .../benchmark/CometArrowWriterBenchmark.scala | 7 +- .../sql/benchmark/CometBenchmarkBase.scala | 4 +- .../CometBroadcastHashJoinBenchmark.scala | 14 +-- ...ometBroadcastNestedLoopJoinBenchmark.scala | 12 +-- .../benchmark/CometCastBooleanBenchmark.scala | 4 +- .../CometCastNumericToNumericBenchmark.scala | 2 +- .../CometCastNumericToStringBenchmark.scala | 2 +- .../CometCastNumericToTemporalBenchmark.scala | 4 +- .../CometCastStringToNumericBenchmark.scala | 2 +- .../CometCastStringToTemporalBenchmark.scala | 4 +- .../CometCastTemporalToNumericBenchmark.scala | 4 +- .../CometCastTemporalToStringBenchmark.scala | 4 +- ...CometCastTemporalToTemporalBenchmark.scala | 4 +- .../CometCodegenDispatchBenchmark.scala | 4 +- .../CometColumnarToRowBenchmark.scala | 46 ++++---- .../CometComparisonExpressionBenchmark.scala | 2 +- .../benchmark/CometConcatWsBenchmark.scala | 2 +- .../CometConditionalExpressionBenchmark.scala | 16 +-- .../CometCsvExpressionBenchmark.scala | 2 +- .../CometDatetimeExpressionBenchmark.scala | 21 ++-- .../sql/benchmark/CometExecBenchmark.scala | 16 +-- .../sql/benchmark/CometExplodeBenchmark.scala | 8 +- .../CometGetJsonObjectBenchmark.scala | 8 +- .../CometHashExpressionBenchmark.scala | 8 +- .../benchmark/CometHashJoinBenchmark.scala | 14 +-- .../benchmark/CometIcebergReadBenchmark.scala | 5 +- .../CometIcebergSystemFunctionBenchmark.scala | 2 +- .../CometInMemoryCacheBenchmark.scala | 15 +-- .../CometJsonExpressionBenchmark.scala | 2 +- .../CometLengthOfJsonArrayBenchmark.scala | 8 +- .../CometOperatorSerdeBenchmark.scala | 4 +- .../CometPaddingExpressionBenchmark.scala | 4 +- .../CometPartitionColumnBenchmark.scala | 2 +- .../CometPredicateExpressionBenchmark.scala | 2 +- .../sql/benchmark/CometReadBenchmark.scala | 39 ++++--- .../sql/benchmark/CometRegExpBenchmark.scala | 2 +- .../CometRegExpExtractBenchmark.scala | 4 +- .../benchmark/CometSequenceBenchmark.scala | 2 +- .../sql/benchmark/CometShuffleBenchmark.scala | 20 ++-- .../CometSortMergeJoinBenchmark.scala | 14 +-- .../CometStringExpressionBenchmark.scala | 2 +- .../benchmark/CometUnBase64Benchmark.scala | 2 +- .../sql/comet/CometPlanStabilitySuite.scala | 63 +---------- .../sql/comet/CometTaskMetricsSuite.scala | 6 +- .../arrow/CometArrowStreamSuite.scala | 4 +- ...lebornConcurrentMaterializationSuite.scala | 4 +- .../CometCelebornLocalFetchFailureSuite.scala | 4 +- .../CometCelebornShuffleFallbackSuite.scala | 2 +- 106 files changed, 482 insertions(+), 426 deletions(-) diff --git a/pom.xml b/pom.xml index f4b2be220ec..6afdba7360c 100644 --- a/pom.xml +++ b/pom.xml @@ -859,29 +859,85 @@ under the License. + - strict-warnings - - - - net.alchim31.maven - scala-maven-plugin - - - -deprecation - -unchecked - -feature - -Xlint:_ - -Ywarn-dead-code - -Ywarn-numeric-widen - -Ywarn-value-discard - -Ywarn-unused:imports,patvars,privates,locals,params,-implicits - -Xfatal-warnings - - - - - + strict-warnings + + + + net.alchim31.maven + scala-maven-plugin + + + scala-compile-first + + + -deprecation + -unchecked + -feature + -Xlint:_ + -Ywarn-dead-code + -Ywarn-numeric-widen + -Ywarn-value-discard + -Ywarn-unused:imports,patvars,privates,locals,-implicits + -Xfatal-warnings + + + + + scala-test-compile-first + + + -deprecation + -unchecked + -feature + -Xlint:_ + -Ywarn-dead-code + -Ywarn-numeric-widen + -Ywarn-unused:imports,patvars,privates,locals,-implicits + -Xfatal-warnings + + + + + + + diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala index e2c132904d5..22c17cb8e3d 100644 --- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala +++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala @@ -86,7 +86,7 @@ class CometExecIterator( private val nativeLib = new Native() private val nativeUtil = new NativeUtil() private val taskAttemptId = TaskContext.get().taskAttemptId() - private val taskCPUs = TaskContext.get().cpus() + private val taskCPUs = TaskContext.get().cpus().toLong private val cometTaskMemoryManager = new CometTaskMemoryManager(id, taskAttemptId) private val plan = { diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index ba41a957288..40334224966 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -465,7 +465,7 @@ object IcebergReflection extends Logging { groups.forEach { group => val groupTasks = groupTasksMethod.invoke(group).asInstanceOf[java.util.Collection[_ <: AnyRef]] - flat.addAll(groupTasks) + val _ = flat.addAll(groupTasks) } Some(flat) } @@ -1688,13 +1688,19 @@ object IcebergReflection extends Logging { private def newDataManifestFile(inputFile: AnyRef, specId: Int): AnyRef = { val inputFileClass = loadClass(ClassNames.INPUT_FILE) val cls = loadClass(ClassNames.GENERIC_MANIFEST_FILE) - val (ctor, args): (java.lang.reflect.Constructor[_], Array[Object]) = + // `Constructor[AnyRef]` rather than `Constructor[_]`: the two `try`/`catch` branches + // would otherwise infer a top-level existential, which `-Xlint:existential` rejects. + val (ctor, args): (java.lang.reflect.Constructor[AnyRef], Array[Object]) = try { - val c = cls.getDeclaredConstructor(inputFileClass, classOf[Int], classOf[Long]) + val c = cls + .getDeclaredConstructor(inputFileClass, classOf[Int], classOf[Long]) + .asInstanceOf[java.lang.reflect.Constructor[AnyRef]] (c, Array[Object](inputFile, Integer.valueOf(specId), java.lang.Long.valueOf(0L))) } catch { case _: NoSuchMethodException => - val c = cls.getDeclaredConstructor(inputFileClass, classOf[Int]) + val c = cls + .getDeclaredConstructor(inputFileClass, classOf[Int]) + .asInstanceOf[java.lang.reflect.Constructor[AnyRef]] (c, Array[Object](inputFile, Integer.valueOf(specId))) } ctor.setAccessible(true) @@ -1730,10 +1736,11 @@ object IcebergReflection extends Logging { } result } finally { - try reader.getClass.getMethod("close").invoke(reader) - catch { - case e: Exception => logWarning(s"Failed to close ManifestReader: ${e.getMessage}") - } + val _ = + try reader.getClass.getMethod("close").invoke(reader) + catch { + case e: Exception => logWarning(s"Failed to close ManifestReader: ${e.getMessage}") + } } } diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index 58eab77e3b5..f2d0a513ee0 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -911,7 +911,7 @@ case class CometExecRule(session: SparkSession) .flatten .toSet if (reasons.nonEmpty) { - withFallbackReasons(op, reasons) + val _ = withFallbackReasons(op, reasons) } } @@ -943,7 +943,7 @@ case class CometExecRule(session: SparkSession) "operator or any of its expressions. Add a withFallbackReason call stating why " + s"conversion failed. Operator:\n$op") } - withFallbackReason(op, s"${op.nodeName} is not supported") + val _ = withFallbackReason(op, s"${op.nodeName} is not supported") } } @@ -972,7 +972,8 @@ case class CometExecRule(session: SparkSession) CometExplainInfo.collectExprTagValues(allExprs, CometExplainInfo.CODEGEN_DISPATCH_EXPRS) appendTagValues(exec, CometExplainInfo.CODEGEN_DISPATCH_EXPRS, routedNames) if (routedNames.nonEmpty && CometConf.COMET_EXPLAIN_CODEGEN_ENABLED.get()) { - withInfo(exec, s"JVM codegen dispatcher: ${routedNames.toSeq.sorted.mkString(", ")}") + val _ = + withInfo(exec, s"JVM codegen dispatcher: ${routedNames.toSeq.sorted.mkString(", ")}") } } diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index e3b6f77eb8e..193af4e684b 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -31,7 +31,7 @@ import scala.jdk.CollectionConverters._ import org.apache.hadoop.conf.Configuration import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.expressions.{Attribute, DynamicPruningExpression, Expression, GenericInternalRow, InputFileBlockLength, InputFileBlockStart, InputFileName, PlanExpression} +import org.apache.spark.sql.catalyst.expressions.{Attribute, DynamicPruningExpression, Expression, GenericInternalRow, InputFileBlockLength, InputFileBlockStart, InputFileName} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.{sideBySide, ArrayBasedMapData, GenericArrayData, MetadataColumnHelper} import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues @@ -486,11 +486,17 @@ case class CometScanRule(session: SparkSession) // which bucket's per-bucket S3 settings the single native FileIO consumes (see // catalogProperties below), so this must run before we build them. The task list is // handed to `extract` below so the reflective accessor runs once per scan. + // The task list is cast to `java.util.List[AnyRef]` rather than left at the + // `java.util.List[_]` that `getTasks` returns: a wildcard element type makes this + // pair infer a top-level existential, which `-Xlint:existential` rejects. The list + // is erased at runtime, so the cast costs nothing. val (icebergTasks, taskValidation) = try { IcebergReflection.getTasks(scanExec.scan) match { case Some(tasks) => - (tasks, CometScanRule.validateIcebergFileScanTasks(tasks, s3CompliantSchemes)) + ( + tasks.asInstanceOf[java.util.List[AnyRef]], + CometScanRule.validateIcebergFileScanTasks(tasks, s3CompliantSchemes)) case None => fallbackReasons += "Iceberg reflection failure: Could not extract FileScanTasks" return withFallbackReasons(scanExec, fallbackReasons.toSet) @@ -1043,9 +1049,6 @@ case class CometScanRule(session: SparkSession) } } - private def isDynamicPruningFilter(e: Expression): Boolean = - e.exists(_.isInstanceOf[PlanExpression[_]]) - /** * Detects AQE DPP (SubqueryAdaptiveBroadcastExec), as opposed to non-AQE DPP. * diff --git a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala index 9249ab280f0..57d609c158b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala @@ -181,7 +181,7 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] { target.foreach { case _: AttributeReference | _: Literal => case node if !(node eq target) => - withCodegenDispatchExpr(expr, CometExplainInfo.exprDisplayName(node)) + val _ = withCodegenDispatchExpr(expr, CometExplainInfo.exprDisplayName(node)) case _ => } Some( diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index be4bc9c3412..1da6937357a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -911,7 +911,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { e.getTagValue(CometExplainInfo.FALLBACK_REASONS).foreach(reasons ++= _) } if (reasons.nonEmpty) { - withFallbackReasons(to, reasons.toSet) + val _ = withFallbackReasons(to, reasons.toSet) } } diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index b368f80320c..98a3757ffc8 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -86,8 +86,8 @@ object CometLiteral extends CometExpressionSerde[Literal] with CometTypeShim wit exprBuilder.setIsNull(false) dataType match { case _: BooleanType => exprBuilder.setBoolVal(value.asInstanceOf[Boolean]) - case _: ByteType => exprBuilder.setByteVal(value.asInstanceOf[Byte]) - case _: ShortType => exprBuilder.setShortVal(value.asInstanceOf[Short]) + case _: ByteType => exprBuilder.setByteVal(value.asInstanceOf[Byte].toInt) + case _: ShortType => exprBuilder.setShortVal(value.asInstanceOf[Short].toInt) case _: IntegerType | _: DateType => exprBuilder.setIntVal(value.asInstanceOf[Int]) case _: LongType | _: TimestampType | _: TimestampNTZType | _: DayTimeIntervalType => exprBuilder.setLongVal(value.asInstanceOf[Long]) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala index ac2538288e2..24add22cb25 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala @@ -353,7 +353,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit val equalityIds = equalityIdsMethod .invoke(deleteFile) .asInstanceOf[java.util.List[Integer]] - equalityIds.forEach(id => deleteBuilder.addEqualityIds(id)) + equalityIds.forEach(id => { val _ = deleteBuilder.addEqualityIds(id) }) } catch { case _: Exception => } @@ -515,7 +515,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit commonBuilder.addPartitionTypePool(partitionTypeJson) idx }) - taskBuilder.setPartitionSpecIdx(specIdx) + val _ = taskBuilder.setPartitionSpecIdx(specIdx) } catch { case e: Exception => logWarning(s"Failed to serialize partition spec to JSON: ${e.getMessage}") @@ -575,7 +575,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit commonBuilder.addPartitionDataPool(partitionDataProto) idx }) - taskBuilder.setPartitionDataIdx(partitionDataIdx) + val _ = taskBuilder.setPartitionDataIdx(partitionDataIdx) } else { // Defensive: ContentScanTask.partition() returns an empty struct (never null) for // unpartitioned tables in practice. If it is ever null we cannot compute values, so diff --git a/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala b/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala index 37d67f55ca4..27ecec8f559 100644 --- a/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala +++ b/spark/src/main/scala/org/apache/comet/shuffle/CelebornShufflePusherFactory.scala @@ -246,7 +246,7 @@ object CelebornShufflePusherFactory { client: AnyRef, taskContext: TaskContext, handle: ShuffleHandle): Unit = { - sparkUtilsClass + val _ = sparkUtilsClass .getMethod( "addFailureListenerIfBarrierTask", shuffleClientClass, @@ -336,7 +336,7 @@ object CelebornShufflePusherFactory { /** Remove task-independent state for one unregistered Celeborn generation. */ def cleanupShuffle(client: AnyRef, celebornShuffleId: Int): Unit = { try { - client.getClass + val _ = client.getClass .getMethod("cleanupShuffle", java.lang.Integer.TYPE) .invoke(client, Int.box(celebornShuffleId)) } catch { @@ -354,7 +354,8 @@ object CelebornShufflePusherFactory { try { val shuffleClientClass = ClassLoaders.loadClass(CELEBORN_SHUFFLE_CLIENT) try { - shuffleClientClass.getMethod("removeInstance", shuffleClientClass).invoke(null, client) + val _ = + shuffleClientClass.getMethod("removeInstance", shuffleClientClass).invoke(null, client) } catch { case _: NoSuchMethodException => // Celeborn 0.6 owns one shared application client and cannot remove a single instance. diff --git a/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala b/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala index ef541fb3e2d..1cab882df93 100644 --- a/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala +++ b/spark/src/main/scala/org/apache/comet/udf/codegen/CometScalaUDFCodegen.scala @@ -296,7 +296,7 @@ object CometScalaUDFCodegen { private[codegen] def recordCompiledSignature( specs: IndexedSeq[ArrowColumnSpec], outputType: DataType): Unit = { - compiledSignatures.add((specs.map(_.vectorClass), outputType)) + val _ = compiledSignatures.add((specs.map(_.vectorClass), outputType)) } /** diff --git a/spark/src/main/scala/org/apache/spark/CometSource.scala b/spark/src/main/scala/org/apache/spark/CometSource.scala index 95d75236166..6581cb2018a 100644 --- a/spark/src/main/scala/org/apache/spark/CometSource.scala +++ b/spark/src/main/scala/org/apache/spark/CometSource.scala @@ -54,9 +54,9 @@ object CometSource extends Source { }) def recordStats(stats: CometCoverageStats): Unit = { - NATIVE_OPERATORS.inc(stats.cometOperators) - SPARK_OPERATORS.inc(stats.sparkOperators) - TRANSITIONS.inc(stats.transitions) + NATIVE_OPERATORS.inc(stats.cometOperators.toLong) + SPARK_OPERATORS.inc(stats.sparkOperators.toLong) + TRANSITIONS.inc(stats.transitions.toLong) QUERIES_PLANNED.inc() } } diff --git a/spark/src/main/scala/org/apache/spark/Plugins.scala b/spark/src/main/scala/org/apache/spark/Plugins.scala index eaeac316655..3e80b55c7f1 100644 --- a/spark/src/main/scala/org/apache/spark/Plugins.scala +++ b/spark/src/main/scala/org/apache/spark/Plugins.scala @@ -187,13 +187,13 @@ object CometDriverPlugin extends Logging { val listeners = sc.conf.get(listenerKey, "") if (listeners.isEmpty) { logInfo(s"Setting $listenerKey=$listenerClass") - sc.conf.set(listenerKey, listenerClass) + val _ = sc.conf.set(listenerKey, listenerClass) } else { val currentListeners = listeners.split(",").map(_.trim) if (!currentListeners.contains(listenerClass)) { val newValue = s"$listeners,$listenerClass" logInfo(s"Setting $listenerKey=$newValue") - sc.conf.set(listenerKey, newValue) + val _ = sc.conf.set(listenerKey, newValue) } } } else { @@ -208,13 +208,13 @@ object CometDriverPlugin extends Logging { val extensions = conf.get(extensionKey, "") if (extensions.isEmpty) { logInfo(s"Setting $extensionKey=$extensionClass") - conf.set(extensionKey, extensionClass) + val _ = conf.set(extensionKey, extensionClass) } else { val currentExtensions = extensions.split(",").map(_.trim) if (!currentExtensions.contains(extensionClass)) { val newValue = s"$extensions,$extensionClass" logInfo(s"Setting $extensionKey=$newValue") - conf.set(extensionKey, newValue) + val _ = conf.set(extensionKey, newValue) } } } diff --git a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala index d3536850f68..887dae05bca 100644 --- a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala +++ b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala @@ -27,7 +27,7 @@ class RowPartition(initialSize: Int) { def addRow(addr: Long, size: Int): Unit = { rowAddresses += addr - rowSizes += size + val _ = rowSizes += size } def getNumRows: Int = if (rowAddresses == null) { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala index d900889e176..4f807dc1c79 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBatchScanExec.scala @@ -76,7 +76,7 @@ case class CometBatchScanExec( override def next(): ColumnarBatch = { val batch = batches.next() - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong batch } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala index 9ae92845c9c..615f7bada26 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala @@ -107,7 +107,7 @@ case class CometBroadcastExchangeExec( private val timeout: Long = conf.broadcastTimeout @transient - private lazy val maxBroadcastRows = 512000000 + private lazy val maxBroadcastRows = 512000000L private def getByteArrayRdd(plan: SparkPlan): RDD[(Long, ChunkedByteBuffer)] = { plan.executeColumnar().mapPartitionsInternal { iter => @@ -195,7 +195,7 @@ case class CometBroadcastExchangeExec( override protected def doPrepare(): Unit = { // Materialize the future. - relationFuture + val _ = relationFuture } override protected def doExecute(): RDD[InternalRow] = { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala index 6d43fe156b2..4d560ef8d3f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometCollectLimitExec.scala @@ -112,7 +112,7 @@ case class CometCollectLimitExec( outputPartitioning, serializer, metrics) - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) new CometShuffledBatchRDD(dep, readMetrics) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala index 2fe870ed069..da1d9ba296f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala @@ -80,7 +80,7 @@ case class CometColumnarToRowExec(child: SparkPlan) val toUnsafe = UnsafeProjection.create(localOutput, localOutput) batches.flatMap { batch => numInputBatches += 1 - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong batch.rowIterator().asScala.map(toUnsafe) } } @@ -111,14 +111,14 @@ case class CometColumnarToRowExec(child: SparkPlan) val numOutputRows = longMetric("numOutputRows") val numInputBatches = longMetric("numInputBatches") val localOutput = this.output - val broadcastColumnar = child.executeBroadcast() + val broadcastColumnar = child.executeBroadcast[Any]() val serializedBatches = broadcastColumnar.value.asInstanceOf[Array[ChunkedByteBuffer]] val toUnsafe = UnsafeProjection.create(localOutput, localOutput) val rows = serializedBatches.iterator .flatMap(CometUtils.decodeBatches(_, this.getClass.getSimpleName)) .flatMap { batch => numInputBatches += 1 - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong batch.rowIterator().asScala.map(toUnsafe) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala index 00a5047625f..c95c0d41e46 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala @@ -373,7 +373,8 @@ object CometIcebergWriteExec { /** Take ownership of the locations the native writer reported. */ def own(written: Seq[String]): Unit = locations = written - override def onTaskFailure(context: TaskContext, error: Throwable): Unit = - IcebergReflection.deleteFilesQuietly(io, locations, describeTask) + override def onTaskFailure(context: TaskContext, error: Throwable): Unit = { + val _ = IcebergReflection.deleteFilesQuietly(io, locations, describeTask) + } } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala index 0f6f531fdc3..103f95f2e53 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala @@ -123,7 +123,7 @@ case class CometInMemoryTableScanExec( serializer .convertCachedBatchToColumnarBatch(filteredBuffers, relationOutput, scanOutput, conf) .map { cb => - numOutputRows += cb.numRows() + numOutputRows += cb.numRows().toLong cb } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala index 8ac9a70de3e..8e6ff6c2ad3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala @@ -91,7 +91,7 @@ case class CometMapInBatchExec( def processPartition(batches: Iterator[ColumnarBatch]): Iterator[ColumnarBatch] = { val context = TaskContext.get() - val counting = batches.map { b => numInputRows += b.numRows(); b } + val counting = batches.map { b => numInputRows += b.numRows().toLong; b } val columnarBatchIter = computeArrowPython( resolvedRunnerInputs, @@ -113,7 +113,7 @@ case class CometMapInBatchExec( outputAttrs.indices.map(i => structVector.getChild(i)).toArray val flattenedBatch = new ColumnarBatch(outputVectors) flattenedBatch.setNumRows(batch.numRows()) - numOutputRows += flattenedBatch.numRows() + numOutputRows += flattenedBatch.numRows().toLong numOutputBatches += 1 flattenedBatch } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala index 0308779ce07..e8f0008c2c6 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala @@ -110,7 +110,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM * be called in a TaskCompletionListener after the iterator is fully consumed. */ def reportScanInputMetrics(ctx: TaskContext): Unit = { - ctx.addTaskCompletionListener[Unit] { _ => + val _ = ctx.addTaskCompletionListener[Unit] { _ => val scanLeaves = leafNodes.filter(_.metrics.contains("bytes_scanned")) if (scanLeaves.nonEmpty) { val totalBytes = scanLeaves.map(_.metrics("bytes_scanned").value).sum @@ -137,7 +137,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM * before this listener runs. */ def reportNativeWriteOutputMetrics(ctx: TaskContext): Unit = { - ctx.addTaskCompletionListener[Unit] { _ => + val _ = ctx.addTaskCompletionListener[Unit] { _ => metrics.get("bytes_written").foreach { m => ctx.taskMetrics().outputMetrics.setBytesWritten(m.value) } @@ -162,7 +162,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM */ def reportSpillMetrics(ctx: TaskContext): Unit = { val seenMetrics = CometMetricNode.taskSeenSpillMetrics(ctx) - ctx.addTaskCompletionListener[Unit] { _ => + val _ = ctx.addTaskCompletionListener[Unit] { _ => val diskBytesSpilled = sumMetricValues("spilled_bytes", seenMetrics.disk) if (diskBytesSpilled > 0L) { ctx.taskMetrics().incDiskBytesSpilled(diskBytesSpilled) @@ -244,7 +244,9 @@ object CometMetricNode { // The task thread is the only registrant for its attempt id, so there is no put race. val created = SeenSpillMetrics(new IdentityHashMap(), new IdentityHashMap()) seenSpillMetricsByTask.put(attemptId, created) - ctx.addTaskCompletionListener[Unit](_ => seenSpillMetricsByTask.remove(attemptId)) + ctx.addTaskCompletionListener[Unit](_ => { + val _ = seenSpillMetricsByTask.remove(attemptId) + }) created } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala index ce800ddf098..0c3eea71f8b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeColumnarToRowExec.scala @@ -97,7 +97,7 @@ case class CometNativeColumnarToRowExec(child: SparkPlan) val numInputBatches = longMetric("numInputBatches") val localSchema = this.schema val batchSize = CometConf.COMET_BATCH_SIZE.get() - val broadcastColumnar = child.executeBroadcast() + val broadcastColumnar = child.executeBroadcast[Any]() val serializedBatches = broadcastColumnar.value.asInstanceOf[Array[org.apache.spark.util.io.ChunkedByteBuffer]] @@ -108,7 +108,7 @@ case class CometNativeColumnarToRowExec(child: SparkPlan) .flatMap(CometUtils.decodeBatches(_, this.getClass.getSimpleName)) .flatMap { batch => numInputBatches += 1 - numOutputRows += batch.numRows() + numOutputRows += batch.numRows().toLong val result = converter.convert(batch) // Wrap iterator to close batch after consumption new Iterator[InternalRow] { @@ -202,7 +202,7 @@ case class CometNativeColumnarToRowExec(child: SparkPlan) batches.flatMap { batch => numInputBatches += 1 val numRows = batch.numRows() - numOutputRows += numRows + numOutputRows += numRows.toLong val startTime = System.nanoTime() val result = converter.convert(batch) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala index f0d10b17667..ced525c1869 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeWriteExec.scala @@ -188,7 +188,7 @@ case class CometNativeWriteExec( val taskAttemptId = org.apache.spark.TaskContext.get().taskAttemptId() // Setup task-level commit protocol if provided - val (workDir, taskContext, commitMsg) = capturedCommitter + val (workDir, taskContext, _) = capturedCommitter .map { committer => val taskContext = createTaskContext(capturedJobTrackerID, partitionId, taskAttemptId.toInt) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala index f88e3d380e9..3e1beec716d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala @@ -228,7 +228,7 @@ case class CometScanExec( driverMetrics("staticFilesSize") = filesSize } if (relation.partitionSchema.nonEmpty) { - driverMetrics("numPartitions") = partitions.length + driverMetrics("numPartitions") = partitions.length.toLong } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala index ed4dc84c9cc..9b74b6ddb69 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometSparkToColumnarExec.scala @@ -105,7 +105,7 @@ case class CometSparkToColumnarExec(child: SparkPlan) CometArrowStream .countingIterator( sparkBatches, - (b: ColumnarBatch) => numInputRows.add(b.numRows())), + (b: ColumnarBatch) => numInputRows.add(b.numRows().toLong)), maxBatchInt, onConversionNs)) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala index 61254794489..419b99aef70 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometSubqueryBroadcastExec.scala @@ -128,7 +128,7 @@ case class CometSubqueryBroadcastExec( val beforeBuild = System.nanoTime() longMetric("collectTime") += (beforeBuild - beforeCollect) / 1000000 - longMetric("numOutputRows") += rows.length + longMetric("numOutputRows") += rows.length.toLong val dataSize = rows.map(_.asInstanceOf[UnsafeRow].getSizeInBytes.toLong).sum longMetric("dataSize") += dataSize SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq) @@ -139,7 +139,7 @@ case class CometSubqueryBroadcastExec( } protected override def doPrepare(): Unit = { - relationFuture + val _ = relationFuture } protected override def doExecute(): RDD[InternalRow] = { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala index 09dd944d93e..4cb5047f7d7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala @@ -160,7 +160,7 @@ case class CometTakeOrderedAndProjectExec( outputPartitioning, serializer, metrics) - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) new CometShuffledBatchRDD(dep, readMetrics) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala index df3c38b3de3..90a8d0dd370 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/IcebergCommitExec.scala @@ -102,7 +102,7 @@ case class IcebergCommitExec( } throw cause } - longMetric("numCommittedMessages").add(messages.length) + longMetric("numCommittedMessages").add(messages.length.toLong) try { messages.foreach(batchWrite.onDataWriterCommit) @@ -132,7 +132,7 @@ case class IcebergCommitExec( .flatMap(IcebergReflection.getTableIO) io match { case Some(fileIO) => - IcebergReflection.deleteFilesQuietly( + val _ = IcebergReflection.deleteFilesQuietly( fileIO, locations, s"job abort, ${completed.length} completed task(s)") diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala index 3048456ea78..3592095a37c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala @@ -121,7 +121,7 @@ class CometBlockStoreShuffleReader[K, C]( // Update the context task metrics for each record read. val metricIter = CompletionIterator[(Any, Any), Iterator[(Any, Any)]]( recordIter.map { record => - readMetrics.incRecordsRead(record._2.numRows()) + readMetrics.incRecordsRead(record._2.numRows().toLong) record }, context.taskMetrics().mergeShuffleReadMetrics()) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala index 71fec591ee8..2fdda64582b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManager.scala @@ -23,6 +23,7 @@ import java.lang.reflect.InvocationTargetException import java.util.Locale import java.util.concurrent.ConcurrentHashMap +import scala.annotation.nowarn import scala.collection.mutable import scala.concurrent.ExecutionContext import scala.jdk.CollectionConverters._ @@ -124,6 +125,7 @@ class CometCelebornShuffleManager private[shuffle] ( } } + @nowarn("msg=references private") override def getWriter[K, V]( handle: ShuffleHandle, mapId: Long, @@ -212,6 +214,7 @@ class CometCelebornShuffleManager private[shuffle] ( } // Spark's final all-mapper reader overload delegates to this mapper-range overload. + @nowarn("msg=references private") override def getReader[K, C]( handle: ShuffleHandle, startMapIndex: Int, @@ -256,9 +259,9 @@ class CometCelebornShuffleManager private[shuffle] ( endPartition), context, metrics, - client => ownedNativeClients.put(client, java.lang.Boolean.TRUE), + client => { val _ = ownedNativeClients.put(client, java.lang.Boolean.TRUE) }, (client, celebornShuffleId) => { - nativeShuffleClients + val _ = nativeShuffleClients .computeIfAbsent(handle.shuffleId, _ => new ConcurrentHashMap[Int, AnyRef]()) .put(celebornShuffleId, client) }, @@ -392,7 +395,7 @@ class CometCelebornShuffleManager private[shuffle] ( conf, handle, context, - client => ownedNativeClients.put(client, java.lang.Boolean.TRUE), + client => { val _ = ownedNativeClients.put(client, java.lang.Boolean.TRUE) }, onGenerationResolved, onGenerationInvalidated, onInvalidationUnsafe) @@ -411,7 +414,7 @@ class CometCelebornShuffleManager private[shuffle] ( } private[shuffle] def removeSizeLimitFallback(shuffleId: Int): Unit = { - sizeLimitFallbacks.remove(shuffleId) + val _ = sizeLimitFallbacks.remove(shuffleId) } private def isLocalNativeHandle(handle: ShuffleHandle): Boolean = handle match { @@ -1070,7 +1073,9 @@ private[shuffle] final class CelebornShuffleGenerationCoordinator( invalidatedGenerations.remove(shuffleId) generationEpochs.remove(shuffleId) claimOwners.retain { case ((ownerShuffleId, _, _, _), _) => ownerShuffleId != shuffleId } - deniedAttempts.retain { case ((ownerShuffleId, _, _, _), _) => ownerShuffleId != shuffleId } + val _ = deniedAttempts.retain { case ((ownerShuffleId, _, _, _), _) => + ownerShuffleId != shuffleId + } } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala index 891290a441e..b7deac55b49 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReader.scala @@ -143,7 +143,7 @@ private[shuffle] final class CometCelebornShuffleReader[K, C]( } val rows = decoder.map { batch: ColumnarBatch => - readMetrics.incRecordsRead(batch.numRows()) + readMetrics.incRecordsRead(batch.numRows().toLong) (0, batch) } val completed = CompletionIterator[(Int, ColumnarBatch), Iterator[(Int, ColumnarBatch)]]( @@ -511,7 +511,7 @@ private[shuffle] object CelebornRawPartitionReader { java.util.Objects.checkFromIndexSize(offset, length, buffer.length) if (length == 0) return 0 val readingHeader = bodyRemaining == 0 - val remaining = if (readingHeader) header.length - headerBytes else bodyRemaining + val remaining = if (readingHeader) (header.length - headerBytes).toLong else bodyRemaining val count = in.read(buffer, offset, math.min(length.toLong, remaining).toInt) if (count < 0) { if (headerBytes != 0 || bodyRemaining != 0) { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala index 21dd7686302..7fe3ac0ef7f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala @@ -72,7 +72,10 @@ class CometNativeShuffleWriter[K, V]( private val OFFSET_LENGTH = 8 var partitionLengths: Array[Long] = _ - var mapStatus: MapStatus = _ + // `private[shuffle]` rather than public: `MapStatus` is `private[spark]`, and a public + // var exposing it trips `-Xlint:inaccessible`. The shuffle package, which includes the + // tests that read it, is the only thing that ever touches this. + private[shuffle] var mapStatus: MapStatus = _ private var stopped = false private lazy val effectivePartitionCount = remoteDestination.map(_.numPartitions).getOrElse(outputPartitioning.numPartitions) @@ -118,7 +121,7 @@ class CometNativeShuffleWriter[K, V]( val dataFile = resolver.getDataFile(shuffleId, mapId) val indexFile = resolver.getIndexFile(shuffleId, mapId) Some( - LocalShuffleOutput( + CometNativeShuffleWriter.LocalShuffleOutput( resolver, dataFile.getPath.replace(".data", ".data.tmp"), indexFile.getPath.replace(".index", ".index.tmp"))) @@ -479,14 +482,19 @@ class CometNativeShuffleWriter[K, V]( } override def getPartitionLengths(): Array[Long] = partitionLengths +} + +private[shuffle] object CometNativeShuffleWriter { - private final case class LocalShuffleOutput( + /** + * Declared here rather than inside the class: as an inner case class every type test against it + * carries an outer reference that cannot be checked at run time, which `-Xlint` reports. + */ + private[shuffle] final case class LocalShuffleOutput( resolver: IndexShuffleBlockResolver, dataFile: String, indexFile: String) -} -private[shuffle] object CometNativeShuffleWriter { private[shuffle] def isSizeLimitFailure(failure: Throwable): Boolean = { var cause = failure val visited = new java.util.IdentityHashMap[Throwable, java.lang.Boolean]() diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala index ee3333337fa..41554ed0532 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala @@ -215,7 +215,7 @@ case class CometShuffleExchangeExec( serializer, metrics) } - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) SQLMetrics.postDriverMetricUpdates( sparkContext, @@ -229,7 +229,7 @@ case class CometShuffleExchangeExec( outputPartitioning, serializer, metrics) - metrics("numPartitions").set(dep.partitioner.numPartitions) + metrics("numPartitions").set(dep.partitioner.numPartitions.toLong) val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) SQLMetrics.postDriverMetricUpdates( sparkContext, @@ -1040,7 +1040,7 @@ object CometShuffleExchangeExec // end up being almost the same regardless of the index. substantially scrambling the // seed by hashing will help. Refer to SPARK-21782 for more details. val partitionId = TaskContext.get().partitionId() - var position = new XORShiftRandom(partitionId).nextInt(numPartitions) + var position = new XORShiftRandom(partitionId.toLong).nextInt(numPartitions) (_: InternalRow) => { // The HashPartitioner will handle the `mod` by the number of partitions position += 1 @@ -1087,7 +1087,7 @@ object CometShuffleExchangeExec row: InternalRow): UnsafeExternalRowSorter.PrefixComputer.Prefix = { // The hashcode generated from the binary form of a [[UnsafeRow]] should not be null. result.isNull = false - result.value = row.hashCode() + result.value = row.hashCode().toLong result } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala index 398ed66b6a5..546a1d40427 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala @@ -21,6 +21,7 @@ package org.apache.spark.sql.comet.execution.shuffle import java.util.concurrent.ConcurrentHashMap +import scala.annotation.nowarn import scala.jdk.CollectionConverters._ import org.apache.spark.ShuffleDependency @@ -164,6 +165,7 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { } } + @nowarn("msg=references private") override def getReader[K, C]( handle: ShuffleHandle, startMapIndex: Int, @@ -216,6 +218,7 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { } /** Get a writer for a given partition. Called on executors by map tasks. */ + @nowarn("msg=references private") override def getWriter[K, V]( handle: ShuffleHandle, mapId: Long, diff --git a/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala b/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala index 15c480b057c..57db09d5f38 100644 --- a/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala +++ b/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala @@ -37,7 +37,7 @@ object ShimCometUnionExec { def unionRDDs[T: ClassTag]( sc: SparkContext, rdds: Seq[RDD[T]], - @annotation.nowarn("cat=unused") outputPartitioning: Partitioning): RDD[T] = { + outputPartitioning: Partitioning): RDD[T] = { sc.union(rdds) } } diff --git a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala index 15c480b057c..57db09d5f38 100644 --- a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala +++ b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometUnionExec.scala @@ -37,7 +37,7 @@ object ShimCometUnionExec { def unionRDDs[T: ClassTag]( sc: SparkContext, rdds: Seq[RDD[T]], - @annotation.nowarn("cat=unused") outputPartitioning: Partitioning): RDD[T] = { + outputPartitioning: Partitioning): RDD[T] = { sc.union(rdds) } } diff --git a/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala index 1e5d1686f96..567999fc1c6 100644 --- a/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala +++ b/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala @@ -22,40 +22,37 @@ package org.apache.comet.shims import java.nio.ByteBuffer import java.nio.charset.{CharacterCodingException, CodingErrorAction, StandardCharsets} -import scala.annotation.nowarn - import org.apache.spark.sql.catalyst.expressions.aggregate.Mode import org.apache.spark.sql.types.{DataType, StructType} import org.apache.spark.unsafe.types.UTF8String trait CometTypeShim { - @nowarn // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. + // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. def isStringCollationType(dt: DataType): Boolean = false // `mode() WITHIN GROUP (ORDER BY ...)` and the deterministic-flag form (which set `reverseOpt`) // are Spark 4.0 features; Spark 3.x `Mode` is always the plain `mode(col)` form. - @nowarn def modeHasUnsupportedOrdering(expr: Mode): Boolean = false - @nowarn // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. + // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. def hasNonDefaultStringCollation(dt: DataType): Boolean = false - @nowarn // Spark 4 feature; collation does not exist in Spark 3.x. + // Spark 4 feature; collation does not exist in Spark 3.x. def hasCollationSupport: Boolean = false - @nowarn // Spark 4 feature; Variant shredding doesn't exist in Spark 3.x. + // Spark 4 feature; Variant shredding doesn't exist in Spark 3.x. def isVariantStruct(s: StructType): Boolean = false - @nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x. + // Spark 4 feature; VariantType doesn't exist in Spark 3.x. def isVariantType(dt: DataType): Boolean = false - @nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x. + // Spark 4 feature; VariantType doesn't exist in Spark 3.x. def containsVariantType(dt: DataType): Boolean = false - @nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x. + // Spark 4 feature; VariantType doesn't exist in Spark 3.x. def variantType: Option[DataType] = None - @nowarn // Spark 4.1 feature; TimeType doesn't exist in Spark 3.x. + // Spark 4.1 feature; TimeType doesn't exist in Spark 3.x. def isTimeType(dt: DataType): Boolean = false /** diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 4ab86b4f373..0ee1b4b5a13 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1262,7 +1262,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { def makeDecimalRDD(num: Int, decimal: DecimalType, useDictionary: Boolean): DataFrame = { val div = if (useDictionary) 5 else num // narrow the space to make it dictionary encoded spark - .range(num) + .range(num.toLong) .map(_ % div) // Parquet doesn't allow column names with spaces, have to add an alias here. // Minus 500 here so that negative decimals are also tested. @@ -1766,7 +1766,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { // must stay native; interval types are unsupported and fall back to Spark. withParquetTable( (1 to 5).map(i => - (i.toByte, i.toShort, i, i.toLong, i.toFloat, i.toDouble, BigDecimal(i * 3, 2))), + (i.toByte, i.toShort, i, i.toLong, i.toFloat, i.toDouble, BigDecimal((i * 3).toLong, 2))), "umt") { checkSparkAnswerAndOperator("SELECT -_1, -_2, -_3, -_4, -_5, -_6, -_7 FROM umt") } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index d92723ecf83..1dfd62ee2e2 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -5902,7 +5902,8 @@ class CometIcebergNativeSuite conflictingField.put("name", "id_as_region") conflictingField.put("transform", "identity") conflictingFields.add(conflictingField) - conflictingSpec.set("fields", conflictingFields) + conflictingSpec + .set[com.fasterxml.jackson.databind.node.ObjectNode]("fields", conflictingFields) specs.add(conflictingSpec) // Round-trip through Iceberg's own parser before writing, so a malformed hand-edit diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala index e7dd961150a..118b2371d53 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala @@ -483,7 +483,7 @@ class CometIcebergSystemFunctionSuite maybeNull(randomDecimal38()), maybeNull(randomString()), maybeNull(randomBinary()), - maybeNull(LocalDate.ofEpochDay(random.nextInt(40000) - 20000)), + maybeNull(LocalDate.ofEpochDay((random.nextInt(40000) - 20000).toLong)), maybeNull(instant(randomMicros())), maybeNull(localDateTime(randomMicros()))) } diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index e96367bfe5c..6447bd848e1 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -2710,28 +2710,7 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { values.map(v => Some(v)) ++ Seq(None) } - private def castFallbackTest( - input: DataFrame, - toType: DataType, - expectedMessage: String): Unit = { - withTempPath { dir => - val data = roundtripParquet(input, dir).coalesce(1) - data.createOrReplaceTempView("t") - - withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) { - val df = data.withColumn("converted", col("a").cast(toType)) - df.collect() - val str = - new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan) - assert(str.contains(expectedMessage)) - } - } - } - - private def castTimestampTest( - input: DataFrame, - toType: DataType, - assertNative: Boolean = false) = { + private def castTimestampTest(input: DataFrame, toType: DataType, assertNative: Boolean) = { withTempPath { dir => val data = roundtripParquet(input, dir).coalesce(1) data.createOrReplaceTempView("t") diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 5b5d43dbe8a..5b32b137501 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -3385,19 +3385,11 @@ class CometExecSuite extends CometTestBase { val df2 = (0 until 50).map(i => (i % 7, i % 11, i.toString)).toDF("i", "j", "k").as("df2") - val BucketedTableTestSpec( - bucketSpecLeft, - numPartitionsLeft, - shuffleLeft, - sortLeft, - numOutputPartitionsLeft) = bucketedTableTestSpecLeft - - val BucketedTableTestSpec( - bucketSpecRight, - numPartitionsRight, - shuffleRight, - sortRight, - numOutputPartitionsRight) = bucketedTableTestSpecRight + val BucketedTableTestSpec(bucketSpecLeft, numPartitionsLeft, _, _, _) = + bucketedTableTestSpecLeft + + val BucketedTableTestSpec(bucketSpecRight, numPartitionsRight, _, _, _) = + bucketedTableTestSpecRight withTable("bucketed_table1", "bucketed_table2") { withBucket(df1.repartition(numPartitionsLeft).write.format("parquet"), bucketSpecLeft) @@ -3585,7 +3577,7 @@ class CometExecSuite extends CometTestBase { withTable("t1") { val numRows = 10 spark - .range(numRows) + .range(numRows.toLong) .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b") .repartition(3) // Move data across multiple partitions .write @@ -3622,7 +3614,7 @@ class CometExecSuite extends CometTestBase { withTable("t1") { val numRows = 10 spark - .range(numRows) + .range(numRows.toLong) .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b") .repartition(3) // Force repartition to test data will come to single partition .write @@ -3653,7 +3645,7 @@ class CometExecSuite extends CometTestBase { withTable("t1") { val numRows = 10 spark - .range(numRows) + .range(numRows.toLong) .selectExpr("if (id % 2 = 0, null, id) AS a", s"$numRows - id AS b") .repartition(3) // Force repartition to test data will come to single partition .write diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index 5af9a37c267..e658b9623ea 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -473,7 +473,7 @@ class CometJoinSuite extends CometTestBase { SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { withParquetTable( - (0 until 100).map(i => (Some(i), i.toLong)) :+ (None, -1L), + (0 until 100).map(i => (Some(i), i.toLong)) :+ ((None, -1L)), "dynamic_probe") { for (build <- Seq( Seq((Some(10), 1L), (None, 2L), (Some(10), 3L), (Some(90), 4L)), diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala index 800264bb141..818be0b1ecd 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala @@ -357,7 +357,7 @@ class CometNativeColumnarToRowSuite extends CometTestBase with AdaptiveSparkPlan if (isNull) null else s"string_$i", if (isNull) null else new Date(baseDate.getTime + i * 24 * 60 * 60 * 1000L), if (isNull) null else new Timestamp(baseTs.getTime + i * 1000L), - if (isNull) null else BigDecimal(i * 100 + i, 2).bigDecimal) + if (isNull) null else BigDecimal((i * 100 + i).toLong, 2).bigDecimal) } val schema = StructType( diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index d5e768f886e..417b105b392 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -428,7 +428,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") makeParquetFileAllPrimitiveTypes(path, dictionaryEnabled = dictionaryEnabled, 1000) - var allTypes: Seq[Int] = (1 to 20) + val allTypes: Seq[Int] = (1 to 20) allTypes.map(i => s"_$i").foreach { c => withSQLConf("parquet.enable.dictionary" -> dictionaryEnabled.toString) { readParquetFile(path.toString) { df => diff --git a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala index 8e5a60d63b0..0b12b9a8a51 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala @@ -108,13 +108,13 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { } } - test("extractObjectStoreOptions - forwards the substituted value of a ${...} reference") { + test(s"extractObjectStoreOptions - forwards the substituted value of a $${...} reference") { // Hadoop's own consumers read values through Configuration#get, which expands a ${...} // reference against another conf entry. Forwarding the raw, unexpanded literal here would // give native a different credential than every Hadoop-side consumer sees. val hadoopConf = new Configuration() hadoopConf.set("my.custom.access.key", "expanded-access-key") - hadoopConf.set("fs.s3a.access.key", "${my.custom.access.key}") + hadoopConf.set("fs.s3a.access.key", s"$${my.custom.access.key}") val options = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/test-object")) @@ -122,19 +122,19 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { } test( - "extractObjectStoreOptions - a cyclic ${...} reference falls back to the raw value " + + s"extractObjectStoreOptions - a cyclic $${...} reference falls back to the raw value " + "instead of throwing") { // Configuration#get raises IllegalStateException once ${...} expansion recurses past // Hadoop's MAX_SUBST bound; a two-key mutual cycle triggers this on every call. Extraction // must still return a full options map rather than aborting for the whole object store. val hadoopConf = new Configuration() - hadoopConf.set("fs.s3a.access.key", "${fs.s3a.secret.key}") - hadoopConf.set("fs.s3a.secret.key", "${fs.s3a.access.key}") + hadoopConf.set("fs.s3a.access.key", s"$${fs.s3a.secret.key}") + hadoopConf.set("fs.s3a.secret.key", s"$${fs.s3a.access.key}") val options = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/test-object")) - assert(options("fs.s3a.access.key") == "${fs.s3a.secret.key}") - assert(options("fs.s3a.secret.key") == "${fs.s3a.access.key}") + assert(options("fs.s3a.access.key") == s"$${fs.s3a.secret.key}") + assert(options("fs.s3a.secret.key") == s"$${fs.s3a.access.key}") } test( diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala index eef77d88246..29ccfcb0162 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala @@ -949,7 +949,7 @@ class CometParquetWriterSuite extends CometTestBase { var iterations = 0 while (capturedPlan.isEmpty && iterations < maxIterations) { - Thread.sleep(checkIntervalMs) + Thread.sleep(checkIntervalMs.toLong) iterations += 1 } diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index d412918b572..552fd98c218 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -736,8 +736,8 @@ abstract class ParquetReadSuite extends CometTestBase { opt match { case Some(i) => record.add(0, i % 2 == 0) - record.add(1, i.toByte) - record.add(2, i.toShort) + record.add(1, i.toByte.toInt) + record.add(2, i.toShort.toInt) record.add(3, i) record.add(4, i.toLong) record.add(5, i.toFloat) @@ -1153,7 +1153,7 @@ abstract class ParquetReadSuite extends CometTestBase { var b = record.addGroup("b") b.add("b1", 1) b.add("b2", 1) - var c = record.addGroup("c") + val c = record.addGroup("c") c.add("c1", 1) c.add("c2", 1) writer.write(record) @@ -1223,7 +1223,7 @@ abstract class ParquetReadSuite extends CometTestBase { var b = record.addGroup("b") b.add("b1", 1) b.add("b2", 1) - var c = record.addGroup("c") + val c = record.addGroup("c") c.add("c1", 1) c.add("c2", 1) writer.write(record) @@ -1900,7 +1900,7 @@ abstract class ParquetReadSuite extends CometTestBase { } private def withId(id: Int) = - new MetadataBuilder().putLong(ParquetUtils.FIELD_ID_METADATA_KEY, id).build() + new MetadataBuilder().putLong(ParquetUtils.FIELD_ID_METADATA_KEY, id.toLong).build() // Based on Spark ParquetIOSuite.test("vectorized reader: array of nested struct") test("array of nested struct with and without field id") { diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala index c0f3c0a6ce4..c848501a71a 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.ServiceLoader +import scala.annotation.nowarn import scala.jdk.CollectionConverters._ import org.scalatest.funsuite.AnyFunSuite @@ -331,6 +332,7 @@ class RecordingClaimingScanContrib extends ClaimingScanContrib { class NotAContribAtAll /** Implements the service but cannot be constructed; `next` throws. */ +@nowarn("msg=dead code") class ThrowingCtorScanContrib extends CometScanContrib { throw new IllegalStateException("contrib constructor blew up") } diff --git a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala index a5b98dd5988..cfe80c032ad 100644 --- a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala @@ -93,10 +93,10 @@ class CometExecIteratorLifecycleSuite extends CometTestBase { // Unique synthetic task attempt ids keep each iteration's pool entry independent. val taskAttemptId = 4200000L + i withTaskContext(taskAttemptId) { - val manager = new CometTaskMemoryManager(i, taskAttemptId) + val manager = new CometTaskMemoryManager(i.toLong, taskAttemptId) val thrown = intercept[Throwable] { nativeLib.createPlan( - i, + i.toLong, Array.empty[Object], emptyPlan, badConfigs, diff --git a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala index dfbe38b6484..92f7d03a5af 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala @@ -75,7 +75,8 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { spills: java.util.LinkedList[org.apache.spark.sql.comet.execution.shuffle.SpillInfo] = new java.util.LinkedList[org.apache.spark.sql.comet.execution.shuffle.SpillInfo](), partitionChecksums: Array[Long] = new Array[Long](10)): SpillSorter = { - val allocator = CometShuffleMemoryAllocator.getInstance(conf, taskMemoryManager, PAGE_SIZE) + val allocator = + CometShuffleMemoryAllocator.getInstance(conf, taskMemoryManager, PAGE_SIZE.toLong) val schema = createTestSchema() val writeMetrics = new ShuffleWriteMetrics() val taskContext = TaskContext.empty() @@ -115,7 +116,11 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { val partitionId = 0 sorter.initialCurrentPage(recordData.length + UAO_SIZE) - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, partitionId) + sorter.insertRecord( + recordData, + Platform.BYTE_ARRAY_OFFSET.toLong, + recordData.length, + partitionId) assert(sorter.numRecords() === 1) } finally { @@ -136,7 +141,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { val partitionId = i % 10 sorter.insertRecord( recordData, - Platform.BYTE_ARRAY_OFFSET, + Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, partitionId) } @@ -153,7 +158,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { try { val recordData = Array[Byte](1, 2, 3, 4) sorter.initialCurrentPage(recordData.length + UAO_SIZE) - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, 0) + sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, 0) assert(sorter.numRecords() === 1) @@ -190,7 +195,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { try { sorter.initialCurrentPage(1024) val recordData = Array[Byte](1, 2, 3, 4) - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, 0) + sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, 0) assert(spillCount.get() === 0, "Spill callback should not be triggered during normal ops") } finally { @@ -225,8 +230,9 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { val sorter = createSpillSorter() try { val initialMemory = sorter.getMemoryUsage() - val allocator = CometShuffleMemoryAllocator.getInstance(conf, taskMemoryManager, PAGE_SIZE) - val newArray = allocator.allocateArray(INITIAL_SIZE * 2) + val allocator = + CometShuffleMemoryAllocator.getInstance(conf, taskMemoryManager, PAGE_SIZE.toLong) + val newArray = allocator.allocateArray((INITIAL_SIZE * 2).toLong) sorter.expandPointerArray(newArray) assert(sorter.getMemoryUsage() >= initialMemory) @@ -248,7 +254,7 @@ class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { for (p <- 0 until numPartitions) { for (_ <- 0 until recordsPerPartition) { - sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET, recordData.length, p) + sorter.insertRecord(recordData, Platform.BYTE_ARRAY_OFFSET.toLong, recordData.length, p) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala index a3437559757..c07006c23d2 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala @@ -99,7 +99,7 @@ trait CometTPCQueryBase extends Logging { // Recover partitions but don't fail if a table is not partitioned. Try { cometSpark.sql(s"ALTER TABLE $tableName RECOVER PARTITIONS") - }.getOrElse { + }.failed.foreach { _ => logInfo(s"Recovering partitions of table $tableName failed") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala index a2bbe415cf5..8fcfff800b1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala @@ -794,7 +794,7 @@ abstract class CometTestBase .withType(schema) // TODO we need to shim this and use withRowGroupSize(Long) with later parquet-hadoop versions to remove // the deprecated warning here - .withRowGroupSize(rowGroupSize.toInt) + .withRowGroupSize(rowGroupSize) .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) .withPageRowCountLimit(pageRowCountLimit) @@ -910,15 +910,15 @@ abstract class CometTestBase opt match { case Some(i) => record.add(0, i % 2 == 0) - record.add(1, i.toByte) - record.add(2, i.toShort) + record.add(1, i.toByte.toInt) + record.add(2, i.toShort.toInt) record.add(3, i) record.add(4, i.toLong) record.add(5, i.toFloat) record.add(6, i.toDouble) record.add(7, i.toString * 48) - record.add(8, (-i).toByte) - record.add(9, (-i).toShort) + record.add(8, (-i).toByte.toInt) + record.add(9, (-i).toShort.toInt) record.add(10, -i) record.add(11, (-i).toLong) record.add(12, i.toString) @@ -939,15 +939,15 @@ abstract class CometTestBase val i = rand.nextLong() val record = new SimpleGroup(schema) record.add(0, i % 2 == 0) - record.add(1, i.toByte) - record.add(2, i.toShort) + record.add(1, i.toByte.toInt) + record.add(2, i.toShort.toInt) record.add(3, i.toInt) record.add(4, i) record.add(5, java.lang.Float.intBitsToFloat(i.toInt)) record.add(6, java.lang.Double.longBitsToDouble(i)) record.add(7, i.toString * 24) - record.add(8, (-i).toByte) - record.add(9, (-i).toShort) + record.add(8, (-i).toByte.toInt) + record.add(9, (-i).toShort.toInt) record.add(10, (-i).toInt) record.add(11, -i) record.add(12, i.toString) @@ -996,7 +996,7 @@ abstract class CometTestBase if (rand.nextBoolean()) { None } else { - Some(getValue(i, div)) + Some(getValue(i.toLong, div.toLong)) } } expected.foreach { opt => @@ -1050,7 +1050,7 @@ abstract class CometTestBase if (rand.nextBoolean()) { None } else { - Some(getValue(i, div)) + Some(getValue(i.toLong, div.toLong)) } } expected.foreach { opt => @@ -1228,7 +1228,7 @@ abstract class CometTestBase val div = if (dictionaryEnabled) 10 else n // maps value to a small range for dict to kick in val expected = (0 until n).map { i => - Some(getValue(i, div)) + Some(getValue(i.toLong, div.toLong)) } expected.foreach { opt => val timestampFormats = List( @@ -1276,7 +1276,7 @@ abstract class CometTestBase def makeDecimalRDD(num: Int, decimal: DecimalType, useDictionary: Boolean): DataFrame = { val div = if (useDictionary) 5 else num // narrow the space to make it dictionary encoded spark - .range(num) + .range(num.toLong) .map(_ % div) // Parquet doesn't allow column names with spaces, have to add an alias here. // Minus 500 here so that negative decimals are also tested. @@ -1456,8 +1456,8 @@ abstract class CometTestBase val record = new SimpleGroup(schema) opt match { case Some(i) => - record.add(0, i.toByte) - record.add(1, i.toShort) + record.add(0, i.toByte.toInt) + record.add(1, i.toShort.toInt) record.add(2, i) record.add(3, i.toLong) record.add(4, rand.nextFloat()) diff --git a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala index e25d4e51e46..6e8ce177d62 100644 --- a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala +++ b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala @@ -65,7 +65,7 @@ object GenTPCHData { // Install the data generators in all nodes // TODO: think a better way to install on each worker node // such as https://stackoverflow.com/a/40876671 - spark.range(0, workers, 1, workers).foreach(worker => installDBGEN(baseDir)(worker)) + spark.range(0, workers.toLong, 1, workers).foreach(worker => installDBGEN(baseDir)(worker)) s"${baseDir}/dbgen" } else { config.dbgenDir @@ -91,7 +91,7 @@ object GenTPCHData { // Clean up if (defaultDbgenDir != null) { - spark.range(0, workers, 1, workers).foreach { _ => + spark.range(0, workers.toLong, 1, workers).foreach { _ => val _ = FileUtils.deleteQuietly(defaultDbgenDir) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala index 2772d37ffbf..f577ed102d5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometAggregateExpressionBenchmark.scala @@ -217,7 +217,7 @@ object CometAggregateExpressionBenchmark extends CometBenchmarkBase { approxCountDistinctAggregates ++ maxMinByAggregates allAggregates.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala index a513aa1a77b..7594ab62fb5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArithmeticBenchmark.scala @@ -44,7 +44,7 @@ object CometArithmeticBenchmark extends CometBenchmarkBase { val name = s"Binary op ${dataType.sql}, dictionary = $useDictionary" val query = s"SELECT c1 ${op.sig} c2 FROM $table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -64,7 +64,7 @@ object CometArithmeticBenchmark extends CometBenchmarkBase { val name = s"Binary op ${dataType.sql}, dictionary = $useDictionary" val query = s"SELECT c1 ${op.sig} c2 FROM $table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala index 624abdebe88..90c247a9996 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala @@ -65,7 +65,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { prepareSortArrayTable(width) { runExpressionBenchmark( s"sort_array int ascending (width=$width)", - values, + values.toLong, "SELECT sort_array(int_arr) FROM parquetV1Table") } } @@ -74,7 +74,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { prepareSortArrayTable(width) { runExpressionBenchmark( s"sort_array int descending (width=$width)", - values, + values.toLong, "SELECT sort_array(int_arr, false) FROM parquetV1Table") } } @@ -83,7 +83,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { prepareSortArrayTable(width) { runExpressionBenchmark( s"element_at(sort_array(int_arr), 1) (width=$width)", - values, + values.toLong, "SELECT element_at(sort_array(int_arr), 1) FROM parquetV1Table") } } @@ -112,7 +112,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { runExpressionBenchmark( "array_position - int array", - values, + values.toLong, "SELECT array_position(int_arr, search_val) FROM parquetV1Table") } } @@ -140,7 +140,7 @@ object CometArrayExpressionBenchmark extends CometBenchmarkBase { runExpressionBenchmark( "array_position - string array", - values, + values.toLong, "SELECT array_position(str_arr, search_val) FROM parquetV1Table") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala index 11033703afc..c8073e04c66 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrowWriterBenchmark.scala @@ -97,7 +97,10 @@ object CometArrowWriterBenchmark extends BenchmarkBase { val root = VectorSchemaRoot.create(arrowSchema, allocator) try { val benchmark = - new Benchmark(s"Spark columnar to Arrow ($numRows rows)", numRows, output = output) + new Benchmark( + s"Spark columnar to Arrow ($numRows rows)", + numRows.toLong, + output = output) benchmark.addCase("on-heap optimized path") { _ => writeBatch(onHeap, bulkCopy = true, root) } @@ -133,7 +136,7 @@ object CometArrowWriterBenchmark extends BenchmarkBase { arrowSchema, Iterator.from(0).map(i => if ((i & 1) == 0) nullRow else row), numRows) - val rowBenchmark = new Benchmark("Spark rows to Arrow", numRows, output = output) + val rowBenchmark = new Benchmark("Spark rows to Arrow", numRows.toLong, output = output) try { rowBenchmark.addCase("fixed-width, no nulls") { _ => noNullRowReader.loadNextBatch() } rowBenchmark.addCase("fixed-width, 50% nulls") { _ => nullableRowReader.loadNextBatch() } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala index 487e4797d12..2a957f2d0a2 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala @@ -103,7 +103,7 @@ trait CometBenchmarkBase // generator, so that results are comparable across runs. Seeding a driver-side `Random` // would not work here: the closure runs per row on the executor. spark - .range(values) + .range(values.toLong) .map(i => if (useDictionary) CometBenchmarkBase.mix64(i) % 5 else CometBenchmarkBase.mix64(i)) .createOrReplaceTempView(tbl) @@ -374,7 +374,7 @@ trait CometBenchmarkBase val div = if (useDictionary) 5 else values spark - .range(values) + .range(values.toLong) .map(_ % div) .select((($"value" - 500) / 100.0) cast decimal as Symbol("dec")) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala index 152d79dbb98..81f5f4d7a39 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala @@ -65,12 +65,12 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id AS k", "id * 10 AS w") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -81,7 +81,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - inner count") { runExpressionBenchmark( "inner count", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -89,7 +89,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - inner projected") { runExpressionBenchmark( "inner projected", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ p.k, p.v, b.w FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -97,7 +97,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - left outer") { runExpressionBenchmark( "left outer", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p LEFT JOIN build b ON p.k = b.k", cometConfigs) } @@ -105,7 +105,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - left semi") { runExpressionBenchmark( "left semi", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p LEFT SEMI JOIN build b ON p.k = b.k", cometConfigs) } @@ -113,7 +113,7 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastHashJoin - right outer") { runExpressionBenchmark( "right outer", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(p) */ count(*) FROM probe p RIGHT JOIN build b ON p.k = b.k", cometConfigs) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala index f8fe6100d4d..dcfc7d44a21 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala @@ -70,12 +70,12 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id * 1000 AS lo", "id * 1000 + 500 AS hi") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -86,7 +86,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - range") { runExpressionBenchmark( "range join (BETWEEN)", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p " + "JOIN build b ON p.k BETWEEN b.lo AND b.hi", cometConfigs) @@ -95,7 +95,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - inequality") { runExpressionBenchmark( "inequality join (>)", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p " + "JOIN build b ON p.k > b.lo", cometConfigs) @@ -104,7 +104,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - left outer with non-equi") { runExpressionBenchmark( "left outer non-equi", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ count(*) FROM probe p " + "LEFT OUTER JOIN build b ON p.k BETWEEN b.lo AND b.hi", cometConfigs) @@ -113,7 +113,7 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase { runBenchmark("BroadcastNestedLoopJoin - range, materialized rows") { runExpressionBenchmark( "range join (BETWEEN, projected)", - probeRows, + probeRows.toLong, "SELECT /*+ BROADCAST(b) */ p.k, p.v, b.lo, b.hi FROM probe p " + "JOIN build b ON p.k BETWEEN b.lo AND b.hi", cometConfigs) diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala index 57b8e88a7b5..8419ade9e63 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala @@ -89,7 +89,7 @@ object CometCastBooleanBenchmark extends CometBenchmarkBase { """)) (boolToStringConfigs ++ boolToNumericConfigs).foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -115,7 +115,7 @@ object CometCastBooleanBenchmark extends CometBenchmarkBase { """)) numericToBoolConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala index 2141b1bea08..12ae85374d0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala @@ -148,7 +148,7 @@ object CometCastNumericToNumericBenchmark extends CometBenchmarkBase { generateConfigs(floatToIntPairs) ++ generateConfigs(decimalPairs) ++ generateConfigs(floatToDecimalPairs)).foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala index 1fd2138c581..cdf719b0406 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala @@ -95,7 +95,7 @@ object CometCastNumericToStringBenchmark extends CometBenchmarkBase { """)) castConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala index ec2d9ab12f1..e54127c9dc6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala @@ -70,7 +70,7 @@ object CometCastNumericToTemporalBenchmark extends CometBenchmarkBase { """)) intToDateConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -92,7 +92,7 @@ object CometCastNumericToTemporalBenchmark extends CometBenchmarkBase { """)) longToTimestampConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala index c71eadad8c4..ac9c172e5b8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala @@ -89,7 +89,7 @@ object CometCastStringToNumericBenchmark extends CometBenchmarkBase { """)) castConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala index 77cc009ae10..31520824b73 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala @@ -71,7 +71,7 @@ object CometCastStringToTemporalBenchmark extends CometBenchmarkBase { // Run date cast benchmarks with the same data dateCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -94,7 +94,7 @@ object CometCastStringToTemporalBenchmark extends CometBenchmarkBase { // Run timestamp cast benchmarks with the same data timestampCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala index 1468cbe086f..6a4b9f81b06 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala @@ -74,7 +74,7 @@ object CometCastTemporalToNumericBenchmark extends CometBenchmarkBase { """)) dateToNumericConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -96,7 +96,7 @@ object CometCastTemporalToNumericBenchmark extends CometBenchmarkBase { """)) timestampToNumericConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala index 1ef3e7711d2..5e77742a345 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala @@ -67,7 +67,7 @@ object CometCastTemporalToStringBenchmark extends CometBenchmarkBase { """)) dateCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -89,7 +89,7 @@ object CometCastTemporalToStringBenchmark extends CometBenchmarkBase { """)) timestampCastConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala index f2e25724873..c030846861a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala @@ -69,7 +69,7 @@ object CometCastTemporalToTemporalBenchmark extends CometBenchmarkBase { """)) dateToTimestampConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -91,7 +91,7 @@ object CometCastTemporalToTemporalBenchmark extends CometBenchmarkBase { """)) timestampToDateConfigs.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala index 53a8fefa7f3..05370a7afc7 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCodegenDispatchBenchmark.scala @@ -294,7 +294,7 @@ object CometCodegenDispatchBenchmark extends CometBenchmarkBase { private def runSteadyState(c: DispatchCase, rows: Int): Unit = { runBenchmark(s"${c.name} -- $rows rows") { - val benchmark = new Benchmark(s"${c.name} -- $rows rows", rows, output = output) + val benchmark = new Benchmark(s"${c.name} -- $rows rows", rows.toLong, output = output) checkPlans(benchmark, c) // The dispatch-off arm goes first so the `Relative` column reads as the speedup this // change buys over the behaviour that shipped before it. @@ -364,7 +364,7 @@ object CometCodegenDispatchBenchmark extends CometBenchmarkBase { private def withCorpus(rows: Int)(f: => Unit): Unit = { withTempPath { dir => withTempTable(tbl, "parquetV1Table") { - spark.range(rows).createOrReplaceTempView(tbl) + spark.range(rows.toLong).createOrReplaceTempView(tbl) prepareTable(dir, spark.sql(corpusQuery)) f } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala index 22e7dbb54dd..1a05bdbcd73 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala @@ -150,13 +150,13 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def primitiveTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Primitive Types", values, output = output) + new Benchmark("Columnar to Row - Primitive Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { // Create a table with various primitive types (includes strings) val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id as long_col", "cast(id as int) as int_col", @@ -182,13 +182,16 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def fixedWidthOnlyBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Fixed Width Only (no strings)", values, output = output) + new Benchmark( + "Columnar to Row - Fixed Width Only (no strings)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { // Create a table with ONLY fixed-width primitive types (no strings!) val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id as long_col", "cast(id as int) as int_col", @@ -214,12 +217,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def stringTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - String Types", values, output = output) + new Benchmark("Columnar to Row - String Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", "concat('short_', cast(id % 100 as string)) as short_str", @@ -239,12 +242,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def structTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Struct Types", values, output = output) + new Benchmark("Columnar to Row - Struct Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Simple struct @@ -279,12 +282,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def arrayTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Array Types", values, output = output) + new Benchmark("Columnar to Row - Array Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Array of primitives @@ -310,12 +313,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def mapTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Map Types", values, output = output) + new Benchmark("Columnar to Row - Map Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Map with string keys and int values @@ -343,12 +346,12 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def complexNestedTypesBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Complex Nested Types", values, output = output) + new Benchmark("Columnar to Row - Complex Nested Types", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", // Array of structs @@ -380,7 +383,7 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def wideRowsBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Wide Rows (50 columns)", values, output = output) + new Benchmark("Columnar to Row - Wide Rows (50 columns)", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -395,7 +398,7 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { } } - val df = spark.range(values).selectExpr(columns: _*) + val df = spark.range(values.toLong).selectExpr(columns: _*) prepareTable(dir, df) val query = "SELECT * FROM parquetV1Table" @@ -411,12 +414,15 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def dictionaryEncodedBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - Dictionary-encoded strings", values, output = output) + new Benchmark( + "Columnar to Row - Dictionary-encoded strings", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id", "concat('val_', cast(id % 1000 as string)) as low_card_str", @@ -446,14 +452,14 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { */ def jvmConsumerBenchmark(values: Int): Unit = { val benchmark = - new Benchmark("Columnar to Row - JVM UDF consumer", values, output = output) + new Benchmark("Columnar to Row - JVM UDF consumer", values.toLong, output = output) spark.udf.register("plus_one", (x: Long) => x + 1) withTempPath { dir => withTempTable("parquetV1Table") { val df = spark - .range(values) + .range(values.toLong) .selectExpr( "id as long_col", "cast(id as int) as int_col", diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala index 56ce2ac2640..060b96b8ea4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometComparisonExpressionBenchmark.scala @@ -85,7 +85,7 @@ object CometComparisonExpressionBenchmark extends CometBenchmarkBase { """)) comparisonExpressions.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala index 6db28b3072b..54391073bbd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala @@ -98,7 +98,7 @@ object CometConcatWsBenchmark extends CometBenchmarkBase { // scalastyle:on println } runBenchmark(name) { - runExpressionBenchmark(name, rows, query, cometConfigs) + runExpressionBenchmark(name, rows.toLong, query, cometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala index 5b3532d5f23..877318e96b4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometConditionalExpressionBenchmark.scala @@ -57,7 +57,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { prepareTestTable(values) { val query = "SELECT CASE WHEN c1 < 0 THEN '<0' WHEN c1 = 0 THEN '=0' ELSE '>0' END FROM parquetV1Table" - runExpressionBenchmark("Case When Literal (3 branches)", values, query) + runExpressionBenchmark("Case When Literal (3 branches)", values.toLong, query) } } @@ -78,7 +78,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { ELSE 'j' END FROM parquetV1Table """ - runExpressionBenchmark("Case When Literal (10 branches)", values, query) + runExpressionBenchmark("Case When Literal (10 branches)", values.toLong, query) } } @@ -87,7 +87,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { // Result expressions are column references, not literals val query = "SELECT CASE WHEN c1 < 0 THEN c3 WHEN c1 = 0 THEN c1 ELSE c3 + c1 END FROM parquetV1Table" - runExpressionBenchmark("Case When Column Result (3 branches)", values, query) + runExpressionBenchmark("Case When Column Result (3 branches)", values.toLong, query) } } @@ -108,14 +108,14 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { ELSE c1 + c2 + c3 END FROM parquetV1Table """ - runExpressionBenchmark("Case When Column Result (10 branches)", values, query) + runExpressionBenchmark("Case When Column Result (10 branches)", values.toLong, query) } } def ifLiteralBenchmark(values: Int): Unit = { prepareTestTable(values) { val query = "SELECT IF(c1 < 0, '<0', '>=0') FROM parquetV1Table" - runExpressionBenchmark("If Literal", values, query) + runExpressionBenchmark("If Literal", values.toLong, query) } } @@ -123,7 +123,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { prepareTestTable(values) { // Result expressions are column references val query = "SELECT IF(c1 < 0, c3, c1 + c3) FROM parquetV1Table" - runExpressionBenchmark("If Column Result", values, query) + runExpressionBenchmark("If Column Result", values.toLong, query) } } @@ -136,7 +136,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { IF(c2 < 75, 'c', 'd'))) FROM parquetV1Table """ - runExpressionBenchmark("Nested If Literal (4 outcomes)", values, query) + runExpressionBenchmark("Nested If Literal (4 outcomes)", values.toLong, query) } } @@ -148,7 +148,7 @@ object CometConditionalExpressionBenchmark extends CometBenchmarkBase { IF(c2 < 75, c1 + c3, c3 * 2))) FROM parquetV1Table """ - runExpressionBenchmark("Nested If Column Result (4 outcomes)", values, query) + runExpressionBenchmark("Nested If Column Result (4 outcomes)", values.toLong, query) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala index 1495b0320e2..3e0e02a4cc8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCsvExpressionBenchmark.scala @@ -63,7 +63,7 @@ object CometCsvExpressionBenchmark extends CometBenchmarkBase { CometConf.getExprAllowIncompatConfigKey( classOf[CsvToStructs]) -> "true") ++ config.extraCometConfigs - runExpressionBenchmark(config.name, values, config.query, extraConfigs) + runExpressionBenchmark(config.name, values.toLong, config.query, extraConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala index 8e2978714aa..1ff82a6d450 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala @@ -45,7 +45,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { Seq("YEAR", "MONTH").foreach { level => val name = s"Date Truncate - $level" val query = s"select trunc(dt, '$level') from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -70,7 +70,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { "MICROSECOND").foreach { level => val name = s"Timestamp Truncate - $level" val query = s"select date_trunc('$level', ts) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -85,7 +85,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { val name = s"Unix Timestamp from Timestamp ($timeZone)" val query = "select unix_timestamp(ts) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -101,7 +101,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { val name = s"Unix Timestamp from Date ($timeZone)" val query = "select unix_timestamp(dt) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -116,7 +116,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { s"select concat(cast(abs(value) % 24 as string), ':', lpad(cast(abs(value) % 60 as string), 2, '0'), ':', lpad(cast(abs(value) % 60 as string), 2, '0')) as s FROM $tbl")) val name = "to_time" val query = "select to_time(s) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -130,7 +130,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { s"select cast(abs(value) % 24 as int) as h, cast(abs(value) % 60 as int) as m, cast(abs(value) % 60 as decimal(16,6)) as s FROM $tbl")) val name = "make_time" val query = "select make_time(h, m, s) from parquetV1Table" - runExpressionBenchmark(name, values, query) + runExpressionBenchmark(name, values.toLong, query) } } } @@ -154,7 +154,7 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { def consumeIntervals(): Unit = { spark.sql(query).queryExecution.toRdd.foreachPartition(_.foreach(_.getInterval(0))) } - val benchmark = new Benchmark("MakeInterval", values, output = output) + val benchmark = new Benchmark("MakeInterval", values.toLong, output = output) val cometConfigs = Map( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", @@ -209,11 +209,14 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase { END AS dow FROM $tbl """)) - runExpressionBenchmark("NextDay", values, "select next_day(dt, dow) from parquetV1Table") + runExpressionBenchmark( + "NextDay", + values.toLong, + "select next_day(dt, dow) from parquetV1Table") if (isSpark40Plus) { runExpressionBenchmark( "NextDay - collated dayOfWeek", - values, + values.toLong, "select next_day(dt, dow collate utf8_lcase) from parquetV1Table") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala index 2e7e143a472..3e25e408004 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala @@ -70,7 +70,10 @@ object CometExecBenchmark extends CometBenchmarkBase { def numericFilterExecBenchmark(values: Int, fractionOfZeros: Double): Unit = { val percentageOfZeros = fractionOfZeros * 100 val benchmark = - new Benchmark(s"Project + Filter Exec ($percentageOfZeros% zeros)", values, output = output) + new Benchmark( + s"Project + Filter Exec ($percentageOfZeros% zeros)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -108,7 +111,7 @@ object CometExecBenchmark extends CometBenchmarkBase { } def subqueryExecBenchmark(values: Int): Unit = { - val benchmark = new Benchmark("Subquery", values, output = output) + val benchmark = new Benchmark("Subquery", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -140,7 +143,7 @@ object CometExecBenchmark extends CometBenchmarkBase { } def sortExecBenchmark(values: Int): Unit = { - val benchmark = new Benchmark("Sort Exec", values, output = output) + val benchmark = new Benchmark("Sort Exec", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -164,7 +167,8 @@ object CometExecBenchmark extends CometBenchmarkBase { } def sampleExecBenchmark(values: Int, fraction: Double): Unit = { - val benchmark = new Benchmark(s"Sample Exec (fraction $fraction)", values, output = output) + val benchmark = + new Benchmark(s"Sample Exec (fraction $fraction)", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -194,7 +198,7 @@ object CometExecBenchmark extends CometBenchmarkBase { } def expandExecBenchmark(values: Int): Unit = { - val benchmark = new Benchmark("Expand Exec", values, output = output) + val benchmark = new Benchmark("Expand Exec", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -232,7 +236,7 @@ object CometExecBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"BloomFilterAggregate Exec (cardinality $cardinality)", - values, + values.toLong, output = output) val funcId_bloom_filter_agg = new FunctionIdentifier("bloom_filter_agg") diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala index aa16417ca2c..86909f839c5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExplodeBenchmark.scala @@ -79,7 +79,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { * Rows of an `rows`-row dataset whose [[arrayColumn]] is neither NULL nor empty, which is how * many rows reach a non-outer generator with something to emit. */ - private def nonEmptyRows(rows: Int): Long = rows - (rows + 9) / 10 - (rows + 8) / 10 + private def nonEmptyRows(rows: Int): Long = (rows - (rows + 9) / 10 - (rows + 8) / 10).toLong /** * A SQL expression for an array of `len` elements of `elementExpr`, where `elementExpr` may @@ -88,7 +88,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { * Elements are wrapped in a never-taken null branch so the array types as `containsNull`; see * [[nullableExpr]]. */ - private def fullArray(elementExpr: String, len: Int, v: String = "x"): String = + private def fullArray(elementExpr: String, len: Int, v: String): String = s"transform(sequence(1, $len), $v -> ${nullableExpr(elementExpr, s"$v = 0")})" /** @@ -214,7 +214,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { /** Writes a view's rows to Parquet and registers it. */ private def createView(dir: File, view: TempView): Unit = { val path = s"${dir.getAbsolutePath}/${view.name}" - spark.range(view.rows).selectExpr(view.columns: _*).write.parquet(path) + spark.range(view.rows.toLong).selectExpr(view.columns: _*).write.parquet(path) spark.read.parquet(path).createOrReplaceTempView(view.name) } @@ -306,7 +306,7 @@ object CometExplodeBenchmark extends CometBenchmarkBase { /** Verifies a case's aggregate under both engines, then times it. */ private def runCase(name: String, benchmarkCase: Case, rows: Int = numRows): Unit = { verifySink(name, benchmarkCase.query, benchmarkCase.expected) - runExpressionBenchmark(name, rows, benchmarkCase.query) + runExpressionBenchmark(name, rows.toLong, benchmarkCase.query) } override def runCometBenchmark(mainArgs: Array[String]): Unit = { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala index fa639453475..6b18aed5dee 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometGetJsonObjectBenchmark.scala @@ -38,7 +38,7 @@ object CometGetJsonObjectBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(numRows) + .range(numRows.toLong) .map { i => val name = s"user_$i" val age = (i % 80 + 18).toInt @@ -76,7 +76,11 @@ object CometGetJsonObjectBenchmark extends CometBenchmarkBase { benchmarks.foreach { config => runBenchmark(config.name) { - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark( + config.name, + v.toLong, + config.query, + config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala index 68ee3c1d1e0..238d8b662b1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashExpressionBenchmark.scala @@ -65,7 +65,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { """)) hashExpressions.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -124,7 +124,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { HashExprConfig("hash_decimal", "SELECT hash(c_decimal) FROM parquetV1Table")) primitiveHashBenchmarks.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -175,7 +175,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { "SELECT hash(c_struct_multi) FROM parquetV1Table")) complexHashBenchmarks.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } @@ -232,7 +232,7 @@ object CometHashExpressionBenchmark extends CometBenchmarkBase { "SELECT hash(c_array_of_struct) FROM parquetV1Table")) nestedHashBenchmarks.foreach { config => - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala index ef2c325c547..8963451d809 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala @@ -72,12 +72,12 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id AS k", "id * 10 AS w") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -88,7 +88,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - inner count") { runExpressionBenchmark( "inner count", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ count(*) FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -96,7 +96,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - inner projected") { runExpressionBenchmark( "inner projected", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ p.k, p.v, b.w FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -104,7 +104,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - left outer") { runExpressionBenchmark( "left outer", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ count(*) FROM probe p LEFT JOIN build b ON p.k = b.k", cometConfigs) } @@ -112,7 +112,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - left semi") { runExpressionBenchmark( "left semi", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(b) */ count(*) FROM probe p LEFT SEMI JOIN build b ON p.k = b.k", cometConfigs) } @@ -120,7 +120,7 @@ object CometHashJoinBenchmark extends CometBenchmarkBase { runBenchmark("ShuffledHashJoin - right outer") { runExpressionBenchmark( "right outer", - probeRows, + probeRows.toLong, "SELECT /*+ SHUFFLE_HASH(p) */ count(*) FROM probe p RIGHT JOIN build b ON p.k = b.k", cometConfigs) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala index b90b893712b..1a171a5adac 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergReadBenchmark.scala @@ -34,7 +34,10 @@ object CometIcebergReadBenchmark extends CometBenchmarkBase { def icebergScanBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"SQL Single ${dataType.sql} Iceberg Column Scan", values, output = output) + new Benchmark( + s"SQL Single ${dataType.sql} Iceberg Column Scan", + values.toLong, + output = output) withTempPath { dir => withTempTable("icebergTable") { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala index 7369a532304..97b11a54f3c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergSystemFunctionBenchmark.scala @@ -154,7 +154,7 @@ object CometIcebergSystemFunctionBenchmark extends CometBenchmarkBase { cases.foreach { case (name, query) => verifyOutputsMatch(name, query) runBenchmark(name) { - runExpressionBenchmark(name, v, query) + runExpressionBenchmark(name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala index 7d171ac26ee..af06f0bc10c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometInMemoryCacheBenchmark.scala @@ -37,7 +37,10 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { private val numRows = 5 * 1000 * 1000 private val cacheTable = "comet_cache_bench" private val sourceTable = "comet_cache_bench_src" - @volatile private var statsResult: (Array[Any], Array[Any], Array[Int]) = _ + // A sink for the benchmarked call's result: written but never read, so that neither the + // compiler nor the JIT can treat `gatherColumnStats` as dead code. Not `private`, because + // a private field that is only ever written is what `-Ywarn-unused:privates` reports. + @volatile var statsResult: (Array[Any], Array[Any], Array[Int]) = _ override def getSparkSession: SparkSession = { val conf = new SparkConf() @@ -75,7 +78,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { withTempTable(sourceTable, cacheTable) { spark - .range(0, numRows, 1, 16) + .range(0, numRows.toLong, 1, 16) .selectExpr( "id", "id % 1000 AS k", @@ -124,15 +127,15 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { var r = 0 while (r < batchSize) { columns(0).putLong(r, r.toLong) - columns(1).putLong(r, r % 1000) - columns(2).putLong(r, r + 1) + columns(1).putLong(r, (r % 1000).toLong) + columns(2).putLong(r, (r + 1).toLong) columns(3).putByteArray(r, s"str_a_${r % 100000}".getBytes(StandardCharsets.UTF_8)) columns(4).putByteArray(r, s"str_b_${r % 7919}".getBytes(StandardCharsets.UTF_8)) columns(5).putByteArray(r, s"str_c_$r".getBytes(StandardCharsets.UTF_8)) r += 1 } val serializer = new ArrowCachedBatchSerializer - val benchmark = new Benchmark("in-memory cache statistics", numRows, output = output) + val benchmark = new Benchmark("in-memory cache statistics", numRows.toLong, output = output) // One case measures this collector across commits; Spark's default cache has its own collector. benchmark.addCase("Comet statistics collector") { _ => var i = 0 @@ -154,7 +157,7 @@ object CometInMemoryCacheBenchmark extends CometBenchmarkBase { verifyPlan(query, nativeCacheEnabled = true) } - val benchmark = new Benchmark(name, numRows, output = output) + val benchmark = new Benchmark(name, numRows.toLong, output = output) benchmark.addCase("Spark cache scan + CometSparkColumnarToColumnar") { _ => withSQLConf(cacheConf(nativeCacheEnabled = false): _*) { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala index 82aae3d7b93..8a422bb27ba 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometJsonExpressionBenchmark.scala @@ -159,7 +159,7 @@ object CometJsonExpressionBenchmark extends CometBenchmarkBase { CometConf.getExprAllowIncompatConfigKey( classOf[StructsToJson]) -> "true") ++ config.extraCometConfigs - runExpressionBenchmark(config.name, values, config.query, extraConfigs) + runExpressionBenchmark(config.name, values.toLong, config.query, extraConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala index 8c09ce01cf1..53acadbc729 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometLengthOfJsonArrayBenchmark.scala @@ -41,7 +41,7 @@ object CometLengthOfJsonArrayBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(numRows) + .range(numRows.toLong) .map { i => val arrayLength = (i % 100).toInt (0 until arrayLength) @@ -61,7 +61,11 @@ object CometLengthOfJsonArrayBenchmark extends CometBenchmarkBase { benchmarks.foreach { config => runBenchmark(config.name) { - runExpressionBenchmark(config.name, v, config.query, config.extraCometConfigs) + runExpressionBenchmark( + config.name, + v.toLong, + config.query, + config.extraCometConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala index 2f3904c5df6..fa54e0dfe3b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometOperatorSerdeBenchmark.scala @@ -219,7 +219,7 @@ object CometOperatorSerdeBenchmark extends CometBenchmarkBase { val iterations = 100 val benchmark = new Benchmark( s"IcebergScan serde ($numPartitions partitions, ${tasks.size()} tasks)", - iterations, + iterations.toLong, output = output) // Benchmark: Convert FileScanTasks to protobuf (the convert() method) @@ -342,7 +342,7 @@ object CometOperatorSerdeBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"CometScanRule apply ($numPartitions partitions)", - iterations, + iterations.toLong, output = output) benchmark.addCase("CometScanRule.apply(sparkPlan)") { _ => diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala index 6fab1ea7a76..58750ee9f4c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPaddingExpressionBenchmark.scala @@ -34,7 +34,7 @@ object CometPaddingExpressionBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(rows) + .range(rows.toLong) .selectExpr( "CAST(id AS STRING) AS s", "CAST(id % 32 + 8 AS INT) AS len", @@ -48,7 +48,7 @@ object CometPaddingExpressionBenchmark extends CometBenchmarkBase { runBenchmark(name) { runExpressionBenchmark( name, - rows, + rows.toLong, s"SELECT $function($arguments) FROM parquetV1Table") } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala index a7d170057f4..cdc93a44ff1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala @@ -39,7 +39,7 @@ object CometPartitionColumnBenchmark extends CometBenchmarkBase { def partitionColumnScanBenchmark(values: Int, numPartitionCols: Int): Unit = { val sqlBenchmark = new Benchmark( s"Partitioned Scan with $numPartitionCols partition column(s)", - values, + values.toLong, output = output) withTempPath { dir => diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala index 6506c5665d9..a7e22d8e90d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometPredicateExpressionBenchmark.scala @@ -38,7 +38,7 @@ object CometPredicateExpressionBenchmark extends CometBenchmarkBase { val query = "select * from parquetV1Table where c1 in ('positive', 'zero')" - runExpressionBenchmark("in Expr", values, query) + runExpressionBenchmark("in Expr", values.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala index 1055240cd73..6b809d381b1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala @@ -77,7 +77,11 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val operators = 1024 val creation = - new Benchmark("Scan I/O SQL metrics: creation", operators, minNumIters = 5, output = output) + new Benchmark( + "Scan I/O SQL metrics: creation", + operators.toLong, + minNumIters = 5, + output = output) creation.addCase("nine metrics per operator") { _ => var count = 0 while (count < operators) { @@ -91,7 +95,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val driverMetrics = createMetrics() val updates = new Benchmark( "Scan I/O SQL metrics: task snapshots", - tasks, + tasks.toLong, minNumIters = 5, output = output) updates.addCase("copy, update and merge nine metrics") { _ => @@ -112,7 +116,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val partitions = spark.sparkContext.parallelize(0 until tasks, tasks) val jobs = new Benchmark( "Scan I/O SQL metrics: 10000-task Spark job", - tasks, + tasks.toLong, minNumIters = 3, output = output) val cases = if (reverseCases) Seq(9, 0) else Seq(0, 9) @@ -131,7 +135,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def numericScanBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"SQL Single ${dataType.sql} Column Scan", values, output = output) + new Benchmark(s"SQL Single ${dataType.sql} Column Scan", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -150,7 +154,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def encryptedScanBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"SQL Single ${dataType.sql} Encrypted Column Scan", values, output = output) + new Benchmark( + s"SQL Single ${dataType.sql} Encrypted Column Scan", + values.toLong, + output = output) val encoder = Base64.getEncoder val footerKey = @@ -191,7 +198,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def decimalScanBenchmark(values: Int, precision: Int, scale: Int): Unit = { val sqlBenchmark = new Benchmark( s"SQL Single Decimal(precision: $precision, scale: $scale) Column Scan", - values, + values.toLong, output = output) withTempPath { dir => @@ -210,7 +217,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def readerBenchmark(values: Int, dataType: DataType): Unit = { val sqlBenchmark = - new Benchmark(s"Parquet reader benchmark for $dataType", values, output = output) + new Benchmark(s"Parquet reader benchmark for $dataType", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -268,7 +275,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def numericFilterScanBenchmark(values: Int, fractionOfZeros: Double): Unit = { val percentageOfZeros = fractionOfZeros * 100 val benchmark = - new Benchmark(s"Numeric Filter Scan ($percentageOfZeros% zeros)", values, output = output) + new Benchmark( + s"Numeric Filter Scan ($percentageOfZeros% zeros)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table", "parquetV2Table") { @@ -286,7 +296,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def stringWithDictionaryScanBenchmark(values: Int): Unit = { val sqlBenchmark = - new Benchmark("String Scan with Dictionary Encoding", values, output = output) + new Benchmark("String Scan with Dictionary Encoding", values.toLong, output = output) withTempPath { dir => withTempTable("parquetV1Table", "parquetV2Table") { @@ -316,7 +326,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def stringWithNullsScanBenchmark(values: Int, fractionOfNulls: Double): Unit = { val percentageOfNulls = fractionOfNulls * 100 val benchmark = - new Benchmark(s"String with Nulls Scan ($percentageOfNulls%)", values, output = output) + new Benchmark( + s"String with Nulls Scan ($percentageOfNulls%)", + values.toLong, + output = output) withTempPath { dir => withTempTable("parquetV1Table") { @@ -337,7 +350,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { def columnsBenchmark(values: Int, width: Int): Unit = { val benchmark = - new Benchmark(s"Single Column Scan from $width columns", values, output = output) + new Benchmark(s"Single Column Scan from $width columns", values.toLong, output = output) withTempPath { dir => withTempTable("t1", "parquetV1Table") { @@ -358,7 +371,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"Large String Filter Scan ($percentageOfZeros% zeros)", - values, + values.toLong, output = output) withTempPath { dir => @@ -380,7 +393,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase { val benchmark = new Benchmark( s"Sorted Lg Str Filter Scan ($percentageOfZeros% zeros)", - values, + values.toLong, output = output) withTempPath { dir => diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala index 5a6741dc2a8..8ab4ecf5996 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpBenchmark.scala @@ -71,7 +71,7 @@ object CometRegExpBenchmark extends CometBenchmarkBase { patterns.foreach { p => val query = s"select c1 rlike '${p.pattern}' from parquetV1Table" runBenchmark(p.name) { - runRLikeModes(p.name, v, query) + runRLikeModes(p.name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala index 26cc7f73f14..1d8d423f060 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala @@ -112,14 +112,14 @@ object CometRegExpExtractBenchmark extends CometBenchmarkBase { val extractQuery = s"select regexp_extract(c1, '${p.pattern}', ${p.idx}) from parquetV1Table" runBenchmark(extractName) { - runModes("RegExpExtract", extractName, v, extractQuery) + runModes("RegExpExtract", extractName, v.toLong, extractQuery) } val extractAllName = s"regexp_extract_all / ${p.name}" val extractAllQuery = s"select regexp_extract_all(c1, '${p.pattern}', ${p.idx}) from parquetV1Table" runBenchmark(extractAllName) { - runModes("RegExpExtractAll", extractAllName, v, extractAllQuery) + runModes("RegExpExtractAll", extractAllName, v.toLong, extractAllQuery) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala index 93f41ccf6b2..1b559ee6ddc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSequenceBenchmark.scala @@ -70,7 +70,7 @@ object CometSequenceBenchmark extends CometBenchmarkBase { sequenceQueries.foreach { case (name, query) => runBenchmark(name) { - runExpressionBenchmark(name, v, query) + runExpressionBenchmark(name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala index 64d9d0237af..db63d2ef942 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala @@ -110,7 +110,9 @@ object CometShuffleBenchmark extends CometBenchmarkBase { def shuffleArrayBenchmark(values: Int, dataType: DataType, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"SQL ${dataType.sql} shuffle on array ($partitionNum Partition)", values) + microBenchmark( + s"SQL ${dataType.sql} shuffle on array ($partitionNum Partition)", + values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { @@ -158,7 +160,9 @@ object CometShuffleBenchmark extends CometBenchmarkBase { def shuffleStructBenchmark(values: Int, dataType: DataType, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"SQL ${dataType.sql} shuffle on struct ($partitionNum Partition)", values) + microBenchmark( + s"SQL ${dataType.sql} shuffle on struct ($partitionNum Partition)", + values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { @@ -213,7 +217,9 @@ object CometShuffleBenchmark extends CometBenchmarkBase { def shuffleDictionaryBenchmark(values: Int, dataType: DataType, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"SQL ${dataType.sql} Dictionary Shuffle($partitionNum Partition)", values) + microBenchmark( + s"SQL ${dataType.sql} Dictionary Shuffle($partitionNum Partition)", + values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { @@ -298,7 +304,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { val benchmark = microBenchmark( s"SQL Wide ($width cols) ${dataType.sql} Shuffle($partitionNum Partition)", - values) + values.toLong) val projection = (1 to width) .map(i => s"CAST(CAST(RAND(1) * 100 AS INTEGER) AS ${dataType.sql}) AS c$i") @@ -367,7 +373,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { val benchmark = microBenchmark( s"SQL Wide ($width cols) ${dataType.sql} Range Partition Shuffle($partitionNum Partition)", - values) + values.toLong) val projection = (1 to width) .map(i => s"CAST(CAST(RAND(1) * 100 AS INTEGER) AS ${dataType.sql}) AS c$i") @@ -434,7 +440,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { numRows: Int, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"Shuffle with nested schema ($name)", numRows) + microBenchmark(s"Shuffle with nested schema ($name)", numRows.toLong) val df = spark.read.parquet(filename) withTempTable("deeplyNestedTable") { df.createOrReplaceTempView("deeplyNestedTable") @@ -496,7 +502,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { values: Int, partitionNum: Int): Unit = { val benchmark = - microBenchmark(s"Nested hash key: $name ($partitionNum Partition)", values) + microBenchmark(s"Nested hash key: $name ($partitionNum Partition)", values.toLong) withTempPath { dir => withTempTable("parquetV1Table") { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala index a429da58350..9cf3fdb5dbd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala @@ -71,12 +71,12 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { withTempPath { dir => withTempTable("probe", "build") { spark - .range(probeRows) + .range(probeRows.toLong) .selectExpr("id AS k", "id % 100 AS v") .write .parquet(s"${dir.getAbsolutePath}/probe") spark - .range(buildRows) + .range(buildRows.toLong) .selectExpr("id AS k", "id * 10 AS w") .write .parquet(s"${dir.getAbsolutePath}/build") @@ -87,7 +87,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - inner count") { runExpressionBenchmark( "inner count", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ count(*) FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -95,7 +95,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - inner projected") { runExpressionBenchmark( "inner projected", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ p.k, p.v, b.w FROM probe p JOIN build b ON p.k = b.k", cometConfigs) } @@ -103,7 +103,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - left outer") { runExpressionBenchmark( "left outer", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ count(*) FROM probe p LEFT JOIN build b ON p.k = b.k", cometConfigs) } @@ -111,7 +111,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - left semi") { runExpressionBenchmark( "left semi", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(b) */ count(*) FROM probe p LEFT SEMI JOIN build b ON p.k = b.k", cometConfigs) } @@ -119,7 +119,7 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase { runBenchmark("SortMergeJoin - right outer") { runExpressionBenchmark( "right outer", - probeRows, + probeRows.toLong, "SELECT /*+ MERGE(p) */ count(*) FROM probe p RIGHT JOIN build b ON p.k = b.k", cometConfigs) } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala index fe3c78dac93..aede98a129d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala @@ -127,7 +127,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase { (stringExpressions ++ collatedStringExpressions).foreach { config => val allConfigs = extraConfigs ++ config.extraCometConfigs runBenchmark(config.name) { - runExpressionBenchmark(config.name, v, config.query, allConfigs) + runExpressionBenchmark(config.name, v.toLong, config.query, allConfigs) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala index 20ebfb9df8f..e70005fa063 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnBase64Benchmark.scala @@ -88,7 +88,7 @@ object CometUnBase64Benchmark extends CometBenchmarkBase { shapes.foreach { s => val query = s"select unbase64(${s.column}) from parquetV1Table" runBenchmark(s.name) { - runUnBase64Modes(s.name, v, query) + runUnBase64Modes(s.name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index 8f309f5cd27..d7d94ce831c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -28,11 +28,9 @@ import org.apache.commons.io.FileUtils import org.apache.spark.SparkContext import org.apache.spark.internal.config.{MEMORY_OFFHEAP_ENABLED, MEMORY_OFFHEAP_SIZE} import org.apache.spark.sql.TPCDSBase -import org.apache.spark.sql.catalyst.expressions.AttributeSet import org.apache.spark.sql.catalyst.util.resourceToString -import org.apache.spark.sql.execution.{ReusedSubqueryExec, SparkPlan, SubqueryBroadcastExec, SubqueryExec} import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite -import org.apache.spark.sql.execution.exchange.{Exchange, ReusedExchangeExec, ValidateRequirements} +import org.apache.spark.sql.execution.exchange.ValidateRequirements import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.TestSparkSession @@ -66,7 +64,6 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa getWorkspaceFilePath("spark", "src", "test", "resources", "tpcds-plan-stability").toFile } - private val referenceRegex = "#\\d+".r private val normalizeRegex = "#\\d+L?".r private val planIdRegex = "plan_id=\\d+".r @@ -182,64 +179,6 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa } } - /** - * Get the simplified plan for a specific SparkPlan. In the simplified plan, the node only has - * its name and all the sorted reference and produced attributes names(without ExprId) and its - * simplified children as well. And we'll only identify the performance sensitive nodes, e.g., - * Exchange, Subquery, in the simplified plan. Given such a identical but simplified plan, we'd - * expect to avoid frequent plan changing and catch the possible meaningful regression. - */ - private def getSimplifiedPlan(plan: SparkPlan): String = { - val exchangeIdMap = new mutable.HashMap[Int, Int]() - val subqueriesMap = new mutable.HashMap[Int, Int]() - - def getId(plan: SparkPlan): Int = plan match { - case exchange: Exchange => - exchangeIdMap.getOrElseUpdate(exchange.id, exchangeIdMap.size + 1) - case ReusedExchangeExec(_, exchange) => - exchangeIdMap.getOrElseUpdate(exchange.id, exchangeIdMap.size + 1) - case subquery: SubqueryExec => - subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1) - case subquery: SubqueryBroadcastExec => - subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1) - case ReusedSubqueryExec(subquery) => - subqueriesMap.getOrElseUpdate(subquery.id, subqueriesMap.size + 1) - case _ => -1 - } - - /** - * Some expression names have ExprId in them due to using things such as - * "sum(sr_return_amt#14)", so we remove all of these using regex - */ - def cleanUpReferences(references: AttributeSet): String = { - referenceRegex.replaceAllIn(references.map(_.name).mkString(","), "") - } - - /** - * Generate a simplified plan as a string Example output: TakeOrderedAndProject - * [c_customer_id] WholeStageCodegen Project [c_customer_id] - */ - def simplifyNode(node: SparkPlan, depth: Int): String = { - val padding = " " * depth - var thisNode = node.nodeName - if (node.references.nonEmpty) { - thisNode += s" [${cleanUpReferences(node.references)}]" - } - if (node.producedAttributes.nonEmpty) { - thisNode += s" [${cleanUpReferences(node.producedAttributes)}]" - } - val id = getId(node) - if (id > 0) { - thisNode += s" #$id" - } - val childrenSimplified = node.children.map(simplifyNode(_, depth + 1)) - val subqueriesSimplified = node.subqueries.map(simplifyNode(_, depth + 1)) - s"$padding$thisNode\n${subqueriesSimplified.mkString("")}${childrenSimplified.mkString("")}" - } - - simplifyNode(plan, 0) - } - private def normalizeIds(plan: String): String = { val map = new mutable.HashMap[String, String]() normalizeRegex diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala index 1c3500426c3..d2fb23c90b9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala @@ -308,11 +308,11 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper { def insert(value: Int): Unit = { val bytes = new Array[Byte](4 + 16) - Platform.putInt(bytes, Platform.BYTE_ARRAY_OFFSET, value) + Platform.putInt(bytes, Platform.BYTE_ARRAY_OFFSET.toLong, value) val row = new UnsafeRow(1) - row.pointTo(bytes, Platform.BYTE_ARRAY_OFFSET + 4, 16) + row.pointTo(bytes, (Platform.BYTE_ARRAY_OFFSET + 4).toLong, 16) row.setInt(0, value) - sorter.insertRecord(bytes, Platform.BYTE_ARRAY_OFFSET, bytes.length, value % 2) + sorter.insertRecord(bytes, Platform.BYTE_ARRAY_OFFSET.toLong, bytes.length, value % 2) } try { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala index d95a4c718e5..767a0a9d9a8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala @@ -122,7 +122,7 @@ class CometArrowStreamSuite extends AnyFunSuite with Matchers { } requiredInput.putInt(i, -i) booleanInput.putBoolean(i, (i & 1) == 0) - val decimal = Decimal(i % 10000, decimalType.precision, decimalType.scale) + val decimal = Decimal((i % 10000).toLong, decimalType.precision, decimalType.scale) decimalInput.putDecimal(i, decimal, decimalType.precision) i += 1 } @@ -147,7 +147,7 @@ class CometArrowStreamSuite extends AnyFunSuite with Matchers { requiredArrow.get(i) shouldBe -i booleanArrow.get(i) shouldBe (if ((i & 1) == 0) 1 else 0) decimalArrow.getObject(i) shouldBe - Decimal(i % 10000, decimalType.precision, decimalType.scale).toJavaBigDecimal + Decimal((i % 10000).toLong, decimalType.precision, decimalType.scale).toJavaBigDecimal i += 1 } // A realloc frees the old buffers, so cumulative allocations would exceed live memory. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala index 60132c984fd..7603c22d784 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala @@ -315,7 +315,9 @@ private[shuffle] class ConcurrentRemoteMapStarts { } } -class CometCelebornConcurrentMaterializationTestManager(conf: SparkConf, isDriver: Boolean) +private[shuffle] class CometCelebornConcurrentMaterializationTestManager( + conf: SparkConf, + isDriver: Boolean) extends CometCelebornFallbackTestShuffleManager(conf, isDriver) { @volatile private[shuffle] var remoteMapStarts: Option[ConcurrentRemoteMapStarts] = None diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala index 85829401d4d..a2f8f1e41e5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala @@ -88,7 +88,9 @@ class CometCelebornLocalFetchFailureSuite extends CometTestBase { } /** Injects one local block loss only after Spark accepts another result partition. */ -class CometCelebornLocalFetchFailureTestManager(conf: SparkConf, isDriver: Boolean) +private[shuffle] class CometCelebornLocalFetchFailureTestManager( + conf: SparkConf, + isDriver: Boolean) extends CometCelebornFallbackTestShuffleManager(conf, isDriver) { // This local-mode fixture keeps state in its SparkContext-owned shuffle manager. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala index 826ac7e86df..de3381febfe 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala @@ -542,7 +542,7 @@ private[shuffle] class CompletedMaterializationJobs extends SparkListener { } } -class CometCelebornFallbackTestShuffleManager(conf: SparkConf, isDriver: Boolean) +private[shuffle] class CometCelebornFallbackTestShuffleManager(conf: SparkConf, isDriver: Boolean) extends CometCelebornShuffleManager( conf, isDriver, From 33f16426f7b7789aa6516c849e570a027f3f96b7 Mon Sep 17 00:00:00 2001 From: Sudarshan Date: Sun, 13 Sep 2026 10:48:35 +0530 Subject: [PATCH 2/5] comment fixes --- .github/workflows/pr_build_linux.yml | 33 +++++++++++++++++++ pom.xml | 9 +++-- .../comet/iceberg/IcebergReflection.scala | 8 ++--- .../apache/comet/rules/CometScanRule.scala | 8 +---- .../operator/CometIcebergNativeScan.scala | 2 +- .../comet/objectstore/NativeConfigSuite.scala | 26 ++++++++++----- .../org/apache/spark/sql/CometTestBase.scala | 2 -- 7 files changed, 61 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 6814d75769b..39aee1f64c0 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -166,6 +166,39 @@ jobs: run: | ./dev/ci/check-working-tree-clean.sh + # Compiles main and test sources with -Pstrict-warnings, which promotes scalac warnings to + # errors, so a change that reintroduces a cleared warning (an Int widened into a Long metric, + # a discarded builder result) fails here rather than accumulating. Spark 3.5 only: the profile + # passes on the Scala 2.12 profiles, and the Scala 2.13 remainder is tracked separately (see + # the strict-warnings profile in pom.xml). + strict-scala-warnings: + needs: lint + name: Strict Scala warnings (Spark 3.5, JDK 17) + runs-on: ubuntu-24.04 + container: + image: amd64/rust + steps: + - uses: actions/checkout@v7 + + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ env.RUST_VERSION }} + jdk-version: 17 + + - name: Cache Maven dependencies + uses: actions/cache@v6 + with: + path: | + ~/.m2/repository + /root/.m2/repository + key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-strict-warnings + restore-keys: | + ${{ runner.os }}-java-maven- + + - name: Compile with strict warnings + run: ./mvnw -B test-compile -Pspark-3.5 -Pstrict-warnings -DskipTests + # Compile-only verification for Spark 4.1. Tests are intentionally skipped: the spark-4.1 # profile is currently a build target only, and several runtime/test failures are tracked # in follow-up PRs. Excluded from lint-java because semanticdb-scalac_2.13.17 is not yet diff --git a/pom.xml b/pom.xml index 6afdba7360c..100b795edb1 100644 --- a/pom.xml +++ b/pom.xml @@ -889,11 +889,10 @@ under the License. `-Xlint:nowarn` and `-Xfatal-warnings` turns into a build failure. `-Ywarn-value-discard` stays on for main sources, where a discarded result is - usually a dropped builder or a swallowed return, but is off for test sources. - Under it a ScalaTest suite reports ~1,250-1,450 warnings, and the two largest - groups are the idiom itself: an `assert(...)` in trailing position discards an - `org.scalatest.Assertion`, and `checkSparkAnswerAndOperator` discards the - `(SparkPlan, SparkPlan)` it returns at all but 30 of its ~1,300 call sites. + usually a dropped builder or a swallowed return, but is off for test sources, + where what it reports is the testing idiom itself: an `assert(...)` in trailing + position discards an `org.scalatest.Assertion`, and helpers such as + `checkSparkAnswerAndOperator` return plans that most callers ignore. --> strict-warnings diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index 40334224966..645b657da9d 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -436,13 +436,13 @@ object IcebergReflection extends Logging { * `taskGroups()`, so for staged scans we flatten the groups instead. Both methods are protected * and require reflection. */ - def getTasks(scan: Any): Option[java.util.List[_]] = + def getTasks(scan: Any): Option[java.util.List[AnyRef]] = if (isStagedScan(scan)) tasksFromTaskGroups(scan) else tasksFromTasksAccessor(scan) - private def tasksFromTasksAccessor(scan: Any): Option[java.util.List[_]] = + private def tasksFromTasksAccessor(scan: Any): Option[java.util.List[AnyRef]] = findMethodInHierarchy(scan.getClass, "tasks") match { case Some(method) => - Some(method.invoke(scan).asInstanceOf[java.util.List[_]]) + Some(method.invoke(scan).asInstanceOf[java.util.List[AnyRef]]) case None => logError( "Iceberg reflection failure: Failed to get tasks from SparkScan: " + @@ -450,7 +450,7 @@ object IcebergReflection extends Logging { None } - private def tasksFromTaskGroups(scan: Any): Option[java.util.List[_]] = + private def tasksFromTaskGroups(scan: Any): Option[java.util.List[AnyRef]] = findMethodInHierarchy(scan.getClass, "taskGroups") match { case Some(method) => try { diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 193af4e684b..fb2582b2335 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -486,17 +486,11 @@ case class CometScanRule(session: SparkSession) // which bucket's per-bucket S3 settings the single native FileIO consumes (see // catalogProperties below), so this must run before we build them. The task list is // handed to `extract` below so the reflective accessor runs once per scan. - // The task list is cast to `java.util.List[AnyRef]` rather than left at the - // `java.util.List[_]` that `getTasks` returns: a wildcard element type makes this - // pair infer a top-level existential, which `-Xlint:existential` rejects. The list - // is erased at runtime, so the cast costs nothing. val (icebergTasks, taskValidation) = try { IcebergReflection.getTasks(scanExec.scan) match { case Some(tasks) => - ( - tasks.asInstanceOf[java.util.List[AnyRef]], - CometScanRule.validateIcebergFileScanTasks(tasks, s3CompliantSchemes)) + (tasks, CometScanRule.validateIcebergFileScanTasks(tasks, s3CompliantSchemes)) case None => fallbackReasons += "Iceberg reflection failure: Could not extract FileScanTasks" return withFallbackReasons(scanExec, fallbackReasons.toSet) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala index 24add22cb25..256a7b09fa3 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala @@ -353,7 +353,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit val equalityIds = equalityIdsMethod .invoke(deleteFile) .asInstanceOf[java.util.List[Integer]] - equalityIds.forEach(id => { val _ = deleteBuilder.addEqualityIds(id) }) + deleteBuilder.addAllEqualityIds(equalityIds) } catch { case _: Exception => } diff --git a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala index 0b12b9a8a51..1cfd1d36b92 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala @@ -30,6 +30,14 @@ import org.apache.comet.CometConf.COMET_S3_COMPLIANT_SCHEMES_KEY class NativeConfigSuite extends AnyFunSuite with Matchers { + /** + * A Hadoop `Configuration` variable reference to `key`, as `Configuration#get` expands it. + * Built by concatenation: written as a literal, it reads to scalac as a string missing its `s` + * interpolator, and adding the `s` (escaping the dollar) is then flagged by scalafix as + * redundant. + */ + private def varRef(key: String): String = "${" + key + "}" + test("extractObjectStoreOptions - multiple cloud provider configurations") { val hadoopConf = new Configuration() // S3A configs @@ -108,13 +116,15 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { } } - test(s"extractObjectStoreOptions - forwards the substituted value of a $${...} reference") { + test( + "extractObjectStoreOptions - forwards the substituted value of a " + + s"${varRef("...")} reference") { // Hadoop's own consumers read values through Configuration#get, which expands a ${...} // reference against another conf entry. Forwarding the raw, unexpanded literal here would // give native a different credential than every Hadoop-side consumer sees. val hadoopConf = new Configuration() hadoopConf.set("my.custom.access.key", "expanded-access-key") - hadoopConf.set("fs.s3a.access.key", s"$${my.custom.access.key}") + hadoopConf.set("fs.s3a.access.key", varRef("my.custom.access.key")) val options = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/test-object")) @@ -122,19 +132,19 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { } test( - s"extractObjectStoreOptions - a cyclic $${...} reference falls back to the raw value " + - "instead of throwing") { + s"extractObjectStoreOptions - a cyclic ${varRef("...")} reference falls back to the raw " + + "value instead of throwing") { // Configuration#get raises IllegalStateException once ${...} expansion recurses past // Hadoop's MAX_SUBST bound; a two-key mutual cycle triggers this on every call. Extraction // must still return a full options map rather than aborting for the whole object store. val hadoopConf = new Configuration() - hadoopConf.set("fs.s3a.access.key", s"$${fs.s3a.secret.key}") - hadoopConf.set("fs.s3a.secret.key", s"$${fs.s3a.access.key}") + hadoopConf.set("fs.s3a.access.key", varRef("fs.s3a.secret.key")) + hadoopConf.set("fs.s3a.secret.key", varRef("fs.s3a.access.key")) val options = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/test-object")) - assert(options("fs.s3a.access.key") == s"$${fs.s3a.secret.key}") - assert(options("fs.s3a.secret.key") == s"$${fs.s3a.access.key}") + assert(options("fs.s3a.access.key") == varRef("fs.s3a.secret.key")) + assert(options("fs.s3a.secret.key") == varRef("fs.s3a.access.key")) } test( diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala index 8fcfff800b1..0156ddfbdbc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala @@ -792,8 +792,6 @@ abstract class CometTestBase .builder(path) .withDictionaryEncoding(dictionaryEnabled) .withType(schema) - // TODO we need to shim this and use withRowGroupSize(Long) with later parquet-hadoop versions to remove - // the deprecated warning here .withRowGroupSize(rowGroupSize) .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) From 380d68fce88c8bcd147b6a41c0f57cbf14b9ab3e Mon Sep 17 00:00:00 2001 From: Sudarshan Date: Sun, 13 Sep 2026 19:31:15 +0530 Subject: [PATCH 3/5] comment fixes --- .github/workflows/pr_build_linux.yml | 3 +++ pom.xml | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 39aee1f64c0..75579cd1419 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -196,6 +196,9 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Compile with strict warnings run: ./mvnw -B test-compile -Pspark-3.5 -Pstrict-warnings -DskipTests diff --git a/pom.xml b/pom.xml index 100b795edb1..51ac77f823c 100644 --- a/pom.xml +++ b/pom.xml @@ -877,16 +877,16 @@ under the License. Two lints are deliberately absent from both lists: - `-Ywarn-unused:params` reports ~90-120 parameters per profile, and essentially - all of them are structurally unfixable rather than dead: `Native.scala` is 64 - `@native` declarations whose parameters have no body to be used in, and the rest - are cross-version shims under `src/main/spark-3.x`, `spark-4.x` and friends that - take a parameter to satisfy the signature of the Spark version they shim. Nor can - those be annotated away one by one: the set differs between Scala 2.12 and 2.13 - (`CometScanContrib.scala` warns under spark-3.5 but not spark-4.0, and vice versa - for `ShimSparkErrorConverter.scala`), so any `@nowarn` that silences one profile - is an unused annotation on the other, which `-Xlint:_` then reports via - `-Xlint:nowarn` and `-Xfatal-warnings` turns into a build failure. + `-Ywarn-unused:params` reports parameters that are unused because a signature + requires them rather than because they are dead: the `@native` declarations in + `Native.scala`, which have no body to use them in; cross-version shims under + `src/main/spark-*` that match the signature of the Spark version they shim; and + overridable defaults and serde helpers whose parameter is part of their public + shape. Suppressing these one by one with `@nowarn` does not work across profiles: + some shared sources warn under Scala 2.12 but not 2.13 (`CometScanContrib.scala`, + for example), so an annotation that silences one profile is unused on the other, + which `-Xlint:_` reports via `-Xlint:nowarn` and `-Xfatal-warnings` turns into a + build failure. `-Ywarn-value-discard` stays on for main sources, where a discarded result is usually a dropped builder or a swallowed return, but is off for test sources, From c0e1648cd12a817e0ab036dd42c124d9d4fdae2b Mon Sep 17 00:00:00 2001 From: Sudarshan Date: Sun, 13 Sep 2026 20:36:36 +0530 Subject: [PATCH 4/5] added issue comments --- .github/workflows/pr_build_linux.yml | 4 ++-- pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 75579cd1419..4a4643380ce 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -169,8 +169,8 @@ jobs: # Compiles main and test sources with -Pstrict-warnings, which promotes scalac warnings to # errors, so a change that reintroduces a cleared warning (an Int widened into a Long metric, # a discarded builder result) fails here rather than accumulating. Spark 3.5 only: the profile - # passes on the Scala 2.12 profiles, and the Scala 2.13 remainder is tracked separately (see - # the strict-warnings profile in pom.xml). + # passes on the Scala 2.12 profiles, and the Scala 2.13 remainder is tracked in + # https://github.com/apache/datafusion-comet/issues/5893. strict-scala-warnings: needs: lint name: Strict Scala warnings (Spark 3.5, JDK 17) diff --git a/pom.xml b/pom.xml index 51ac77f823c..50c8dbd48f3 100644 --- a/pom.xml +++ b/pom.xml @@ -867,8 +867,8 @@ under the License. still report warnings that 2.12 does not raise at all, dominated by `-Xlint:nonlocal-return` (a `return` inside a closure, which the compiler implements by throwing) and non-exhaustive matches. Clearing those means - restructuring control flow rather than annotating it, so they are left for a - follow-up rather than silenced here. + restructuring control flow rather than annotating it, so they are tracked in + https://github.com/apache/datafusion-comet/issues/5893 rather than silenced here. `args` is configured per execution rather than on the plugin, because main and test sources warrant different flags (see `-Ywarn-value-discard` below). An From 57a5a7a2896bb02f69e075a626ac4996d678fd8b Mon Sep 17 00:00:00 2001 From: Sudarshan Date: Wed, 23 Sep 2026 20:47:43 +0530 Subject: [PATCH 5/5] error fixes --- docs/source/contributor-guide/ci.md | 2 +- docs/source/contributor-guide/development.md | 14 ++++++++++++-- .../comet/parquet/CometParquetWriterSuite.scala | 2 +- .../comet/serde/SerdeRegistrationSuite.scala | 8 ++++---- .../sql/benchmark/CometBinaryLengthBenchmark.scala | 2 +- .../sql/benchmark/CometIcebergWriteBenchmark.scala | 2 +- .../benchmark/CometUnixTimestampBenchmark.scala | 6 +++--- 7 files changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/source/contributor-guide/ci.md b/docs/source/contributor-guide/ci.md index cea032370e9..d9cd1992723 100644 --- a/docs/source/contributor-guide/ci.md +++ b/docs/source/contributor-guide/ci.md @@ -409,7 +409,7 @@ actionlint --shellcheck=off ``` A new test suite has to be registered in the workflow files by hand; see -[Register New Test Suites in CI](development.md#5-register-new-test-suites-in-ci). A new job in +[Register New Test Suites in CI](development.md#6-register-new-test-suites-in-ci). A new job in `ci.yml` needs an entry in both `FILTERS` and `POLICY` in `dev/ci/compute-changes.py`, a case in `dev/ci/check-ci-config.py`, and a line in `required_checks.needs`. The check will tell you which of those is missing. diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index b5752e08630..4e7dddbdbd1 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -600,7 +600,17 @@ cargo clippy --color=never --all-targets --workspace -- -D warnings Make sure to resolve any Clippy warnings before submitting your pull request, as the CI/CD pipeline will fail if warnings are present. -### 4. Run Tests +### 4. Compile With Strict Scala Warnings (Recommended) + +The `Strict Scala warnings` job runs on every pull request and in the merge queue. It compiles the main and test sources with scalac warnings promoted to errors, so anything it reports fails the build — an `Int` widened into a `Long` metric, or a discarded builder result, for example. Reproduce it locally with: + +```sh +./mvnw test-compile -Pspark-3.5 -Pstrict-warnings -DskipTests +``` + +Use the Spark 3.5 profile: it is the one the job runs, and the default build profile will not reproduce it. The default is Spark 4.1 on Scala 2.13, where `-Pstrict-warnings` still fails on warnings unrelated to your change (tracked in [#5893](https://github.com/apache/datafusion-comet/issues/5893)), and where the compiler reports a different set — an adapted argument list, for instance, is flagged under Scala 2.12 but not under 2.13. + +### 5. Run Tests Run the relevant tests for your changes: @@ -615,7 +625,7 @@ make test-rust make test-jvm ``` -### 5. Register New Test Suites in CI +### 6. Register New Test Suites in CI Comet's CI does not automatically discover test suites. Instead, test suites are explicitly listed in the GitHub Actions workflow files so they can be grouped by category and run as separate parallel diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala index c46524e2917..babe86b3c72 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala @@ -1752,5 +1752,5 @@ class RecordingStatsTracker extends WriteTaskStatsTracker { override def closeFile(filePath: String): Unit = {} override def newRow(filePath: String, row: InternalRow): Unit = rows += ((filePath, row)) override def getFinalStats(taskCommitTime: Long): WriteTaskStats = - BasicWriteTaskStats(Seq.empty, 0, 0, rows.size) + BasicWriteTaskStats(Seq.empty, 0, 0, rows.size.toLong) } diff --git a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala index 6b82b2fcfbb..300d19081f1 100644 --- a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala @@ -30,10 +30,10 @@ class SerdeRegistrationSuite extends AnyFunSuite { test("version shims register only classes the shared serde maps do not") { import QueryPlanSerde._ val overlaps = Seq( - "math" -> (baseMathExpressions, sparkVersionSpecificMathExpressions), - "map" -> (baseMapExpressions, sparkVersionSpecificMapExpressions), - "string" -> (baseStringExpressions, sparkVersionSpecificStringExpressions), - "misc" -> (baseMiscExpressions, sparkVersionSpecificMiscExpressions)) + "math" -> ((baseMathExpressions, sparkVersionSpecificMathExpressions)), + "map" -> ((baseMapExpressions, sparkVersionSpecificMapExpressions)), + "string" -> ((baseStringExpressions, sparkVersionSpecificStringExpressions)), + "misc" -> ((baseMiscExpressions, sparkVersionSpecificMiscExpressions))) .flatMap { case (group, (base, shim)) => base.keySet.intersect(shim.keySet).map(cls => s"$group: ${cls.getSimpleName}") } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBinaryLengthBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBinaryLengthBenchmark.scala index 04e65ab354b..23699ccdd3e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBinaryLengthBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBinaryLengthBenchmark.scala @@ -57,7 +57,7 @@ object CometBinaryLengthBenchmark extends CometBenchmarkBase { cases.foreach { case (name, query) => runBenchmark(name) { - runExpressionBenchmark(name, v, query) + runExpressionBenchmark(name, v.toLong, query) } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala index 55f14d9613a..509bcbf8796 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala @@ -328,7 +328,7 @@ object CometIcebergWriteBenchmark extends CometBenchmarkBase { // better-jitted measurement. val benchmark = new Benchmark( workload.title, - values, + values.toLong, // The iceberg-java baseline is the least repeatable of the three arms - it carries a fifth of // its own runtime as spread between iterations, against a few percent for the two Comet arms // - and it is also the divisor of every `Relative` figure. Five iterations rather than the diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnixTimestampBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnixTimestampBenchmark.scala index 85f2a30b14a..20d2aec2aa7 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnixTimestampBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometUnixTimestampBenchmark.scala @@ -34,7 +34,7 @@ object CometUnixTimestampBenchmark extends CometBenchmarkBase { prepareTable( dir, spark - .range(rows) + .range(rows.toLong) .selectExpr( "timestamp_seconds(id) AS ts", "date_format(timestamp_seconds(id), 'yyyy-MM-dd HH:mm:ss') AS s", @@ -48,7 +48,7 @@ object CometUnixTimestampBenchmark extends CometBenchmarkBase { runBenchmark(name) { runExpressionBenchmark( name, - rows, + rows.toLong, s"SELECT unix_timestamp($arguments) FROM parquetV1Table") } } @@ -57,7 +57,7 @@ object CometUnixTimestampBenchmark extends CometBenchmarkBase { runBenchmark(name) { runExpressionBenchmark( name, - rows, + rows.toLong, s"SELECT unix_timestamp($arguments) AS u, count(*) FROM parquetV1Table GROUP BY u") } }