From fcf00db5ec51c9a782edeada2b0894760f9876e5 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 7 Jul 2026 14:34:10 -0600 Subject: [PATCH 1/3] feat: add comet_version() SQL function Register a nullary SQL function via `SparkSessionExtensions.injectFunction` that returns the Comet build version string. This gives Spark users a supported way to query which Comet version is loaded from any language binding, e.g. `SELECT comet_version()`, without having to inspect JAR manifests or driver logs. The function returns the same value as the JVM `COMET_VERSION` constant (populated from `comet-git-info.properties` at build time). --- .../comet/CometSparkSessionExtensions.scala | 25 +++++++++++++++++++ .../CometSparkSessionExtensionsSuite.scala | 16 ++++++++++++ 2 files changed, 41 insertions(+) diff --git a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala index f117372b3c..16364d198b 100644 --- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala +++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala @@ -25,11 +25,15 @@ import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.network.util.ByteUnit import org.apache.spark.sql.{SparkSession, SparkSessionExtensions} +import org.apache.spark.sql.catalyst.FunctionIdentifier +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo, Literal} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreeNode import org.apache.spark.sql.comet._ import org.apache.spark.sql.execution._ import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.StringType +import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometConf._ import org.apache.comet.rules.{CometExecRule, CometPlanAdaptiveDynamicPruningFilters, CometReuseSubquery, CometScanRule, CometSpark34AqeDppFallbackRule, EliminateRedundantTransitions, RevertNativeForTransitionHeavyStages} @@ -99,6 +103,7 @@ class CometSparkSessionExtensions extensions.injectQueryStagePrepRule { session => CometExecRule(session) } injectQueryStageOptimizerRuleShim(extensions, CometPlanAdaptiveDynamicPruningFilters) injectQueryStageOptimizerRuleShim(extensions, CometReuseSubquery) + extensions.injectFunction(CometSparkSessionExtensions.cometVersionFunction) } case class CometScanColumnar(session: SparkSession) extends ColumnarRule { @@ -119,6 +124,26 @@ class CometSparkSessionExtensions object CometSparkSessionExtensions extends Logging { lazy val isBigEndian: Boolean = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN) + /** + * SQL function `comet_version()` returning the Comet build version as a string. Registered via + * `SparkSessionExtensions.injectFunction` so users can query the loaded Comet version, e.g. + * `SELECT comet_version()`. + */ + private[comet] val cometVersionFunction + : (FunctionIdentifier, ExpressionInfo, Seq[Expression] => Expression) = { + val name = "comet_version" + val versionLiteral = Literal(UTF8String.fromString(COMET_VERSION), StringType) + val info = new ExpressionInfo(classOf[Literal].getCanonicalName, null, name) + val builder = (args: Seq[Expression]) => { + if (args.nonEmpty) { + throw new IllegalArgumentException( + s"Function $name does not accept arguments (got ${args.length})") + } + versionLiteral + } + (FunctionIdentifier(name), info, builder) + } + /** * Checks whether Comet extension should be loaded for Spark. */ diff --git a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala index 268fdf94eb..7f61cbac22 100644 --- a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala @@ -130,4 +130,20 @@ class CometSparkSessionExtensionsSuite extends CometTestBase { getCometShuffleMemorySize(conf, sqlConf) == getBytesFromMib((1024 * 0.2).toLong)) } + + test("comet_version() returns the Comet build version") { + withSQLConf(CometConf.COMET_EXEC_ENABLED.key -> "false") { + val result = spark.sql("SELECT comet_version()").collect() + assert(result.length == 1) + val version = result(0).getString(0) + assert(version == COMET_VERSION, s"Expected $COMET_VERSION but got $version") + } + } + + test("comet_version() rejects arguments") { + val ex = intercept[Exception] { + spark.sql("SELECT comet_version(1)").collect() + } + assert(ex.getMessage.contains("comet_version")) + } } From 70cf347d6b8814dd3e4972ee89c5d5821abdb293 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 7 Jul 2026 16:25:07 -0600 Subject: [PATCH 2/3] fix: populate ExpressionInfo for comet_version to satisfy Spark ExpressionInfoSuite --- .../comet/CometSparkSessionExtensions.scala | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala index 16364d198b..9d9dc63e09 100644 --- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala +++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala @@ -133,7 +133,21 @@ object CometSparkSessionExtensions extends Logging { : (FunctionIdentifier, ExpressionInfo, Seq[Expression] => Expression) = { val name = "comet_version" val versionLiteral = Literal(UTF8String.fromString(COMET_VERSION), StringType) - val info = new ExpressionInfo(classOf[Literal].getCanonicalName, null, name) + // Populate the full ExpressionInfo so that Spark's ExpressionInfoSuite (SPARK-32870, and the + // example-output check) is satisfied for this injected function. The example output is derived + // from COMET_VERSION so it matches the value the function actually returns across releases. + val info = new ExpressionInfo( + classOf[Literal].getCanonicalName, + "", + name, + s"_FUNC_() - Returns the Apache DataFusion Comet version.", + "", + s"\n Examples:\n > SELECT _FUNC_();\n $COMET_VERSION\n ", + "", + "misc_funcs", + "1.0.0", + "", + "") val builder = (args: Seq[Expression]) => { if (args.nonEmpty) { throw new IllegalArgumentException( From 30d608ad1c64afe9bc984c96bccb37fb92b3d9fe Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 7 Jul 2026 17:16:09 -0600 Subject: [PATCH 3/3] fix: remove redundant string interpolator in comet_version ExpressionInfo --- .../scala/org/apache/comet/CometSparkSessionExtensions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala index 9d9dc63e09..5af23382cf 100644 --- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala +++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala @@ -140,7 +140,7 @@ object CometSparkSessionExtensions extends Logging { classOf[Literal].getCanonicalName, "", name, - s"_FUNC_() - Returns the Apache DataFusion Comet version.", + "_FUNC_() - Returns the Apache DataFusion Comet version.", "", s"\n Examples:\n > SELECT _FUNC_();\n $COMET_VERSION\n ", "",