fix: gate case conversion on standard per-expression allowIncompatible

docs/source/contributor-guide/expression-audits/predicate_funcs.md

@@ -94,7 +94,7 @@
 ## ilike
 
 - Spark 3.4.3 (audited 2026-05-27): identical to 3.5.8.
-- Spark 3.5.8 (audited 2026-05-27): baseline. `ILike(left, right, escapeChar) extends RuntimeReplaceable`; the analyzer rewrites to `Like(Lower(left), Lower(right), escapeChar)`. Comet handles via `CometLike` and `CometLower` (case-conversion path, gated by `spark.comet.caseConversion.enabled=false` by default).
+- Spark 3.5.8 (audited 2026-07-07): baseline. `ILike(left, right, escapeChar) extends RuntimeReplaceable`; the analyzer rewrites to `Like(Lower(left), Lower(right), escapeChar)`. Comet handles via `CometLike` and `CometLower` (the case-conversion path is Spark-compatible by default via codegen dispatch, with a faster native path available behind `spark.comet.expression.Lower.allowIncompatible=true`).
 - Spark 4.0.1 (audited 2026-05-27): identical to 3.5.8.
 - Spark 4.1.1 (audited 2026-05-27): identical to 4.0.1.
 

spark/src/main/scala/org/apache/comet/CometConf.scala

@@ -779,14 +779,6 @@ object CometConf extends ShimCometConf {
       .toSequence
       .createWithDefault(Seq("Range,InMemoryTableScan,RDDScan,OneRowRelation"))
 
-  val COMET_CASE_CONVERSION_ENABLED: ConfigEntry[Boolean] =
-    conf("spark.comet.caseConversion.enabled")
-      .category(CATEGORY_EXEC)
-      .doc("Java uses locale-specific rules when converting strings to upper or lower case and " +
-        "Rust does not, so we disable upper and lower by default.")
-      .booleanConf
-      .createWithDefault(false)
-
   val COMET_PARQUET_UNSIGNED_SMALL_INT_CHECK: ConfigEntry[Boolean] =
     conf("spark.comet.scan.unsignedSmallIntSafetyCheck")
       .category(CATEGORY_SCAN)

spark/src/main/scala/org/apache/comet/serde/strings.scala

@@ -56,18 +56,16 @@ class CometCaseConversionBase[T <: Expression](function: String)
   override def getIncompatibleReasons(): Seq[String] =
     Seq("Results can vary depending on locale and character set")
 
-  override def nativeOptInConfigKeyOverride: Option[String] =
-    Some(CometConf.COMET_CASE_CONVERSION_ENABLED.key)
-
   override def getSupportLevel(expr: T): SupportLevel =
-    if (!CometConf.COMET_CASE_CONVERSION_ENABLED.get()) {
-      Compatible(nativeOptIn = Some(NativeOptIn(CometConf.COMET_CASE_CONVERSION_ENABLED.key)))
+    if (!CometConf.isExprAllowIncompat(getExprConfigName(expr))) {
+      Compatible(nativeOptIn =
+        Some(NativeOptIn(CometConf.getExprAllowIncompatConfigKey(getExprConfigName(expr)))))
     } else {
       Compatible()
     }
 
   override def convert(expr: T, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = {
-    if (CometConf.COMET_CASE_CONVERSION_ENABLED.get()) {
+    if (CometConf.isExprAllowIncompat(getExprConfigName(expr))) {
       // Native scalar function: faster but does not match Spark for locale-specific characters
       // (e.g. Turkish dotted/dotless I). Opt-in.
       super.convert(expr, inputs, binding)

spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/extended.txt

@@ -9,7 +9,7 @@ CometNativeColumnarToRow
    :                    +- CometExchange
    :                       +- CometHashAggregate
    :                          +- CometProject
-   :                             +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
+   :                             +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
    :                                :- CometProject
    :                                :  +- CometBroadcastHashJoin
    :                                :     :- CometProject
@@ -51,7 +51,7 @@ CometNativeColumnarToRow
                +- CometExchange
                   +- CometHashAggregate
                      +- CometProject
-                        +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
+                        +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
                            :- CometProject
                            :  +- CometBroadcastHashJoin
                            :     :- CometProject

spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/extended.txt

@@ -9,7 +9,7 @@ CometNativeColumnarToRow
    :                    +- CometExchange
    :                       +- CometHashAggregate
    :                          +- CometProject
-   :                             +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
+   :                             +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
    :                                :- CometProject
    :                                :  +- CometBroadcastHashJoin
    :                                :     :- CometProject
@@ -51,7 +51,7 @@ CometNativeColumnarToRow
                +- CometExchange
                   +- CometHashAggregate
                      +- CometProject
-                        +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
+                        +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
                            :- CometProject
                            :  +- CometBroadcastHashJoin
                            :     :- CometProject

spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/extended.txt

@@ -11,7 +11,7 @@ CometNativeColumnarToRow
          :                    +- CometExchange
          :                       +- CometHashAggregate
          :                          +- CometProject
-         :                             +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
+         :                             +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
          :                                :- CometProject
          :                                :  +- CometBroadcastHashJoin
          :                                :     :- CometProject
@@ -53,7 +53,7 @@ CometNativeColumnarToRow
                      +- CometExchange
                         +- CometHashAggregate
                            +- CometProject
-                              +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.caseConversion.enabled=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
+                              +-  CometBroadcastHashJoin [COMET-INFO: A native implementation of Upper is available. Set spark.comet.expression.Upper.allowIncompatible=true to enable it. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)]
                                  :- CometProject
                                  :  +- CometBroadcastHashJoin
                                  :     :- CometProject

spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala

@@ -3378,14 +3378,14 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
     }
   }
 
-  test("upper shows native opt-in via caseConversion config key") {
+  test("upper shows native opt-in via per-expression allowIncompatible config key") {
     withTable("t_upper") {
       spark.sql("create table t_upper(s string) using parquet")
       spark.sql("insert into t_upper values ('abc'), ('xyz')")
       val df = spark.sql("select upper(s) as u from t_upper")
       val explain = new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan)
       assert(explain.contains("native implementation of Upper"))
-      assert(explain.contains("spark.comet.caseConversion.enabled"))
+      assert(explain.contains("spark.comet.expression.Upper.allowIncompatible"))
     }
   }