From 68d9a2c3a7fb9885cdd93f2aaf19c124e033671b Mon Sep 17 00:00:00 2001 From: David Mollitor Date: Thu, 10 Sep 2026 18:45:05 +0000 Subject: [PATCH] [SPARK-59413][SQL] Treat the Collate passthrough as cheap in CollapseProject.isCheap `CollapseProject.isCheap` decides whether an expression is cheap enough to duplicate/inline. It did not recognize the `Collate` expression, so `Collate(cheapChild)` was treated as non-cheap and blocked from safe duplication/inlining across every `isCheap`-gated rule (CollapseProject inlining, the multi-LIKE `LikeSimplification` rules, the `FilterExec` CSE gate, `RewriteWithExpression`). `Collate` (the SQL `collate(expr, 'COLLATION')` function / `expr COLLATE COLLATION` syntax) is a pure pass-through: its `eval` and codegen delegate directly to the child and it never evaluates its collation argument -- it only re-tags the collation carried on the result type. It is deterministic and, at runtime, exactly as cheap to duplicate as the child it wraps. Teach `isCheap` to look through `Collate` to its value child. Only the value child is inspected -- the collation argument is a constant `ResolvedCollation` marker that is never evaluated -- so a non-cheap value child (e.g. `Collate(substring(col), ...)`) correctly stays non-cheap, preserving the SPARK-40228 guard. This is behavior-preserving: `Collate` returns the child's value unchanged, so duplicating `Collate(cheapChild)` is semantically identical to duplicating the child. It unblocks the collated-column case of the gated `startsAndEndsWith` `LikeSimplification` rewrite (SPARK-59371). Generated-by: Claude Opus 4.8 Co-authored-by: Isaac --- .../spark/sql/catalyst/optimizer/Optimizer.scala | 4 ++++ .../catalyst/optimizer/CollapseProjectSuite.scala | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index f1484ff02e154..24a7daf29beec 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -1677,6 +1677,10 @@ object CollapseProject extends Rule[LogicalPlan] with AliasHelper { } // Alias and ExtractValue are very cheap. case _: Alias | _: ExtractValue => e.children.forall(isCheap) + // `Collate` only re-tags the collation in the type; at runtime it is a pass-through + // (eval/genCode delegate to the child) and never evaluates its collation argument, so it is + // as cheap to duplicate as its value child. + case c: Collate => isCheap(c.child) case _ => false } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseProjectSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseProjectSuite.scala index 25c310ab7300e..d0d62695ea9cf 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseProjectSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseProjectSuite.scala @@ -21,7 +21,7 @@ import org.apache.spark.api.python.PythonEvalType import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ -import org.apache.spark.sql.catalyst.expressions.{Alias, CreateArray, Expression, GetArrayItem, PythonUDF, Rand, UpdateFields} +import org.apache.spark.sql.catalyst.expressions.{Alias, Collate, CreateArray, Expression, GetArrayItem, Literal, PythonUDF, Rand, ResolvedCollation, Substring, UpdateFields} import org.apache.spark.sql.catalyst.plans.PlanTest import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.RuleExecutor @@ -364,4 +364,17 @@ class CollapseProjectSuite extends PlanTest { comparePlans(optimized, expected) } + + test("SPARK-59413: Collate is cheap when its value child is cheap") { + // `Collate` is a pure passthrough: `eval` delegates to the child and the collation argument + // is never evaluated, so `isCheap` mirrors the value child's cheapness. + val col = $"s".string + assert(CollapseProject.isCheap(col)) + assert(CollapseProject.isCheap(Collate(col, ResolvedCollation("UTF8_LCASE")))) + + // A non-cheap value child stays non-cheap through `Collate` (the SPARK-40228 guard is intact). + val nonCheap = Substring(col, Literal(1), Literal(5)) + assert(!CollapseProject.isCheap(nonCheap)) + assert(!CollapseProject.isCheap(Collate(nonCheap, ResolvedCollation("UTF8_LCASE")))) + } }