From 393fe6f09e0461550495c0e0150280e2c765b07d Mon Sep 17 00:00:00 2001 From: Ganesha S Date: Thu, 17 Sep 2026 16:53:43 +0000 Subject: [PATCH 1/2] [SPARK-59600][SQL] Bind the resolved function owner across star preprocessing and function resolution Direct-star preprocessing and `ResolveFunctions` resolved a routed SQL/JSON call's owner independently, so a temp/persistent shadow dropped between the two phases could expand the star for the shadow while resolution fell through to the stock built-in, skipping `INVALID_USAGE_OF_STAR_OR_REGEX`. Resolve the owner once in a single ordered PATH pass and bind the winner on `UnresolvedFunction.boundOwner` so both phases share one decision; `resolveFunction` then resolves only that candidate. `count(*)` and injectFunction-replacement handling are unchanged. Tested in `JsonArraySuite` for both the fixed-point and single-pass analyzers. Co-authored-by: Isaac --- .../sql/catalyst/analysis/Analyzer.scala | 85 ++++++++------- .../catalyst/analysis/FunctionRegistry.scala | 4 +- .../analysis/FunctionResolution.scala | 102 ++++++++++++++++-- .../resolver/FunctionResolverUtils.scala | 44 ++++---- .../sql/catalyst/analysis/unresolved.scala | 7 +- .../spark/sql/classic/DataFrameWriterV2.scala | 2 +- .../command/CreateSQLFunctionCommand.scala | 2 +- .../org/apache/spark/sql/JsonArraySuite.scala | 39 ++++++- 8 files changed, 212 insertions(+), 73 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 3543326b84198..04dcd223275ad 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -2117,43 +2117,52 @@ class Analyzer( def expandStarExpression(expr: Expression, child: LogicalPlan): Expression = { expr.transformUp { case f: UnresolvedFunction if containsStar(f.arguments) => - // A routed SQL/JSON function (json_array(*)) forbids a direct star argument -- a bare `*` - // or a qualified `t.*` -- so reject it rather than expand below. A star nested in another - // expression (json_array(array(*))) is expanded bottom-up before we get here, so only a - // direct star reaches this guard. - if (functionResolution.resolvesToStarDisallowedSqlJsonFunction(f.nameParts)) { - throw QueryCompilationErrors.invalidStarUsageError( - s"expression `${f.prettyName}`", extractStar(f.arguments)) - } - // The count owner probe can hit an external functionExists lookup on a persistent-first - // PATH, so compute it once (lazily, after the cheap star-shape check) and reuse it for - // the count(*) rewrite and the count(tbl.*) guard, as `FunctionResolverUtils` does. - lazy val resolvesToCountBuiltin = matchesFunctionName(f.nameParts, "count") - if (!f.isDistinct && isCountStarExpansionAllowed(f.arguments) && resolvesToCountBuiltin) { - // Transform COUNT(*) into COUNT(1). - // We do not normalize the name to "count"; we keep the original name parts - // (e.g. builtin.count, system.builtin.count) so that resolution still sees - // the same qualification. - f.copy(arguments = Seq(Literal(1))) - } else { - // SPECIAL CASE: We want to block count(tblName.*) because in spark, count(tblName.*) - // will be expanded while count(*) will be converted to count(1). They will produce - // different results and confuse users if there are any null values. For - // count(t1.*, t2.*), it is still allowed, since it's well-defined in spark. - if (!conf.allowStarWithSingleTableIdentifierInCount && - resolvesToCountBuiltin && - f.arguments.length == 1) { - f.arguments.foreach { - case u: UnresolvedStar if u.isQualifiedByTable(child.output, resolver) => - throw QueryCompilationErrors - .singleTableStarInCountNotAllowedError(u.target.get.mkString(".")) - case _ => // do nothing + // Only a direct star (bare `*` or qualified `t.*`) reaches here: a star nested in another + // expression (json_array(array(*))) is expanded bottom-up first. For a routed SQL/JSON + // call, resolve the owner once and bind a shadow so `ResolveFunctions` cannot later fall + // through to the stock built-in after the star is expanded away. + functionResolution.selectRoutedSqlJsonDirectStarOwner(f.nameParts) match { + case RoutedSqlJsonStarOwner.RejectStockBuiltin => + throw QueryCompilationErrors.invalidStarUsageError( + s"expression `${f.prettyName}`", extractStar(f.arguments)) + case RoutedSqlJsonStarOwner.BindShadowOwner(candidate) => + f.copy( + arguments = f.arguments.flatMap { + case s: Star => expand(s, child) + case o => o :: Nil + }, + boundOwner = Some(candidate)) + case RoutedSqlJsonStarOwner.NoBinding => + // The count owner probe can hit an external functionExists lookup on a + // persistent-first PATH, so compute it once (lazily) and reuse it for the count(*) + // rewrite and the count(tbl.*) guard, as `FunctionResolverUtils` does. + lazy val resolvesToCountBuiltin = matchesFunctionName(f.nameParts, "count") + if (!f.isDistinct && isCountStarExpansionAllowed(f.arguments) && + resolvesToCountBuiltin) { + // Transform COUNT(*) into COUNT(1). + // We do not normalize the name to "count"; we keep the original name parts + // (e.g. builtin.count, system.builtin.count) so that resolution still sees + // the same qualification. + f.copy(arguments = Seq(Literal(1))) + } else { + // SPECIAL CASE: block count(tblName.*). In spark count(tblName.*) is expanded while + // count(*) is converted to count(1); they produce different results and confuse + // users when there are null values. count(t1.*, t2.*) stays allowed (well-defined). + if (!conf.allowStarWithSingleTableIdentifierInCount && + resolvesToCountBuiltin && + f.arguments.length == 1) { + f.arguments.foreach { + case u: UnresolvedStar if u.isQualifiedByTable(child.output, resolver) => + throw QueryCompilationErrors + .singleTableStarInCountNotAllowedError(u.target.get.mkString(".")) + case _ => // do nothing + } + } + f.copy(arguments = f.arguments.flatMap { + case s: Star => expand(s, child) + case o => o :: Nil + }) } - } - f.copy(arguments = f.arguments.flatMap { - case s: Star => expand(s, child) - case o => o :: Nil - }) } case c: CreateNamedStruct if containsStar(c.valExprs) => val newChildren = c.children.grouped(2).flatMap { @@ -2312,7 +2321,7 @@ class Analyzer( val externalFunctionNameSet = new mutable.HashSet[Seq[String]]() plan.resolveExpressionsWithPruning(_.containsAnyPattern(UNRESOLVED_FUNCTION)) { - case f @ UnresolvedFunction(nameParts, _, _, _, _, _, _) => + case f @ UnresolvedFunction(nameParts, _, _, _, _, _, _, _) => // For builtin/temp functions, we can do a quick check without catalog lookup val quickCheck = if (nameParts.size == 1 || FunctionResolution.sessionNamespaceKind(nameParts).isDefined) { @@ -2499,7 +2508,7 @@ class Analyzer( q.transformExpressionsUpWithPruning( _.containsAnyPattern(UNRESOLVED_FUNCTION, GENERATOR), ruleId) { - case u @ UnresolvedFunction(nameParts, arguments, _, _, _, _, _) + case u @ UnresolvedFunction(nameParts, arguments, _, _, _, _, _, _) if functionResolution.hasLambdaAndResolvedArguments(arguments) => withPosition(u) { functionResolution.resolveFunction(u) match { case func: HigherOrderFunction => func diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 16dd70bcaf769..49c4a8c7b7d92 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -966,8 +966,8 @@ object FunctionRegistry { /** * Names of the clause-free SQL/JSON functions that `AstBuilder` routes through - * function resolution. This is the shared list backing the star guard in - * [[FunctionResolution.resolvesToStarDisallowedSqlJsonFunction]], which derives its set from here + * function resolution. This is the shared list backing the star owner check in + * [[FunctionResolution.selectRoutedSqlJsonDirectStarOwner]], which derives its set from here * so a newly routed function is covered automatically. It is NOT a single source of truth for * the whole feature: two sibling lists still need a matching manual entry when a function is * added or removed -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala index 4f4dc6e909ddd..6897b2f59b680 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala @@ -190,6 +190,20 @@ class FunctionResolution( def resolveFunction(unresolvedFunc: UnresolvedFunction): Expression = { withPosition(unresolvedFunc) { + // Preprocessing already bound this call's owner (a shadow of a routed SQL/JSON built-in) and + // expanded its star. Resolve only that candidate so a shadow dropped since then fails here + // instead of falling through the PATH to the stock built-in. + unresolvedFunc.boundOwner match { + case Some(candidate) => + return resolveFunctionCandidate(candidate, unresolvedFunc).getOrElse { + throw QueryCompilationErrors.unresolvedRoutineError( + unresolvedFunc.nameParts, + sqlResolutionPathEntriesForAnalysis.map(toSQLId), + unresolvedFunc.origin) + } + case None => + } + // Internal functions resolve via the internal registry when the parser marks them as // internal; they are not resolved via the search path. if (unresolvedFunc.isInternal && unresolvedFunc.nameParts.size == 1) { @@ -456,20 +470,68 @@ class FunctionResolution( } } - // All routed SQL/JSON functions (JSON_ARRAY, JSON_VALUE, JSON_QUERY, JSON_EXISTS) forbid a direct - // star argument (a bare `*` or a qualified `t.*`). Derived from the single registry list so a - // newly routed function is covered without editing this file too. + // All routed SQL/JSON functions forbid a direct star argument (a bare `*` or a qualified + // `t.*`). Derived from the single registry list so a newly routed function is covered without + // editing this file too. private val starDisallowedSqlJsonFunctions = FunctionRegistry.routedSqlJsonFunctionNames /** - * True if `nameParts` resolves to Spark's stock built-in routed SQL/JSON function that forbids a - * direct star. The `isStockBuiltinFunction` check excludes an `injectFunction` replacement of - * the name, whose expanded star is passed through rather than rejected. + * Resolves, once, who owns a routed SQL/JSON call (e.g. `json_array(*)`) that carries a direct + * star, so direct-star preprocessing and the later [[resolveFunction]] consume a single SQL PATH + * decision. Otherwise each phase probes ownership independently, and a temp/persistent shadow + * dropped between them lets preprocessing expand the star for the shadow while resolution falls + * through to the stock built-in, which no longer sees a [[Star]] and skips + * `INVALID_USAGE_OF_STAR_OR_REGEX`. See [[RoutedSqlJsonStarOwner]] for outcomes. + */ + def selectRoutedSqlJsonDirectStarOwner(nameParts: Seq[String]): RoutedSqlJsonStarOwner = { + routedSqlJsonBuiltinName(nameParts) match { + case None => RoutedSqlJsonStarOwner.NoBinding + case Some(name) => routedSqlJsonStarOwnerFromPath(nameParts, name) + } + } + + /** + * One ordered pass over the SQL PATH candidates: bind the first shadow that owns the call ahead + * of the stock `system.builtin`; reach `system.builtin` first and the stock builder owns it + * (reject the star, unless an injectFunction replacement holds the slot -- keep pass-through). + * A single pass avoids two independent probes disagreeing when a shadow is dropped between them, + * which could bind the stock built-in with the [[Star]] already expanded away. */ - def resolvesToStarDisallowedSqlJsonFunction(nameParts: Seq[String]): Boolean = - starDisallowedSqlJsonFunctions.exists { name => - functionNameResolvesToBuiltin(nameParts, name) && - v1SessionCatalog.isStockBuiltinFunction(name) + private def routedSqlJsonStarOwnerFromPath( + nameParts: Seq[String], + name: String): RoutedSqlJsonStarOwner = { + for (candidate <- resolutionCandidates(nameParts)) { + if (isSystemBuiltinCandidate(candidate)) { + return if (v1SessionCatalog.isStockBuiltinFunction(name)) { + RoutedSqlJsonStarOwner.RejectStockBuiltin + } else { + RoutedSqlJsonStarOwner.NoBinding + } + } else if (candidateOwnsFunction(candidate)) { + return RoutedSqlJsonStarOwner.BindShadowOwner(candidate) + } + } + RoutedSqlJsonStarOwner.NoBinding + } + + /** The routed SQL/JSON built-in name this unqualified/`builtin`-qualified call could refer to. */ + private def routedSqlJsonBuiltinName(nameParts: Seq[String]): Option[String] = + starDisallowedSqlJsonFunctions.find { name => + FunctionResolution.isUnqualifiedOrBuiltinFunctionName(nameParts, name) + } + + /** Whether `candidate` is the stock `system.builtin.` terminus of the PATH walk. */ + private def isSystemBuiltinCandidate(candidate: Seq[String]): Boolean = + isSystemCatalogQualified(candidate) && + candidate(1).equalsIgnoreCase(CatalogManager.BUILTIN_NAMESPACE) + + /** Whether `candidate` owns a function, branching as [[resolveFunctionCandidate]] does. */ + private def candidateOwnsFunction(candidate: Seq[String]): Boolean = + if (isSystemCatalogQualified(candidate)) { + lookupBuiltinOrTempFunction(candidate, None).isDefined || + lookupBuiltinOrTempTableFunction(candidate).isDefined + } else { + persistentFunctionExists(candidate) } private def persistentFunctionExists(nameParts: Seq[String]): Boolean = { @@ -941,3 +1003,23 @@ object FunctionResolution { (nameParts.length == 1 || maybeBuiltinFunctionName(nameParts)) } } + +/** + * Who owns a routed SQL/JSON call carrying a direct star, resolved once during star preprocessing + * (see [[FunctionResolution.selectRoutedSqlJsonDirectStarOwner]]) so later resolution reuses it. + */ +sealed trait RoutedSqlJsonStarOwner +object RoutedSqlJsonStarOwner { + /** Spark's stock routed SQL/JSON builder owns the call; reject the direct star. */ + case object RejectStockBuiltin extends RoutedSqlJsonStarOwner + + /** + * A temp/persistent shadow owns the call. `candidate` is the winning SQL PATH candidate; bind it + * via [[UnresolvedFunction.boundOwner]] so later resolution resolves it (or fails), never the + * stock built-in. + */ + case class BindShadowOwner(candidate: Seq[String]) extends RoutedSqlJsonStarOwner + + /** Not a routed built-in, or an injectFunction replacement owns the slot: expand the star. */ + case object NoBinding extends RoutedSqlJsonStarOwner +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FunctionResolverUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FunctionResolverUtils.scala index a0f79e1415361..9911234393c02 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FunctionResolverUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FunctionResolverUtils.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.analysis.resolver import org.apache.spark.sql.catalyst.analysis.{ FunctionResolution, ResolvedStar, + RoutedSqlJsonStarOwner, Star, UnresolvedFunction, UnresolvedStar @@ -61,30 +62,35 @@ trait FunctionResolverUtils { case _ => false } - // Whether the call resolves to the builtin `count` (distinct-agnostic). This owner probe can - // hit an external FunctionCatalog.functionExists lookup on a persistent-first SQL PATH, so - // compute it once and reuse it for both the count(*) normalization and the count(tbl.*) guard. - // Lazy so the non-star and SQL/JSON direct-star paths never pay for it. + // Distinct-agnostic probe for the builtin `count`. Lazy so only the count(*)/count(tbl.*) paths + // pay for the (possibly external) lookup; reused by the rewrite and the single-table guard. lazy val resolvesToCountBuiltin = functionResolution.functionNameResolvesToBuiltin(unresolvedFunction.nameParts, "count") - if (functionContainsDirectStarInArguments && - functionResolution.resolvesToStarDisallowedSqlJsonFunction(unresolvedFunction.nameParts)) { - // A direct star argument -- a bare `*` or a qualified `t.*` -- is rejected in a routed - // SQL/JSON function; a star nested in another expression (json_array(array(*))) is expanded - // there and count(*) is rewritten to count(1), so both stay valid arguments. - throw QueryCompilationErrors.invalidStarUsageError( - s"expression `${unresolvedFunction.prettyName}`", extractStar(unresolvedFunction.arguments)) - } else if (!functionContainsDirectStarInArguments) { + if (!functionContainsDirectStarInArguments) { unresolvedFunction - } else if (!unresolvedFunction.isDistinct && resolvesToCountBuiltin && - hasSingleSimpleStarArgument(unresolvedFunction)) { - normalizeCountExpression(unresolvedFunction) } else { - assertSingleTableStarNotInCountFunction(unresolvedFunction, resolvesToCountBuiltin) - unresolvedFunction.copy( - arguments = expressionResolver.expandStarExpressions(unresolvedFunction.arguments) - ) + // Resolve the routed SQL/JSON owner once and mirror the fixed-point analyzer. + functionResolution.selectRoutedSqlJsonDirectStarOwner(unresolvedFunction.nameParts) match { + case RoutedSqlJsonStarOwner.RejectStockBuiltin => + throw QueryCompilationErrors.invalidStarUsageError( + s"expression `${unresolvedFunction.prettyName}`", + extractStar(unresolvedFunction.arguments)) + case RoutedSqlJsonStarOwner.BindShadowOwner(candidate) => + unresolvedFunction.copy( + arguments = expressionResolver.expandStarExpressions(unresolvedFunction.arguments), + boundOwner = Some(candidate)) + case RoutedSqlJsonStarOwner.NoBinding => + if (!unresolvedFunction.isDistinct && resolvesToCountBuiltin && + hasSingleSimpleStarArgument(unresolvedFunction)) { + normalizeCountExpression(unresolvedFunction) + } else { + assertSingleTableStarNotInCountFunction(unresolvedFunction, resolvesToCountBuiltin) + unresolvedFunction.copy( + arguments = expressionResolver.expandStarExpressions(unresolvedFunction.arguments) + ) + } + } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala index 79215115c7031..144cf9d1d3a76 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala @@ -394,6 +394,10 @@ case class UnresolvedGenerator(name: FunctionIdentifier, children: Seq[Expressio * Represents an unresolved function that is being invoked. The analyzer will resolve the function * arguments first, then look up the function by name and arguments, and return an expression that * can be evaluated to get the result of this function invocation. + * + * `boundOwner`, when set, is the fully-qualified candidate that direct-star preprocessing selected + * as this call's owner; later resolution binds to exactly it instead of re-walking the SQL PATH + * (see [[FunctionResolution.selectRoutedSqlJsonDirectStarOwner]]). */ case class UnresolvedFunction( nameParts: Seq[String], @@ -402,7 +406,8 @@ case class UnresolvedFunction( filter: Option[Expression] = None, ignoreNulls: Option[Boolean] = None, orderingWithinGroup: Seq[SortOrder] = Seq.empty, - isInternal: Boolean = false) + isInternal: Boolean = false, + boundOwner: Option[Seq[String]] = None) extends Expression with Unevaluable { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/DataFrameWriterV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/DataFrameWriterV2.scala index a8cc7c79f7fc0..eb8131a555321 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/DataFrameWriterV2.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/DataFrameWriterV2.scala @@ -267,7 +267,7 @@ private object PartitionTransform { private val NAMES = Seq(name) def unapply(e: Expression): Option[Seq[Expression]] = e match { - case UnresolvedFunction(NAMES, children, false, None, None, Nil, true) => Option(children) + case UnresolvedFunction(NAMES, children, false, None, None, Nil, true, _) => Option(children) case _ => None } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionCommand.scala index a597087085b42..3c60f9c578f0b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionCommand.scala @@ -426,7 +426,7 @@ case class CreateSQLFunctionCommand( def checkExpression(expression: Expression, path: Seq[FunctionIdentifier]): Unit = { expression.foreach { case s: SubqueryExpression => checkPlan(s.plan, path) - case u @ UnresolvedFunction(nameParts, arguments, _, _, _, _, _) => + case u @ UnresolvedFunction(nameParts, arguments, _, _, _, _, _, _) => val funcId = nameParts.asFunctionIdentifier val info = catalog.lookupFunctionInfo(funcId) if (isSQLFunction(info.getClassName)) { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonArraySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonArraySuite.scala index 20f5987869af2..2e8da431883f7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JsonArraySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonArraySuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql import org.apache.spark.SparkRuntimeException import org.apache.spark.sql.catalyst.FunctionIdentifier -import org.apache.spark.sql.catalyst.analysis.{NoSuchNamespaceException, Star} +import org.apache.spark.sql.catalyst.analysis.{NoSuchFunctionException, NoSuchNamespaceException, Star} import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch import org.apache.spark.sql.catalyst.expressions.{Cast, Collate, JsonArray, JsonConstructorNullBehavior, JsonQuery, JsonQueryBehavior, JsonQueryQuotes, JsonQueryWrapper, Literal, ResolvedCollation} import org.apache.spark.sql.catalyst.plans.logical.{Project, Range} @@ -814,6 +814,32 @@ class JsonArraySuite extends QueryTest with SharedSparkSession { } } + test("SPARK-59600: a routed JSON_ARRAY shadow dropped between preprocessing and resolution " + + "fails cleanly instead of resolving to the built-in") { + // Star preprocessing selects the persistent shadow (functionExists = true) and expands the + // direct star for it; by resolution time the shadow is gone (loadFunction throws). Binding the + // selected owner makes resolution fail on that candidate rather than falling through to the + // stock built-in, which -- with the star already expanded away -- would otherwise silently + // accept json_array(*). A deterministic stand-in for a concurrent DROP FUNCTION between phases. + withSQLConf( + SQLConf.PATH_ENABLED.key -> "true", + "spark.sql.catalog.vanishing_cat" -> classOf[VanishingShadowFunctionCatalog].getName) { + try { + sql("SET PATH = vanishing_cat.some_ns, system.builtin") + Seq(false, true).foreach { singlePass => + withSQLConf(SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED.key -> singlePass.toString) { + val e = intercept[AnalysisException] { + sql("SELECT json_array(*) FROM VALUES (1, 'x') AS t(a, b)").queryExecution.analyzed + } + assert(e.getCondition == "UNRESOLVED_ROUTINE", s"singlePass=$singlePass") + } + } + } finally { + sql("SET PATH = DEFAULT_PATH") + } + } + } + test("default collation recurses into a nested JSON_ARRAY value") { // Col a (parser-built nested, direct grammar path) and col b (flat routed built-in) both // adopt the table default UTF8_LCASE collation. @@ -1004,3 +1030,14 @@ class MissingNamespaceFunctionCatalog extends InMemoryCatalog { override def loadFunction(ident: Identifier): UnboundFunction = throw new NoSuchNamespaceException(ident.namespace) } + +/** + * A [[org.apache.spark.sql.connector.catalog.FunctionCatalog]] that reports a function as existing + * but fails to load it -- a deterministic stand-in for a shadow dropped between star preprocessing + * and routine resolution. + */ +class VanishingShadowFunctionCatalog extends InMemoryCatalog { + override def functionExists(ident: Identifier): Boolean = true + override def loadFunction(ident: Identifier): UnboundFunction = + throw new NoSuchFunctionException(ident) +} From ea04a26bbeefb38a69ae30bf511d06744bb4f23b Mon Sep 17 00:00:00 2001 From: Ganesha S Date: Sun, 20 Sep 2026 16:12:30 +0000 Subject: [PATCH 2/2] [SPARK-59600][SQL] Address review nits: boundOwner doc and lazy resolution candidates - boundOwner may hold a relative persistent-catalog candidate that only gets qualified during later resolution, so the doc no longer calls it "fully-qualified". - resolutionCandidates builds single-part PATH candidates lazily so an early-returning consumer (built-in hit, routed direct-star owner walk) skips concatenating the unused PATH suffix. Co-authored-by: Isaac --- .../spark/sql/catalyst/analysis/FunctionResolution.scala | 4 +++- .../org/apache/spark/sql/catalyst/analysis/unresolved.scala | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala index 6897b2f59b680..ebaecdb117e83 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala @@ -128,7 +128,9 @@ class FunctionResolution( private def resolutionCandidates(nameParts: Seq[String]): Seq[Seq[String]] = { if (nameParts.size == 1) { - sqlResolutionPathEntriesForAnalysis.map(_ ++ nameParts) + // Built lazily so an early-returning consumer (built-in hit, routed direct-star owner walk) + // skips concatenating the unused PATH suffix; consumers walk candidates in order. + sqlResolutionPathEntriesForAnalysis.to(LazyList).map(_ ++ nameParts) } else if (nameParts.size == 2 && FunctionResolution.sessionNamespaceKind(nameParts).isDefined) { val systemCandidate = CatalogManager.SYSTEM_CATALOG_NAME +: nameParts diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala index 144cf9d1d3a76..6765af018b2ab 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala @@ -395,8 +395,9 @@ case class UnresolvedGenerator(name: FunctionIdentifier, children: Seq[Expressio * arguments first, then look up the function by name and arguments, and return an expression that * can be evaluated to get the result of this function invocation. * - * `boundOwner`, when set, is the fully-qualified candidate that direct-star preprocessing selected - * as this call's owner; later resolution binds to exactly it instead of re-walking the SQL PATH + * `boundOwner`, when set, is the SQL PATH candidate that direct-star preprocessing selected as this + * call's owner (a relative persistent-catalog candidate stays relative until later resolution + * qualifies it); later resolution binds to exactly it instead of re-walking the SQL PATH * (see [[FunctionResolution.selectRoutedSqlJsonDirectStarOwner]]). */ case class UnresolvedFunction(