diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java index a651e8ee7b426a..fd29b0f2dad1f1 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.common.Pair; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -35,6 +37,8 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -276,6 +280,9 @@ private static void validateFieldsList(List fields) { } } + // an escaped character, or an unescaped @ + private static final Pattern ESCAPED_OR_AT = Pattern.compile("\\\\.|@"); + private static String buildFieldPath(SearchParser.FieldPathContext ctx) { if (ctx == null) { throw new RuntimeException("Invalid field query: missing field path"); @@ -289,13 +296,49 @@ private static String buildFieldPath(SearchParser.FieldPathContext ctx) { } String segment = segments.get(i).getText(); if (segment.startsWith("\"") && segment.endsWith("\"")) { - segment = segment.substring(1, segment.length() - 1); + // An @ inside quotes is part of the field name; see splitAnalyzerSelector. + segment = ESCAPED_OR_AT.matcher(segment.substring(1, segment.length() - 1)) + .replaceAll(m -> Matcher.quoteReplacement(m.group().equals("@") ? "\\@" : m.group())); } fullPath.append(segment); } return fullPath.toString(); } + /** + * Splits a DSL field reference {@code path[@analyzer]} into the field path and the analyzer name (null when + * no analyzer is selected). The split is purely syntactic and never depends on the table schema: the last + * unescaped {@code @} that follows a non-empty path segment selects the analyzer, while {@code \@}, an + * {@code @} inside a quoted segment (escaped by buildFieldPath) and an {@code @} that starts a segment + * ({@code v.@timestamp}) belong to the field name. + */ + public static Pair splitAnalyzerSelector(String fieldReference) { + StringBuilder path = new StringBuilder(); + int selector = -1; + for (int i = 0; i < fieldReference.length(); i++) { + char c = fieldReference.charAt(i); + if (c == '\\' && i + 1 < fieldReference.length()) { + char escaped = fieldReference.charAt(++i); + if (escaped != '@') { + path.append(c); + } + path.append(escaped); + continue; + } + if (c == '@' && i > 0 && fieldReference.charAt(i - 1) != '.') { + selector = path.length(); + } + path.append(c); + } + if (selector < 0) { + return Pair.of(path.toString(), null); + } + if (selector == path.length() - 1) { + throw new SearchDslSyntaxException("SEARCH analyzer selector must be field@analyzer: " + fieldReference); + } + return Pair.of(path.substring(0, selector), path.substring(selector + 1)); + } + private static String normalizeNestedFieldPath(String fieldPath, @Nullable String nestedPath) { if (nestedPath == null || nestedPath.isEmpty()) { return fieldPath; @@ -1303,6 +1346,13 @@ public String getNestedPath() { return nestedPath; } + /** + * Sets the nested path (used for field name normalization). + */ + public void setNestedPath(String nestedPath) { + this.nestedPath = nestedPath; + } + /** * Returns whether the field was explicitly specified in the DSL syntax. */ @@ -1365,6 +1415,17 @@ public static class QsFieldBinding { @JsonProperty("slotIndex") private final int slotIndex; + @JsonProperty("analyzerName") + private String analyzerName; + + public String getAnalyzerName() { + return analyzerName; + } + + public void setAnalyzerName(String analyzerName) { + this.analyzerName = analyzerName; + } + @JsonCreator public QsFieldBinding(@JsonProperty("fieldName") String fieldName, @JsonProperty("slotIndex") int slotIndex) { @@ -1390,7 +1451,7 @@ public int getSlotIndex() { @Override public int hashCode() { - return Objects.hash(fieldName, slotIndex); + return Objects.hash(fieldName, slotIndex, analyzerName); } @Override @@ -1403,7 +1464,8 @@ public boolean equals(Object o) { } QsFieldBinding that = (QsFieldBinding) o; return slotIndex == that.slotIndex - && Objects.equals(fieldName, that.fieldName); + && Objects.equals(fieldName, that.fieldName) + && Objects.equals(analyzerName, that.analyzerName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 95d305f8d68547..ea0cebcef5f9df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -250,7 +250,7 @@ public Expr visitMatch(Match match, PlanTranslatorContext context) { // column/table reference (e.g., after CTE inlining or join projection remapping), // we gracefully fall back to invertedIndex = null. The BE can still evaluate MATCH // correctly without inverted index (slow path), or the PushDownProject / - // PushDownMatchProjectionAsVirtualColumn rules may have already pushed the expression + // PushDownIndexSearchAsVirtualColumn rules may have already pushed the expression // down for storage-level index evaluation (fast path). Index invertedIndex = null; String analyzer = match.getAnalyzer().orElse(null); @@ -698,19 +698,25 @@ public Expr visitSearchExpression(SearchExpression searchExpression, // Look up the inverted index for each field (needed for variant subcolumn analyzer) Index invertedIndex = null; + String analyzer = searchExpression.getQsPlan().getFieldBindings() + .get(fieldIndexes.size()).getAnalyzerName(); if (slotExpr instanceof SlotReference) { SlotReference slot = (SlotReference) slotExpr; OlapTable olapTbl = getOlapTableDirectly(slot); if (olapTbl != null) { Column column = slot.getOriginalColumn().orElse(null); if (column != null) { - invertedIndex = olapTbl.getInvertedIndex(column, slot.getSubPath()); + invertedIndex = olapTbl.getInvertedIndex(column, slot.getSubPath(), analyzer); } } } - if (invertedIndex == null) { + if (invertedIndex == null && analyzer == null) { invertedIndex = getInvertedIndexFromTranslatedSlot(translatedSlot, context); } + if (analyzer != null && invertedIndex == null) { + throw new AnalysisException("No inverted index found for SEARCH analyzer '" + analyzer + + "' on " + slotExpr.toSql()); + } fieldIndexes.add(invertedIndex); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index db6ed38ac41ac9..992e9d7f0eb498 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -54,6 +54,7 @@ import org.apache.doris.nereids.rules.analysis.SubqueryToApply; import org.apache.doris.nereids.rules.rewrite.AdjustNullable; import org.apache.doris.nereids.rules.rewrite.MergeFilters; +import org.apache.doris.nereids.rules.rewrite.RewriteSearchToSlots; import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalView; @@ -169,6 +170,8 @@ private static List buildAnalyzerJobs() { ), // run CheckSearchUsage before CheckAnalysis to detect search() in GROUP BY before it gets optimized bottomUp(new CheckSearchUsage()), + // Bind SEARCH dependencies before predicate movement and column pruning. + bottomUp(new RewriteSearchToSlots()), // run CheckAnalysis before EliminateGroupByConstant in order to report error message correctly like bellow // select SUM(lo_tax) FROM lineorder group by 1; // errCode = 2, detailMessage = GROUP BY expression must not contain aggregate functions: sum(lo_tax) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 7a3d6495204297..398bf06eb3fd5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -132,11 +132,11 @@ import org.apache.doris.nereids.rules.rewrite.PushDownEncodeSlot; import org.apache.doris.nereids.rules.rewrite.PushDownFilterIntoSchemaScan; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject; +import org.apache.doris.nereids.rules.rewrite.PushDownIndexSearchAsVirtualColumn; import org.apache.doris.nereids.rules.rewrite.PushDownJoinOnAssertNumRows; import org.apache.doris.nereids.rules.rewrite.PushDownLimit; import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughUnion; -import org.apache.doris.nereids.rules.rewrite.PushDownMatchProjectionAsVirtualColumn; import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit; import org.apache.doris.nereids.rules.rewrite.PushDownScoreTopNIntoOlapScan; import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin; @@ -154,7 +154,6 @@ import org.apache.doris.nereids.rules.rewrite.ReduceAggregateChildOutputRows; import org.apache.doris.nereids.rules.rewrite.ReorderJoin; import org.apache.doris.nereids.rules.rewrite.RewriteCteChildren; -import org.apache.doris.nereids.rules.rewrite.RewriteSearchToSlots; import org.apache.doris.nereids.rules.rewrite.RewriteSimpleAggToConstantRule; import org.apache.doris.nereids.rules.rewrite.SaltJoin; import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; @@ -797,7 +796,7 @@ public class Rewriter extends AbstractBatchJobExecutor { custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, EliminateUnnecessaryProject::new), topDown(new PushDownVectorTopNIntoOlapScan()), topDown(new PushDownVirtualColumnsIntoOlapScan()), - topDown(new PushDownMatchProjectionAsVirtualColumn()), + topDown(new PushDownIndexSearchAsVirtualColumn()), topic("score optimize", topDown(new PushDownScoreTopNIntoOlapScan(), new CheckScoreUsage()) @@ -929,12 +928,6 @@ private static List getWholeTreeRewriteJobs( custom(RuleType.DISTINCT_AGG_STRATEGY_SELECTOR, () -> DistinctAggStrategySelector.INSTANCE)))); - // Rewrite search function before VariantSubPathPruning - // so that ElementAt expressions from search can be processed - rewriteJobs.addAll(jobs( - bottomUp(new RewriteSearchToSlots()) - )); - if (needSubPathPushDown) { rewriteJobs.addAll(jobs( topic("variant element_at push down", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index db263edb7750f0..0e1da87971965d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotNotFromChildren; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; @@ -50,6 +51,7 @@ import org.apache.commons.lang3.StringUtils; import org.roaringbitmap.RoaringBitmap; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -65,10 +67,41 @@ public Rule build() { checkUnexpectedExpression(plan); checkMetricTypeIsUsedCorrectly(plan); checkMatchIsUsedCorrectly(plan); + checkSearchIsUsedCorrectly(plan); return null; }).toRule(RuleType.CHECK_ANALYSIS); } + /** + * The execution invariant of SEARCH, checked in this one place for every way a rewrite can move it: BE + * evaluates a SearchExpression only with inverted indexes inside an OLAP scan, so it must be a conjunct of + * the filter on the scan or a scan virtual column, and every child must still bind an index field. + */ + private void checkSearchIsUsedCorrectly(Plan plan) { + // Scan virtual columns are not part of LogicalOlapScan.getExpressions(). + List expressions = plan instanceof LogicalOlapScan + ? ((LogicalOlapScan) plan).getVirtualColumns() : plan.getExpressions(); + boolean evaluatedByScan = plan instanceof LogicalOlapScan + || (plan instanceof LogicalFilter && plan.child(0) instanceof LogicalOlapScan); + for (Expression expression : expressions) { + for (SearchExpression search : expression.collect(SearchExpression.class::isInstance)) { + if (!search.bindsOnlyFields()) { + // e.g. LEFT JOIN ... ON false, or an outer join converted to an anti join + throw new AnalysisException("SEARCH on the null-generating side of an outer join is not " + + "supported: its field is always NULL in " + search.toSql()); + } + if (!evaluatedByScan) { + throw new AnalysisException("SEARCH must be evaluated by an OLAP scan, but " + search.toSql() + + " remains in " + plan.getType() + ". Unless it is a WHERE conjunct that reaches the " + + "scan, all its fields must come from one DUP_KEYS or merge-on-write UNIQUE_KEYS " + + "table reference that is neither on the null-generating side of an outer join nor " + + "below a LIMIT, TopN, aggregate or window; combine separate SEARCH expressions with " + + "SQL AND/OR to search several tables"); + } + } + } + } + private void checkUnexpectedExpression(Plan plan) { boolean isGenerate = plan instanceof Generate; boolean isAgg = plan instanceof LogicalAggregate; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java index 40fc84e41f2d22..3066c79e5583fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -36,9 +37,13 @@ import java.util.List; /** - * Check search expression usage - search() can only be used in WHERE filters on single-table OLAP scans. + * Check that search() is used in WHERE filters over OLAP tables, including joins of them. * This rule validates that search() expressions only appear in supported contexts. * Must run in analysis phase before search() gets optimized away. + * + *

This is a placement check on the analyzed plan and only a necessary condition: predicate push down may + * still move the filter anywhere below. Whether the search can be evaluated by a scan is decided by + * PushDownIndexSearchAsVirtualColumn and verified on the final plan by CheckAfterRewrite. */ public class CheckSearchUsage implements AnalysisRuleFactory { private static final Logger LOG = LogManager.getLogger(CheckSearchUsage.class); @@ -66,13 +71,13 @@ private void checkPlanRecursively(Plan plan) { for (Expression expr : agg.getGroupByExpressions()) { if (containsSearchExpression(expr)) { throw new AnalysisException("search() cannot appear in GROUP BY expressions; " - + "search predicates are only supported in WHERE filters on single-table scans"); + + "search predicates are only supported in WHERE filters over OLAP tables"); } } for (Expression expr : agg.getOutputExpressions()) { if (containsSearchExpression(expr)) { throw new AnalysisException("search() cannot appear in aggregate output expressions; " - + "search predicates are only supported in WHERE filters on single-table scans"); + + "search predicates are only supported in WHERE filters over OLAP tables"); } } } @@ -82,7 +87,6 @@ private void checkPlanRecursively(Plan plan) { LogicalProject project = (LogicalProject) plan; for (Expression expr : project.getProjects()) { if (containsSearchExpression(expr)) { - // Only allow if it's the project directly above a filter->scan pattern throw new AnalysisException("search() can only appear in WHERE filters on OLAP scans; " + "projection of search() is not supported"); } @@ -99,15 +103,13 @@ private void validateSearchUsage(Plan plan) { LOG.debug("validateSearchUsage: {}", plan.treeString()); if (plan instanceof LogicalFilter) { Plan child = plan.child(0); - if (!isSingleTableScanPipeline(child)) { - throw new AnalysisException("search() predicate only supports filtering directly on a single " - + "table scan; remove joins, subqueries, or additional operators between search() " - + "and the target table"); + if (!isOlapScanPipeline(child)) { + throw new AnalysisException("search() predicates require an OLAP scan pipeline"); } } else if (!(plan instanceof LogicalProject)) { - // search() can only appear in LogicalFilter or specific LogicalProject nodes - throw new AnalysisException("search() predicates are only supported inside WHERE filters on " - + "single-table scans"); + // a projection gets its own message in checkPlanRecursively + throw new AnalysisException("search() predicates are only supported inside WHERE filters over " + + "OLAP tables"); } } @@ -132,12 +134,17 @@ private boolean containsSearchExpression(Expression expression) { return false; } - private boolean isSingleTableScanPipeline(Plan plan) { + // Every relation below the filter must be an OLAP scan; unary nodes in between are not restricted here. + private boolean isOlapScanPipeline(Plan plan) { Plan current = plan; while (true) { if (current instanceof LogicalOlapScan) { return true; } + if (current instanceof LogicalJoin) { + return isOlapScanPipeline(current.child(0)) + && isOlapScanPipeline(current.child(1)); + } if (current.arity() != 1) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 0f87bdd2aaf54a..76b6343e2cc823 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.ai.AIFunction; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; @@ -275,7 +276,7 @@ private static boolean shouldSkipFold(Expression expr) { } // Search function should always not be folded to constant. - if (expr instanceof Search) { + if (expr instanceof Search || expr instanceof SearchExpression) { return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java index 12570f9503533f..3bb88d7f3040a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -493,7 +494,7 @@ private boolean canReplaceExpression(Expression expression) { // "https://doris.apache.org/docs/sql-manual/basic-element/operators/conditional-operators // /full-text-search-operators", the match function require left is a slot, not a literal. - if (expression instanceof Match) { + if (expression instanceof Match || expression instanceof SearchExpression) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java index c2ca99f0b61b9c..d8cd37ca999176 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Like; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -214,6 +215,11 @@ public static Set infer(Set inputs) { || input.getInputSlots().size() != 1) { continue; } + // A SEARCH is evaluated with the inverted indexes of the columns it binds, not from their values, + // so it does not hold for another column that is merely equal. + if (input.containsType(SearchExpression.class)) { + continue; + } input.accept(PredicatesCollector.INSTANCE, exprPredicates); } Set inferPredicates = new LinkedHashSet<>(inputs); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java index 8688db6a7d1486..b7dbc00c2895d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -226,6 +227,11 @@ private Plan inferNewPredicate(Plan plan, Set expressions) { // query semantics (see EXCEPT/INTERSECT regression cases). continue; } + if (expr.containsType(SearchExpression.class)) { + // A SEARCH is bound to the inverted indexes of its own scan. The slot substitution of the + // SetOp visitors would rebind it to the column of a sibling branch, where BE finds no index. + continue; + } Set slots = expr.getInputSlots(); if (!slots.isEmpty() && planOutputs.containsAll(slots)) { predicates.add(expr); @@ -255,6 +261,10 @@ private Plan inferNewPredicateRemoveUselessIsNull(Plan plan, Set exp // predicates into a subtree that did not already evaluate them. continue; } + if (expr.containsType(SearchExpression.class)) { + // See inferNewPredicate: a SEARCH stays on the scan whose indexes it binds. + continue; + } Set slots = expr.getInputSlots(); if (slots.isEmpty() || !planOutputs.containsAll(slots)) { continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumn.java new file mode 100644 index 00000000000000..26cee48e81e571 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumn.java @@ -0,0 +1,245 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.KeysType; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Materialize index searches (MATCH and SEARCH expressions) as virtual columns on OlapScan, so that BE evaluates + * them with inverted indexes during the scan and the consumer reads the resulting boolean. + * + *

Consumers are projections, residual filters (a filter directly on a scan is evaluated by the scan itself) + * and join conditions. All of them go through the same steps: collect every index search in the consumer's + * expressions, however deeply nested, {@link #materialize} it in the child that produces its inputs, and + * replace it with the virtual column slot. + * + *

{@link #materialize} is the only place that decides where an index search may be computed. It descends + * through Project, Filter and Join down to one OlapScan that outputs all inputs, and refuses to cross the + * NULL-extended side of an outer join unless the expression is NULL for NULL input. A MATCH that cannot be + * materialized stays where it is; a SEARCH that is left outside a scan is rejected by CheckAfterRewrite. + * + *

The rule type keeps its original name because it is referenced by disable_nereids_rules. + * + * Example transformation: + * Before: + * Project[a, b, col MATCH_ANY 'hello'] + * └── OlapScan[table] + * + * After: + * Project[a, b, virtual_slot_ref] + * └── OlapScan[table, virtual_columns=[(col MATCH_ANY 'hello') as alias]] + */ +public class PushDownIndexSearchAsVirtualColumn implements RewriteRuleFactory { + + private boolean canPushDown(LogicalOlapScan scan) { + boolean dupTblOrMOW = scan.getTable().getKeysType() == KeysType.DUP_KEYS + || (scan.getTable().getTableProperty() != null + && scan.getTable().getTableProperty().getEnableUniqueKeyMergeOnWrite()); + return dupTblOrMOW; + } + + @Override + public List buildRules() { + return ImmutableList.of( + logicalProject().thenApply(ctx -> pushDownFromProject(ctx.root, ctx.cascadesContext)) + .toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), + // A filter directly on the scan is pushed into the scan and evaluated there. + logicalFilter().when(filter -> !(filter.child() instanceof LogicalOlapScan)) + .thenApply(ctx -> pushDownFromFilter(ctx.root, ctx.cascadesContext)) + .toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), + logicalJoin().thenApply(ctx -> pushDownFromJoin(ctx.root, ctx.cascadesContext)) + .toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN) + ); + } + + private boolean isIndexSearch(Expression expression) { + return expression instanceof Match || expression instanceof SearchExpression; + } + + /** + * Materialize every index search of the consumer's expressions in one of the children, which are updated in + * place. Returns the replacement of each materialized search. + */ + private Map materializeAll(List expressions, + List children, CascadesContext context) { + Map replacements = new LinkedHashMap<>(); + for (Expression expression : expressions) { + for (Expression search : expression.collect(e -> isIndexSearch((Expression) e))) { + if (replacements.containsKey(search)) { + continue; + } + for (int i = 0; i < children.size(); i++) { + Pair result = materialize(search, children.get(i), context); + if (result != null) { + children.set(i, result.first); + replacements.put(search, result.second); + break; + } + } + } + } + return replacements; + } + + private Plan pushDownFromProject(LogicalProject project, CascadesContext context) { + List children = new ArrayList<>(project.children()); + Map replacements = materializeAll(project.getProjects(), children, context); + if (replacements.isEmpty()) { + return null; + } + List projects = new ArrayList<>(); + for (NamedExpression expression : project.getProjects()) { + projects.add((NamedExpression) ExpressionUtils.replace(expression, replacements)); + } + return project.withProjectsAndChild(projects, children.get(0)); + } + + private Plan pushDownFromFilter(LogicalFilter filter, CascadesContext context) { + List children = new ArrayList<>(filter.children()); + Map replacements = materializeAll(filter.getExpressions(), children, context); + if (replacements.isEmpty()) { + return null; + } + // Hide additional scan values from the original filter's consumers. + return new LogicalProject<>(ImmutableList.copyOf(filter.getOutput()), + new LogicalFilter<>(ExpressionUtils.replace(filter.getConjuncts(), replacements), + children.get(0))); + } + + private Plan pushDownFromJoin(LogicalJoin join, CascadesContext context) { + // Join conditions consume child values before this join's NULL extension, so they are materialized + // directly in the children. This also handles WHERE predicates moved into an inner join by rewriting. + List children = new ArrayList<>(join.children()); + Map replacements = materializeAll(join.getExpressions(), children, context); + if (replacements.isEmpty()) { + return null; + } + Plan rewritten = join.withConjunctsChildren( + ExpressionUtils.replace(join.getHashJoinConjuncts(), replacements), + ExpressionUtils.replace(join.getOtherJoinConjuncts(), replacements), + ExpressionUtils.replace(join.getMarkJoinConjuncts(), replacements), + children.get(0), children.get(1), join.getJoinReorderContext()); + return new LogicalProject<>(ImmutableList.copyOf(join.getOutput()), rewritten); + } + + /** + * Compute the index search in the OlapScan below the plan that produces all its inputs, reusing an equal + * virtual column of that scan. Returns the rewritten plan and the slot, part of its output, that carries the + * value; or null when the search cannot be computed below the plan. + */ + private Pair materialize(Expression expression, Plan plan, CascadesContext context) { + Set inputs = expression.getInputSlots(); + if (inputs.isEmpty() || !plan.getOutputSet().containsAll(inputs)) { + return null; + } + if (plan instanceof LogicalOlapScan) { + LogicalOlapScan scan = (LogicalOlapScan) plan; + if (!canPushDown(scan)) { + return null; + } + for (NamedExpression column : scan.getVirtualColumns()) { + if (column instanceof Alias && ((Alias) column).child().equals(expression)) { + return Pair.of(scan, column.toSlot()); + } + } + Alias alias = new Alias(expression); + return Pair.of(scan.appendVirtualColumns(ImmutableList.of(alias)), alias.toSlot()); + } + if (plan instanceof LogicalProject) { + LogicalProject project = (LogicalProject) plan; + if (project.containsNoneMovableFunction()) { + return null; + } + // Like PushDownFilterThroughProject: inlining a volatile producer would evaluate it a second time. + Map aliasToProducer = project.getAliasToProducer(); + if (inputs.stream().map(aliasToProducer::get) + .anyMatch(producer -> producer != null && producer.containsVolatileExpression())) { + return null; + } + Expression rewritten = ExpressionUtils.replace(expression, aliasToProducer); + Pair result = materialize(rewritten, project.child(), context); + if (result == null) { + return null; + } + List projects = new ArrayList<>(project.getProjects()); + // A reused virtual column may already pass through this project. + if (!project.getOutputSet().contains(result.second)) { + projects.add(result.second); + } + return Pair.of(project.withProjectsAndChild(projects, result.first), result.second); + } + if (plan instanceof LogicalFilter) { + Pair result = materialize(expression, plan.child(0), context); + return result == null ? null : Pair.of(plan.withChildren(result.first), result.second); + } + if (plan instanceof LogicalJoin) { + LogicalJoin join = (LogicalJoin) plan; + for (int side = 0; side < 2; side++) { + if (!join.child(side).getOutputSet().containsAll(inputs)) { + continue; + } + boolean nullExtended = side == 0 ? join.getJoinType().isLeftSideNullable() + : join.getJoinType().isRightSideNullable(); + // The join turns a value computed below it into NULL for NULL-extended rows, so only + // an expression that is itself NULL for NULL input may be computed there. SEARCH has + // DSL-level existence and negation semantics and is not NULL-propagating. + if (nullExtended && !ExpressionUtils.isNullPropagating(expression, context)) { + return null; + } + Pair result = materialize(expression, join.child(side), context); + if (result == null) { + return null; + } + List children = new ArrayList<>(join.children()); + children.set(side, result.first); + Plan newJoin = join.withChildren(children); + for (Slot output : newJoin.getOutput()) { + if (output.getExprId().equals(result.second.getExprId())) { + return Pair.of(newJoin, output); + } + } + return null; + } + } + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java deleted file mode 100644 index 53e93dfa0fc644..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java +++ /dev/null @@ -1,138 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.catalog.KeysType; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Match; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; - -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -/** - * Push down MATCH expressions in projections as virtual columns on OlapScan. - * This allows the BE to evaluate MATCH using inverted index during scan. - * - * Example transformation: - * Before: - * Project[a, b, col MATCH_ANY 'hello'] - * └── OlapScan[table] - * - * After: - * Project[a, b, virtual_slot_ref] - * └── OlapScan[table, virtual_columns=[(col MATCH_ANY 'hello') as alias]] - */ -public class PushDownMatchProjectionAsVirtualColumn implements RewriteRuleFactory { - - private boolean canPushDown(LogicalOlapScan scan) { - boolean dupTblOrMOW = scan.getTable().getKeysType() == KeysType.DUP_KEYS - || (scan.getTable().getTableProperty() != null - && scan.getTable().getTableProperty().getEnableUniqueKeyMergeOnWrite()); - return dupTblOrMOW; - } - - @Override - public List buildRules() { - return ImmutableList.of( - // Pattern 1: Project -> OlapScan - logicalProject(logicalOlapScan().when(this::canPushDown)) - .then(project -> { - LogicalOlapScan scan = project.child(); - return pushDown(project, scan, newScan -> newScan); - }).toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), - // Pattern 2: Project -> Filter -> OlapScan - logicalProject(logicalFilter(logicalOlapScan().when(this::canPushDown))) - .then(project -> { - LogicalFilter filter = project.child(); - LogicalOlapScan scan = filter.child(); - return pushDown(project, scan, - newScan -> filter.withChildren(newScan)); - }).toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN) - ); - } - - /** - * Extract MATCH projections and push them as virtual columns on the scan. - * @param childRebuilder rebuilds the project's child tree with the new scan - */ - private LogicalProject pushDown( - LogicalProject project, LogicalOlapScan scan, - Function childRebuilder) { - List projections = project.getProjects(); - List virtualColumns = new ArrayList<>(); - Map replaceMap = new HashMap<>(); - - for (NamedExpression projection : projections) { - Expression matchExpr = unwrapMatch(projection); - if (matchExpr != null && !replaceMap.containsKey(matchExpr)) { - Alias alias = new Alias(matchExpr); - replaceMap.put(matchExpr, alias.toSlot()); - virtualColumns.add(alias); - } - } - - if (virtualColumns.isEmpty()) { - return null; - } - - ImmutableList.Builder newProjections = ImmutableList.builder(); - for (NamedExpression projection : projections) { - Expression matchExpr = unwrapMatch(projection); - if (matchExpr != null && replaceMap.containsKey(matchExpr)) { - Expression slot = replaceMap.get(matchExpr); - if (projection instanceof Alias) { - newProjections.add(new Alias(((Alias) projection).getExprId(), - slot, ((Alias) projection).getName())); - } else { - newProjections.add((NamedExpression) slot); - } - } else { - newProjections.add(projection); - } - } - - LogicalOlapScan newScan = scan.appendVirtualColumns(virtualColumns); - return (LogicalProject) project.withProjectsAndChild( - newProjections.build(), childRebuilder.apply(newScan)); - } - - /** - * Unwrap a Match expression from a projection. - * Returns the Match expression if the projection is a Match directly or an Alias wrapping a Match. - * Returns null otherwise. - */ - private Expression unwrapMatch(NamedExpression projection) { - if (projection instanceof Alias && ((Alias) projection).child() instanceof Match) { - return ((Alias) projection).child(); - } - return null; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java index 86322c99762085..9dd387724826b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.rules.Rule; @@ -36,6 +37,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ArrayListMultimap; @@ -142,7 +144,7 @@ private Plan pushDownJoinExpressions(MatchingContext> ct private Plan pushDownFilterExpressions(MatchingContext>> ctx) { LogicalFilter> filter = ctx.root; LogicalJoin join = filter.child(); - PushdownProjectHelper pushdownProjectHelper = new PushdownProjectHelper(ctx.statementContext, join); + PushdownProjectHelper pushdownProjectHelper = new PushdownProjectHelper(ctx.cascadesContext, join); Pair> pushPredicates = pushdownProjectHelper.pushDownExpressions(filter.getConjuncts()); if (!pushPredicates.first) { @@ -197,7 +199,7 @@ private Plan defaultPushDownProject(MatchingContext project = ctx.root; C child = project.child(); PushdownProjectHelper pushdownProjectHelper - = new PushdownProjectHelper(ctx.statementContext, child); + = new PushdownProjectHelper(ctx.cascadesContext, child); Pair> pushProjects = pushdownProjectHelper.pushDownExpressions(project.getProjects()); @@ -341,11 +343,23 @@ private static List replaceSlot( private static class PushdownProjectHelper { private final Plan plan; private final StatementContext statementContext; + // needed to push through a join only, see canComputeInChild + private final CascadesContext cascadesContext; private final Map oldExprToNewExpr; private final Multimap childToPushDownProjects; + public PushdownProjectHelper(CascadesContext cascadesContext, Plan plan) { + this(cascadesContext.getStatementContext(), cascadesContext, plan); + } + public PushdownProjectHelper(StatementContext statementContext, Plan plan) { + this(statementContext, null, plan); + } + + private PushdownProjectHelper(StatementContext statementContext, CascadesContext cascadesContext, + Plan plan) { this.statementContext = statementContext; + this.cascadesContext = cascadesContext; this.plan = plan; this.oldExprToNewExpr = new LinkedHashMap<>(); this.childToPushDownProjects = ArrayListMultimap.create(); @@ -391,7 +405,7 @@ public Optional pushDownExpression(E expression) { List children = plan.children(); for (int i = 0; i < children.size(); i++) { Plan child = children.get(i); - if (child.getOutputSet().containsAll(e.getInputSlots())) { + if (child.getOutputSet().containsAll(e.getInputSlots()) && canComputeInChild(i, e)) { Alias alias = new Alias(statementContext.getNextExprId(), e); Slot slot = alias.toSlot(); childToPushDownProjects.put(child, alias); @@ -410,6 +424,19 @@ public Optional pushDownExpression(E expression) { } } + // A filter or project above an outer join reads NULL for rows of the NULL-extended side, so an + // expression computed inside that child must itself be NULL for NULL input: nvl(col, 'x') MATCH 'x' + // is TRUE above the join but would be padded to NULL when computed below it. + private boolean canComputeInChild(int childIndex, Expression expression) { + if (!(plan instanceof LogicalJoin)) { + return true; + } + LogicalJoin join = (LogicalJoin) plan; + boolean nullExtended = childIndex == 0 ? join.getJoinType().isLeftSideNullable() + : join.getJoinType().isRightSideNullable(); + return !nullExtended || ExpressionUtils.isNullPropagating(expression, cascadesContext); + } + public List buildNewChildren() { if (childToPushDownProjects.isEmpty()) { return plan.children(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java index ca5d2d93b292aa..87b0c6f21faa14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java @@ -22,50 +22,66 @@ import org.apache.doris.catalog.Index; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.info.IndexType; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; +import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Search; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; +import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; /** - * Rewrite search function to add proper slot reference children. + * Bind the fields of a search() function: resolve each DSL field reference to a slot of the filter's child and + * produce a SearchExpression whose children are those slots (or variant subcolumns of them). * This is crucial for BE's "action on slot" detection in normalize conjuncts. + * + *

The steps are kept apart on purpose. SearchDslParser owns the DSL syntax (field path, {@code @analyzer} + * selector, escapes). Field names resolve exactly like SQL column references, so {@code alias.field} selects a + * relation of a join before {@code column.subcolumn} is tried. Index validation and the field names sent to BE + * use the physical column behind the slot, never its output alias. Where a SEARCH may be evaluated is not + * decided here: PushDownIndexSearchAsVirtualColumn places it and CheckAfterRewrite verifies the final plan. */ public class RewriteSearchToSlots extends OneRewriteRuleFactory { private static final Logger LOG = LogManager.getLogger(RewriteSearchToSlots.class); @Override public Rule build() { - return logicalFilter(logicalOlapScan()) + return logicalFilter() .when(filter -> ExpressionUtils.containsTypes(filter.getExpressions(), Search.class)) - .then(this::rewriteSearchExpressions) + .thenApply(ctx -> rewriteSearchExpressions(ctx.root, ctx.cascadesContext)) .toRule(RuleType.REWRITE_SEARCH_TO_SLOTS); } - private Plan rewriteSearchExpressions(LogicalFilter filter) { + private Plan rewriteSearchExpressions(LogicalFilter filter, CascadesContext cascadesContext) { List newExpressions = new ArrayList<>(); for (Expression expr : filter.getExpressions()) { - Expression rewritten = rewriteExpression(expr, filter.child()); + Expression rewritten = rewriteExpression(expr, filter.child(), cascadesContext); newExpressions.add(rewritten); } @@ -76,14 +92,14 @@ private Plan rewriteSearchExpressions(LogicalFilter filter) { return filter; } - private Expression rewriteExpression(Expression expr, LogicalOlapScan scan) { + private Expression rewriteExpression(Expression expr, Plan child, CascadesContext cascadesContext) { if (expr instanceof Search) { - return rewriteSearch((Search) expr, scan); + return rewriteSearch((Search) expr, child, cascadesContext); } // Recursively process children List newChildren = expr.children().stream() - .map(child -> rewriteExpression(child, scan)) + .map(c -> rewriteExpression(c, child, cascadesContext)) .collect(Collectors.toList()); if (!newChildren.equals(expr.children())) { @@ -93,7 +109,7 @@ private Expression rewriteExpression(Expression expr, LogicalOlapScan scan) { return expr; } - private Expression rewriteSearch(Search search, LogicalOlapScan scan) { + private Expression rewriteSearch(Search search, Plan child, CascadesContext cascadesContext) { try { // Parse DSL to get field bindings SearchDslParser.QsPlan qsPlan = search.getQsPlan(); @@ -103,65 +119,41 @@ private Expression rewriteSearch(Search search, LogicalOlapScan scan) { } Map normalizedFields = new HashMap<>(); + // physical field -> the inverted index its analyzer selects + Map fieldIndexes = new HashMap<>(); + Scope scope = new Scope(child.getOutput()); + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(child, scope, cascadesContext, false, false); // Create slot reference children from field bindings List slotChildren = new ArrayList<>(); for (SearchDslParser.QsFieldBinding binding : qsPlan.getFieldBindings()) { - String originalFieldName = binding.getFieldName(); - Expression childExpr; - String normalizedFieldName; - - if (originalFieldName.contains(".")) { - int firstDotPos = originalFieldName.indexOf('.'); - String parentFieldName = originalFieldName.substring(0, firstDotPos); - String subcolumnPath = originalFieldName.substring(firstDotPos + 1); - - // Find parent slot - Slot parentSlot = findSlotByName(parentFieldName, scan); - if (parentSlot == null) { - throw new AnalysisException(String.format( - "Parent field '%s' not found in table for search: %s", - parentFieldName, search.getDslString())); - } - - // Verify it's a variant type - if (!parentSlot.getDataType().isVariantType()) { - throw new AnalysisException(String.format( - "Field '%s' is not VARIANT type for subcolumn access: %s", - parentFieldName, search.getDslString())); - } - String normalizedParentFieldName = parentSlot.getName(); - - // Check the parent variant column has at least one INVERTED index. The concrete - // subcolumn binding is resolved per-segment in BE, so we only enforce the parent - // level here. See function_search.cpp is_variant_sub branch. - checkInvertedIndexExists(scan.getTable(), normalizedParentFieldName, - search.getDslString(), true); - - // Create ElementAt expression for variant subcolumn + String fieldReference = binding.getFieldName(); + Pair pathAndAnalyzer = SearchDslParser.splitAnalyzerSelector(fieldReference); + binding.setAnalyzerName(pathAndAnalyzer.second); + + Pair> field = resolveField(pathAndAnalyzer.first, child, analyzer, + scope, search.getDslString()); + SlotReference slot = field.first; + List subPath = field.second; + Index index = checkInvertedIndex(slot, subPath, binding.getAnalyzerName(), search.getDslString()); + + String normalizedFieldName = physicalFieldName(slot, subPath); + Expression childExpr = slot; + if (!subPath.isEmpty()) { // This will be converted to an extracted column slot by VariantSubPathPruning rule // If the subcolumn doesn't exist, ElementAt will remain and BE will handle it gracefully - childExpr = new ElementAt(parentSlot, new StringLiteral(subcolumnPath)); - normalizedFieldName = normalizedParentFieldName + "." + subcolumnPath; - - LOG.info( - "Created ElementAt expression for variant subcolumn: parent='{}', " - + "subcolumn='{}', field_name='{}'", - normalizedParentFieldName, subcolumnPath, normalizedFieldName); - } else { - // Normal field - find slot directly - Slot slot = findSlotByName(originalFieldName, scan); - if (slot == null) { - throw new AnalysisException(String.format( - "Field '%s' not found in table for search: %s", - originalFieldName, search.getDslString())); - } - checkInvertedIndexExists(scan.getTable(), slot.getName(), search.getDslString(), false); - childExpr = slot; - normalizedFieldName = slot.getName(); + childExpr = new ElementAt(slot, new StringLiteral(String.join(".", subPath))); } - normalizedFields.put(originalFieldName, normalizedFieldName); + // BE keeps one index per field, so two references to a field must select the same index. + // Comparing the selected indexes reuses the analyzer identity of the index lookup itself. + if (fieldIndexes.containsKey(normalizedFieldName) + && fieldIndexes.get(normalizedFieldName) != index) { + throw new AnalysisException("SEARCH supports one analyzer per field; use separate SEARCH " + + "expressions for different analyzers on " + normalizedFieldName); + } + fieldIndexes.put(normalizedFieldName, index); + normalizedFields.put(fieldReference, normalizedFieldName); binding.setFieldName(normalizedFieldName); slotChildren.add(childExpr); } @@ -169,7 +161,12 @@ private Expression rewriteSearch(Search search, LogicalOlapScan scan) { LOG.info("Rewriting search function: dsl='{}' with {} slot children", search.getDslString(), slotChildren.size()); - normalizePlanFields(qsPlan.getRoot(), normalizedFields); + normalizePlanFields(qsPlan.getRoot(), normalizedFields, + nestedPath -> { + Pair> nested = resolveField(nestedPath, child, analyzer, scope, + search.getDslString()); + return physicalFieldName(nested.first, nested.second); + }); // Create SearchExpression with slot children return new SearchExpression(search.getDslString(), qsPlan, slotChildren); @@ -180,77 +177,117 @@ private Expression rewriteSearch(Search search, LogicalOlapScan scan) { } /** - * Ensure the column referenced by a Lucene-syntax SEARCH predicate has an inverted index. - * Without this check the BE path would silently fall back to an empty bitmap (i.e. all FALSE), - * which is indistinguishable from "no rows matched" to the user. Throw at planning time so the - * behavior is consistent with referencing a non-existent column. - * - * @param table table backing the LogicalOlapScan - * @param columnName column name (parent column name when isVariantParent) - * @param dsl original DSL, used in the error message - * @param isVariantParent true when {@code columnName} is the parent of a variant subcolumn - * access (e.g. {@code msg.body}); for that case any INVERTED index on - * the parent column is accepted because the concrete subcolumn binding - * is resolved per-segment in BE. + * Resolve a DSL field path against the child's output with the SQL name resolution rules + * (ExpressionAnalyzer#bindSlotByScope): {@code field}, {@code alias.field}, {@code db.tbl.field}, each + * optionally followed by a variant subcolumn path. Returns the slot and the subcolumn path (empty for a + * plain column). */ - private void checkInvertedIndexExists(OlapTable table, String columnName, String dsl, - boolean isVariantParent) { - Column column = table.getColumn(columnName); - if (column == null) { - // Field existence is already validated by findSlotByName; if we reach here the schema - // changed concurrently. Surface a clear error rather than fall through. - throw new AnalysisException(String.format( - "Column '%s' not found in table '%s' for search: %s", - columnName, table.getName(), dsl)); + private Pair> resolveField(String fieldPath, Plan child, + ExpressionAnalyzer analyzer, Scope scope, String dsl) { + List nameParts = Arrays.asList(fieldPath.split("\\.", -1)); + if (nameParts.stream().anyMatch(String::isEmpty)) { + throw new AnalysisException(String.format("Invalid field '%s' for search: %s", fieldPath, dsl)); + } + List candidates = analyzer.bindSlotByScope(new UnboundSlot(nameParts), scope) + .stream().distinct().collect(Collectors.toList()); + if (candidates.isEmpty()) { + throw new AnalysisException(String.format("Field '%s' not found in table for search: %s", + fieldPath, dsl)); + } + if (candidates.size() > 1) { + throw new AnalysisException(String.format("Ambiguous field '%s' in search(); qualify it with its " + + "table alias, as in a SQL column reference: %s", fieldPath, dsl)); } - if (isVariantParent) { - for (Index index : table.getIndexes()) { - if (index.getIndexType() != IndexType.INVERTED) { - continue; - } - List columns = index.getColumns(); - if (columns != null && !columns.isEmpty() - && columnName.equalsIgnoreCase(columns.get(0))) { - return; - } + // A nested reference is bound as Alias(element_at(element_at(slot, 'a'), 'b')). + Expression bound = candidates.get(0) instanceof Alias ? candidates.get(0).child(0) : candidates.get(0); + List subPath = new ArrayList<>(); + while (bound instanceof ElementAt) { + subPath.add(0, ((StringLiteral) bound.child(1)).getStringValue()); + bound = bound.child(0); + } + SlotReference boundSlot = (SlotReference) bound; + if (!subPath.isEmpty() && !boundSlot.getDataType().isVariantType()) { + throw new AnalysisException(String.format( + "Field '%s' is not VARIANT type for subcolumn access: %s", boundSlot.getName(), dsl)); + } + // bindSlotByScope renames the slot to the spelling of the reference; keep the child's own slot. + for (Slot output : child.getOutput()) { + if (output.getExprId().equals(boundSlot.getExprId())) { + return Pair.of((SlotReference) output, subPath); } - } else if (table.getInvertedIndex(column, null) != null) { - return; } + throw new AnalysisException(String.format("Field '%s' not found in table for search: %s", fieldPath, dsl)); + } - throw new AnalysisException(String.format( - "Field '%s' has no inverted index, cannot be used in search: %s. " - + "Create an inverted index on the column first " - + "(ALTER TABLE ... ADD INDEX ... USING INVERTED).", - columnName, dsl)); + // BE looks fields up in the tablet schema, so it gets the physical column name, never an output alias. + private String physicalFieldName(SlotReference slot, List subPath) { + String columnName = slot.getOriginalColumn().map(Column::getName).orElse(slot.getName()); + return subPath.isEmpty() ? columnName : columnName + "." + String.join(".", subPath); } - private Slot findSlotByName(String fieldName, LogicalOlapScan scan) { - // Direct match only - variant subcolumns are handled by caller - for (Slot slot : scan.getOutput()) { - if (slot.getName().equalsIgnoreCase(fieldName)) { - return slot; - } + /** + * Ensure the physical column referenced by a Lucene-syntax SEARCH predicate has an inverted index, and return + * the index the analyzer selects (OlapTable#getInvertedIndex, the lookup the translator sends to BE; null + * when a variant subcolumn gets its index per segment in BE). + * Without this check the BE path would silently fall back to an empty bitmap (i.e. all FALSE), + * which is indistinguishable from "no rows matched" to the user. Throw at planning time so the + * behavior is consistent with referencing a non-existent column. + * + * @param slot resolved field; its original table and column identify the physical column, whatever + * alias the slot carries + * @param subPath variant subcolumn path, empty for a plain column. For a subcolumn any INVERTED index on + * the parent column is accepted because the concrete subcolumn binding is resolved + * per-segment in BE. See function_search.cpp is_variant_sub branch. + * @param analyzer analyzer selected by {@code field@analyzer}, or null + * @param dsl original DSL, used in the error message + */ + private Index checkInvertedIndex(SlotReference slot, List subPath, String analyzer, String dsl) { + if (!(slot.getOriginalTable().orElse(null) instanceof OlapTable) || !slot.getOriginalColumn().isPresent()) { + throw new AnalysisException("search() requires a field from an OLAP table: " + slot.toSql()); + } + OlapTable table = (OlapTable) slot.getOriginalTable().get(); + Column column = slot.getOriginalColumn().get(); + Index index = table.getInvertedIndex(column, subPath, analyzer); + if (index == null && analyzer != null) { + throw new AnalysisException(String.format( + "No inverted index found for SEARCH analyzer '%s' on field '%s': %s", + analyzer, column.getName(), dsl)); } - return null; + boolean hasIndex = index != null; + if (!subPath.isEmpty()) { + hasIndex = table.getIndexes().stream().anyMatch(i -> i.getIndexType() == IndexType.INVERTED + && i.getColumns() != null && !i.getColumns().isEmpty() + && column.getName().equalsIgnoreCase(i.getColumns().get(0))); + } + if (!hasIndex) { + throw new AnalysisException(String.format( + "Field '%s' has no inverted index, cannot be used in search: %s. " + + "Create an inverted index on the column first " + + "(ALTER TABLE ... ADD INDEX ... USING INVERTED).", + column.getName(), dsl)); + } + return index; } - private void normalizePlanFields(SearchDslParser.QsNode node, Map normalized) { + private void normalizePlanFields(SearchDslParser.QsNode node, Map normalized, + Function nestedPathNormalizer) { if (node == null) { return; } - if (node.getField() != null) { - for (Map.Entry entry : normalized.entrySet()) { - if (entry.getKey().equalsIgnoreCase(node.getField())) { - node.setField(entry.getValue()); - break; - } - } + // Variant subcolumn paths are case sensitive, so match the reference exactly as the binding was named. + if (node.getField() != null && !node.getField().isEmpty()) { + // The parser names every binding after a node's field. + Preconditions.checkState(normalized.containsKey(node.getField()), + "SEARCH field %s has no binding", node.getField()); + node.setField(normalized.get(node.getField())); + } + if (node.getNestedPath() != null) { + node.setNestedPath(nestedPathNormalizer.apply(node.getNestedPath())); } if (node.getChildren() != null) { for (SearchDslParser.QsNode child : node.getChildren()) { - normalizePlanFields(child, normalized); + normalizePlanFields(child, normalized, nestedPathNormalizer); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java index 5767d72beddc85..b4011a9539e4d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java @@ -20,6 +20,8 @@ import org.apache.doris.analysis.SearchDslParser; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -30,6 +32,11 @@ /** * SearchExpression represents a search query with bound slot references. * This is created by RewriteSearchToSlots rule from Search scalar function. + * + *

Each child binds one DSL field, in the order of the QsPlan field bindings: a slot, or element_at on a slot + * for a variant subcolumn. BE evaluates the expression only with the inverted indexes of those fields inside an + * OLAP scan, never row by row. Rewrites are free to move the expression; CheckAfterRewrite verifies on the final + * plan that it sits in a scan (filter conjunct or virtual column) and that {@link #bindsOnlyFields()} holds. */ public class SearchExpression extends Expression { private final String dslString; @@ -72,24 +79,48 @@ public boolean foldable() { @Override public SearchExpression withChildren(List children) { - // Validate that all children are SlotReference or ElementAt (for variant subcolumns) + // Rewrites may replace a field with NULL: null-rejection inference does so on a temporary copy, and + // NULL padding of an outer join side (e.g. ON false, outer-to-anti join) does so in the plan. + // Such a SEARCH no longer binds an index field; CheckAfterRewrite rejects it via bindsOnlyFields. for (Expression child : children) { - if (!(child instanceof SlotReference || child instanceof ElementAt)) { + if (!(child instanceof SlotReference || child instanceof ElementAt + || child instanceof NullLiteral)) { throw new IllegalArgumentException( - "SearchExpression children must be SlotReference or ElementAt instances"); + "SEARCH field binding must be a slot, subcolumn, or NULL, found " + + child.getClass().getSimpleName()); } } return new SearchExpression(dslString, qsPlan, children); } + /** + * Whether every child still binds an index field: a slot, or a variant subcolumn of a slot. + */ + public boolean bindsOnlyFields() { + return children().stream().allMatch(SearchExpression::isFieldBinding); + } + + private static boolean isFieldBinding(Expression expression) { + Expression current = expression; + while (current instanceof ElementAt) { + current = current.child(0); + } + return current instanceof SlotReference; + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitSearchExpression(this, context); } + @Override + public String computeToSql() { + return "search(" + new StringLiteral(dslString).toSql() + ")"; + } + @Override public String toString() { - return "search('" + dslString + "')"; + return computeToSql(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 00f5caa8364e7f..65490bfb096756 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -945,6 +945,17 @@ private static boolean isFalseOrNull(Expression expression) { return expression.isNullLiteral() || BooleanLiteral.FALSE.equals(expression); } + /** + * Whether the expression is NULL when all its input slots are NULL, e.g. {@code col MATCH_ANY 'x'} or + * {@code element_at(v, 'k')}, but not {@code nvl(col, 'x') MATCH_ANY 'x'}. Only such an expression can be + * computed below the NULL-extended side of an outer join, because the join pads its value with NULL. + */ + public static boolean isNullPropagating(Expression expression, CascadesContext cascadesContext) { + Set inputSlots = expression.getInputSlots(); + return !inputSlots.isEmpty() + && matchesWhenSlotsAreNull(expression, inputSlots, cascadesContext, Expression::isNullLiteral); + } + /** * infer notNulls slot from predicate */ @@ -980,7 +991,8 @@ private static Set inferNotNullSlots(Set predicates, Set } inputSlots = mergedInputSlots.get(); for (Slot slot : candidateSlots) { - if (matchesWhenSlotIsNull(predicate, slot, cascadesContext, nullInputResultPredicate)) { + if (matchesWhenSlotsAreNull(predicate, ImmutableSet.of(slot), cascadesContext, + nullInputResultPredicate)) { notNullSlots.add(slot); } } @@ -1000,11 +1012,12 @@ private static Set collectNotNullInferenceTargetSlots(Set expr return targetSlots; } - private static boolean matchesWhenSlotIsNull(Expression expression, Slot slot, CascadesContext cascadesContext, - Predicate nullInputResultPredicate) { + private static boolean matchesWhenSlotsAreNull(Expression expression, Set slots, + CascadesContext cascadesContext, Predicate nullInputResultPredicate) { Map replaceMap = new HashMap<>(); - Literal nullLiteral = new NullLiteral(slot.getDataType()); - replaceMap.put(slot, nullLiteral); + for (Slot slot : slots) { + replaceMap.put(slot, new NullLiteral(slot.getDataType())); + } Expression evalExpr = FoldConstantRule.evaluate( ExpressionUtils.replace(expression, replaceMap), new ExpressionRewriteContext(cascadesContext)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java index ab9a98660e840c..5cd52f5f3b4a87 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java @@ -28,7 +28,7 @@ /** * Unit tests for CheckSearchUsage rule. * This test validates that search() function can only be used in WHERE clauses - * on single-table OLAP scans, and is rejected in other contexts. + * over OLAP scans and joins, and is rejected in other contexts. */ public class CheckSearchUsageTest extends TestWithFeService implements MemoPatternMatchSupported { @@ -89,8 +89,7 @@ public void testSearchInGroupByRejected() { Assertions.assertTrue( exception.getMessage().contains("search()") && (exception.getMessage().contains("GROUP BY") - || exception.getMessage().contains("WHERE filters") - || exception.getMessage().contains("single-table")), + || exception.getMessage().contains("WHERE filters")), "Expected error about search() usage restrictions, got: " + exception.getMessage()); } @@ -160,7 +159,7 @@ public void testSearchInHavingRejected() { } @Test - public void testSearchWithJoinRejected() { + public void testSearchWithJoinAllowed() { // Create second table for join test try { createTable("CREATE TABLE test_search_table2 (\n" @@ -174,19 +173,12 @@ public void testSearchWithJoinRejected() { // Table might already exist from previous test } - // Invalid: search() in WHERE with JOIN + // SEARCH depends only on title from the first input. String sql = "SELECT t1.id FROM test_search_table t1 " + "JOIN test_search_table2 t2 ON t1.id = t2.id " + "WHERE search('title:hello')"; - AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> { - PlanChecker.from(connectContext).analyze(sql); - }); - - Assertions.assertTrue( - exception.getMessage().contains("search()") - && exception.getMessage().contains("single"), - "Expected error about single table, got: " + exception.getMessage()); + Assertions.assertDoesNotThrow(() -> PlanChecker.from(connectContext).analyze(sql)); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumnTest.java similarity index 95% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumnTest.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumnTest.java index 55735021e78480..a91a5cf395279c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumnTest.java @@ -46,9 +46,9 @@ import java.util.stream.Collectors; /** - * Test for PushDownMatchProjectionAsVirtualColumn rule. + * Test for PushDownIndexSearchAsVirtualColumn rule. */ -public class PushDownMatchProjectionAsVirtualColumnTest implements MemoPatternMatchSupported { +public class PushDownIndexSearchAsVirtualColumnTest implements MemoPatternMatchSupported { @Test void testPushDownMatchProjection() { @@ -62,7 +62,7 @@ void testPushDownMatchProjection() { ImmutableList.of(idSlot, new Alias(matchExpr, "m")), scan); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); // Verify plan structure @@ -105,7 +105,7 @@ void testPushDownMatchProjectionWithFilter() { new LogicalFilter<>(ImmutableSet.of(filterPred), scan)); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); // Verify plan structure: Project -> Filter -> OlapScan @@ -141,7 +141,7 @@ void testNoMatchExpressionNoChange() { ImmutableList.of(idSlot), scan); PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .matches( logicalProject( logicalOlapScan().when(s -> s.getVirtualColumns().isEmpty()) @@ -163,7 +163,7 @@ void testDuplicateMatchDedup() { scan); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); Assertions.assertInstanceOf(LogicalProject.class, root); @@ -201,7 +201,7 @@ void testMultipleDistinctMatchExpressions() { scan); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); Assertions.assertInstanceOf(LogicalProject.class, root); @@ -245,7 +245,7 @@ void testAppendToExistingVirtualColumns() { ImmutableList.of(idSlot, new Alias(matchExpr, "m")), scanWithVc); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); Assertions.assertInstanceOf(LogicalProject.class, root); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownProjectTest.java index 0a0b425e81766a..3e0fd1228640e2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownProjectTest.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.PreferPushDownProject; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.JoinType; @@ -240,4 +241,49 @@ public void shouldRewritePreferPushDownProjectInOrFilterToSlot() { && !rewrittenPredicate.anyMatch(PreferPushDownProject.class::isInstance); })); } + + @Test + public void shouldNotPushNonNullPropagatingExpressionIntoNullExtendedSide() { + LogicalPlan rStudent = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.student, + ImmutableList.of("")); + LogicalPlan rScore = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.score, + ImmutableList.of("")); + // nvl(NULL, 1) is 1, so the MATCH is not NULL for rows that the left join NULL-extends. + Expression nonNullPropagating = new MatchAny( + new Nvl(rScore.getOutput().get(2), Literal.of(1)), Literal.of("abc")); + Expression leftSidePredicate = new GreaterThan(rStudent.getOutput().get(0), Literal.of(60)); + + LogicalPlan plan = new LogicalPlanBuilder(rStudent) + .joinEmptyOn(rScore, JoinType.LEFT_OUTER_JOIN) + .filter(new Or(nonNullPropagating, leftSidePredicate)) + .build(); + + PlanChecker.from(connectContext, plan) + .applyTopDown(new PushDownProject()) + .matchesFromRoot(logicalFilter(logicalJoin(logicalOlapScan(), logicalOlapScan())) + .when(filter -> filter.getConjuncts().iterator().next() + .anyMatch(PreferPushDownProject.class::isInstance))); + } + + @Test + public void shouldPushNullPropagatingExpressionIntoNullExtendedSide() { + LogicalPlan rStudent = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.student, + ImmutableList.of("")); + LogicalPlan rScore = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.score, + ImmutableList.of("")); + Expression nullPropagating = new MatchAny( + new Add(rScore.getOutput().get(2), Literal.of(1)), Literal.of("abc")); + Expression leftSidePredicate = new GreaterThan(rStudent.getOutput().get(0), Literal.of(60)); + + LogicalPlan plan = new LogicalPlanBuilder(rStudent) + .joinEmptyOn(rScore, JoinType.LEFT_OUTER_JOIN) + .filter(new Or(nullPropagating, leftSidePredicate)) + .build(); + + PlanChecker.from(connectContext, plan) + .applyTopDown(new PushDownProject()) + .matchesFromRoot(logicalFilter(logicalJoin(logicalOlapScan(), logicalProject(logicalOlapScan()))) + .when(filter -> !filter.getConjuncts().iterator().next() + .anyMatch(PreferPushDownProject.class::isInstance))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java index a72b6ff8dd0a08..9883eaf9f9e8b8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.TableIndexes; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.IndexType; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.trees.expressions.Expression; @@ -35,6 +36,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Search; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.util.PlanConstructor; @@ -244,10 +246,10 @@ public void testRewriteSearchHandlesCaseInsensitiveField() throws Exception { Search searchFunc = new Search(new StringLiteral("NAME:alice")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); - Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan); + Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan, null); Assertions.assertInstanceOf(SearchExpression.class, rewritten); SearchExpression searchExpression = (SearchExpression) rewritten; @@ -268,10 +270,10 @@ public void testRewriteSearchHandlesCaseInsensitiveVariantParentField() throws E Search searchFunc = new Search(new StringLiteral("V.foo:bar")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); - Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan); + Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan, null); Assertions.assertInstanceOf(SearchExpression.class, rewritten); SearchExpression searchExpression = (SearchExpression) rewritten; @@ -293,11 +295,11 @@ public void testRewriteSearchThrowsWhenFieldMissing() throws Exception { Search searchFunc = new Search(new StringLiteral("unknown_field:value")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); InvocationTargetException thrown = Assertions.assertThrows(InvocationTargetException.class, - () -> rewriteMethod.invoke(rewriteRule, searchFunc, scan)); + () -> rewriteMethod.invoke(rewriteRule, searchFunc, scan, null)); Assertions.assertNotNull(thrown.getCause()); Assertions.assertInstanceOf(AnalysisException.class, thrown.getCause()); Assertions.assertTrue(thrown.getCause().getMessage().contains("unknown_field")); @@ -312,11 +314,11 @@ public void testRewriteSearchThrowsWhenColumnHasNoInvertedIndex() throws Excepti Search searchFunc = new Search(new StringLiteral("name:alice")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); InvocationTargetException thrown = Assertions.assertThrows(InvocationTargetException.class, - () -> rewriteMethod.invoke(rewriteRule, searchFunc, scan)); + () -> rewriteMethod.invoke(rewriteRule, searchFunc, scan, null)); Assertions.assertNotNull(thrown.getCause()); Assertions.assertInstanceOf(AnalysisException.class, thrown.getCause()); Assertions.assertTrue(thrown.getCause().getMessage().contains("inverted index"), @@ -331,10 +333,10 @@ public void testRewriteSearchSucceedsWhenColumnHasInvertedIndex() throws Excepti Search searchFunc = new Search(new StringLiteral("name:alice")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); - Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan); + Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan, null); Assertions.assertInstanceOf(SearchExpression.class, rewritten); SearchExpression searchExpression = (SearchExpression) rewritten; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java new file mode 100644 index 00000000000000..d22b03bffdaa15 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java @@ -0,0 +1,418 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.analysis.SearchDslParser.QsFieldBinding; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +public class SearchJoinDocumentTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("search_join_document"); + connectContext.setDatabase("search_join_document"); + // Mocked tables have no rowsets. Preserve scans so these tests exercise + // field binding, virtual columns and translation rather than empty plans. + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + createTable("CREATE TABLE objects (id BIGINT, v VARIANT, " + + "INDEX idx_v(v) USING INVERTED PROPERTIES('parser'='english')) " + + "DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); + createTable("CREATE TABLE lists (object_id BIGINT, list_id BIGINT) " + + "DUPLICATE KEY(object_id) DISTRIBUTED BY HASH(object_id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); + createTable("CREATE TABLE associations (to_id BIGINT, from_id BIGINT) " + + "DUPLICATE KEY(to_id) DISTRIBUTED BY HASH(to_id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); + createTable("CREATE TABLE notes (id BIGINT, title TEXT, " + + "INDEX idx_title(title) USING INVERTED PROPERTIES('parser'='english')) " + + "DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); + createTable("CREATE TABLE docs (id BIGINT, title TEXT, " + + "INDEX idx_title_en(title) USING INVERTED PROPERTIES('parser'='english'), " + + "INDEX idx_title_uni(title) USING INVERTED PROPERTIES('parser'='unicode')) " + + "DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); + } + + @Test + public void testDocumentSearchAboveTwoLeftJoins() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM (SELECT id, v FROM objects) o " + + "LEFT JOIN lists l ON o.id=l.object_id " + + "LEFT JOIN associations a ON o.id=a.to_id " + + "WHERE search('john', '{\"default_field\":\"v.string_17\",\"mode\":\"lucene\"}')"); + } + + @Test + public void testDocumentMatchExistsOr() { + PlanChecker.from(connectContext).checkPlannerResult( + "WITH contacts AS (SELECT id, CAST(v['string_8'] AS VARCHAR) firstname FROM objects), " + + "members AS (SELECT object_id FROM lists) " + + "SELECT id, firstname FROM contacts c WHERE " + + "(firstname MATCH_ANY 'keyur patel' AND EXISTS " + + "(SELECT 1 FROM members l WHERE l.object_id=c.id)) OR c.id>1 LIMIT 10"); + } + + @Test + public void testSearchOrAssociation() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o LEFT JOIN lists l ON o.id=l.object_id " + + "WHERE search('v.string_8:john') OR l.list_id=12", planner -> { + Assertions.assertTrue(planner.getPhysicalPlan().anyMatch(plan -> + plan instanceof PhysicalOlapScan && ((PhysicalOlapScan) plan).getVirtualColumns() + .stream().anyMatch(column -> column.anyMatch(e -> e instanceof SearchExpression))), + planner.getPhysicalPlan().treeString()); + }); + } + + @Test + public void testMatchResidualUsesVirtualColumn() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o FULL OUTER JOIN lists l ON o.id=l.object_id " + + "WHERE (CAST(o.v['string_8'] AS VARCHAR) MATCH_ANY 'john' " + + "AND CAST(o.v['string_17'] AS VARCHAR) MATCH_ALL 'smith') OR l.list_id=12", planner -> { + Assertions.assertTrue(planner.getPhysicalPlan().anyMatch(plan -> + plan instanceof PhysicalOlapScan && ((PhysicalOlapScan) plan).getVirtualColumns() + .stream().anyMatch(column -> column.anyMatch(e -> e instanceof Match))), + planner.getPhysicalPlan().treeString()); + }); + } + + @Test + public void testAmbiguousSearchFieldRejected() { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze( + "SELECT a.id FROM objects a JOIN objects b ON a.id=b.id WHERE search('v.string_8:john')")); + Assertions.assertTrue(exception.getMessage().contains("Ambiguous field 'v.string_8'"), + exception.getMessage()); + } + + @Test + public void testSelfJoinFieldQualifiedByTableAlias() { + // alias.field resolves like a SQL column reference, so each side of a self join can be searched. + for (String side : new String[] {"a", "b"}) { + String otherSide = side.equals("a") ? "b" : "a"; + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT a.id FROM notes a JOIN notes b ON a.id=b.id " + + "WHERE search('" + side + ".title:john') OR " + otherSide + ".id=1", planner -> { + List scans = scansWithSearchVirtualColumn(planner.getPhysicalPlan()); + Assertions.assertEquals(1, scans.size(), planner.getPhysicalPlan().treeString()); + SearchExpression search = searchExpressions(planner.getPhysicalPlan()).get(0); + Assertions.assertTrue(((PhysicalOlapScan) scans.get(0)).getOutputSet() + .containsAll(search.getInputSlots())); + Assertions.assertEquals(ImmutableList.of(side), + search.getInputSlots().stream().map(slot -> slot.getQualifier() + .get(slot.getQualifier().size() - 1)).collect(Collectors.toList())); + }); + } + // alias.variant.subcolumn + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT a.id FROM objects a JOIN objects b ON a.id=b.id WHERE search('b.v.string_8:john')", + planner -> Assertions.assertEquals("v.string_8", searchExpressions(planner.getPhysicalPlan()) + .get(0).getQsPlan().getFieldBindings().get(0).getFieldName())); + } + + @Test + public void testFieldAliasBindsPhysicalColumn() { + // The DSL names the visible column; index validation and the field sent to BE use the physical column. + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT n.id FROM (SELECT id, title AS headline FROM notes) n " + + "LEFT JOIN lists l ON n.id=l.object_id WHERE search('headline:john') OR l.list_id=12", + planner -> assertSingleBinding(planner.getPhysicalPlan(), "title")); + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM (SELECT id, v AS props FROM objects) o " + + "JOIN lists l ON o.id=l.object_id WHERE search('props.string_8:john')", + planner -> assertSingleBinding(planner.getPhysicalPlan(), "v.string_8")); + // An alias that hides a column without an inverted index must not pass as the indexed column. + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze( + "SELECT id FROM (SELECT o.id, l.list_id AS title FROM notes o JOIN lists l ON o.id=l.object_id) t " + + "WHERE search('title:john')")); + Assertions.assertTrue(exception.getMessage().contains("'list_id' has no inverted index"), + exception.getMessage()); + } + + @Test + public void testNestedPathBindsPhysicalColumn() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM (SELECT id, v AS props FROM objects) o JOIN lists l ON o.id=l.object_id " + + "WHERE search('NESTED(props.items, name:john)')", planner -> { + SearchExpression search = searchExpressions(planner.getPhysicalPlan()).get(0); + Assertions.assertEquals("v.items", search.getQsPlan().getRoot().getNestedPath()); + Assertions.assertEquals("v.items.name", + search.getQsPlan().getFieldBindings().get(0).getFieldName()); + }); + } + + @Test + public void testSearchIsNotInferredForEqualColumn() { + // a.title = b.title lets predicates on a.title be inferred for b.title, but a SEARCH is bound to the + // inverted index of a.title and must stay there. + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT a.id FROM notes a JOIN docs b ON a.title=b.title " + + "WHERE search('a.title:john') OR a.title='x'", planner -> { + List searches = searchExpressions(planner.getPhysicalPlan()); + Assertions.assertEquals(1, searches.size(), planner.getPhysicalPlan().treeString()); + Slot field = searches.get(0).getInputSlots().iterator().next(); + Assertions.assertEquals("a", field.getQualifier().get(field.getQualifier().size() - 1)); + }); + } + + @Test + public void testSearchIsNotInferredThroughSetOperation() { + PlanChecker.from(connectContext).checkPlannerResult( + "(SELECT title FROM notes WHERE search('title:john')) INTERSECT (SELECT title FROM docs)", + planner -> Assertions.assertEquals(1, searchExpressions(planner.getPhysicalPlan()).size(), + planner.getPhysicalPlan().treeString())); + } + + @Test + public void testAnalyzerSelectorUsesIndexIdentity() { + // The selector is matched like the index lookup: case-insensitively, and the default analyzer of a + // field is the index it resolves to. + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT id FROM docs WHERE search('title@English:john AND title@english:smith')"); + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT id FROM notes WHERE search('title:john AND title@english:smith')"); + AnalysisException conflict = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze( + "SELECT id FROM docs WHERE search('title@english:john AND title@unicode:smith')")); + Assertions.assertTrue(conflict.getMessage().contains("one analyzer per field"), conflict.getMessage()); + AnalysisException missing = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze( + "SELECT id FROM docs WHERE search('title@chinese:john')")); + Assertions.assertTrue(missing.getMessage().contains("No inverted index found for SEARCH analyzer"), + missing.getMessage()); + } + + @Test + public void testNestedIndexSearchInProjectionUsesVirtualColumn() { + // Direct scans and joins share one materialization path, so a wrapped MATCH is handled by both. + for (String from : new String[] {"notes n", "notes n LEFT JOIN lists l ON n.id=l.object_id"}) { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT n.id, CASE WHEN n.title MATCH_ANY 'john' THEN 1 ELSE 0 END FROM " + from, + planner -> { + Assertions.assertTrue(hasMatchVirtualColumn(planner.getPhysicalPlan()), + planner.getPhysicalPlan().treeString()); + assertNoIndexSearchOutsideScan(planner.getPhysicalPlan()); + }); + } + } + + @Test + public void testVirtualColumnReusedBySelectAndWhere() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT n.id, n.title MATCH_ANY 'john' FROM notes n LEFT JOIN lists l ON n.id=l.object_id " + + "WHERE n.title MATCH_ANY 'john' OR l.list_id=12", planner -> { + Plan plan = planner.getPhysicalPlan(); + List scans = new ArrayList<>(); + collectPlans(plan, scans); + scans.removeIf(node -> !(node instanceof PhysicalOlapScan) + || ((PhysicalOlapScan) node).getVirtualColumns().isEmpty()); + Assertions.assertEquals(1, scans.size(), plan.treeString()); + Assertions.assertEquals(1, ((PhysicalOlapScan) scans.get(0)).getVirtualColumns().size(), + plan.treeString()); + assertNoIndexSearchOutsideScan(plan); + List nodes = new ArrayList<>(); + collectPlans(plan, nodes); + for (Plan node : nodes) { + List output = node.getOutput(); + Assertions.assertEquals(new HashSet<>(output).size(), output.size(), + "duplicate output slot in " + node + "\n" + plan.treeString()); + } + }); + } + + @Test + public void testIndexSearchNotReachingScanIsRejected() { + // The filter cannot move below the TopN and materialization does not cross it. + String from = "FROM (SELECT id, title FROM notes ORDER BY id LIMIT 10) t WHERE "; + assertRejected("SELECT id " + from + "search('title:john')", "SEARCH must be evaluated by an OLAP scan"); + assertRejected("SELECT id " + from + "title MATCH_ANY 'john'", "only support in olapScan filter"); + // Fields of two tables can never be evaluated by one scan. + assertRejected("SELECT n.id FROM notes n JOIN objects o ON n.id=o.id " + + "WHERE search('title:john OR v.string_8:john')", "SEARCH must be evaluated by an OLAP scan"); + } + + @Test + public void testPushDownIsIdempotent() { + PlanChecker checker = PlanChecker.from(connectContext).analyze( + "SELECT n.id, CASE WHEN n.title MATCH_ANY 'john' THEN 1 ELSE 0 END FROM notes n " + + "LEFT JOIN lists l ON n.id=l.object_id WHERE search('title:smith') OR l.list_id=12").rewrite(); + String rewritten = checker.getPlan().treeString(); + Assertions.assertTrue(checker.getPlan().anyMatch(node -> node instanceof LogicalOlapScan + && ((LogicalOlapScan) node).getVirtualColumns().size() == 2), rewritten); + Assertions.assertEquals(rewritten, + checker.applyTopDown(new PushDownIndexSearchAsVirtualColumn()).getPlan().treeString()); + } + + @Test + public void testSearchWithFieldSelector() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o LEFT JOIN lists l ON o.id=l.object_id " + + "WHERE search('v.string_8@english:john')"); + } + + @Test + public void testSearchConstantPredicatePreservesField() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o LEFT JOIN lists l ON o.id=l.object_id " + + "WHERE CAST(o.v['string_8'] AS VARCHAR)='john' " + + "AND (search('v.string_8:john') OR l.list_id=12)"); + } + + @Test + public void testQuotedAtSignIsLiteralVariantPath() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT id FROM objects WHERE search('\"v.email@work\":john')"); + } + + @Test + public void testSearchOrMovedIntoInnerJoin() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o JOIN lists l ON o.id=l.object_id " + + "WHERE search('v.string_8:john') OR l.list_id=12", planner -> { + Assertions.assertTrue(planner.getPhysicalPlan().anyMatch(plan -> + plan instanceof PhysicalOlapScan && ((PhysicalOlapScan) plan).getVirtualColumns() + .stream().anyMatch(column -> column.anyMatch(e -> e instanceof SearchExpression))), + planner.getPhysicalPlan().treeString()); + }); + } + + @Test + public void testNullPropagatingMatchOnNullExtendedSideUsesVirtualColumn() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT l.object_id FROM lists l LEFT JOIN objects o ON l.object_id=o.id " + + "WHERE CAST(o.v['string_8'] AS VARCHAR) MATCH_ANY 'john' OR l.list_id=12", planner -> { + Assertions.assertTrue(hasMatchVirtualColumn(planner.getPhysicalPlan()), + planner.getPhysicalPlan().treeString()); + }); + } + + @Test + public void testNonNullPropagatingMatchStaysAboveOuterJoin() { + // nvl(NULL, 'john') matches, so the MATCH cannot be computed below the NULL-extended side. + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT l.object_id FROM lists l LEFT JOIN objects o ON l.object_id=o.id " + + "WHERE nvl(CAST(o.v['string_8'] AS VARCHAR), 'john') MATCH_ANY 'john' OR l.list_id=12", + planner -> { + Assertions.assertFalse(hasMatchVirtualColumn(planner.getPhysicalPlan()), + planner.getPhysicalPlan().treeString()); + }); + } + + @Test + public void testSearchOnNullPaddedSideRejected() { + // LEFT JOIN ... ON false becomes Project(l.*, NULL AS o.*). + assertNullSideSearchRejected("SELECT l.object_id FROM lists l LEFT JOIN notes n ON false " + + "WHERE NOT search('title:john') OR l.list_id=8"); + // LEFT JOIN ... WHERE n.id IS NULL becomes a left anti join with NULL aliases for n.*. + assertNullSideSearchRejected("SELECT l.object_id FROM lists l LEFT JOIN notes n ON l.object_id=n.id " + + "WHERE n.id IS NULL AND (NOT search('title:john') OR l.list_id=8)"); + // A derived table exposing both sides lets materialize() inline NULL into a scan virtual column. + assertNullSideSearchRejected("SELECT s.id FROM (SELECT o.id, o.v, n.title FROM objects o " + + "LEFT JOIN notes n ON false) s JOIN lists l ON s.id=l.object_id " + + "WHERE NOT search('title:john OR v.string_8:hello') OR l.list_id=1"); + } + + private void collectPlans(Plan plan, List plans) { + plans.add(plan); + plan.children().forEach(child -> collectPlans(child, plans)); + } + + private List scanEvaluatedExpressions(Plan node) { + return node instanceof PhysicalOlapScan + ? new ArrayList<>(((PhysicalOlapScan) node).getVirtualColumns()) : new ArrayList<>(); + } + + private List scansWithSearchVirtualColumn(Plan plan) { + List plans = new ArrayList<>(); + collectPlans(plan, plans); + return plans.stream().filter(node -> scanEvaluatedExpressions(node).stream() + .anyMatch(column -> column.anyMatch(e -> e instanceof SearchExpression))) + .collect(Collectors.toList()); + } + + private List searchExpressions(Plan plan) { + List plans = new ArrayList<>(); + collectPlans(plan, plans); + List result = new ArrayList<>(); + for (Plan node : plans) { + List expressions = scanEvaluatedExpressions(node); + expressions.addAll(node.getExpressions()); + expressions.forEach(e -> result.addAll(e.collect(SearchExpression.class::isInstance))); + } + return result; + } + + private void assertSingleBinding(Plan plan, String physicalField) { + List searches = searchExpressions(plan); + Assertions.assertEquals(1, searches.size(), plan.treeString()); + List bindings = searches.get(0).getQsPlan().getFieldBindings(); + Assertions.assertEquals(1, bindings.size()); + Assertions.assertEquals(physicalField, bindings.get(0).getFieldName()); + Assertions.assertEquals(physicalField, searches.get(0).getQsPlan().getRoot().getField()); + } + + // Every MATCH/SEARCH of these plans can be materialized, so none may be left for row evaluation. + private void assertNoIndexSearchOutsideScan(Plan plan) { + List plans = new ArrayList<>(); + collectPlans(plan, plans); + for (Plan node : plans) { + Assertions.assertFalse(node.getExpressions().stream().anyMatch(expression -> expression.anyMatch( + e -> e instanceof Match || e instanceof SearchExpression)), plan.treeString()); + } + } + + private void assertRejected(String sql, String message) { + Throwable thrown = Assertions.assertThrows(Throwable.class, + () -> PlanChecker.from(connectContext).checkPlannerResult(sql)); + Throwable cause = thrown; + while (cause != null && (cause.getMessage() == null || !cause.getMessage().contains(message))) { + cause = cause.getCause(); + } + Assertions.assertNotNull(cause, sql + " failed with: " + thrown); + } + + private boolean hasMatchVirtualColumn(Plan plan) { + return plan.anyMatch(node -> node instanceof PhysicalOlapScan && ((PhysicalOlapScan) node).getVirtualColumns() + .stream().anyMatch(column -> column.anyMatch(e -> e instanceof Match))); + } + + private void assertNullSideSearchRejected(String sql) { + assertRejected(sql, "null-generating side of an outer join"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java index b1397cf8ce6f68..96111f32873360 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java @@ -18,6 +18,9 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.analysis.SearchDslParser; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.StringType; @@ -136,6 +139,26 @@ public void testToString() { String str = searchExpr.toString(); Assertions.assertEquals("search('title:hello')", str); + Assertions.assertEquals(str, searchExpr.toSql()); + } + + @Test + public void testNullFieldIsNotAFieldBinding() { + SlotReference title = createTestSlot("title"); + SearchExpression search = new SearchExpression("title:hello", createTestPlan(), + Collections.singletonList(title)); + Assertions.assertTrue(search.bindsOnlyFields()); + + // Null-rejection inference and outer join NULL padding substitute NULL for the field. + SearchExpression nullField = search.withChildren(Collections.singletonList(NullLiteral.INSTANCE)); + Assertions.assertFalse(nullField.foldable()); + Assertions.assertFalse(nullField.bindsOnlyFields()); + + StringLiteral key = new StringLiteral("name"); + Assertions.assertTrue(search.withChildren( + Collections.singletonList(new ElementAt(title, key))).bindsOnlyFields()); + Assertions.assertFalse(search.withChildren( + Collections.singletonList(new ElementAt(NullLiteral.INSTANCE, key))).bindsOnlyFields()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java index c078e56912118e..3575d454970bb4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.SearchDslParser.QsNode; import org.apache.doris.analysis.SearchDslParser.QsOccur; import org.apache.doris.analysis.SearchDslParser.QsPlan; +import org.apache.doris.common.Pair; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -2814,4 +2815,42 @@ public void testMultiFieldMatchAllDocsPreservesOccurWithAndOperator() { Assertions.assertTrue(foundMatchAllWithMust, "Should contain MATCH_ALL_DOCS node with MUST occur"); } + + private void assertFieldReference(String reference, String path, String analyzer) { + Pair result = SearchDslParser.splitAnalyzerSelector(reference); + Assertions.assertEquals(path, result.first, reference); + Assertions.assertEquals(analyzer, result.second, reference); + } + + @Test + public void testAnalyzerSelectorIsSyntactic() { + assertFieldReference("title", "title", null); + assertFieldReference("title@exact", "title", "exact"); + assertFieldReference("v.name@exact", "v.name", "exact"); + // Only the last unescaped @ is the selector. + assertFieldReference("a@b@exact", "a@b", "exact"); + // Escaped @ (also produced for quoted segments) is part of the name. + assertFieldReference("name\\@literal", "name@literal", null); + assertFieldReference("name\\@literal@exact", "name@literal", "exact"); + assertFieldReference("name\\\\@exact", "name\\\\", "exact"); + // An @ that starts a path segment cannot follow a field, so it is part of the name. + assertFieldReference("@timestamp", "@timestamp", null); + assertFieldReference("v.@timestamp", "v.@timestamp", null); + assertFieldReference("v.@timestamp@exact", "v.@timestamp", "exact"); + Assertions.assertThrows(SearchDslParser.SearchDslSyntaxException.class, + () -> SearchDslParser.splitAnalyzerSelector("title@")); + } + + @Test + public void testQuotedAtSignIsNotAnalyzerSelector() { + QsPlan quoted = SearchDslParser.parseDsl("\"name@literal\":john"); + assertFieldReference(quoted.getFieldBindings().get(0).getFieldName(), "name@literal", null); + + // An @ that is already escaped inside the quotes is not escaped twice. + QsPlan escaped = SearchDslParser.parseDsl("\"name\\@literal\":john"); + assertFieldReference(escaped.getFieldBindings().get(0).getFieldName(), "name@literal", null); + + QsPlan unquoted = SearchDslParser.parseDsl("name@literal:john"); + assertFieldReference(unquoted.getFieldBindings().get(0).getFieldName(), "name", "literal"); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java index a0016dfdf5da38..37018414799078 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java @@ -19,11 +19,14 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableTest; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MatchAny; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.generator.Explode; @@ -35,18 +38,22 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode; import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.StringType; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; @@ -295,6 +302,22 @@ public void testUnnest() { Assertions.assertTrue(ExpressionUtils.convertUnnest(unnest) instanceof ExplodeBitmapOuter); } + @Test + public void testIsNullPropagating() { + CascadesContext context = createCascadesContext("select 1"); + SlotReference column = new SlotReference("c", StringType.INSTANCE, true, Lists.newArrayList()); + StringLiteral term = new StringLiteral("x"); + Assertions.assertTrue(ExpressionUtils.isNullPropagating(column, context)); + Assertions.assertTrue(ExpressionUtils.isNullPropagating(new MatchAny(column, term), context)); + Assertions.assertTrue(ExpressionUtils.isNullPropagating( + new MatchAny(new Cast(new ElementAt(column, term), StringType.INSTANCE), term), context)); + // nvl turns a NULL input into a value, so the MATCH above it is not NULL for NULL input. + Assertions.assertFalse(ExpressionUtils.isNullPropagating( + new MatchAny(new Nvl(column, term), term), context)); + // Without an input slot the value does not depend on the NULL-extended row at all. + Assertions.assertFalse(ExpressionUtils.isNullPropagating(new MatchAny(term, term), context)); + } + private void assertExpect(List originalExpressions, List shuttledExpressions, String... expectExpressions) { diff --git a/regression-test/data/search/test_crm_search_analyzers.out b/regression-test/data/search/test_crm_search_analyzers.out new file mode 100644 index 00000000000000..3f3e7d67787a90 --- /dev/null +++ b/regression-test/data/search/test_crm_search_analyzers.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !document_10_exact -- +1 +2 + +-- !document_11_single -- +2 + +-- !document_11_mixed -- +1 + +-- !document_11_fields -- +4 + +-- !document_11_separate_search -- +1 +2 + +-- !document_11_selector_case -- +1 +2 + +-- !document_13_ordinary_in -- +1 + +-- !literal_column_at -- +1 + +-- !literal_variant_at -- +1 + diff --git a/regression-test/data/search/test_crm_search_join_document.out b/regression-test/data/search/test_crm_search_join_document.out new file mode 100644 index 00000000000000..fcec5f1bdf09fa --- /dev/null +++ b/regression-test/data/search/test_crm_search_join_document.out @@ -0,0 +1,253 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !document_01 -- +1 + +-- !document_02 -- +1 3 john jane keyur patel hr + +-- !document_03 -- +1 3 john jane keyur patel hr +2 4 john +3 5 john +4 6 \N +5 7 \N + +-- !document_04 -- +1 3 john jane keyur patel hr + +-- !document_05 -- +1 john patel +1 john patel +1 john patel +1 john patel +1 john patel +1 john patel +3 john + +-- !document_06 -- +101 +102 + +-- !document_07 -- +-1 nobody [1, 2] +0 nobody \N +1 john jane keyur patel hr [1, 2] +1 john jane keyur patel hr [1, 2] +1 john jane keyur patel hr [1, 2] +2 john [3] +2 john [3] +4 \N \N +5 \N \N + +-- !document_08 -- +1 john jane keyur patel hr +2 john + +-- !document_09 -- +-1 +0 +1 +1 +2 +2 +3 +4 +5 + +-- !document_10 -- +-1 +0 +1 +1 +2 +2 +3 +4 +5 + +-- !document_11 -- +-1 +0 +1 +2 +3 +4 +5 + +-- !document_12 -- +1 +2 +3 +4 +5 + +-- !document_13 -- +1 +2 + +-- !document_14_original -- +\N \N +\N 8 +1 1 +1 1 +4 4 +5 5 +6 6 + +-- !document_14_alternative -- +\N \N +\N 8 +1 1 +1 1 +2 \N +3 \N +4 4 +5 5 +6 6 +7 \N + +-- !search_join_or -- +-1 12 +0 12 +1 12 +1 12 +1 123 +1 456 +2 12 +2 12 +2 123 +3 \N +4 12 +5 12 + +-- !search_two_joins_or -- +-1 12 \N +0 12 \N +1 12 1 +1 12 1 +1 12 2 +1 12 2 +1 123 2 +1 456 1 +1 456 2 +2 12 \N +2 12 \N +4 12 \N +5 12 \N + +-- !search_join_not -- +-1 12 +0 12 +1 12 +1 12 +2 12 +2 12 +4 12 +5 12 + +-- !mow_match_join -- +1 false 1 +1 false 1 +2 true \N + +-- !mow_search_join -- +1 1 +1 1 +2 \N + +-- !search_exists_or -- +-1 +0 +1 +2 +3 +4 +5 + +-- !separate_search_two_tables -- +1 1 +2 2 +3 3 + +-- !search_self_join_left -- +1 1 +2 2 +3 3 +6 6 + +-- !search_self_join_right -- +1 1 +2 2 +3 3 +7 7 + +-- !search_renamed_column -- +\N \N +1 1 +1 1 +2 \N +4 4 +6 6 + +-- !search_renamed_variant -- +1 1 +1 1 +2 \N + +-- !match_case_scan -- +\N hit +1 hit +2 hit +3 miss +4 miss +5 miss +6 hit +7 miss + +-- !match_case_join -- +\N \N hit +1 1 hit +1 1 hit +2 \N hit +3 \N miss +4 4 miss +5 5 miss +6 6 hit +7 \N miss + +-- !match_case_null_side -- +\N null +1 hit +1 hit +4 miss +5 null +6 hit +8 null + +-- !search_not_inferred_for_equal_column -- +\N 10 +1 10 +2 20 +6 20 + +-- !search_not_inferred_through_intersect -- +hello +hello world + +-- !nullside_nonstrict_match_where -- +\N +1 +1 +5 +6 +8 + +-- !nullside_nonstrict_match_select -- +\N true +1 true +1 true +4 false +5 true +6 true +8 true + diff --git a/regression-test/data/search/test_crm_search_variant_topn.out b/regression-test/data/search/test_crm_search_variant_topn.out new file mode 100644 index 00000000000000..fa2d42fcd5de39 --- /dev/null +++ b/regression-test/data/search/test_crm_search_variant_topn.out @@ -0,0 +1,307 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !document_12_numeric_order -- +-2 \N \N +-1 \N \N +0 payload-0 0 +1 payload-1 2 +2 payload-2 4 +3 payload-3 6 +4 payload-4 8 +5 payload-5 10 +6 payload-6 12 +7 payload-7 14 +8 payload-8 16 +9 payload-9 18 +10 payload-10 20 +11 payload-11 22 +12 payload-12 24 +13 payload-13 26 +14 payload-14 28 +15 payload-15 30 +16 payload-16 32 +17 payload-17 34 +18 payload-18 36 +19 payload-19 38 +20 payload-20 40 +21 payload-21 42 +22 payload-22 44 +23 payload-23 46 +24 payload-24 48 +25 payload-25 50 +26 payload-26 52 +27 payload-27 54 +28 payload-28 56 +29 payload-29 58 +30 payload-30 60 +31 payload-31 62 +32 payload-32 64 +33 payload-33 66 +34 payload-34 68 +35 payload-35 70 +36 payload-36 72 +37 payload-37 74 +38 payload-38 76 +39 payload-39 78 +40 payload-40 80 +41 payload-41 82 +42 payload-42 84 +43 payload-43 86 +44 payload-44 88 +45 payload-45 90 +46 payload-46 92 +47 payload-47 94 +48 payload-48 96 +49 payload-49 98 +50 payload-50 100 +51 payload-51 102 +52 payload-52 104 +53 payload-53 106 +54 payload-54 108 +55 payload-55 110 +56 payload-56 112 +57 payload-57 114 +58 payload-58 116 +59 payload-59 118 +60 payload-60 120 +61 payload-61 122 +62 payload-62 124 +63 payload-63 126 +64 payload-64 128 +65 payload-65 130 +66 payload-66 132 +67 payload-67 134 +68 payload-68 136 +69 payload-69 138 +70 payload-70 140 +71 payload-71 142 +72 payload-72 144 +73 payload-73 146 +74 payload-74 148 +75 payload-75 150 +76 payload-76 152 +77 payload-77 154 +78 payload-78 156 +79 payload-79 158 +80 payload-80 160 +81 payload-81 162 +82 payload-82 164 +83 payload-83 166 +84 payload-84 168 +85 payload-85 170 +86 payload-86 172 +87 payload-87 174 +88 payload-88 176 +89 payload-89 178 +90 payload-90 180 +91 payload-91 182 +92 payload-92 184 +93 payload-93 186 +94 payload-94 188 +95 payload-95 190 +96 payload-96 192 +97 payload-97 194 + +-- !document_12_typed -- +241 -2 \N \N +242 -1 \N \N +0 0 payload-0 0 +1 1 payload-1 2 +2 2 payload-2 4 +3 3 payload-3 6 +4 4 payload-4 8 +5 5 payload-5 10 +6 6 payload-6 12 +7 7 payload-7 14 +8 8 payload-8 16 +9 9 payload-9 18 +10 10 payload-10 20 +11 11 payload-11 22 +12 12 payload-12 24 +13 13 payload-13 26 +14 14 payload-14 28 +15 15 payload-15 30 +16 16 payload-16 32 +17 17 payload-17 34 +18 18 payload-18 36 +19 19 payload-19 38 +20 20 payload-20 40 +21 21 payload-21 42 +22 22 payload-22 44 +23 23 payload-23 46 +24 24 payload-24 48 +25 25 payload-25 50 +26 26 payload-26 52 +27 27 payload-27 54 +28 28 payload-28 56 +29 29 payload-29 58 +30 30 payload-30 60 +31 31 payload-31 62 +32 32 payload-32 64 +33 33 payload-33 66 +34 34 payload-34 68 +35 35 payload-35 70 +36 36 payload-36 72 +37 37 payload-37 74 +38 38 payload-38 76 +39 39 payload-39 78 +40 40 payload-40 80 +41 41 payload-41 82 +42 42 payload-42 84 +43 43 payload-43 86 +44 44 payload-44 88 +45 45 payload-45 90 +46 46 payload-46 92 +47 47 payload-47 94 +48 48 payload-48 96 +49 49 payload-49 98 +50 50 payload-50 100 +51 51 payload-51 102 +52 52 payload-52 104 +53 53 payload-53 106 +54 54 payload-54 108 +55 55 payload-55 110 +56 56 payload-56 112 +57 57 payload-57 114 +58 58 payload-58 116 +59 59 payload-59 118 +60 60 payload-60 120 +61 61 payload-61 122 +62 62 payload-62 124 +63 63 payload-63 126 +64 64 payload-64 128 +65 65 payload-65 130 +66 66 payload-66 132 +67 67 payload-67 134 +68 68 payload-68 136 +69 69 payload-69 138 +70 70 payload-70 140 +71 71 payload-71 142 +72 72 payload-72 144 +73 73 payload-73 146 +74 74 payload-74 148 +75 75 payload-75 150 +76 76 payload-76 152 +77 77 payload-77 154 +78 78 payload-78 156 +79 79 payload-79 158 +80 80 payload-80 160 +81 81 payload-81 162 +82 82 payload-82 164 +83 83 payload-83 166 +84 84 payload-84 168 +85 85 payload-85 170 +86 86 payload-86 172 +87 87 payload-87 174 +88 88 payload-88 176 +89 89 payload-89 178 +90 90 payload-90 180 +91 91 payload-91 182 +92 92 payload-92 184 +93 93 payload-93 186 +94 94 payload-94 188 +95 95 payload-95 190 +96 96 payload-96 192 +97 97 payload-97 194 + +-- !document_12_typed_eager -- +241 -2 \N \N +242 -1 \N \N +0 0 payload-0 0 +1 1 payload-1 2 +2 2 payload-2 4 +3 3 payload-3 6 +4 4 payload-4 8 +5 5 payload-5 10 +6 6 payload-6 12 +7 7 payload-7 14 +8 8 payload-8 16 +9 9 payload-9 18 +10 10 payload-10 20 +11 11 payload-11 22 +12 12 payload-12 24 +13 13 payload-13 26 +14 14 payload-14 28 +15 15 payload-15 30 +16 16 payload-16 32 +17 17 payload-17 34 +18 18 payload-18 36 +19 19 payload-19 38 +20 20 payload-20 40 +21 21 payload-21 42 +22 22 payload-22 44 +23 23 payload-23 46 +24 24 payload-24 48 +25 25 payload-25 50 +26 26 payload-26 52 +27 27 payload-27 54 +28 28 payload-28 56 +29 29 payload-29 58 +30 30 payload-30 60 +31 31 payload-31 62 +32 32 payload-32 64 +33 33 payload-33 66 +34 34 payload-34 68 +35 35 payload-35 70 +36 36 payload-36 72 +37 37 payload-37 74 +38 38 payload-38 76 +39 39 payload-39 78 +40 40 payload-40 80 +41 41 payload-41 82 +42 42 payload-42 84 +43 43 payload-43 86 +44 44 payload-44 88 +45 45 payload-45 90 +46 46 payload-46 92 +47 47 payload-47 94 +48 48 payload-48 96 +49 49 payload-49 98 +50 50 payload-50 100 +51 51 payload-51 102 +52 52 payload-52 104 +53 53 payload-53 106 +54 54 payload-54 108 +55 55 payload-55 110 +56 56 payload-56 112 +57 57 payload-57 114 +58 58 payload-58 116 +59 59 payload-59 118 +60 60 payload-60 120 +61 61 payload-61 122 +62 62 payload-62 124 +63 63 payload-63 126 +64 64 payload-64 128 +65 65 payload-65 130 +66 66 payload-66 132 +67 67 payload-67 134 +68 68 payload-68 136 +69 69 payload-69 138 +70 70 payload-70 140 +71 71 payload-71 142 +72 72 payload-72 144 +73 73 payload-73 146 +74 74 payload-74 148 +75 75 payload-75 150 +76 76 payload-76 152 +77 77 payload-77 154 +78 78 payload-78 156 +79 79 payload-79 158 +80 80 payload-80 160 +81 81 payload-81 162 +82 82 payload-82 164 +83 83 payload-83 166 +84 84 payload-84 168 +85 85 payload-85 170 +86 86 payload-86 172 +87 87 payload-87 174 +88 88 payload-88 176 +89 89 payload-89 178 +90 90 payload-90 180 +91 91 payload-91 182 +92 92 payload-92 184 +93 93 payload-93 186 +94 94 payload-94 188 +95 95 payload-95 190 +96 96 payload-96 192 +97 97 payload-97 194 + diff --git a/regression-test/data/search/test_search_usage_restrictions.out b/regression-test/data/search/test_search_usage_restrictions.out index 6fc83963beb6de..98c913845ad12e 100644 --- a/regression-test/data/search/test_search_usage_restrictions.out +++ b/regression-test/data/search/test_search_usage_restrictions.out @@ -8,6 +8,9 @@ -- !valid_with_limit -- +-- !valid_join -- +1 + -- !valid_subquery -- -- !valid_multi_field -- diff --git a/regression-test/suites/search/test_crm_search_analyzers.groovy b/regression-test/suites/search/test_crm_search_analyzers.groovy new file mode 100644 index 00000000000000..aeb841b7b1f175 --- /dev/null +++ b/regression-test/suites/search/test_crm_search_analyzers.groovy @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_crm_search_analyzers") { + // The pipeline randomizes these defaults. A VARIANT subcolumn stored in the sparse or doc column has no + // inverted index, so SEARCH finds nothing in it and MATCH fails without enable_match_without_inverted_index. + sql "set default_variant_enable_doc_mode = false" + sql "set default_variant_enable_typed_paths_to_sparse = false" + sql "set default_variant_max_subcolumns_count = 0" + // Chapters X and XI: a normalized keyword index and a full-text index. + sql "DROP TABLE IF EXISTS crm_search_analyzers" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS crm_doc_text" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS crm_doc_exact" + sql "DROP INVERTED INDEX TOKEN_FILTER IF EXISTS crm_doc_nfkc" + sql """CREATE INVERTED INDEX TOKEN_FILTER crm_doc_nfkc + PROPERTIES("type"="icu_normalizer", "name"="nfkc")""" + sql """CREATE INVERTED INDEX ANALYZER crm_doc_text + PROPERTIES("tokenizer"="standard", "token_filter"="crm_doc_nfkc,lowercase")""" + sql """CREATE INVERTED INDEX ANALYZER crm_doc_exact + PROPERTIES("tokenizer"="keyword", "token_filter"="crm_doc_nfkc,lowercase")""" + // Analyzer policies reach BE asynchronously, as in the existing multi-analyzer suites. + sleep(10000) + sql """CREATE TABLE crm_search_analyzers ( + id BIGINT, name TEXT, title TEXT, + INDEX idx_name_text(name) USING INVERTED + PROPERTIES("analyzer"="crm_doc_text", "support_phrase"="true"), + INDEX idx_name_exact(name) USING INVERTED PROPERTIES("analyzer"="crm_doc_exact"), + INDEX idx_title_text(title) USING INVERTED + PROPERTIES("analyzer"="crm_doc_text", "support_phrase"="true") + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1", "inverted_index_storage_format"="V2")""" + sql """INSERT INTO crm_search_analyzers VALUES + (1,'John Smith','Senior Software Engineer'), + (2,'ＪＯＨＮ Smith','machine learning'), + (3,'John Smith Junior','software'), + (4,'Other','john smith'),(5,NULL,NULL)""" + order_qt_document_10_exact """ + SELECT id FROM crm_search_analyzers + WHERE name MATCH 'John Smith' USING ANALYZER crm_doc_exact + """ + order_qt_document_11_single """ + SELECT id FROM crm_search_analyzers + WHERE search('title@crm_doc_text:"machine learning"') + """ + order_qt_document_11_mixed """ + SELECT id FROM crm_search_analyzers + WHERE search('name@crm_doc_exact:"John Smith" AND title@crm_doc_text:software') + """ + order_qt_document_11_fields """ + SELECT id FROM crm_search_analyzers + WHERE search('john smith', '{"fields":["name@crm_doc_exact","title@crm_doc_text"]}') + """ + // Chapter XI's same-field dual-analyzer example is outside the per-field P2 scope. + test { + sql """SELECT id FROM crm_search_analyzers + WHERE search('name@crm_doc_text:John AND name@crm_doc_exact:"John Smith"')""" + exception "one analyzer per field" + } + order_qt_document_11_separate_search """ + SELECT id FROM crm_search_analyzers + WHERE search('name@crm_doc_text:John') + AND search('name@crm_doc_exact:"John Smith"') + """ + test { + sql """SELECT id FROM crm_search_analyzers WHERE search('name@does_not_exist:John')""" + exception "No inverted index found for SEARCH analyzer" + } + // The selector names an analyzer the way the index lookup does, ignoring case. + order_qt_document_11_selector_case """ + SELECT id FROM crm_search_analyzers + WHERE search('name@CRM_DOC_EXACT:"John Smith" AND name@crm_doc_exact:"john smith"') + """ + // Chapter XIII is outside P0-P2. Ordinary IN must keep SQL equality semantics. + order_qt_document_13_ordinary_in """ + SELECT id FROM crm_search_analyzers WHERE name IN ('John Smith','Mason Jackson') + """ + test { + sql """SELECT id FROM crm_search_analyzers + WHERE name@crm_doc_exact IN ('John Smith','Mason Jackson')""" + exception "mismatched input '@'" + } + test { + sql """SELECT id FROM crm_search_analyzers + WHERE name USING analyzer 'crm_doc_exact' IN ('John Smith','Mason Jackson')""" + exception "mismatched input 'USING'" + } + // A quoted @ remains part of the physical field name. + sql "DROP TABLE IF EXISTS crm_search_literal_fields" + sql """CREATE TABLE crm_search_literal_fields (id INT, `name@literal` TEXT, v VARIANT, + INDEX idx_name(`name@literal`) USING INVERTED PROPERTIES("parser"="english"), + INDEX idx_v(v) USING INVERTED PROPERTIES("parser"="english")) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_literal_fields VALUES + (1,'john',parse_to_variant('{"email@work":"john"}')),(2,'jane',parse_to_variant('{"email@work":"jane"}'))""" + order_qt_literal_column_at """ + SELECT id FROM crm_search_literal_fields WHERE search('"name@literal":john') + """ + order_qt_literal_variant_at """ + SELECT id FROM crm_search_literal_fields WHERE search('"v.email@work":john') + """ + // An unquoted @ always selects an analyzer, whatever columns the table has. + test { + sql """SELECT id FROM crm_search_literal_fields WHERE search('name@literal:john')""" + exception "Field 'name' not found" + } + +} diff --git a/regression-test/suites/search/test_crm_search_join_document.groovy b/regression-test/suites/search/test_crm_search_join_document.groovy new file mode 100644 index 00000000000000..c3bbc8c8275ac2 --- /dev/null +++ b/regression-test/suites/search/test_crm_search_join_document.groovy @@ -0,0 +1,1039 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Source: query string and search+join examples, revision 23. +suite("test_crm_search_join_document") { + sql "set enable_match_without_inverted_index = false" + // The pipeline randomizes these defaults. A VARIANT subcolumn stored in the sparse or doc column has no + // inverted index, so SEARCH finds nothing in it and MATCH fails without enable_match_without_inverted_index. + sql "set default_variant_enable_doc_mode = false" + sql "set default_variant_enable_typed_paths_to_sparse = false" + sql "set default_variant_max_subcolumns_count = 0" + + sql "DROP TABLE IF EXISTS crm_search_objects" + sql """CREATE TABLE crm_search_objects ( + OBJECTID BIGINT, PORTALID BIGINT, OBJECTTYPEID STRING, OBJECTIDHASH INT, + DELETED BOOLEAN, INGESTIONTIMESTAMP BIGINT, PROCESSEDTIMESTAMP BIGINT, + VERSION BIGINT, OVERFLOWPROPERTIES VARIANT, + INDEX idx_properties(OVERFLOWPROPERTIES) USING INVERTED PROPERTIES("parser"="english") + ) DUPLICATE KEY(OBJECTID) DISTRIBUTED BY HASH(OBJECTID) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_objects VALUES + (-1,865815822,'0-1',0,false,1,1,1,parse_to_variant('{"string_0":"nobody","string_8":"nobody","string_9":"nobody","string_17":"nobody","number_array_1":[1,2]}')), + (0,865815822,'0-1',0,false,2,2,1,parse_to_variant('{"string_0":"nobody","string_8":"nobody","string_9":"nobody","string_17":"nobody"}')), + (1,865815822,'0-1',0,false,3,3,1,parse_to_variant('{"string_0":"john","string_8":"john jane keyur patel hr","string_9":"jane","string_17":"john patel","number_array_1":[1,2]}')), + (2,865815822,'0-1',0,false,4,4,1,parse_to_variant('{"string_0":"nobody","string_8":"john","string_9":"jane","string_17":"patel","number_array_1":[3]}')), + (3,865815822,'0-1',0,false,5,5,1,parse_to_variant('{"string_0":"john","string_8":"john","string_17":"john"}')), + (4,865815822,'0-1',0,false,6,6,1,parse_to_variant('{}')), + (5,865815822,'0-1',0,false,7,7,1,NULL), + (6,865815822,'0-1',0,true,8,8,1,parse_to_variant('{"string_8":"john"}')), + (7,7,'0-1',0,false,9,9,1,parse_to_variant('{"string_8":"john"}')), + (101,865815822,'0-3',0,false,10,10,1,parse_to_variant('{"string_3":"naive"}')), + (102,865815822,'0-3',0,false,11,11,1,parse_to_variant('{"string_3":"other"}')), + (103,865815822,'0-3',0,false,12,12,1,parse_to_variant('{"string_3":"naive"}')), + (201,865815822,'0-2',0,false,13,13,1,parse_to_variant('{}'))""" + sql "DROP TABLE IF EXISTS crm_search_lists" + sql """CREATE TABLE crm_search_lists ( + OBJECTID BIGINT, PORTALID BIGINT, OBJECTTYPEID STRING, + OBJECTIDHASH INT, LISTID BIGINT, DELETED BOOLEAN + ) DUPLICATE KEY(OBJECTID) DISTRIBUTED BY HASH(OBJECTID) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_lists VALUES + (-1,865815822,'0-1',0,12,false),(0,865815822,'0-1',0,12,false), + (1,865815822,'0-1',0,456,false),(1,865815822,'0-1',0,123,false), + (1,865815822,'0-1',0,12,false),(2,865815822,'0-1',0,123,false), + (2,865815822,'0-1',0,12,false),(4,865815822,'0-1',0,12,false), + (5,865815822,'0-1',0,12,false),(99,865815822,'0-1',0,12,false), + (1,1,'0-1',0,12,false),(2,1,'0-1',1,12,false), + (NULL,1,'0-1',0,12,false)""" + sql "DROP TABLE IF EXISTS crm_search_associations" + sql """CREATE TABLE crm_search_associations ( + FROMOBJECTID BIGINT, TOOBJECTID BIGINT, PORTALID BIGINT, + FROMOBJECTTYPEID STRING, FROMOBJECTIDHASH INT, + COMBINEDASSOCIATIONTYPEID STRING, DELETED BOOLEAN + ) DUPLICATE KEY(FROMOBJECTID) DISTRIBUTED BY HASH(FROMOBJECTID) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_associations VALUES + (201,101,865815822,'0-2',0,'0-342',false), + (1,102,865815822,'0-1',0,'0-4',false), + (1,1,865815822,'0-1',0,'0-123',false), + (2,1,865815822,'0-1',0,'0-123',false), + (1,10,1,'0-1',0,'0-1',false), + (2,11,1,'0-1',2,'0-1',false), + (NULL,12,1,'0-1',0,'0-1',false)""" + + // Document example 0: 一、多列IN语法报错SQL（初始问题：多字段元组IN子查询不支持） / 报错原始SQL + test { + sql """WITH lists AS ( + SELECT + portalId, + objectTypeId, + objectId, + objectIdHash + FROM + crm_search_lists + where + portalId = 1 + and objectTypeId = '0-1' + and objectIdHash IN (0, 1, 2) +), +associations AS ( + SELECT + portalId, + fromObjectTypeId, + fromObjectIdHash, + fromObjectId, + toObjectId + FROM + crm_search_associations + where + portalId = 1 + and combinedAssociationTypeId = '0-1' + and fromObjectIdHash IN (0, 1, 2) +) +SELECT + objectId +FROM + lists +WHERE + (portalId, objectTypeId, objectIdHash, objectId) IN ( + SELECT + portalId, + fromObjectTypeId, + fromObjectIdHash, + fromObjectId + FROM + associations + )""" + exception "mismatched input ','" + } + + // Document example 1: 一、多列IN语法报错SQL（初始问题：多字段元组IN子查询不支持） / 官方替代方案：EXISTS改写（无语法报错） + order_qt_document_01 """ +WITH lists AS ( + SELECT + portalId, + objectTypeId, + objectId, + objectIdHash + FROM + crm_search_lists + WHERE + portalId = 1 + AND objectTypeId = '0-1' + AND objectIdHash IN (0, 1, 2) +), +associations AS ( + SELECT + portalId, + fromObjectTypeId, + fromObjectIdHash, + fromObjectId, + toObjectId + FROM + crm_search_associations + WHERE + portalId = 1 + AND combinedAssociationTypeId = '0-1' + AND fromObjectIdHash IN (0, 1, 2) +) +SELECT + l.objectId +FROM + lists l +WHERE + EXISTS ( + SELECT 1 + FROM associations a + WHERE + a.portalId = l.portalId + AND a.fromObjectTypeId = l.objectTypeId + AND a.fromObjectIdHash = l.objectIdHash + AND a.fromObjectId = l.objectId + ) + """ + + // Document example 2: 二、OR+EXISTS触发Mark Join报错（SlotReference异常） / 2.1 正常可执行SQL（无OR，无报错） + order_qt_document_02 """ +WITH contacts AS ( + select + OBJECTID, + INGESTIONTIMESTAMP, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as firstname + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +lists AS ( + select + PORTALID as PORTALID, + LISTID as LISTID, + OBJECTID as OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) +), +results AS ( + select + contacts.OBJECTID, + contacts.INGESTIONTIMESTAMP, + contacts.firstname + from + contacts + where + ( + (firstname MATCH_ANY 'keyur patel') + and EXISTS ( + SELECT + 1 + FROM + lists l + WHERE + l.OBJECTID = contacts.OBJECTID + ) + ) +) +SELECT + * +FROM + results +LIMIT + 10 + """ + + // Document example 3: 二、OR+EXISTS触发Mark Join报错（SlotReference异常） / 2.2 报错SQL（新增OR触发Mark Join，报SlotReference缺失列） + order_qt_document_03 """ +WITH contacts AS ( + select + OBJECTID, + INGESTIONTIMESTAMP, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as firstname + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +lists AS ( + select + PORTALID as PORTALID, + LISTID as LISTID, + OBJECTID as OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) +), +results AS ( + select + contacts.OBJECTID, + contacts.INGESTIONTIMESTAMP, + contacts.firstname + from + contacts + where + ( + (firstname MATCH_ANY 'keyur patel') + and EXISTS ( + SELECT + 1 + FROM + lists l + WHERE + l.OBJECTID = contacts.OBJECTID + ) + ) + OR contacts.OBJECTID > 1 +) +SELECT + * +FROM + results +LIMIT + 10 + """ + + // Document example 4: 二、OR+EXISTS触发Mark Join报错（SlotReference异常） / 2.3 临时规避写法（MATCH过滤下推至内层CTE，避免外层OR关联） + order_qt_document_04 """ +WITH contacts AS ( + SELECT + OBJECTID, + INGESTIONTIMESTAMP, + CAST(OVERFLOWPROPERTIES['string_8'] AS VARCHAR) AS firstname + FROM + crm_search_objects + WHERE + PORTALID = 865815822 + AND OBJECTTYPEID = '0-1' + AND OBJECTIDHASH IN (0) + AND DELETED = FALSE + -- MATCH过滤下推至内层，规避外层JOIN OR逻辑 + AND CAST(OVERFLOWPROPERTIES['string_8'] AS VARCHAR) MATCH_ANY 'keyur patel' +), +lists AS ( + SELECT + PORTALID, + LISTID, + OBJECTID + FROM + crm_search_lists + WHERE + PORTALID = 865815822 + AND OBJECTTYPEID = '0-1' + AND DELETED = FALSE +), +results AS ( + SELECT + c.OBJECTID, + c.INGESTIONTIMESTAMP, + c.firstname + FROM + contacts c + WHERE + EXISTS ( + SELECT 1 + FROM lists l + WHERE l.OBJECTID = c.OBJECTID + ) + OR c.OBJECTID > 1 +) +SELECT * +FROM results +LIMIT 10 + """ + + // Document example 5: 三、SEARCH函数JOIN限制报错SQL（search不能出现在多表关联后过滤） / 报错SQL（含多表LEFT JOIN + SEARCH函数，FE直接拦截） + order_qt_document_05 """ +select + objects_base.OBJECTID, + cast( + objects_base.OVERFLOWPROPERTIES ['string_17'] as VARCHAR + ) +from + ( + select + PORTALID, + OBJECTTYPEID, + OBJECTID, + DELETED, + INGESTIONTIMESTAMP, + PROCESSEDTIMESTAMP, + VERSION, + OVERFLOWPROPERTIES + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) + ) as objects_base + left outer join ( + select + * + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) + ) as lists_base on objects_base.OBJECTID = lists_base.OBJECTID + left outer join ( + select + * + from + crm_search_associations + where + ( + PORTALID = 865815822 + and DELETED = false + and COMBINEDASSOCIATIONTYPEID in ('0-123') + ) + ) as associations_base on objects_base.OBJECTID = associations_base.TOOBJECTID +where + search( + "john", + '{"default_field":"OVERFLOWPROPERTIES.string_17","mode":"lucene"}' + ) + """ + + // Document example 6: 四、PR#60839性能退化测试样例（多层CTE+多OR+MATCH） / + order_qt_document_06 """ +with objects_0_3 as ( + select + OBJECTID as _OBJECTID, + INGESTIONTIMESTAMP as _INGESTIONTIMESTAMP, + cast(OVERFLOWPROPERTIES ['string_3'] as VARCHAR) as dealname + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-3' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +objects_xo_0_0_2 as ( + select + OBJECTID as _OBJECTID, + INGESTIONTIMESTAMP as _INGESTIONTIMESTAMP + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-2' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +objects_xo_1_0_1 as ( + select + OBJECTID as _OBJECTID, + INGESTIONTIMESTAMP as _INGESTIONTIMESTAMP + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +xo_assoc_0_0_342 as ( + select + FROMOBJECTID as FROMOBJECTID, + TOOBJECTID as TOOBJECTID + from + crm_search_associations + where + ( + PORTALID = 865815822 + and DELETED = false + and COMBINEDASSOCIATIONTYPEID in ('0-342') + ) +), +xo_assoc_1_0_4 as ( + select + FROMOBJECTID as FROMOBJECTID, + TOOBJECTID as TOOBJECTID + from + crm_search_associations + where + ( + PORTALID = 865815822 + and DELETED = false + and COMBINEDASSOCIATIONTYPEID in ('0-4') + ) +), +secondary_0_0_2 as ( + select + objects_xo_0_0_2._OBJECTID as _OBJECTID + from + objects_xo_0_0_2 +), +xo_result_0 as ( + select + distinct a.TOOBJECTID as _OBJECTID + from + xo_assoc_0_0_342 as a + join secondary_0_0_2 as s on a.FROMOBJECTID = s._OBJECTID +), +secondary_1_0_1 as ( + select + objects_xo_1_0_1._OBJECTID as _OBJECTID + from + objects_xo_1_0_1 +), +xo_result_1 as ( + select + distinct a.TOOBJECTID as _OBJECTID + from + xo_assoc_1_0_4 as a + join secondary_1_0_1 as s on a.FROMOBJECTID = s._OBJECTID +), +results as ( + select + objects_0_3._OBJECTID as _OBJECTID + from + objects_0_3 + left join xo_result_0 xo0 on objects_0_3._OBJECTID = xo0._OBJECTID + left join xo_result_1 xo1 on objects_0_3._OBJECTID = xo1._OBJECTID + where + ( + (objects_0_3.dealname MATCH_ALL 'naive') + and xo0._OBJECTID is not null + ) + or (xo1._OBJECTID is not null) +) +select + * +from + results + """ + + // Document example 7: 五、Variant列JOIN+MATCH混合查询（验证子列裁剪） / + order_qt_document_07 """ +select + `contacts`.OBJECTID, + cast( + `contacts`.OVERFLOWPROPERTIES ['string_8'] as VARCHAR + ), + cast( + `contacts`.OVERFLOWPROPERTIES ['number_array_1'] as ARRAY + ) +from + ( + select + OBJECTID, + OBJECTIDHASH, + OVERFLOWPROPERTIES + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) + ) as `contacts` + left outer join ( + select + OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) + ) as `contact_lists` on `contacts`.OBJECTID = `contact_lists`.OBJECTID +where + ( + ( + cast( + `contacts`.OVERFLOWPROPERTIES ['string_8'] as VARCHAR + ) match_all 'john' + ) + and contact_lists.OBJECTID > 0 + ) + OR contact_lists.OBJECTID is not null + """ + + // Document example 8: 六、OR多条件JOIN执行计划样例SQL（可下推部分过滤） / + // The source references agg_lists without defining it; supply distinct member IDs. + order_qt_document_08 """ +WITH agg_lists AS (SELECT DISTINCT OBJECTID FROM crm_search_lists) +select + obj.OBJECTID as OBJECTID, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as objects_0_1__firstname + from + crm_search_objects as obj + left outer join agg_lists on obj.OBJECTID = agg_lists.OBJECTID + where + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + AND ( + ( + ( + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'john' + ) + AND agg_lists.OBJECTID = 2 + ) + OR ( + agg_lists.OBJECTID = 1 + AND cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'jane' + ) + ) + """ + + // Document example 9: 七、PR#61092两种写法对比（是否预提取MATCH为虚拟列） / 7.1 原始写法（MATCH写在WHERE，跨JOIN OR无法下推索引） + order_qt_document_09 """ +WITH objects AS ( +SELECT + objectId, + CAST(overflowProperties['string_0'] AS VARCHAR) firstName +FROM crm_search_objects +), +lists AS ( + SELECT objectId FROM crm_search_lists WHERE listId = 12 +) +SELECT + o.objectId +FROM objects o LEFT JOIN lists l ON o.objectId = l.objectId +WHERE firstName MATCH_ANY 'john' OR l.objectId IS NOT NULL + """ + + // Document example 10: 七、PR#61092两种写法对比（是否预提取MATCH为虚拟列） / 7.2 优化写法（内层CTE预计算MATCH布尔虚拟列，触发索引fast path） + // Move the source misplaced alias t1 onto the inner derived table. + order_qt_document_10 """ +WITH objects AS ( + SELECT objectId, firstName, firstName MATCH_ANY 'john' AS firstNameFilter1 + FROM ( + SELECT + objectId, + CAST(overflowProperties['string_0'] AS VARCHAR) firstName + FROM crm_search_objects + ) t1 +), +lists AS ( + SELECT objectId FROM crm_search_lists WHERE listId = 12 +) +SELECT + o.objectId +FROM objects o LEFT JOIN lists l ON o.objectId = l.objectId +WHERE firstNameFilter1 OR l.objectId IS NOT NULL + """ + + // Document example 11: 八、多层CTE复杂OR场景（两种版本，区分能否下推过滤） / 8.1 无法下推过滤版本 + order_qt_document_11 """ +with lists as ( + select + PORTALID as PORTALID, + LISTID as LISTID, + OBJECTID as OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) +), +agg_lists as ( + select + OBJECTID, + array_agg(LISTID) as listIds + from + lists + group by + OBJECTID +), +fg_1 as ( + select + obj.OBJECTID as OBJECTID, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as objects_0_1__firstname + from + crm_search_objects as obj + where + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + AND ( + ( + ( + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'john' + ) + OR ( + cast(OVERFLOWPROPERTIES ['string_9'] as VARCHAR) MATCH_ALL 'jane' + ) + OR EXISTS ( + SELECT + 1 + FROM + agg_lists l + WHERE + l.OBJECTID = obj.OBJECTID + ) + ) + ) +) +select + OBJECTID +from + fg_1 + """ + + // Document example 12: 八、多层CTE复杂OR场景（两种版本，区分能否下推过滤） / 8.2 可下推过滤版本（新增同表OBJECTID>0约束拆分OR） + order_qt_document_12 """ +with lists as ( + select + PORTALID as PORTALID, + LISTID as LISTID, + OBJECTID as OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) +), +agg_lists as ( + select + OBJECTID, + array_agg(LISTID) as listIds + from + lists + group by + OBJECTID +), +fg_1 as ( + select + obj.OBJECTID as OBJECTID, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as objects_0_1__firstname + from + crm_search_objects as obj + where + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + AND ( + ( + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'john' + ) + OR ( + cast(OVERFLOWPROPERTIES ['string_9'] as VARCHAR) MATCH_ALL 'jane' + ) + OR ( + OBJECTID > 0 + and EXISTS ( + SELECT + 1 + FROM + agg_lists l + WHERE + l.OBJECTID = obj.OBJECTID + ) + ) + ) +) +select + OBJECTID +from + fg_1 + """ + + // Document example 13: 九、多条件AND+OR嵌套JOIN查询（带array_intersect数组过滤） / + // The source final SELECT uses a missing OBJECTID alias; select the declared alias. + order_qt_document_13 """ +with `agg_lists` as ( + select + OBJECTID, + OBJECTTYPEID, + OBJECTIDHASH, + array_agg(LISTID) as `listIds` + from + ( + select + PORTALID, + LISTID, + OBJECTID, + OBJECTTYPEID, + OBJECTIDHASH + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + and LISTID IN (456,123) + and OBJECTIDHASH IN (0) + ) + ) as `lists` + group by + OBJECTID, + OBJECTTYPEID, + OBJECTIDHASH +), +`results` as ( + select + `objects_0_1`.`OBJECTID` as `objects_0_1___OBJECTID`, + `objects_0_1`.`INGESTIONTIMESTAMP` as `objects_0_1___INGESTIONTIMESTAMP`, + `objects_0_1`.`OBJECTTYPEID` as `objects_0_1___OBJECTTYPEID`, + `objects_0_1`.`OBJECTIDHASH` as `objects_0_1___OBJECTIDHASH`, + cast(`objects_0_1`.OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as `objects_0_1__firstname`, + cast( + `objects_0_1`.OVERFLOWPROPERTIES ['string_17'] as VARCHAR + ) as `objects_0_1__lastname`, + `agg_lists`.`OBJECTID` as `agg_lists__OBJECTID`, + `agg_lists`.`OBJECTTYPEID` as `agg_lists__OBJECTTYPEID`, + `agg_lists`.`OBJECTIDHASH` as `agg_lists__OBJECTIDHASH`, + `agg_lists`.`listIds` as `agg_lists__listIds` + from + ( + select + OBJECTID, + INGESTIONTIMESTAMP, + OBJECTTYPEID, + OBJECTIDHASH, + OVERFLOWPROPERTIES + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) + ) as `objects_0_1` + left outer join `agg_lists` on ( + ( + `objects_0_1`.`OBJECTID` = `agg_lists`.`OBJECTID` + ) + and ( + `objects_0_1`.`OBJECTTYPEID` = `agg_lists`.`OBJECTTYPEID` + ) + and ( + `objects_0_1`.`OBJECTIDHASH` = `agg_lists`.`OBJECTIDHASH` + ) + ) + where + ( + ( + ( + cast(`objects_0_1`.OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'hr' + ) + and ( + array_size( + array_intersect(`agg_lists`.`listIds`, array(456)) + ) = 1 + ) + ) + or ( + ( + array_size( + array_intersect(`agg_lists`.`listIds`, array(123)) + ) = 1 + ) + and ( + cast( + `objects_0_1`.OVERFLOWPROPERTIES ['string_17'] as VARCHAR + ) MATCH_ALL 'patel' + ) + ) + ) +) +select objects_0_1___OBJECTID from `results` + """ + + // Chapter XIV: these two predicates are intentionally different. + sql "DROP TABLE IF EXISTS crm_search_full_a" + sql """CREATE TABLE crm_search_full_a (k1 INT, content TEXT, + INDEX idx_content(content) USING INVERTED PROPERTIES("parser"="english")) + DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "DROP TABLE IF EXISTS crm_search_full_b" + sql """CREATE TABLE crm_search_full_b (k1 INT) + DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_full_a VALUES + (1,'hello world'),(2,'hello'),(3,'world'),(4,'neither'), + (5,NULL),(6,'hello'),(7,'world'),(NULL,'hello world')""" + sql "INSERT INTO crm_search_full_b VALUES (1),(1),(4),(5),(6),(8),(NULL)" + order_qt_document_14_original """ + SELECT a.k1, b.k1 FROM crm_search_full_a a FULL OUTER JOIN crm_search_full_b b + ON a.k1=b.k1 WHERE b.k1>2 + OR (a.content MATCH_ALL 'hello' AND a.content MATCH_ALL 'world') + OR (a.content MATCH_ALL 'hello' AND b.k1>5) + """ + order_qt_document_14_alternative """ + SELECT a.k1, b.k1 FROM crm_search_full_a a FULL OUTER JOIN crm_search_full_b b + ON a.k1=b.k1 WHERE (b.k1>2 AND b.k1 IS NOT NULL) + OR a.content MATCH_ALL 'hello' OR a.content MATCH_ALL 'world' + """ + + // SEARCH must not remove rows selected solely by the association branch. + order_qt_search_join_or """ + SELECT o.OBJECTID, l.LISTID FROM crm_search_objects o + LEFT JOIN crm_search_lists l ON o.OBJECTID=l.OBJECTID + WHERE o.PORTALID=865815822 AND o.OBJECTTYPEID='0-1' AND NOT o.DELETED + AND (search('OVERFLOWPROPERTIES.string_8:john') OR l.LISTID=12) + """ + order_qt_search_two_joins_or """ + SELECT o.OBJECTID, l.LISTID, a.FROMOBJECTID FROM crm_search_objects o + LEFT JOIN crm_search_lists l ON o.OBJECTID=l.OBJECTID + LEFT JOIN crm_search_associations a ON o.OBJECTID=a.TOOBJECTID + WHERE o.PORTALID=865815822 AND o.OBJECTTYPEID='0-1' AND NOT o.DELETED + AND ((search('OVERFLOWPROPERTIES.string_8:john') AND l.LISTID=456) + OR a.FROMOBJECTID=2 OR l.LISTID=12) + """ + order_qt_search_join_not """ + SELECT o.OBJECTID, l.LISTID FROM crm_search_objects o + LEFT JOIN crm_search_lists l ON o.OBJECTID=l.OBJECTID + WHERE o.PORTALID=865815822 AND o.OBJECTTYPEID='0-1' AND NOT o.DELETED + AND (NOT search('OVERFLOWPROPERTIES.string_8:john') OR l.LISTID=12) + """ + + // The virtual column rule also supports MOW; old indexed values must not leak. + sql "DROP TABLE IF EXISTS crm_search_mow" + sql """CREATE TABLE crm_search_mow (id BIGINT, v VARIANT, + INDEX idx_v(v) USING INVERTED PROPERTIES("parser"="english")) + UNIQUE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1", "enable_unique_key_merge_on_write"="true")""" + sql """INSERT INTO crm_search_mow VALUES + (1,parse_to_variant('{"name":"john"}')),(2,parse_to_variant('{"name":"other"}')),(3,NULL)""" + sql """INSERT INTO crm_search_mow VALUES + (1,parse_to_variant('{"name":"other"}')),(2,parse_to_variant('{"name":"john"}'))""" + order_qt_mow_match_join """ + SELECT o.id, CAST(o.v['name'] AS STRING) MATCH_ANY 'john', l.k1 + FROM crm_search_mow o LEFT JOIN crm_search_full_b l ON o.id=l.k1 + WHERE CAST(o.v['name'] AS STRING) MATCH_ANY 'john' OR l.k1=1 + """ + order_qt_mow_search_join """ + SELECT o.id, l.k1 + FROM crm_search_mow o LEFT JOIN crm_search_full_b l ON o.id=l.k1 + WHERE search('v.name:john') OR l.k1=1 + """ + order_qt_search_exists_or """ + SELECT o.OBJECTID FROM crm_search_objects o + WHERE o.PORTALID=865815822 AND o.OBJECTTYPEID='0-1' AND NOT o.DELETED + AND (search('OVERFLOWPROPERTIES.string_8:john') OR EXISTS ( + SELECT 1 FROM crm_search_lists l WHERE l.OBJECTID=o.OBJECTID AND l.LISTID=12)) + """ + order_qt_separate_search_two_tables """ + SELECT o.OBJECTID, m.id FROM crm_search_objects o + JOIN crm_search_mow m ON o.OBJECTID=m.id + WHERE search('OVERFLOWPROPERTIES.string_8:john') OR search('v.name:john') + """ + + // A SEARCH field resolves like a SQL column reference: a table alias picks one side of a self join, + // and an output alias still searches the physical column and its index. + order_qt_search_self_join_left """ + SELECT a.k1, b.k1 FROM crm_search_full_a a JOIN crm_search_full_a b ON a.k1=b.k1 + WHERE search('a.content:hello') OR b.k1=3 + """ + order_qt_search_self_join_right """ + SELECT a.k1, b.k1 FROM crm_search_full_a a JOIN crm_search_full_a b ON a.k1=b.k1 + WHERE search('b.content:world') OR a.k1=2 + """ + test { + sql """SELECT a.k1 FROM crm_search_full_a a JOIN crm_search_full_a b ON a.k1=b.k1 + WHERE search('content:hello')""" + exception "Ambiguous field 'content'" + } + // b has k1=1 twice: the join's duplicate rows must survive the virtual column. + order_qt_search_renamed_column """ + SELECT t.k1, b.k1 FROM (SELECT k1, content AS body FROM crm_search_full_a) t + LEFT JOIN crm_search_full_b b ON t.k1=b.k1 + WHERE search('body:hello') OR b.k1=4 + """ + order_qt_search_renamed_variant """ + SELECT o.id, l.k1 FROM (SELECT id, v AS props FROM crm_search_mow) o + LEFT JOIN crm_search_full_b l ON o.id=l.k1 + WHERE search('props.name:john') OR l.k1=1 + """ + // A MATCH wrapped in another expression is materialized the same way above a scan and above a join; + // on the null-generating side it stays NULL for rows without a join partner. + order_qt_match_case_scan """ + SELECT k1, CASE WHEN content MATCH_ANY 'hello' THEN 'hit' ELSE 'miss' END + FROM crm_search_full_a + """ + order_qt_match_case_join """ + SELECT a.k1, b.k1, CASE WHEN a.content MATCH_ANY 'hello' THEN 'hit' ELSE 'miss' END + FROM crm_search_full_a a LEFT JOIN crm_search_full_b b ON a.k1=b.k1 + """ + order_qt_match_case_null_side """ + SELECT b.k1, CASE WHEN a.content MATCH_ANY 'hello' THEN 'hit' + WHEN (a.content MATCH_ANY 'hello') IS NULL THEN 'null' ELSE 'miss' END + FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON b.k1=a.k1 + """ + // a.content = n.name lets predicates on a.content be inferred for n.name, but a SEARCH is bound to the + // inverted index of a.content: copied to n.name, which has no index, it would drop every joined row. + sql "DROP TABLE IF EXISTS crm_search_names" + sql """CREATE TABLE crm_search_names (id INT, name TEXT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO crm_search_names VALUES (10,'hello world'),(20,'hello'),(30,'world')" + order_qt_search_not_inferred_for_equal_column """ + SELECT a.k1, n.id FROM crm_search_full_a a JOIN crm_search_names n ON a.content=n.name + WHERE search('a.content:hello') OR a.content='zzz' + """ + // The same holds for the positional inference of INTERSECT: the SEARCH stays in its own branch. + order_qt_search_not_inferred_through_intersect """ + (SELECT content FROM crm_search_full_a WHERE search('content:hello')) + INTERSECT (SELECT name FROM crm_search_names) + """ + // A SEARCH that cannot reach one scan is rejected instead of being evaluated without an index. + test { + sql """SELECT k1 FROM (SELECT k1, content FROM crm_search_full_a ORDER BY k1 LIMIT 3) t + WHERE search('content:hello')""" + exception "SEARCH must be evaluated by an OLAP scan" + } + test { + sql """SELECT a.k1 FROM crm_search_full_a a JOIN crm_search_mow m ON a.k1=m.id + WHERE search('content:hello OR v.name:john')""" + exception "SEARCH must be evaluated by an OLAP scan" + } + + // Keep SEARCH on the null-generating side gated until its full DSL NULL contract is established. + test { + sql """SELECT b.k1 FROM crm_search_full_b b LEFT JOIN crm_search_mow m ON b.k1=m.id + WHERE search('NOT v.name:john') OR b.k1=8""" + exception "SEARCH must be evaluated by an OLAP scan" + } + // Rewrites that NULL-pad that side (ON false, outer join to anti join) must not bypass the gate. + test { + sql """SELECT b.k1 FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON false + WHERE NOT search('content:hello') OR b.k1=8""" + exception "null-generating side of an outer join" + } + test { + sql """SELECT b.k1 FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON b.k1=a.k1 + WHERE a.k1 IS NULL AND (NOT search('content:hello') OR b.k1=8)""" + exception "null-generating side of an outer join" + } + + // nvl(NULL, 'hello') matches: a MATCH with such an operand must stay above the outer join, + // so rows without a join partner (b.k1 = 8 and NULL) are kept and project TRUE. + sql "set enable_match_without_inverted_index = true" + order_qt_nullside_nonstrict_match_where """ + SELECT b.k1 FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON b.k1=a.k1 + WHERE nvl(a.content, 'hello') MATCH_ANY 'hello' OR b.k1=100 + """ + order_qt_nullside_nonstrict_match_select """ + SELECT b.k1, nvl(a.content, 'hello') MATCH_ANY 'hello' + FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON b.k1=a.k1 + """ + sql "set enable_match_without_inverted_index = false" + +} diff --git a/regression-test/suites/search/test_crm_search_variant_topn.groovy b/regression-test/suites/search/test_crm_search_variant_topn.groovy new file mode 100644 index 00000000000000..89ba1079ad4515 --- /dev/null +++ b/regression-test/suites/search/test_crm_search_variant_topn.groovy @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_crm_search_variant_topn") { + // The pipeline randomizes these defaults. A VARIANT subcolumn stored in the sparse or doc column has no + // inverted index, so SEARCH finds nothing in it and MATCH fails without enable_match_without_inverted_index. + sql "set default_variant_enable_doc_mode = false" + sql "set default_variant_enable_typed_paths_to_sparse = false" + sql "set default_variant_max_subcolumns_count = 0" + // Chapter XII. More than LIMIT matching rows and absent payload paths. + sql "DROP TABLE IF EXISTS crm_search_products" + sql """CREATE TABLE crm_search_products (id BIGINT, v VARIANT, + INDEX idx_v(v) USING INVERTED PROPERTIES("parser"="english")) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_products + SELECT number, parse_to_variant(CONCAT('{"name":"apple","price":', CAST(number AS STRING), + ',"a":"payload-',CAST(number AS STRING),'","b":',CAST(number*2 AS STRING),'}')) + FROM numbers("number"="240")""" + sql """INSERT INTO crm_search_products VALUES + (240,parse_to_variant('{"name":"banana","price":-100,"a":"excluded"}')), + (241,parse_to_variant('{"name":"apple","price":-2}')), + (242,parse_to_variant('{"name":"apple","price":-1,"a":null}'))""" + // The source's raw VARIANT sort key has no SQL ordering contract. + // Preserve the original error and make the numeric ordering explicit below. + test { + sql """SELECT v['price'], v['a'], v['b'] FROM crm_search_products + WHERE v['name'] MATCH_ANY 'apple' ORDER BY v['price'] ASC LIMIT 100""" + exception "variant column must use with specific function" + } + qt_document_12_numeric_order """ + SELECT v['price'], v['a'], v['b'] FROM crm_search_products + WHERE v['name'] MATCH_ANY 'apple' ORDER BY CAST(v['price'] AS BIGINT) ASC LIMIT 100 + """ + def typedQuery = """ + SELECT id, CAST(v['price'] AS BIGINT), CAST(v['a'] AS STRING), CAST(v['b'] AS BIGINT) FROM crm_search_products + WHERE CAST(v['name'] AS STRING) MATCH_ANY 'apple' + ORDER BY CAST(v['price'] AS BIGINT), id LIMIT 100 + """ + explain { + sql typedQuery + contains "MaterializeNode" + } + qt_document_12_typed typedQuery + qt_document_12_typed_eager """ + SELECT /*+ SET_VAR(topn_lazy_materialization_threshold=-1) */ + id, CAST(v['price'] AS BIGINT), CAST(v['a'] AS STRING), CAST(v['b'] AS BIGINT) + FROM crm_search_products WHERE CAST(v['name'] AS STRING) MATCH_ANY 'apple' + ORDER BY CAST(v['price'] AS BIGINT), id LIMIT 100 + """ +} diff --git a/regression-test/suites/search/test_search_usage_restrictions.groovy b/regression-test/suites/search/test_search_usage_restrictions.groovy index de510db6ad2759..3f2cb513822fa1 100644 --- a/regression-test/suites/search/test_search_usage_restrictions.groovy +++ b/regression-test/suites/search/test_search_usage_restrictions.groovy @@ -86,7 +86,7 @@ suite("test_search_usage_restrictions", "p0") { // Test 5: search() in GROUP BY should fail test { sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ count(*) FROM ${tableName} GROUP BY search('title:Learning')" - exception "predicates are only supported inside WHERE filters on single-table scans" + exception "predicates are only supported inside WHERE filters over OLAP tables" } // Test 6: search() in SELECT then GROUP BY alias should fail @@ -110,18 +110,15 @@ suite("test_search_usage_restrictions", "p0") { // Test 9: search() in HAVING clause should fail test { sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ category, count(*) FROM ${tableName} GROUP BY category HAVING search('title:Learning')" - exception "predicates are only supported inside WHERE filters on single-table scans" + exception "predicates are only supported inside WHERE filters over OLAP tables" } - // Test 10: search() with JOIN should fail (not single table) - test { - sql """ - SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ t1.id FROM ${tableName} t1 - JOIN ${tableName2} t2 ON t1.id = t2.id - WHERE search('title:Learning') - """ - exception "single" - } + // Test 10: SEARCH fields belong to t1 even though the WHERE is above a join. + order_qt_valid_join """ + SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ t1.id FROM ${tableName} t1 + JOIN ${tableName2} t2 ON t1.id = t2.id + WHERE search('content:tutorial') + """ // Test 11: search() in ORDER BY should fail test {