From c6ab661ee885f1daa34bc61c59b6fd2049c727db Mon Sep 17 00:00:00 2001 From: lihangyu Date: Mon, 14 Sep 2026 11:59:28 +0800 Subject: [PATCH 1/5] [feature](nereids) Support SEARCH in joins and per-field analyzers --- .../doris/analysis/SearchDslParser.java | 20 ++- .../glue/translator/ExpressionTranslator.java | 10 +- .../doris/nereids/jobs/executor/Analyzer.java | 3 + .../rules/analysis/CheckAfterRewrite.java | 8 + .../rules/analysis/CheckSearchUsage.java | 15 +- .../rules/rewrite/ConstantPropagation.java | 3 +- ...ushDownMatchProjectionAsVirtualColumn.java | 162 +++++++++++++++++- .../rules/rewrite/RewriteSearchToSlots.java | 73 ++++++-- .../trees/expressions/SearchExpression.java | 18 +- 9 files changed, 280 insertions(+), 32 deletions(-) diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java index a651e8ee7b426a..82ebceda51b394 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java @@ -289,7 +289,9 @@ private static String buildFieldPath(SearchParser.FieldPathContext ctx) { } String segment = segments.get(i).getText(); if (segment.startsWith("\"") && segment.endsWith("\"")) { - segment = segment.substring(1, segment.length() - 1); + // Preserve a literal @ in quoted field names until slot binding, + // where an unquoted @ selects the field's analyzer. + segment = segment.substring(1, segment.length() - 1).replace("@", "\\@"); } fullPath.append(segment); } @@ -1365,6 +1367,17 @@ public static class QsFieldBinding { @JsonProperty("slotIndex") private final int slotIndex; + @JsonProperty("analyzerName") + private String analyzerName; + + public String getAnalyzerName() { + return analyzerName; + } + + public void setAnalyzerName(String analyzerName) { + this.analyzerName = analyzerName; + } + @JsonCreator public QsFieldBinding(@JsonProperty("fieldName") String fieldName, @JsonProperty("slotIndex") int slotIndex) { @@ -1390,7 +1403,7 @@ public int getSlotIndex() { @Override public int hashCode() { - return Objects.hash(fieldName, slotIndex); + return Objects.hash(fieldName, slotIndex, analyzerName); } @Override @@ -1403,7 +1416,8 @@ public boolean equals(Object o) { } QsFieldBinding that = (QsFieldBinding) o; return slotIndex == that.slotIndex - && Objects.equals(fieldName, that.fieldName); + && Objects.equals(fieldName, that.fieldName) + && Objects.equals(analyzerName, that.analyzerName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 95d305f8d68547..7bded19895a282 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -698,19 +698,25 @@ public Expr visitSearchExpression(SearchExpression searchExpression, // Look up the inverted index for each field (needed for variant subcolumn analyzer) Index invertedIndex = null; + String analyzer = searchExpression.getQsPlan().getFieldBindings() + .get(fieldIndexes.size()).getAnalyzerName(); if (slotExpr instanceof SlotReference) { SlotReference slot = (SlotReference) slotExpr; OlapTable olapTbl = getOlapTableDirectly(slot); if (olapTbl != null) { Column column = slot.getOriginalColumn().orElse(null); if (column != null) { - invertedIndex = olapTbl.getInvertedIndex(column, slot.getSubPath()); + invertedIndex = olapTbl.getInvertedIndex(column, slot.getSubPath(), analyzer); } } } - if (invertedIndex == null) { + if (invertedIndex == null && analyzer == null) { invertedIndex = getInvertedIndexFromTranslatedSlot(translatedSlot, context); } + if (analyzer != null && invertedIndex == null) { + throw new AnalysisException("No inverted index found for SEARCH analyzer '" + analyzer + + "' on " + slotExpr.toSql()); + } fieldIndexes.add(invertedIndex); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index db6ed38ac41ac9..992e9d7f0eb498 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -54,6 +54,7 @@ import org.apache.doris.nereids.rules.analysis.SubqueryToApply; import org.apache.doris.nereids.rules.rewrite.AdjustNullable; import org.apache.doris.nereids.rules.rewrite.MergeFilters; +import org.apache.doris.nereids.rules.rewrite.RewriteSearchToSlots; import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalView; @@ -169,6 +170,8 @@ private static List buildAnalyzerJobs() { ), // run CheckSearchUsage before CheckAnalysis to detect search() in GROUP BY before it gets optimized bottomUp(new CheckSearchUsage()), + // Bind SEARCH dependencies before predicate movement and column pruning. + bottomUp(new RewriteSearchToSlots()), // run CheckAnalysis before EliminateGroupByConstant in order to report error message correctly like bellow // select SUM(lo_tax) FROM lineorder group by 1; // errCode = 2, detailMessage = GROUP BY expression must not contain aggregate functions: sum(lo_tax) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index db263edb7750f0..957204f1c8754e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotNotFromChildren; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; @@ -65,6 +66,13 @@ public Rule build() { checkUnexpectedExpression(plan); checkMetricTypeIsUsedCorrectly(plan); checkMatchIsUsedCorrectly(plan); + if (!(plan instanceof LogicalOlapScan) + && !(plan instanceof LogicalFilter && plan.child(0) instanceof LogicalOlapScan) + && plan.getExpressions().stream().anyMatch(expression -> + expression.anyMatch(e -> e instanceof SearchExpression))) { + throw new AnalysisException("SEARCH must be evaluated by an OLAP scan; " + + "unsupported expression placement in " + plan.getType()); + } return null; }).toRule(RuleType.CHECK_ANALYSIS); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java index 40fc84e41f2d22..e65f912fa468fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -36,7 +37,7 @@ import java.util.List; /** - * Check search expression usage - search() can only be used in WHERE filters on single-table OLAP scans. + * Check that search() is used in WHERE filters over OLAP tables. * This rule validates that search() expressions only appear in supported contexts. * Must run in analysis phase before search() gets optimized away. */ @@ -99,10 +100,8 @@ private void validateSearchUsage(Plan plan) { LOG.debug("validateSearchUsage: {}", plan.treeString()); if (plan instanceof LogicalFilter) { Plan child = plan.child(0); - if (!isSingleTableScanPipeline(child)) { - throw new AnalysisException("search() predicate only supports filtering directly on a single " - + "table scan; remove joins, subqueries, or additional operators between search() " - + "and the target table"); + if (!isOlapScanPipeline(child)) { + throw new AnalysisException("search() predicates require an OLAP scan pipeline"); } } else if (!(plan instanceof LogicalProject)) { // search() can only appear in LogicalFilter or specific LogicalProject nodes @@ -132,12 +131,16 @@ private boolean containsSearchExpression(Expression expression) { return false; } - private boolean isSingleTableScanPipeline(Plan plan) { + private boolean isOlapScanPipeline(Plan plan) { Plan current = plan; while (true) { if (current instanceof LogicalOlapScan) { return true; } + if (current instanceof LogicalJoin) { + return isOlapScanPipeline(current.child(0)) + && isOlapScanPipeline(current.child(1)); + } if (current.arity() != 1) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java index 12570f9503533f..3bb88d7f3040a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -493,7 +494,7 @@ private boolean canReplaceExpression(Expression expression) { // "https://doris.apache.org/docs/sql-manual/basic-element/operators/conditional-operators // /full-text-search-operators", the match function require left is a slot, not a literal. - if (expression instanceof Match) { + if (expression instanceof Match || expression instanceof SearchExpression) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java index 53e93dfa0fc644..1ada75da11c18c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java @@ -18,28 +18,37 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.catalog.KeysType; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; /** - * Push down MATCH expressions in projections as virtual columns on OlapScan. - * This allows the BE to evaluate MATCH using inverted index during scan. + * Materialize MATCH and SEARCH expressions as virtual columns on OlapScan. + * Projections and residual Filter/Join conditions consume the resulting booleans, + * while BE evaluates the search expressions using inverted indexes during scan. * * Example transformation: * Before: @@ -75,7 +84,15 @@ public List buildRules() { LogicalOlapScan scan = filter.child(); return pushDown(project, scan, newScan -> filter.withChildren(newScan)); - }).toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN) + }).toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), + logicalJoin().then(this::pushDownJoin) + .toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), + logicalFilter().when(filter -> !(filter.child() instanceof LogicalOlapScan)) + .then(this::pushDownResidual) + .toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), + logicalProject().when(project -> !(project.child() instanceof LogicalOlapScan)) + .then(this::pushDownResidual) + .toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN) ); } @@ -124,13 +141,150 @@ private LogicalProject pushDown( newProjections.build(), childRebuilder.apply(newScan)); } + private boolean isIndexSearch(Expression expression) { + return expression instanceof Match || expression instanceof SearchExpression; + } + + private Plan pushDownResidual(Plan plan) { + Plan child = plan.child(0); + Map replacements = new LinkedHashMap<>(); + for (Expression expression : plan.getExpressions()) { + for (Expression search : expression.collect(e -> isIndexSearch((Expression) e))) { + if (replacements.containsKey(search)) { + continue; + } + Pair result = materialize(search, child); + if (result != null) { + child = result.first; + replacements.put(search, result.second); + } + } + } + if (replacements.isEmpty()) { + return null; + } + if (plan instanceof LogicalFilter) { + Set conjuncts = new LinkedHashSet<>(); + for (Expression expression : ((LogicalFilter) plan).getConjuncts()) { + conjuncts.add(ExpressionUtils.replace(expression, replacements)); + } + // Hide additional scan values from the original filter's consumers. + return new LogicalProject<>(ImmutableList.copyOf(plan.getOutput()), + new LogicalFilter<>(conjuncts, child)); + } + LogicalProject project = (LogicalProject) plan; + List projects = new ArrayList<>(); + for (NamedExpression expression : project.getProjects()) { + projects.add((NamedExpression) ExpressionUtils.replace(expression, replacements)); + } + return project.withProjectsAndChild(projects, child); + } + + private Plan pushDownJoin(LogicalJoin join) { + List children = new ArrayList<>(join.children()); + Map replacements = new LinkedHashMap<>(); + for (Expression expression : join.getExpressions()) { + for (Expression search : expression.collect(e -> isIndexSearch((Expression) e))) { + if (replacements.containsKey(search)) { + continue; + } + for (int side = 0; side < children.size(); side++) { + // Join conditions consume child values before this join's NULL extension. + // This also handles WHERE predicates moved into an inner join by rewriting. + Pair result = materialize(search, children.get(side)); + if (result != null) { + children.set(side, result.first); + replacements.put(search, result.second); + break; + } + } + } + } + if (replacements.isEmpty()) { + return null; + } + Plan rewritten = join.withConjunctsChildren( + ExpressionUtils.replace(join.getHashJoinConjuncts(), replacements), + ExpressionUtils.replace(join.getOtherJoinConjuncts(), replacements), + ExpressionUtils.replace(join.getMarkJoinConjuncts(), replacements), + children.get(0), children.get(1), join.getJoinReorderContext()); + return new LogicalProject<>(ImmutableList.copyOf(join.getOutput()), rewritten); + } + + private Pair materialize(Expression expression, Plan plan) { + Set inputs = expression.getInputSlots(); + if (inputs.isEmpty() || !plan.getOutputSet().containsAll(inputs)) { + return null; + } + if (plan instanceof LogicalOlapScan) { + LogicalOlapScan scan = (LogicalOlapScan) plan; + if (!canPushDown(scan)) { + return null; + } + for (NamedExpression column : scan.getVirtualColumns()) { + if (column instanceof Alias && ((Alias) column).child().equals(expression)) { + return Pair.of(scan, column.toSlot()); + } + } + Alias alias = new Alias(expression); + return Pair.of(scan.appendVirtualColumns(ImmutableList.of(alias)), alias.toSlot()); + } + if (plan instanceof LogicalProject) { + LogicalProject project = (LogicalProject) plan; + if (project.containsNoneMovableFunction()) { + return null; + } + Expression rewritten = ExpressionUtils.replace(expression, project.getAliasToProducer()); + Pair result = materialize(rewritten, project.child()); + if (result == null) { + return null; + } + List projects = new ArrayList<>(project.getProjects()); + projects.add(result.second); + return Pair.of(project.withProjectsAndChild(projects, result.first), result.second); + } + if (plan instanceof LogicalFilter) { + Pair result = materialize(expression, plan.child(0)); + return result == null ? null : Pair.of(plan.withChildren(result.first), result.second); + } + if (plan instanceof LogicalJoin) { + LogicalJoin join = (LogicalJoin) plan; + for (int side = 0; side < 2; side++) { + if (!join.child(side).getOutputSet().containsAll(inputs)) { + continue; + } + boolean nullExtended = side == 0 ? join.getJoinType().isLeftSideNullable() + : join.getJoinType().isRightSideNullable(); + // SEARCH has DSL-level existence and negation semantics. Do not assume + // every DSL node is NULL-propagating across an outer join. + if (nullExtended && expression instanceof SearchExpression) { + return null; + } + Pair result = materialize(expression, join.child(side)); + if (result == null) { + return null; + } + List children = new ArrayList<>(join.children()); + children.set(side, result.first); + Plan newJoin = join.withChildren(children); + for (Slot output : newJoin.getOutput()) { + if (output.getExprId().equals(result.second.getExprId())) { + return Pair.of(newJoin, output); + } + } + return null; + } + } + return null; + } + /** * Unwrap a Match expression from a projection. * Returns the Match expression if the projection is a Match directly or an Alias wrapping a Match. * Returns null otherwise. */ private Expression unwrapMatch(NamedExpression projection) { - if (projection instanceof Alias && ((Alias) projection).child() instanceof Match) { + if (projection instanceof Alias && isIndexSearch(((Alias) projection).child())) { return ((Alias) projection).child(); } return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java index ca5d2d93b292aa..c31a77bf8253a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Search; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; @@ -42,8 +43,12 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; /** @@ -55,13 +60,13 @@ public class RewriteSearchToSlots extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalFilter(logicalOlapScan()) + return logicalFilter() .when(filter -> ExpressionUtils.containsTypes(filter.getExpressions(), Search.class)) .then(this::rewriteSearchExpressions) .toRule(RuleType.REWRITE_SEARCH_TO_SLOTS); } - private Plan rewriteSearchExpressions(LogicalFilter filter) { + private Plan rewriteSearchExpressions(LogicalFilter filter) { List newExpressions = new ArrayList<>(); for (Expression expr : filter.getExpressions()) { @@ -76,7 +81,7 @@ private Plan rewriteSearchExpressions(LogicalFilter filter) { return filter; } - private Expression rewriteExpression(Expression expr, LogicalOlapScan scan) { + private Expression rewriteExpression(Expression expr, Plan scan) { if (expr instanceof Search) { return rewriteSearch((Search) expr, scan); } @@ -93,7 +98,7 @@ private Expression rewriteExpression(Expression expr, LogicalOlapScan scan) { return expr; } - private Expression rewriteSearch(Search search, LogicalOlapScan scan) { + private Expression rewriteSearch(Search search, Plan scan) { try { // Parse DSL to get field bindings SearchDslParser.QsPlan qsPlan = search.getQsPlan(); @@ -103,11 +108,27 @@ private Expression rewriteSearch(Search search, LogicalOlapScan scan) { } Map normalizedFields = new HashMap<>(); + Map fieldAnalyzers = new HashMap<>(); + Set> qualifiers = new HashSet<>(); // Create slot reference children from field bindings List slotChildren = new ArrayList<>(); for (SearchDslParser.QsFieldBinding binding : qsPlan.getFieldBindings()) { - String originalFieldName = binding.getFieldName(); + String bindingName = binding.getFieldName(); + String originalFieldName = bindingName; + int analyzerSeparator = bindingName.lastIndexOf('@'); + while (analyzerSeparator > 0 && bindingName.charAt(analyzerSeparator - 1) == '\\') { + analyzerSeparator = bindingName.lastIndexOf('@', analyzerSeparator - 1); + } + if (analyzerSeparator >= 0 && findSlotByName(bindingName, scan) == null) { + originalFieldName = bindingName.substring(0, analyzerSeparator); + String analyzer = bindingName.substring(analyzerSeparator + 1); + if (originalFieldName.isEmpty() || analyzer.isEmpty()) { + throw new AnalysisException("SEARCH analyzer selector must be field@analyzer: " + bindingName); + } + binding.setAnalyzerName(analyzer); + } + originalFieldName = originalFieldName.replace("\\@", "@"); Expression childExpr; String normalizedFieldName; @@ -135,7 +156,7 @@ private Expression rewriteSearch(Search search, LogicalOlapScan scan) { // Check the parent variant column has at least one INVERTED index. The concrete // subcolumn binding is resolved per-segment in BE, so we only enforce the parent // level here. See function_search.cpp is_variant_sub branch. - checkInvertedIndexExists(scan.getTable(), normalizedParentFieldName, + checkInvertedIndexExists(tableForSlot(parentSlot, scan), normalizedParentFieldName, search.getDslString(), true); // Create ElementAt expression for variant subcolumn @@ -156,12 +177,26 @@ private Expression rewriteSearch(Search search, LogicalOlapScan scan) { "Field '%s' not found in table for search: %s", originalFieldName, search.getDslString())); } - checkInvertedIndexExists(scan.getTable(), slot.getName(), search.getDslString(), false); + checkInvertedIndexExists(tableForSlot(slot, scan), slot.getName(), search.getDslString(), false); childExpr = slot; normalizedFieldName = slot.getName(); } - normalizedFields.put(originalFieldName, normalizedFieldName); + for (Slot input : childExpr.getInputSlots()) { + qualifiers.add(input.getQualifier()); + } + if (qualifiers.size() > 1) { + throw new AnalysisException("Each SEARCH expression must reference fields from one table; " + + "combine separate SEARCH expressions with SQL AND/OR"); + } + String fieldKey = normalizedFieldName.toLowerCase(Locale.ROOT); + if (fieldAnalyzers.containsKey(fieldKey) + && !Objects.equals(fieldAnalyzers.get(fieldKey), binding.getAnalyzerName())) { + throw new AnalysisException("SEARCH supports one analyzer per field; use separate SEARCH " + + "expressions for different analyzers on " + normalizedFieldName); + } + fieldAnalyzers.put(fieldKey, binding.getAnalyzerName()); + normalizedFields.put(bindingName, normalizedFieldName); binding.setFieldName(normalizedFieldName); slotChildren.add(childExpr); } @@ -226,14 +261,28 @@ private void checkInvertedIndexExists(OlapTable table, String columnName, String columnName, dsl)); } - private Slot findSlotByName(String fieldName, LogicalOlapScan scan) { - // Direct match only - variant subcolumns are handled by caller + private Slot findSlotByName(String fieldName, Plan scan) { + Slot result = null; for (Slot slot : scan.getOutput()) { if (slot.getName().equalsIgnoreCase(fieldName)) { - return slot; + if (result != null) { + throw new AnalysisException("Ambiguous field '" + fieldName + "' in search()"); + } + result = slot; } } - return null; + return result; + } + + private OlapTable tableForSlot(Slot slot, Plan plan) { + if (plan instanceof LogicalOlapScan) { + return ((LogicalOlapScan) plan).getTable(); + } + if (slot instanceof SlotReference + && ((SlotReference) slot).getOriginalTable().orElse(null) instanceof OlapTable) { + return (OlapTable) ((SlotReference) slot).getOriginalTable().get(); + } + throw new AnalysisException("search() requires a field from an OLAP table: " + slot.toSql()); } private void normalizePlanFields(SearchDslParser.QsNode node, Map normalized) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java index 5767d72beddc85..1f7bba1e3b932e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java @@ -20,6 +20,8 @@ import org.apache.doris.analysis.SearchDslParser; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -72,11 +74,14 @@ public boolean foldable() { @Override public SearchExpression withChildren(List children) { - // Validate that all children are SlotReference or ElementAt (for variant subcolumns) + // Null-rejection inference temporarily replaces input slots with NULL. + // Such symbolic expressions are not execution-time field bindings. for (Expression child : children) { - if (!(child instanceof SlotReference || child instanceof ElementAt)) { + if (!(child instanceof SlotReference || child instanceof ElementAt + || child instanceof NullLiteral)) { throw new IllegalArgumentException( - "SearchExpression children must be SlotReference or ElementAt instances"); + "SEARCH field binding must be a slot, subcolumn, or inference NULL, found " + + child.getClass().getSimpleName()); } } return new SearchExpression(dslString, qsPlan, children); @@ -87,9 +92,14 @@ public R accept(ExpressionVisitor visitor, C context) { return visitor.visitSearchExpression(this, context); } + @Override + public String computeToSql() { + return "search(" + new StringLiteral(dslString).toSql() + ")"; + } + @Override public String toString() { - return "search('" + dslString + "')"; + return computeToSql(); } @Override From 3606a1a2af1b2a3632f0c6d57e89e44959f55205 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Mon, 14 Sep 2026 12:00:02 +0800 Subject: [PATCH 2/5] [test](search) Cover CRM search and join document examples --- .../rules/analysis/CheckSearchUsageTest.java | 15 +- .../rewrite/RewriteSearchToSlotsTest.java | 11 +- .../rules/rewrite/SearchJoinDocumentTest.java | 132 +++ .../expressions/SearchExpressionTest.java | 12 + .../data/search/test_crm_search_analyzers.out | 27 + .../search/test_crm_search_join_document.out | 171 ++++ .../search/test_crm_search_variant_topn.out | 307 ++++++ .../search/test_search_usage_restrictions.out | 3 + .../search/test_crm_search_analyzers.groovy | 106 ++ .../test_crm_search_join_document.groovy | 941 ++++++++++++++++++ .../test_crm_search_variant_topn.groovy | 60 ++ .../test_search_usage_restrictions.groovy | 15 +- 12 files changed, 1775 insertions(+), 25 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java create mode 100644 regression-test/data/search/test_crm_search_analyzers.out create mode 100644 regression-test/data/search/test_crm_search_join_document.out create mode 100644 regression-test/data/search/test_crm_search_variant_topn.out create mode 100644 regression-test/suites/search/test_crm_search_analyzers.groovy create mode 100644 regression-test/suites/search/test_crm_search_join_document.groovy create mode 100644 regression-test/suites/search/test_crm_search_variant_topn.groovy diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java index ab9a98660e840c..40ce885224140b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java @@ -28,7 +28,7 @@ /** * Unit tests for CheckSearchUsage rule. * This test validates that search() function can only be used in WHERE clauses - * on single-table OLAP scans, and is rejected in other contexts. + * over OLAP scans and joins, and is rejected in other contexts. */ public class CheckSearchUsageTest extends TestWithFeService implements MemoPatternMatchSupported { @@ -160,7 +160,7 @@ public void testSearchInHavingRejected() { } @Test - public void testSearchWithJoinRejected() { + public void testSearchWithJoinAllowed() { // Create second table for join test try { createTable("CREATE TABLE test_search_table2 (\n" @@ -174,19 +174,12 @@ public void testSearchWithJoinRejected() { // Table might already exist from previous test } - // Invalid: search() in WHERE with JOIN + // SEARCH depends only on title from the first input. String sql = "SELECT t1.id FROM test_search_table t1 " + "JOIN test_search_table2 t2 ON t1.id = t2.id " + "WHERE search('title:hello')"; - AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> { - PlanChecker.from(connectContext).analyze(sql); - }); - - Assertions.assertTrue( - exception.getMessage().contains("search()") - && exception.getMessage().contains("single"), - "Expected error about single table, got: " + exception.getMessage()); + Assertions.assertDoesNotThrow(() -> PlanChecker.from(connectContext).analyze(sql)); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java index a72b6ff8dd0a08..46cb3c8b036fba 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Search; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.util.PlanConstructor; @@ -244,7 +245,7 @@ public void testRewriteSearchHandlesCaseInsensitiveField() throws Exception { Search searchFunc = new Search(new StringLiteral("NAME:alice")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class); rewriteMethod.setAccessible(true); Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan); @@ -268,7 +269,7 @@ public void testRewriteSearchHandlesCaseInsensitiveVariantParentField() throws E Search searchFunc = new Search(new StringLiteral("V.foo:bar")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class); rewriteMethod.setAccessible(true); Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan); @@ -293,7 +294,7 @@ public void testRewriteSearchThrowsWhenFieldMissing() throws Exception { Search searchFunc = new Search(new StringLiteral("unknown_field:value")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class); rewriteMethod.setAccessible(true); InvocationTargetException thrown = Assertions.assertThrows(InvocationTargetException.class, @@ -312,7 +313,7 @@ public void testRewriteSearchThrowsWhenColumnHasNoInvertedIndex() throws Excepti Search searchFunc = new Search(new StringLiteral("name:alice")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class); rewriteMethod.setAccessible(true); InvocationTargetException thrown = Assertions.assertThrows(InvocationTargetException.class, @@ -331,7 +332,7 @@ public void testRewriteSearchSucceedsWhenColumnHasInvertedIndex() throws Excepti Search searchFunc = new Search(new StringLiteral("name:alice")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, LogicalOlapScan.class); + "rewriteSearch", Search.class, Plan.class); rewriteMethod.setAccessible(true); Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java new file mode 100644 index 00000000000000..f56957ddedd49c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SearchJoinDocumentTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("search_join_document"); + connectContext.setDatabase("search_join_document"); + // Mocked tables have no rowsets. Preserve scans so these tests exercise + // field binding, virtual columns and translation rather than empty plans. + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + createTable("CREATE TABLE objects (id BIGINT, v VARIANT, " + + "INDEX idx_v(v) USING INVERTED PROPERTIES('parser'='english')) " + + "DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); + createTable("CREATE TABLE lists (object_id BIGINT, list_id BIGINT) " + + "DUPLICATE KEY(object_id) DISTRIBUTED BY HASH(object_id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); + createTable("CREATE TABLE associations (to_id BIGINT, from_id BIGINT) " + + "DUPLICATE KEY(to_id) DISTRIBUTED BY HASH(to_id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); + } + + @Test + public void testDocumentSearchAboveTwoLeftJoins() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM (SELECT id, v FROM objects) o " + + "LEFT JOIN lists l ON o.id=l.object_id " + + "LEFT JOIN associations a ON o.id=a.to_id " + + "WHERE search('john', '{\"default_field\":\"v.string_17\",\"mode\":\"lucene\"}')"); + } + + @Test + public void testDocumentMatchExistsOr() { + PlanChecker.from(connectContext).checkPlannerResult( + "WITH contacts AS (SELECT id, CAST(v['string_8'] AS VARCHAR) firstname FROM objects), " + + "members AS (SELECT object_id FROM lists) " + + "SELECT id, firstname FROM contacts c WHERE " + + "(firstname MATCH_ANY 'keyur patel' AND EXISTS " + + "(SELECT 1 FROM members l WHERE l.object_id=c.id)) OR c.id>1 LIMIT 10"); + } + + @Test + public void testSearchOrAssociation() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o LEFT JOIN lists l ON o.id=l.object_id " + + "WHERE search('v.string_8:john') OR l.list_id=12", planner -> { + Assertions.assertTrue(planner.getPhysicalPlan().anyMatch(plan -> + plan instanceof PhysicalOlapScan && ((PhysicalOlapScan) plan).getVirtualColumns() + .stream().anyMatch(column -> column.anyMatch(e -> e instanceof SearchExpression))), + planner.getPhysicalPlan().treeString()); + }); + } + + @Test + public void testMatchResidualUsesVirtualColumn() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o FULL OUTER JOIN lists l ON o.id=l.object_id " + + "WHERE (CAST(o.v['string_8'] AS VARCHAR) MATCH_ANY 'john' " + + "AND CAST(o.v['string_17'] AS VARCHAR) MATCH_ALL 'smith') OR l.list_id=12", planner -> { + Assertions.assertTrue(planner.getPhysicalPlan().anyMatch(plan -> + plan instanceof PhysicalOlapScan && ((PhysicalOlapScan) plan).getVirtualColumns() + .stream().anyMatch(column -> column.anyMatch(e -> e instanceof Match))), + planner.getPhysicalPlan().treeString()); + }); + } + + @Test + public void testAmbiguousSearchFieldRejected() { + Assertions.assertThrows(AnalysisException.class, () -> PlanChecker.from(connectContext).analyze( + "SELECT a.id FROM objects a JOIN objects b ON a.id=b.id WHERE search('v.string_8:john')")); + } + + @Test + public void testSearchWithFieldSelector() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o LEFT JOIN lists l ON o.id=l.object_id " + + "WHERE search('v.string_8@english:john')"); + } + + @Test + public void testSearchConstantPredicatePreservesField() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o LEFT JOIN lists l ON o.id=l.object_id " + + "WHERE CAST(o.v['string_8'] AS VARCHAR)='john' " + + "AND (search('v.string_8:john') OR l.list_id=12)"); + } + + @Test + public void testQuotedAtSignIsLiteralVariantPath() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT id FROM objects WHERE search('\"v.email@work\":john')"); + } + + @Test + public void testSearchOrMovedIntoInnerJoin() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM objects o JOIN lists l ON o.id=l.object_id " + + "WHERE search('v.string_8:john') OR l.list_id=12", planner -> { + Assertions.assertTrue(planner.getPhysicalPlan().anyMatch(plan -> + plan instanceof PhysicalOlapScan && ((PhysicalOlapScan) plan).getVirtualColumns() + .stream().anyMatch(column -> column.anyMatch(e -> e instanceof SearchExpression))), + planner.getPhysicalPlan().treeString()); + }); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java index b1397cf8ce6f68..90623a98a3e1db 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.analysis.SearchDslParser; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.StringType; @@ -136,6 +137,17 @@ public void testToString() { String str = searchExpr.toString(); Assertions.assertEquals("search('title:hello')", str); + Assertions.assertEquals(str, searchExpr.toSql()); + } + + @Test + public void testSymbolicNullChildForNullRejectionInference() { + SearchExpression search = new SearchExpression("title:hello", createTestPlan(), + Collections.singletonList(createTestSlot("title"))); + SearchExpression symbolic = search.withChildren(Collections.singletonList(NullLiteral.INSTANCE)); + Assertions.assertEquals(NullLiteral.INSTANCE, symbolic.child(0)); + Assertions.assertFalse(symbolic.foldable()); + Assertions.assertEquals(search.getQsPlan(), symbolic.getQsPlan()); } @Test diff --git a/regression-test/data/search/test_crm_search_analyzers.out b/regression-test/data/search/test_crm_search_analyzers.out new file mode 100644 index 00000000000000..e0e3b89f4a67e8 --- /dev/null +++ b/regression-test/data/search/test_crm_search_analyzers.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !document_10_exact -- +1 +2 + +-- !document_11_single -- +2 + +-- !document_11_mixed -- +1 + +-- !document_11_fields -- +4 + +-- !document_11_separate_search -- +1 +2 + +-- !document_13_ordinary_in -- +1 + +-- !literal_column_at -- +1 + +-- !literal_variant_at -- +1 + diff --git a/regression-test/data/search/test_crm_search_join_document.out b/regression-test/data/search/test_crm_search_join_document.out new file mode 100644 index 00000000000000..48ec32c04e51d1 --- /dev/null +++ b/regression-test/data/search/test_crm_search_join_document.out @@ -0,0 +1,171 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !document_01 -- +1 + +-- !document_02 -- +1 3 john jane keyur patel hr + +-- !document_03 -- +1 3 john jane keyur patel hr +2 4 john +3 5 john +4 6 \N +5 7 \N + +-- !document_04 -- +1 3 john jane keyur patel hr + +-- !document_05 -- +1 john patel +1 john patel +1 john patel +1 john patel +1 john patel +1 john patel +3 john + +-- !document_06 -- +101 +102 + +-- !document_07 -- +-1 nobody [1, 2] +0 nobody \N +1 john jane keyur patel hr [1, 2] +1 john jane keyur patel hr [1, 2] +1 john jane keyur patel hr [1, 2] +2 john [3] +2 john [3] +4 \N \N +5 \N \N + +-- !document_08 -- +1 john jane keyur patel hr +2 john + +-- !document_09 -- +-1 +0 +1 +1 +2 +2 +3 +4 +5 + +-- !document_10 -- +-1 +0 +1 +1 +2 +2 +3 +4 +5 + +-- !document_11 -- +-1 +0 +1 +2 +3 +4 +5 + +-- !document_12 -- +1 +2 +3 +4 +5 + +-- !document_13 -- +1 +2 + +-- !document_14_original -- +\N \N +\N 8 +1 1 +1 1 +4 4 +5 5 +6 6 + +-- !document_14_alternative -- +\N \N +\N 8 +1 1 +1 1 +2 \N +3 \N +4 4 +5 5 +6 6 +7 \N + +-- !search_join_or -- +-1 12 +0 12 +1 12 +1 12 +1 123 +1 456 +2 12 +2 12 +2 123 +3 \N +4 12 +5 12 + +-- !search_two_joins_or -- +-1 12 \N +0 12 \N +1 12 1 +1 12 1 +1 12 2 +1 12 2 +1 123 2 +1 456 1 +1 456 2 +2 12 \N +2 12 \N +4 12 \N +5 12 \N + +-- !search_join_not -- +-1 12 +0 12 +1 12 +1 12 +2 12 +2 12 +4 12 +5 12 + +-- !mow_match_join -- +1 false 1 +1 false 1 +2 true \N + +-- !mow_search_join -- +1 1 +1 1 +2 \N + +-- !search_exists_or -- +-1 +0 +1 +2 +3 +4 +5 + +-- !separate_search_two_tables -- +1 1 +2 2 +3 3 + diff --git a/regression-test/data/search/test_crm_search_variant_topn.out b/regression-test/data/search/test_crm_search_variant_topn.out new file mode 100644 index 00000000000000..fa2d42fcd5de39 --- /dev/null +++ b/regression-test/data/search/test_crm_search_variant_topn.out @@ -0,0 +1,307 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !document_12_numeric_order -- +-2 \N \N +-1 \N \N +0 payload-0 0 +1 payload-1 2 +2 payload-2 4 +3 payload-3 6 +4 payload-4 8 +5 payload-5 10 +6 payload-6 12 +7 payload-7 14 +8 payload-8 16 +9 payload-9 18 +10 payload-10 20 +11 payload-11 22 +12 payload-12 24 +13 payload-13 26 +14 payload-14 28 +15 payload-15 30 +16 payload-16 32 +17 payload-17 34 +18 payload-18 36 +19 payload-19 38 +20 payload-20 40 +21 payload-21 42 +22 payload-22 44 +23 payload-23 46 +24 payload-24 48 +25 payload-25 50 +26 payload-26 52 +27 payload-27 54 +28 payload-28 56 +29 payload-29 58 +30 payload-30 60 +31 payload-31 62 +32 payload-32 64 +33 payload-33 66 +34 payload-34 68 +35 payload-35 70 +36 payload-36 72 +37 payload-37 74 +38 payload-38 76 +39 payload-39 78 +40 payload-40 80 +41 payload-41 82 +42 payload-42 84 +43 payload-43 86 +44 payload-44 88 +45 payload-45 90 +46 payload-46 92 +47 payload-47 94 +48 payload-48 96 +49 payload-49 98 +50 payload-50 100 +51 payload-51 102 +52 payload-52 104 +53 payload-53 106 +54 payload-54 108 +55 payload-55 110 +56 payload-56 112 +57 payload-57 114 +58 payload-58 116 +59 payload-59 118 +60 payload-60 120 +61 payload-61 122 +62 payload-62 124 +63 payload-63 126 +64 payload-64 128 +65 payload-65 130 +66 payload-66 132 +67 payload-67 134 +68 payload-68 136 +69 payload-69 138 +70 payload-70 140 +71 payload-71 142 +72 payload-72 144 +73 payload-73 146 +74 payload-74 148 +75 payload-75 150 +76 payload-76 152 +77 payload-77 154 +78 payload-78 156 +79 payload-79 158 +80 payload-80 160 +81 payload-81 162 +82 payload-82 164 +83 payload-83 166 +84 payload-84 168 +85 payload-85 170 +86 payload-86 172 +87 payload-87 174 +88 payload-88 176 +89 payload-89 178 +90 payload-90 180 +91 payload-91 182 +92 payload-92 184 +93 payload-93 186 +94 payload-94 188 +95 payload-95 190 +96 payload-96 192 +97 payload-97 194 + +-- !document_12_typed -- +241 -2 \N \N +242 -1 \N \N +0 0 payload-0 0 +1 1 payload-1 2 +2 2 payload-2 4 +3 3 payload-3 6 +4 4 payload-4 8 +5 5 payload-5 10 +6 6 payload-6 12 +7 7 payload-7 14 +8 8 payload-8 16 +9 9 payload-9 18 +10 10 payload-10 20 +11 11 payload-11 22 +12 12 payload-12 24 +13 13 payload-13 26 +14 14 payload-14 28 +15 15 payload-15 30 +16 16 payload-16 32 +17 17 payload-17 34 +18 18 payload-18 36 +19 19 payload-19 38 +20 20 payload-20 40 +21 21 payload-21 42 +22 22 payload-22 44 +23 23 payload-23 46 +24 24 payload-24 48 +25 25 payload-25 50 +26 26 payload-26 52 +27 27 payload-27 54 +28 28 payload-28 56 +29 29 payload-29 58 +30 30 payload-30 60 +31 31 payload-31 62 +32 32 payload-32 64 +33 33 payload-33 66 +34 34 payload-34 68 +35 35 payload-35 70 +36 36 payload-36 72 +37 37 payload-37 74 +38 38 payload-38 76 +39 39 payload-39 78 +40 40 payload-40 80 +41 41 payload-41 82 +42 42 payload-42 84 +43 43 payload-43 86 +44 44 payload-44 88 +45 45 payload-45 90 +46 46 payload-46 92 +47 47 payload-47 94 +48 48 payload-48 96 +49 49 payload-49 98 +50 50 payload-50 100 +51 51 payload-51 102 +52 52 payload-52 104 +53 53 payload-53 106 +54 54 payload-54 108 +55 55 payload-55 110 +56 56 payload-56 112 +57 57 payload-57 114 +58 58 payload-58 116 +59 59 payload-59 118 +60 60 payload-60 120 +61 61 payload-61 122 +62 62 payload-62 124 +63 63 payload-63 126 +64 64 payload-64 128 +65 65 payload-65 130 +66 66 payload-66 132 +67 67 payload-67 134 +68 68 payload-68 136 +69 69 payload-69 138 +70 70 payload-70 140 +71 71 payload-71 142 +72 72 payload-72 144 +73 73 payload-73 146 +74 74 payload-74 148 +75 75 payload-75 150 +76 76 payload-76 152 +77 77 payload-77 154 +78 78 payload-78 156 +79 79 payload-79 158 +80 80 payload-80 160 +81 81 payload-81 162 +82 82 payload-82 164 +83 83 payload-83 166 +84 84 payload-84 168 +85 85 payload-85 170 +86 86 payload-86 172 +87 87 payload-87 174 +88 88 payload-88 176 +89 89 payload-89 178 +90 90 payload-90 180 +91 91 payload-91 182 +92 92 payload-92 184 +93 93 payload-93 186 +94 94 payload-94 188 +95 95 payload-95 190 +96 96 payload-96 192 +97 97 payload-97 194 + +-- !document_12_typed_eager -- +241 -2 \N \N +242 -1 \N \N +0 0 payload-0 0 +1 1 payload-1 2 +2 2 payload-2 4 +3 3 payload-3 6 +4 4 payload-4 8 +5 5 payload-5 10 +6 6 payload-6 12 +7 7 payload-7 14 +8 8 payload-8 16 +9 9 payload-9 18 +10 10 payload-10 20 +11 11 payload-11 22 +12 12 payload-12 24 +13 13 payload-13 26 +14 14 payload-14 28 +15 15 payload-15 30 +16 16 payload-16 32 +17 17 payload-17 34 +18 18 payload-18 36 +19 19 payload-19 38 +20 20 payload-20 40 +21 21 payload-21 42 +22 22 payload-22 44 +23 23 payload-23 46 +24 24 payload-24 48 +25 25 payload-25 50 +26 26 payload-26 52 +27 27 payload-27 54 +28 28 payload-28 56 +29 29 payload-29 58 +30 30 payload-30 60 +31 31 payload-31 62 +32 32 payload-32 64 +33 33 payload-33 66 +34 34 payload-34 68 +35 35 payload-35 70 +36 36 payload-36 72 +37 37 payload-37 74 +38 38 payload-38 76 +39 39 payload-39 78 +40 40 payload-40 80 +41 41 payload-41 82 +42 42 payload-42 84 +43 43 payload-43 86 +44 44 payload-44 88 +45 45 payload-45 90 +46 46 payload-46 92 +47 47 payload-47 94 +48 48 payload-48 96 +49 49 payload-49 98 +50 50 payload-50 100 +51 51 payload-51 102 +52 52 payload-52 104 +53 53 payload-53 106 +54 54 payload-54 108 +55 55 payload-55 110 +56 56 payload-56 112 +57 57 payload-57 114 +58 58 payload-58 116 +59 59 payload-59 118 +60 60 payload-60 120 +61 61 payload-61 122 +62 62 payload-62 124 +63 63 payload-63 126 +64 64 payload-64 128 +65 65 payload-65 130 +66 66 payload-66 132 +67 67 payload-67 134 +68 68 payload-68 136 +69 69 payload-69 138 +70 70 payload-70 140 +71 71 payload-71 142 +72 72 payload-72 144 +73 73 payload-73 146 +74 74 payload-74 148 +75 75 payload-75 150 +76 76 payload-76 152 +77 77 payload-77 154 +78 78 payload-78 156 +79 79 payload-79 158 +80 80 payload-80 160 +81 81 payload-81 162 +82 82 payload-82 164 +83 83 payload-83 166 +84 84 payload-84 168 +85 85 payload-85 170 +86 86 payload-86 172 +87 87 payload-87 174 +88 88 payload-88 176 +89 89 payload-89 178 +90 90 payload-90 180 +91 91 payload-91 182 +92 92 payload-92 184 +93 93 payload-93 186 +94 94 payload-94 188 +95 95 payload-95 190 +96 96 payload-96 192 +97 97 payload-97 194 + diff --git a/regression-test/data/search/test_search_usage_restrictions.out b/regression-test/data/search/test_search_usage_restrictions.out index 6fc83963beb6de..98c913845ad12e 100644 --- a/regression-test/data/search/test_search_usage_restrictions.out +++ b/regression-test/data/search/test_search_usage_restrictions.out @@ -8,6 +8,9 @@ -- !valid_with_limit -- +-- !valid_join -- +1 + -- !valid_subquery -- -- !valid_multi_field -- diff --git a/regression-test/suites/search/test_crm_search_analyzers.groovy b/regression-test/suites/search/test_crm_search_analyzers.groovy new file mode 100644 index 00000000000000..cd60d1928521e2 --- /dev/null +++ b/regression-test/suites/search/test_crm_search_analyzers.groovy @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_crm_search_analyzers") { + // Chapters X and XI: a normalized keyword index and a full-text index. + sql "DROP TABLE IF EXISTS crm_search_analyzers" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS crm_doc_text" + sql "DROP INVERTED INDEX ANALYZER IF EXISTS crm_doc_exact" + sql "DROP INVERTED INDEX TOKEN_FILTER IF EXISTS crm_doc_nfkc" + sql """CREATE INVERTED INDEX TOKEN_FILTER crm_doc_nfkc + PROPERTIES("type"="icu_normalizer", "name"="nfkc")""" + sql """CREATE INVERTED INDEX ANALYZER crm_doc_text + PROPERTIES("tokenizer"="standard", "token_filter"="crm_doc_nfkc,lowercase")""" + sql """CREATE INVERTED INDEX ANALYZER crm_doc_exact + PROPERTIES("tokenizer"="keyword", "token_filter"="crm_doc_nfkc,lowercase")""" + // Analyzer policies reach BE asynchronously, as in the existing multi-analyzer suites. + sleep(10000) + sql """CREATE TABLE crm_search_analyzers ( + id BIGINT, name TEXT, title TEXT, + INDEX idx_name_text(name) USING INVERTED + PROPERTIES("analyzer"="crm_doc_text", "support_phrase"="true"), + INDEX idx_name_exact(name) USING INVERTED PROPERTIES("analyzer"="crm_doc_exact"), + INDEX idx_title_text(title) USING INVERTED + PROPERTIES("analyzer"="crm_doc_text", "support_phrase"="true") + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1", "inverted_index_storage_format"="V2")""" + sql """INSERT INTO crm_search_analyzers VALUES + (1,'John Smith','Senior Software Engineer'), + (2,'ＪＯＨＮ Smith','machine learning'), + (3,'John Smith Junior','software'), + (4,'Other','john smith'),(5,NULL,NULL)""" + order_qt_document_10_exact """ + SELECT id FROM crm_search_analyzers + WHERE name MATCH 'John Smith' USING ANALYZER crm_doc_exact + """ + order_qt_document_11_single """ + SELECT id FROM crm_search_analyzers + WHERE search('title@crm_doc_text:"machine learning"') + """ + order_qt_document_11_mixed """ + SELECT id FROM crm_search_analyzers + WHERE search('name@crm_doc_exact:"John Smith" AND title@crm_doc_text:software') + """ + order_qt_document_11_fields """ + SELECT id FROM crm_search_analyzers + WHERE search('john smith', '{"fields":["name@crm_doc_exact","title@crm_doc_text"]}') + """ + // Chapter XI's same-field dual-analyzer example is outside the per-field P2 scope. + test { + sql """SELECT id FROM crm_search_analyzers + WHERE search('name@crm_doc_text:John AND name@crm_doc_exact:"John Smith"')""" + exception "one analyzer per field" + } + order_qt_document_11_separate_search """ + SELECT id FROM crm_search_analyzers + WHERE search('name@crm_doc_text:John') + AND search('name@crm_doc_exact:"John Smith"') + """ + test { + sql """SELECT id FROM crm_search_analyzers WHERE search('name@does_not_exist:John')""" + exception "No inverted index found for SEARCH analyzer" + } + // Chapter XIII is outside P0-P2. Ordinary IN must keep SQL equality semantics. + order_qt_document_13_ordinary_in """ + SELECT id FROM crm_search_analyzers WHERE name IN ('John Smith','Mason Jackson') + """ + test { + sql """SELECT id FROM crm_search_analyzers + WHERE name@crm_doc_exact IN ('John Smith','Mason Jackson')""" + exception "mismatched input '@'" + } + test { + sql """SELECT id FROM crm_search_analyzers + WHERE name USING analyzer 'crm_doc_exact' IN ('John Smith','Mason Jackson')""" + exception "mismatched input 'USING'" + } + // A quoted @ remains part of the physical field name. + sql "DROP TABLE IF EXISTS crm_search_literal_fields" + sql """CREATE TABLE crm_search_literal_fields (id INT, `name@literal` TEXT, v VARIANT, + INDEX idx_name(`name@literal`) USING INVERTED PROPERTIES("parser"="english"), + INDEX idx_v(v) USING INVERTED PROPERTIES("parser"="english")) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_literal_fields VALUES + (1,'john',parse_to_variant('{"email@work":"john"}')),(2,'jane',parse_to_variant('{"email@work":"jane"}'))""" + order_qt_literal_column_at """ + SELECT id FROM crm_search_literal_fields WHERE search('"name@literal":john') + """ + order_qt_literal_variant_at """ + SELECT id FROM crm_search_literal_fields WHERE search('"v.email@work":john') + """ + +} diff --git a/regression-test/suites/search/test_crm_search_join_document.groovy b/regression-test/suites/search/test_crm_search_join_document.groovy new file mode 100644 index 00000000000000..fad0826c2f5598 --- /dev/null +++ b/regression-test/suites/search/test_crm_search_join_document.groovy @@ -0,0 +1,941 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Source: query string and search+join examples, revision 23. +suite("test_crm_search_join_document") { + sql "set enable_match_without_inverted_index = false" + + sql "DROP TABLE IF EXISTS crm_search_objects" + sql """CREATE TABLE crm_search_objects ( + OBJECTID BIGINT, PORTALID BIGINT, OBJECTTYPEID STRING, OBJECTIDHASH INT, + DELETED BOOLEAN, INGESTIONTIMESTAMP BIGINT, PROCESSEDTIMESTAMP BIGINT, + VERSION BIGINT, OVERFLOWPROPERTIES VARIANT, + INDEX idx_properties(OVERFLOWPROPERTIES) USING INVERTED PROPERTIES("parser"="english") + ) DUPLICATE KEY(OBJECTID) DISTRIBUTED BY HASH(OBJECTID) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_objects VALUES + (-1,865815822,'0-1',0,false,1,1,1,parse_to_variant('{"string_0":"nobody","string_8":"nobody","string_9":"nobody","string_17":"nobody","number_array_1":[1,2]}')), + (0,865815822,'0-1',0,false,2,2,1,parse_to_variant('{"string_0":"nobody","string_8":"nobody","string_9":"nobody","string_17":"nobody"}')), + (1,865815822,'0-1',0,false,3,3,1,parse_to_variant('{"string_0":"john","string_8":"john jane keyur patel hr","string_9":"jane","string_17":"john patel","number_array_1":[1,2]}')), + (2,865815822,'0-1',0,false,4,4,1,parse_to_variant('{"string_0":"nobody","string_8":"john","string_9":"jane","string_17":"patel","number_array_1":[3]}')), + (3,865815822,'0-1',0,false,5,5,1,parse_to_variant('{"string_0":"john","string_8":"john","string_17":"john"}')), + (4,865815822,'0-1',0,false,6,6,1,parse_to_variant('{}')), + (5,865815822,'0-1',0,false,7,7,1,NULL), + (6,865815822,'0-1',0,true,8,8,1,parse_to_variant('{"string_8":"john"}')), + (7,7,'0-1',0,false,9,9,1,parse_to_variant('{"string_8":"john"}')), + (101,865815822,'0-3',0,false,10,10,1,parse_to_variant('{"string_3":"naive"}')), + (102,865815822,'0-3',0,false,11,11,1,parse_to_variant('{"string_3":"other"}')), + (103,865815822,'0-3',0,false,12,12,1,parse_to_variant('{"string_3":"naive"}')), + (201,865815822,'0-2',0,false,13,13,1,parse_to_variant('{}'))""" + sql "DROP TABLE IF EXISTS crm_search_lists" + sql """CREATE TABLE crm_search_lists ( + OBJECTID BIGINT, PORTALID BIGINT, OBJECTTYPEID STRING, + OBJECTIDHASH INT, LISTID BIGINT, DELETED BOOLEAN + ) DUPLICATE KEY(OBJECTID) DISTRIBUTED BY HASH(OBJECTID) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_lists VALUES + (-1,865815822,'0-1',0,12,false),(0,865815822,'0-1',0,12,false), + (1,865815822,'0-1',0,456,false),(1,865815822,'0-1',0,123,false), + (1,865815822,'0-1',0,12,false),(2,865815822,'0-1',0,123,false), + (2,865815822,'0-1',0,12,false),(4,865815822,'0-1',0,12,false), + (5,865815822,'0-1',0,12,false),(99,865815822,'0-1',0,12,false), + (1,1,'0-1',0,12,false),(2,1,'0-1',1,12,false), + (NULL,1,'0-1',0,12,false)""" + sql "DROP TABLE IF EXISTS crm_search_associations" + sql """CREATE TABLE crm_search_associations ( + FROMOBJECTID BIGINT, TOOBJECTID BIGINT, PORTALID BIGINT, + FROMOBJECTTYPEID STRING, FROMOBJECTIDHASH INT, + COMBINEDASSOCIATIONTYPEID STRING, DELETED BOOLEAN + ) DUPLICATE KEY(FROMOBJECTID) DISTRIBUTED BY HASH(FROMOBJECTID) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_associations VALUES + (201,101,865815822,'0-2',0,'0-342',false), + (1,102,865815822,'0-1',0,'0-4',false), + (1,1,865815822,'0-1',0,'0-123',false), + (2,1,865815822,'0-1',0,'0-123',false), + (1,10,1,'0-1',0,'0-1',false), + (2,11,1,'0-1',2,'0-1',false), + (NULL,12,1,'0-1',0,'0-1',false)""" + + // Document example 0: 一、多列IN语法报错SQL（初始问题：多字段元组IN子查询不支持） / 报错原始SQL + test { + sql """WITH lists AS ( + SELECT + portalId, + objectTypeId, + objectId, + objectIdHash + FROM + crm_search_lists + where + portalId = 1 + and objectTypeId = '0-1' + and objectIdHash IN (0, 1, 2) +), +associations AS ( + SELECT + portalId, + fromObjectTypeId, + fromObjectIdHash, + fromObjectId, + toObjectId + FROM + crm_search_associations + where + portalId = 1 + and combinedAssociationTypeId = '0-1' + and fromObjectIdHash IN (0, 1, 2) +) +SELECT + objectId +FROM + lists +WHERE + (portalId, objectTypeId, objectIdHash, objectId) IN ( + SELECT + portalId, + fromObjectTypeId, + fromObjectIdHash, + fromObjectId + FROM + associations + )""" + exception "mismatched input ','" + } + + // Document example 1: 一、多列IN语法报错SQL（初始问题：多字段元组IN子查询不支持） / 官方替代方案：EXISTS改写（无语法报错） + order_qt_document_01 """ +WITH lists AS ( + SELECT + portalId, + objectTypeId, + objectId, + objectIdHash + FROM + crm_search_lists + WHERE + portalId = 1 + AND objectTypeId = '0-1' + AND objectIdHash IN (0, 1, 2) +), +associations AS ( + SELECT + portalId, + fromObjectTypeId, + fromObjectIdHash, + fromObjectId, + toObjectId + FROM + crm_search_associations + WHERE + portalId = 1 + AND combinedAssociationTypeId = '0-1' + AND fromObjectIdHash IN (0, 1, 2) +) +SELECT + l.objectId +FROM + lists l +WHERE + EXISTS ( + SELECT 1 + FROM associations a + WHERE + a.portalId = l.portalId + AND a.fromObjectTypeId = l.objectTypeId + AND a.fromObjectIdHash = l.objectIdHash + AND a.fromObjectId = l.objectId + ) + """ + + // Document example 2: 二、OR+EXISTS触发Mark Join报错（SlotReference异常） / 2.1 正常可执行SQL（无OR，无报错） + order_qt_document_02 """ +WITH contacts AS ( + select + OBJECTID, + INGESTIONTIMESTAMP, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as firstname + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +lists AS ( + select + PORTALID as PORTALID, + LISTID as LISTID, + OBJECTID as OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) +), +results AS ( + select + contacts.OBJECTID, + contacts.INGESTIONTIMESTAMP, + contacts.firstname + from + contacts + where + ( + (firstname MATCH_ANY 'keyur patel') + and EXISTS ( + SELECT + 1 + FROM + lists l + WHERE + l.OBJECTID = contacts.OBJECTID + ) + ) +) +SELECT + * +FROM + results +LIMIT + 10 + """ + + // Document example 3: 二、OR+EXISTS触发Mark Join报错（SlotReference异常） / 2.2 报错SQL（新增OR触发Mark Join，报SlotReference缺失列） + order_qt_document_03 """ +WITH contacts AS ( + select + OBJECTID, + INGESTIONTIMESTAMP, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as firstname + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +lists AS ( + select + PORTALID as PORTALID, + LISTID as LISTID, + OBJECTID as OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) +), +results AS ( + select + contacts.OBJECTID, + contacts.INGESTIONTIMESTAMP, + contacts.firstname + from + contacts + where + ( + (firstname MATCH_ANY 'keyur patel') + and EXISTS ( + SELECT + 1 + FROM + lists l + WHERE + l.OBJECTID = contacts.OBJECTID + ) + ) + OR contacts.OBJECTID > 1 +) +SELECT + * +FROM + results +LIMIT + 10 + """ + + // Document example 4: 二、OR+EXISTS触发Mark Join报错（SlotReference异常） / 2.3 临时规避写法（MATCH过滤下推至内层CTE，避免外层OR关联） + order_qt_document_04 """ +WITH contacts AS ( + SELECT + OBJECTID, + INGESTIONTIMESTAMP, + CAST(OVERFLOWPROPERTIES['string_8'] AS VARCHAR) AS firstname + FROM + crm_search_objects + WHERE + PORTALID = 865815822 + AND OBJECTTYPEID = '0-1' + AND OBJECTIDHASH IN (0) + AND DELETED = FALSE + -- MATCH过滤下推至内层，规避外层JOIN OR逻辑 + AND CAST(OVERFLOWPROPERTIES['string_8'] AS VARCHAR) MATCH_ANY 'keyur patel' +), +lists AS ( + SELECT + PORTALID, + LISTID, + OBJECTID + FROM + crm_search_lists + WHERE + PORTALID = 865815822 + AND OBJECTTYPEID = '0-1' + AND DELETED = FALSE +), +results AS ( + SELECT + c.OBJECTID, + c.INGESTIONTIMESTAMP, + c.firstname + FROM + contacts c + WHERE + EXISTS ( + SELECT 1 + FROM lists l + WHERE l.OBJECTID = c.OBJECTID + ) + OR c.OBJECTID > 1 +) +SELECT * +FROM results +LIMIT 10 + """ + + // Document example 5: 三、SEARCH函数JOIN限制报错SQL（search不能出现在多表关联后过滤） / 报错SQL（含多表LEFT JOIN + SEARCH函数，FE直接拦截） + order_qt_document_05 """ +select + objects_base.OBJECTID, + cast( + objects_base.OVERFLOWPROPERTIES ['string_17'] as VARCHAR + ) +from + ( + select + PORTALID, + OBJECTTYPEID, + OBJECTID, + DELETED, + INGESTIONTIMESTAMP, + PROCESSEDTIMESTAMP, + VERSION, + OVERFLOWPROPERTIES + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) + ) as objects_base + left outer join ( + select + * + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) + ) as lists_base on objects_base.OBJECTID = lists_base.OBJECTID + left outer join ( + select + * + from + crm_search_associations + where + ( + PORTALID = 865815822 + and DELETED = false + and COMBINEDASSOCIATIONTYPEID in ('0-123') + ) + ) as associations_base on objects_base.OBJECTID = associations_base.TOOBJECTID +where + search( + "john", + '{"default_field":"OVERFLOWPROPERTIES.string_17","mode":"lucene"}' + ) + """ + + // Document example 6: 四、PR#60839性能退化测试样例（多层CTE+多OR+MATCH） / + order_qt_document_06 """ +with objects_0_3 as ( + select + OBJECTID as _OBJECTID, + INGESTIONTIMESTAMP as _INGESTIONTIMESTAMP, + cast(OVERFLOWPROPERTIES ['string_3'] as VARCHAR) as dealname + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-3' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +objects_xo_0_0_2 as ( + select + OBJECTID as _OBJECTID, + INGESTIONTIMESTAMP as _INGESTIONTIMESTAMP + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-2' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +objects_xo_1_0_1 as ( + select + OBJECTID as _OBJECTID, + INGESTIONTIMESTAMP as _INGESTIONTIMESTAMP + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) +), +xo_assoc_0_0_342 as ( + select + FROMOBJECTID as FROMOBJECTID, + TOOBJECTID as TOOBJECTID + from + crm_search_associations + where + ( + PORTALID = 865815822 + and DELETED = false + and COMBINEDASSOCIATIONTYPEID in ('0-342') + ) +), +xo_assoc_1_0_4 as ( + select + FROMOBJECTID as FROMOBJECTID, + TOOBJECTID as TOOBJECTID + from + crm_search_associations + where + ( + PORTALID = 865815822 + and DELETED = false + and COMBINEDASSOCIATIONTYPEID in ('0-4') + ) +), +secondary_0_0_2 as ( + select + objects_xo_0_0_2._OBJECTID as _OBJECTID + from + objects_xo_0_0_2 +), +xo_result_0 as ( + select + distinct a.TOOBJECTID as _OBJECTID + from + xo_assoc_0_0_342 as a + join secondary_0_0_2 as s on a.FROMOBJECTID = s._OBJECTID +), +secondary_1_0_1 as ( + select + objects_xo_1_0_1._OBJECTID as _OBJECTID + from + objects_xo_1_0_1 +), +xo_result_1 as ( + select + distinct a.TOOBJECTID as _OBJECTID + from + xo_assoc_1_0_4 as a + join secondary_1_0_1 as s on a.FROMOBJECTID = s._OBJECTID +), +results as ( + select + objects_0_3._OBJECTID as _OBJECTID + from + objects_0_3 + left join xo_result_0 xo0 on objects_0_3._OBJECTID = xo0._OBJECTID + left join xo_result_1 xo1 on objects_0_3._OBJECTID = xo1._OBJECTID + where + ( + (objects_0_3.dealname MATCH_ALL 'naive') + and xo0._OBJECTID is not null + ) + or (xo1._OBJECTID is not null) +) +select + * +from + results + """ + + // Document example 7: 五、Variant列JOIN+MATCH混合查询（验证子列裁剪） / + order_qt_document_07 """ +select + `contacts`.OBJECTID, + cast( + `contacts`.OVERFLOWPROPERTIES ['string_8'] as VARCHAR + ), + cast( + `contacts`.OVERFLOWPROPERTIES ['number_array_1'] as ARRAY + ) +from + ( + select + OBJECTID, + OBJECTIDHASH, + OVERFLOWPROPERTIES + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) + ) as `contacts` + left outer join ( + select + OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) + ) as `contact_lists` on `contacts`.OBJECTID = `contact_lists`.OBJECTID +where + ( + ( + cast( + `contacts`.OVERFLOWPROPERTIES ['string_8'] as VARCHAR + ) match_all 'john' + ) + and contact_lists.OBJECTID > 0 + ) + OR contact_lists.OBJECTID is not null + """ + + // Document example 8: 六、OR多条件JOIN执行计划样例SQL（可下推部分过滤） / + // The source references agg_lists without defining it; supply distinct member IDs. + order_qt_document_08 """ +WITH agg_lists AS (SELECT DISTINCT OBJECTID FROM crm_search_lists) +select + obj.OBJECTID as OBJECTID, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as objects_0_1__firstname + from + crm_search_objects as obj + left outer join agg_lists on obj.OBJECTID = agg_lists.OBJECTID + where + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + AND ( + ( + ( + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'john' + ) + AND agg_lists.OBJECTID = 2 + ) + OR ( + agg_lists.OBJECTID = 1 + AND cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'jane' + ) + ) + """ + + // Document example 9: 七、PR#61092两种写法对比（是否预提取MATCH为虚拟列） / 7.1 原始写法（MATCH写在WHERE，跨JOIN OR无法下推索引） + order_qt_document_09 """ +WITH objects AS ( +SELECT + objectId, + CAST(overflowProperties['string_0'] AS VARCHAR) firstName +FROM crm_search_objects +), +lists AS ( + SELECT objectId FROM crm_search_lists WHERE listId = 12 +) +SELECT + o.objectId +FROM objects o LEFT JOIN lists l ON o.objectId = l.objectId +WHERE firstName MATCH_ANY 'john' OR l.objectId IS NOT NULL + """ + + // Document example 10: 七、PR#61092两种写法对比（是否预提取MATCH为虚拟列） / 7.2 优化写法（内层CTE预计算MATCH布尔虚拟列，触发索引fast path） + // Move the source misplaced alias t1 onto the inner derived table. + order_qt_document_10 """ +WITH objects AS ( + SELECT objectId, firstName, firstName MATCH_ANY 'john' AS firstNameFilter1 + FROM ( + SELECT + objectId, + CAST(overflowProperties['string_0'] AS VARCHAR) firstName + FROM crm_search_objects + ) t1 +), +lists AS ( + SELECT objectId FROM crm_search_lists WHERE listId = 12 +) +SELECT + o.objectId +FROM objects o LEFT JOIN lists l ON o.objectId = l.objectId +WHERE firstNameFilter1 OR l.objectId IS NOT NULL + """ + + // Document example 11: 八、多层CTE复杂OR场景（两种版本，区分能否下推过滤） / 8.1 无法下推过滤版本 + order_qt_document_11 """ +with lists as ( + select + PORTALID as PORTALID, + LISTID as LISTID, + OBJECTID as OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) +), +agg_lists as ( + select + OBJECTID, + array_agg(LISTID) as listIds + from + lists + group by + OBJECTID +), +fg_1 as ( + select + obj.OBJECTID as OBJECTID, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as objects_0_1__firstname + from + crm_search_objects as obj + where + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + AND ( + ( + ( + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'john' + ) + OR ( + cast(OVERFLOWPROPERTIES ['string_9'] as VARCHAR) MATCH_ALL 'jane' + ) + OR EXISTS ( + SELECT + 1 + FROM + agg_lists l + WHERE + l.OBJECTID = obj.OBJECTID + ) + ) + ) +) +select + OBJECTID +from + fg_1 + """ + + // Document example 12: 八、多层CTE复杂OR场景（两种版本，区分能否下推过滤） / 8.2 可下推过滤版本（新增同表OBJECTID>0约束拆分OR） + order_qt_document_12 """ +with lists as ( + select + PORTALID as PORTALID, + LISTID as LISTID, + OBJECTID as OBJECTID + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + ) +), +agg_lists as ( + select + OBJECTID, + array_agg(LISTID) as listIds + from + lists + group by + OBJECTID +), +fg_1 as ( + select + obj.OBJECTID as OBJECTID, + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as objects_0_1__firstname + from + crm_search_objects as obj + where + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + AND ( + ( + cast(OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'john' + ) + OR ( + cast(OVERFLOWPROPERTIES ['string_9'] as VARCHAR) MATCH_ALL 'jane' + ) + OR ( + OBJECTID > 0 + and EXISTS ( + SELECT + 1 + FROM + agg_lists l + WHERE + l.OBJECTID = obj.OBJECTID + ) + ) + ) +) +select + OBJECTID +from + fg_1 + """ + + // Document example 13: 九、多条件AND+OR嵌套JOIN查询（带array_intersect数组过滤） / + // The source final SELECT uses a missing OBJECTID alias; select the declared alias. + order_qt_document_13 """ +with `agg_lists` as ( + select + OBJECTID, + OBJECTTYPEID, + OBJECTIDHASH, + array_agg(LISTID) as `listIds` + from + ( + select + PORTALID, + LISTID, + OBJECTID, + OBJECTTYPEID, + OBJECTIDHASH + from + crm_search_lists + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and DELETED = false + and LISTID IN (456,123) + and OBJECTIDHASH IN (0) + ) + ) as `lists` + group by + OBJECTID, + OBJECTTYPEID, + OBJECTIDHASH +), +`results` as ( + select + `objects_0_1`.`OBJECTID` as `objects_0_1___OBJECTID`, + `objects_0_1`.`INGESTIONTIMESTAMP` as `objects_0_1___INGESTIONTIMESTAMP`, + `objects_0_1`.`OBJECTTYPEID` as `objects_0_1___OBJECTTYPEID`, + `objects_0_1`.`OBJECTIDHASH` as `objects_0_1___OBJECTIDHASH`, + cast(`objects_0_1`.OVERFLOWPROPERTIES ['string_8'] as VARCHAR) as `objects_0_1__firstname`, + cast( + `objects_0_1`.OVERFLOWPROPERTIES ['string_17'] as VARCHAR + ) as `objects_0_1__lastname`, + `agg_lists`.`OBJECTID` as `agg_lists__OBJECTID`, + `agg_lists`.`OBJECTTYPEID` as `agg_lists__OBJECTTYPEID`, + `agg_lists`.`OBJECTIDHASH` as `agg_lists__OBJECTIDHASH`, + `agg_lists`.`listIds` as `agg_lists__listIds` + from + ( + select + OBJECTID, + INGESTIONTIMESTAMP, + OBJECTTYPEID, + OBJECTIDHASH, + OVERFLOWPROPERTIES + from + crm_search_objects + where + ( + PORTALID = 865815822 + and OBJECTTYPEID = '0-1' + and OBJECTIDHASH IN (0) + and DELETED = false + ) + ) as `objects_0_1` + left outer join `agg_lists` on ( + ( + `objects_0_1`.`OBJECTID` = `agg_lists`.`OBJECTID` + ) + and ( + `objects_0_1`.`OBJECTTYPEID` = `agg_lists`.`OBJECTTYPEID` + ) + and ( + `objects_0_1`.`OBJECTIDHASH` = `agg_lists`.`OBJECTIDHASH` + ) + ) + where + ( + ( + ( + cast(`objects_0_1`.OVERFLOWPROPERTIES ['string_8'] as VARCHAR) MATCH_ALL 'hr' + ) + and ( + array_size( + array_intersect(`agg_lists`.`listIds`, array(456)) + ) = 1 + ) + ) + or ( + ( + array_size( + array_intersect(`agg_lists`.`listIds`, array(123)) + ) = 1 + ) + and ( + cast( + `objects_0_1`.OVERFLOWPROPERTIES ['string_17'] as VARCHAR + ) MATCH_ALL 'patel' + ) + ) + ) +) +select objects_0_1___OBJECTID from `results` + """ + + // Chapter XIV: these two predicates are intentionally different. + sql "DROP TABLE IF EXISTS crm_search_full_a" + sql """CREATE TABLE crm_search_full_a (k1 INT, content TEXT, + INDEX idx_content(content) USING INVERTED PROPERTIES("parser"="english")) + DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "DROP TABLE IF EXISTS crm_search_full_b" + sql """CREATE TABLE crm_search_full_b (k1 INT) + DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_full_a VALUES + (1,'hello world'),(2,'hello'),(3,'world'),(4,'neither'), + (5,NULL),(6,'hello'),(7,'world'),(NULL,'hello world')""" + sql "INSERT INTO crm_search_full_b VALUES (1),(1),(4),(5),(6),(8),(NULL)" + order_qt_document_14_original """ + SELECT a.k1, b.k1 FROM crm_search_full_a a FULL OUTER JOIN crm_search_full_b b + ON a.k1=b.k1 WHERE b.k1>2 + OR (a.content MATCH_ALL 'hello' AND a.content MATCH_ALL 'world') + OR (a.content MATCH_ALL 'hello' AND b.k1>5) + """ + order_qt_document_14_alternative """ + SELECT a.k1, b.k1 FROM crm_search_full_a a FULL OUTER JOIN crm_search_full_b b + ON a.k1=b.k1 WHERE (b.k1>2 AND b.k1 IS NOT NULL) + OR a.content MATCH_ALL 'hello' OR a.content MATCH_ALL 'world' + """ + + // SEARCH must not remove rows selected solely by the association branch. + order_qt_search_join_or """ + SELECT o.OBJECTID, l.LISTID FROM crm_search_objects o + LEFT JOIN crm_search_lists l ON o.OBJECTID=l.OBJECTID + WHERE o.PORTALID=865815822 AND o.OBJECTTYPEID='0-1' AND NOT o.DELETED + AND (search('OVERFLOWPROPERTIES.string_8:john') OR l.LISTID=12) + """ + order_qt_search_two_joins_or """ + SELECT o.OBJECTID, l.LISTID, a.FROMOBJECTID FROM crm_search_objects o + LEFT JOIN crm_search_lists l ON o.OBJECTID=l.OBJECTID + LEFT JOIN crm_search_associations a ON o.OBJECTID=a.TOOBJECTID + WHERE o.PORTALID=865815822 AND o.OBJECTTYPEID='0-1' AND NOT o.DELETED + AND ((search('OVERFLOWPROPERTIES.string_8:john') AND l.LISTID=456) + OR a.FROMOBJECTID=2 OR l.LISTID=12) + """ + order_qt_search_join_not """ + SELECT o.OBJECTID, l.LISTID FROM crm_search_objects o + LEFT JOIN crm_search_lists l ON o.OBJECTID=l.OBJECTID + WHERE o.PORTALID=865815822 AND o.OBJECTTYPEID='0-1' AND NOT o.DELETED + AND (NOT search('OVERFLOWPROPERTIES.string_8:john') OR l.LISTID=12) + """ + + // The virtual column rule also supports MOW; old indexed values must not leak. + sql "DROP TABLE IF EXISTS crm_search_mow" + sql """CREATE TABLE crm_search_mow (id BIGINT, v VARIANT, + INDEX idx_v(v) USING INVERTED PROPERTIES("parser"="english")) + UNIQUE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1", "enable_unique_key_merge_on_write"="true")""" + sql """INSERT INTO crm_search_mow VALUES + (1,parse_to_variant('{"name":"john"}')),(2,parse_to_variant('{"name":"other"}')),(3,NULL)""" + sql """INSERT INTO crm_search_mow VALUES + (1,parse_to_variant('{"name":"other"}')),(2,parse_to_variant('{"name":"john"}'))""" + order_qt_mow_match_join """ + SELECT o.id, CAST(o.v['name'] AS STRING) MATCH_ANY 'john', l.k1 + FROM crm_search_mow o LEFT JOIN crm_search_full_b l ON o.id=l.k1 + WHERE CAST(o.v['name'] AS STRING) MATCH_ANY 'john' OR l.k1=1 + """ + order_qt_mow_search_join """ + SELECT o.id, l.k1 + FROM crm_search_mow o LEFT JOIN crm_search_full_b l ON o.id=l.k1 + WHERE search('v.name:john') OR l.k1=1 + """ + order_qt_search_exists_or """ + SELECT o.OBJECTID FROM crm_search_objects o + WHERE o.PORTALID=865815822 AND o.OBJECTTYPEID='0-1' AND NOT o.DELETED + AND (search('OVERFLOWPROPERTIES.string_8:john') OR EXISTS ( + SELECT 1 FROM crm_search_lists l WHERE l.OBJECTID=o.OBJECTID AND l.LISTID=12)) + """ + order_qt_separate_search_two_tables """ + SELECT o.OBJECTID, m.id FROM crm_search_objects o + JOIN crm_search_mow m ON o.OBJECTID=m.id + WHERE search('OVERFLOWPROPERTIES.string_8:john') OR search('v.name:john') + """ + + // Keep SEARCH on the null-generating side gated until its full DSL NULL contract is established. + test { + sql """SELECT b.k1 FROM crm_search_full_b b LEFT JOIN crm_search_mow m ON b.k1=m.id + WHERE search('NOT v.name:john') OR b.k1=8""" + exception "SEARCH must be evaluated by an OLAP scan" + } + +} diff --git a/regression-test/suites/search/test_crm_search_variant_topn.groovy b/regression-test/suites/search/test_crm_search_variant_topn.groovy new file mode 100644 index 00000000000000..c900e01b83c06d --- /dev/null +++ b/regression-test/suites/search/test_crm_search_variant_topn.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_crm_search_variant_topn") { + // Chapter XII. More than LIMIT matching rows and absent payload paths. + sql "DROP TABLE IF EXISTS crm_search_products" + sql """CREATE TABLE crm_search_products (id BIGINT, v VARIANT, + INDEX idx_v(v) USING INVERTED PROPERTIES("parser"="english")) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO crm_search_products + SELECT number, parse_to_variant(CONCAT('{"name":"apple","price":', CAST(number AS STRING), + ',"a":"payload-',CAST(number AS STRING),'","b":',CAST(number*2 AS STRING),'}')) + FROM numbers("number"="240")""" + sql """INSERT INTO crm_search_products VALUES + (240,parse_to_variant('{"name":"banana","price":-100,"a":"excluded"}')), + (241,parse_to_variant('{"name":"apple","price":-2}')), + (242,parse_to_variant('{"name":"apple","price":-1,"a":null}'))""" + // The source's raw VARIANT sort key has no SQL ordering contract. + // Preserve the original error and make the numeric ordering explicit below. + test { + sql """SELECT v['price'], v['a'], v['b'] FROM crm_search_products + WHERE v['name'] MATCH_ANY 'apple' ORDER BY v['price'] ASC LIMIT 100""" + exception "variant column must use with specific function" + } + qt_document_12_numeric_order """ + SELECT v['price'], v['a'], v['b'] FROM crm_search_products + WHERE v['name'] MATCH_ANY 'apple' ORDER BY CAST(v['price'] AS BIGINT) ASC LIMIT 100 + """ + def typedQuery = """ + SELECT id, CAST(v['price'] AS BIGINT), CAST(v['a'] AS STRING), CAST(v['b'] AS BIGINT) FROM crm_search_products + WHERE CAST(v['name'] AS STRING) MATCH_ANY 'apple' + ORDER BY CAST(v['price'] AS BIGINT), id LIMIT 100 + """ + explain { + sql typedQuery + contains "MaterializeNode" + } + qt_document_12_typed typedQuery + qt_document_12_typed_eager """ + SELECT /*+ SET_VAR(topn_lazy_materialization_threshold=-1) */ + id, CAST(v['price'] AS BIGINT), CAST(v['a'] AS STRING), CAST(v['b'] AS BIGINT) + FROM crm_search_products WHERE CAST(v['name'] AS STRING) MATCH_ANY 'apple' + ORDER BY CAST(v['price'] AS BIGINT), id LIMIT 100 + """ +} diff --git a/regression-test/suites/search/test_search_usage_restrictions.groovy b/regression-test/suites/search/test_search_usage_restrictions.groovy index de510db6ad2759..5f2c13c6a6b140 100644 --- a/regression-test/suites/search/test_search_usage_restrictions.groovy +++ b/regression-test/suites/search/test_search_usage_restrictions.groovy @@ -113,15 +113,12 @@ suite("test_search_usage_restrictions", "p0") { exception "predicates are only supported inside WHERE filters on single-table scans" } - // Test 10: search() with JOIN should fail (not single table) - test { - sql """ - SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ t1.id FROM ${tableName} t1 - JOIN ${tableName2} t2 ON t1.id = t2.id - WHERE search('title:Learning') - """ - exception "single" - } + // Test 10: SEARCH fields belong to t1 even though the WHERE is above a join. + order_qt_valid_join """ + SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ t1.id FROM ${tableName} t1 + JOIN ${tableName2} t2 ON t1.id = t2.id + WHERE search('content:tutorial') + """ // Test 11: search() in ORDER BY should fail test { From afbdca47f39f06ed42efa2de316fe97d7756f670 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Thu, 17 Sep 2026 22:34:31 +0800 Subject: [PATCH 3/5] [fix](nereids) Keep expressions that are not NULL for NULL input above an outer join's NULL side ### What problem does this PR solve? Issue Number: None Related PR: #67932 Problem Summary: PushDownProject pushes every PreferPushDownProject expression (MATCH, element_at, ...) used by a filter or project above a join into the child that outputs its slots. When that child is the NULL-extended side of an outer join, the join pads the pushed value with NULL, but the same expression evaluated above the join can be non-NULL for NULL input. Reproduce with SELECT b.k1 FROM b LEFT JOIN a ON b.k1 = a.k1 WHERE nvl(a.content, 'hello') MATCH_ANY 'hello' OR b.k1 = 100 Rows of b without a join partner satisfy the predicate (nvl(NULL, 'hello') matches), yet they were dropped because the MATCH was computed inside a and then padded with NULL. The same happened to such an expression in the SELECT list, which returned NULL instead of TRUE. The fix adds ExpressionUtils.isNullPropagating, built on the existing replace-slots-with-NULL-and-fold inference (matchesWhenSlotsAreNull, generalized from a single slot), and PushDownProject only pushes an expression into a NULL-extended join child when it is NULL for NULL input. The scan virtual column rule of the related PR uses the same helper. SearchExpression is excluded from BE constant folding like Search, because that inference replaces its slots with NULL literals. ### Release note Fix wrong results when a MATCH (or another pushed-down expression) whose operand turns NULL into a value, such as nvl(col, 'x') MATCH_ANY 'x', is evaluated over the NULL-extended side of an outer join. ### Check List (For Author) - Test: Unit Test (PushDownProjectTest, ExpressionUtilsTest) and Regression test (search/test_crm_search_join_document: nullside_nonstrict_match_where, nullside_nonstrict_match_select) - Behavior changed: No - Does this need documentation: No Co-Authored-By: Claude Fable 5.1 --- .../rules/FoldConstantRuleOnBE.java | 3 +- .../rules/rewrite/PushDownProject.java | 33 +++++++++++-- .../doris/nereids/util/ExpressionUtils.java | 23 ++++++++-- .../rules/rewrite/PushDownProjectTest.java | 46 +++++++++++++++++++ .../nereids/util/ExpressionUtilsTest.java | 23 ++++++++++ 5 files changed, 119 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 0f87bdd2aaf54a..76b6343e2cc823 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.ai.AIFunction; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; @@ -275,7 +276,7 @@ private static boolean shouldSkipFold(Expression expr) { } // Search function should always not be folded to constant. - if (expr instanceof Search) { + if (expr instanceof Search || expr instanceof SearchExpression) { return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java index 86322c99762085..9dd387724826b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.rules.Rule; @@ -36,6 +37,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ArrayListMultimap; @@ -142,7 +144,7 @@ private Plan pushDownJoinExpressions(MatchingContext> ct private Plan pushDownFilterExpressions(MatchingContext>> ctx) { LogicalFilter> filter = ctx.root; LogicalJoin join = filter.child(); - PushdownProjectHelper pushdownProjectHelper = new PushdownProjectHelper(ctx.statementContext, join); + PushdownProjectHelper pushdownProjectHelper = new PushdownProjectHelper(ctx.cascadesContext, join); Pair> pushPredicates = pushdownProjectHelper.pushDownExpressions(filter.getConjuncts()); if (!pushPredicates.first) { @@ -197,7 +199,7 @@ private Plan defaultPushDownProject(MatchingContext project = ctx.root; C child = project.child(); PushdownProjectHelper pushdownProjectHelper - = new PushdownProjectHelper(ctx.statementContext, child); + = new PushdownProjectHelper(ctx.cascadesContext, child); Pair> pushProjects = pushdownProjectHelper.pushDownExpressions(project.getProjects()); @@ -341,11 +343,23 @@ private static List replaceSlot( private static class PushdownProjectHelper { private final Plan plan; private final StatementContext statementContext; + // needed to push through a join only, see canComputeInChild + private final CascadesContext cascadesContext; private final Map oldExprToNewExpr; private final Multimap childToPushDownProjects; + public PushdownProjectHelper(CascadesContext cascadesContext, Plan plan) { + this(cascadesContext.getStatementContext(), cascadesContext, plan); + } + public PushdownProjectHelper(StatementContext statementContext, Plan plan) { + this(statementContext, null, plan); + } + + private PushdownProjectHelper(StatementContext statementContext, CascadesContext cascadesContext, + Plan plan) { this.statementContext = statementContext; + this.cascadesContext = cascadesContext; this.plan = plan; this.oldExprToNewExpr = new LinkedHashMap<>(); this.childToPushDownProjects = ArrayListMultimap.create(); @@ -391,7 +405,7 @@ public Optional pushDownExpression(E expression) { List children = plan.children(); for (int i = 0; i < children.size(); i++) { Plan child = children.get(i); - if (child.getOutputSet().containsAll(e.getInputSlots())) { + if (child.getOutputSet().containsAll(e.getInputSlots()) && canComputeInChild(i, e)) { Alias alias = new Alias(statementContext.getNextExprId(), e); Slot slot = alias.toSlot(); childToPushDownProjects.put(child, alias); @@ -410,6 +424,19 @@ public Optional pushDownExpression(E expression) { } } + // A filter or project above an outer join reads NULL for rows of the NULL-extended side, so an + // expression computed inside that child must itself be NULL for NULL input: nvl(col, 'x') MATCH 'x' + // is TRUE above the join but would be padded to NULL when computed below it. + private boolean canComputeInChild(int childIndex, Expression expression) { + if (!(plan instanceof LogicalJoin)) { + return true; + } + LogicalJoin join = (LogicalJoin) plan; + boolean nullExtended = childIndex == 0 ? join.getJoinType().isLeftSideNullable() + : join.getJoinType().isRightSideNullable(); + return !nullExtended || ExpressionUtils.isNullPropagating(expression, cascadesContext); + } + public List buildNewChildren() { if (childToPushDownProjects.isEmpty()) { return plan.children(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 00f5caa8364e7f..65490bfb096756 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -945,6 +945,17 @@ private static boolean isFalseOrNull(Expression expression) { return expression.isNullLiteral() || BooleanLiteral.FALSE.equals(expression); } + /** + * Whether the expression is NULL when all its input slots are NULL, e.g. {@code col MATCH_ANY 'x'} or + * {@code element_at(v, 'k')}, but not {@code nvl(col, 'x') MATCH_ANY 'x'}. Only such an expression can be + * computed below the NULL-extended side of an outer join, because the join pads its value with NULL. + */ + public static boolean isNullPropagating(Expression expression, CascadesContext cascadesContext) { + Set inputSlots = expression.getInputSlots(); + return !inputSlots.isEmpty() + && matchesWhenSlotsAreNull(expression, inputSlots, cascadesContext, Expression::isNullLiteral); + } + /** * infer notNulls slot from predicate */ @@ -980,7 +991,8 @@ private static Set inferNotNullSlots(Set predicates, Set } inputSlots = mergedInputSlots.get(); for (Slot slot : candidateSlots) { - if (matchesWhenSlotIsNull(predicate, slot, cascadesContext, nullInputResultPredicate)) { + if (matchesWhenSlotsAreNull(predicate, ImmutableSet.of(slot), cascadesContext, + nullInputResultPredicate)) { notNullSlots.add(slot); } } @@ -1000,11 +1012,12 @@ private static Set collectNotNullInferenceTargetSlots(Set expr return targetSlots; } - private static boolean matchesWhenSlotIsNull(Expression expression, Slot slot, CascadesContext cascadesContext, - Predicate nullInputResultPredicate) { + private static boolean matchesWhenSlotsAreNull(Expression expression, Set slots, + CascadesContext cascadesContext, Predicate nullInputResultPredicate) { Map replaceMap = new HashMap<>(); - Literal nullLiteral = new NullLiteral(slot.getDataType()); - replaceMap.put(slot, nullLiteral); + for (Slot slot : slots) { + replaceMap.put(slot, new NullLiteral(slot.getDataType())); + } Expression evalExpr = FoldConstantRule.evaluate( ExpressionUtils.replace(expression, replaceMap), new ExpressionRewriteContext(cascadesContext)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownProjectTest.java index 0a0b425e81766a..3e0fd1228640e2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownProjectTest.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.PreferPushDownProject; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.JoinType; @@ -240,4 +241,49 @@ public void shouldRewritePreferPushDownProjectInOrFilterToSlot() { && !rewrittenPredicate.anyMatch(PreferPushDownProject.class::isInstance); })); } + + @Test + public void shouldNotPushNonNullPropagatingExpressionIntoNullExtendedSide() { + LogicalPlan rStudent = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.student, + ImmutableList.of("")); + LogicalPlan rScore = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.score, + ImmutableList.of("")); + // nvl(NULL, 1) is 1, so the MATCH is not NULL for rows that the left join NULL-extends. + Expression nonNullPropagating = new MatchAny( + new Nvl(rScore.getOutput().get(2), Literal.of(1)), Literal.of("abc")); + Expression leftSidePredicate = new GreaterThan(rStudent.getOutput().get(0), Literal.of(60)); + + LogicalPlan plan = new LogicalPlanBuilder(rStudent) + .joinEmptyOn(rScore, JoinType.LEFT_OUTER_JOIN) + .filter(new Or(nonNullPropagating, leftSidePredicate)) + .build(); + + PlanChecker.from(connectContext, plan) + .applyTopDown(new PushDownProject()) + .matchesFromRoot(logicalFilter(logicalJoin(logicalOlapScan(), logicalOlapScan())) + .when(filter -> filter.getConjuncts().iterator().next() + .anyMatch(PreferPushDownProject.class::isInstance))); + } + + @Test + public void shouldPushNullPropagatingExpressionIntoNullExtendedSide() { + LogicalPlan rStudent = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.student, + ImmutableList.of("")); + LogicalPlan rScore = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.score, + ImmutableList.of("")); + Expression nullPropagating = new MatchAny( + new Add(rScore.getOutput().get(2), Literal.of(1)), Literal.of("abc")); + Expression leftSidePredicate = new GreaterThan(rStudent.getOutput().get(0), Literal.of(60)); + + LogicalPlan plan = new LogicalPlanBuilder(rStudent) + .joinEmptyOn(rScore, JoinType.LEFT_OUTER_JOIN) + .filter(new Or(nullPropagating, leftSidePredicate)) + .build(); + + PlanChecker.from(connectContext, plan) + .applyTopDown(new PushDownProject()) + .matchesFromRoot(logicalFilter(logicalJoin(logicalOlapScan(), logicalProject(logicalOlapScan()))) + .when(filter -> !filter.getConjuncts().iterator().next() + .anyMatch(PreferPushDownProject.class::isInstance))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java index a0016dfdf5da38..37018414799078 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java @@ -19,11 +19,14 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableTest; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MatchAny; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.generator.Explode; @@ -35,18 +38,22 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode; import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.StringType; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; @@ -295,6 +302,22 @@ public void testUnnest() { Assertions.assertTrue(ExpressionUtils.convertUnnest(unnest) instanceof ExplodeBitmapOuter); } + @Test + public void testIsNullPropagating() { + CascadesContext context = createCascadesContext("select 1"); + SlotReference column = new SlotReference("c", StringType.INSTANCE, true, Lists.newArrayList()); + StringLiteral term = new StringLiteral("x"); + Assertions.assertTrue(ExpressionUtils.isNullPropagating(column, context)); + Assertions.assertTrue(ExpressionUtils.isNullPropagating(new MatchAny(column, term), context)); + Assertions.assertTrue(ExpressionUtils.isNullPropagating( + new MatchAny(new Cast(new ElementAt(column, term), StringType.INSTANCE), term), context)); + // nvl turns a NULL input into a value, so the MATCH above it is not NULL for NULL input. + Assertions.assertFalse(ExpressionUtils.isNullPropagating( + new MatchAny(new Nvl(column, term), term), context)); + // Without an input slot the value does not depend on the NULL-extended row at all. + Assertions.assertFalse(ExpressionUtils.isNullPropagating(new MatchAny(term, term), context)); + } + private void assertExpect(List originalExpressions, List shuttledExpressions, String... expectExpressions) { From 7b9071b2fb424504e028aace739e53df733cf31d Mon Sep 17 00:00:00 2001 From: lihangyu Date: Thu, 17 Sep 2026 22:34:51 +0800 Subject: [PATCH 4/5] [fix](nereids) Converge SEARCH field binding, scan materialization and execution checks ### What problem does this PR solve? Issue Number: None Related PR: #67932 Problem Summary: Review of SEARCH in joins found correctness and maintainability problems that share a few causes: several places decided the same thing differently, and checks looked at plan shapes instead of what BE executes. Correctness 1. NULL padding could bypass the outer join guard. `LEFT JOIN a ON false`, an outer join converted to an anti join, and alias inlining of `NULL AS col` replace a SEARCH field with a NULL literal; the expression then sat on the preserved scan where BE found no indexed field and evaluated it as FALSE, so `NOT search(...) OR b.k1 = 8` returned every row. CheckAfterRewrite now verifies, in one place and also for scan virtual columns, that every SearchExpression is evaluated by a scan and still binds only index fields. 2. Predicate inference copied a SEARCH to an equal column. With `a JOIN n ON a.content = n.name WHERE search('a.content:hello') OR a.content = 'zzz'`, InferPredicateByReplace produced `search[n.name] OR n.name = 'zzz'` for n, whose column has no inverted index, and the query returned no rows instead of four. The positional inference of INTERSECT/EXCEPT in InferPredicates rebinds a pulled-up SEARCH the same way (an INTERSECT with a SEARCH branch returned no rows). A SEARCH is bound to the indexes of its own columns, so it is neither an input of equality inference nor cloned into a sibling branch, like volatile expressions. 3. `field@analyzer` was read as a column named `field@analyzer` whenever such a column existed. The selector is now purely syntactic and owned by SearchDslParser.splitAnalyzerSelector: the last unescaped `@` after a non-empty path segment selects the analyzer; `\@`, an `@` inside a quoted segment and an `@` that starts a segment (`v.@timestamp`) belong to the name. 4. Index validation and the field names sent to BE used the slot's output name. After `SELECT content AS body` or `v AS props` a valid SEARCH failed with "Column not found", or validated another column that happened to carry the alias name. Names are resolved by their visible name; validation, NESTED paths and BE field names use the slot's original table and column. Generality and maintainability 5. One materialization path. Project over a scan only unwrapped `Alias(match)` while residual filters and joins collected searches recursively, so `CASE WHEN col MATCH ... END` behaved differently by plan shape. The rule (renamed PushDownIndexSearchAsVirtualColumn; the rule type keeps its name for disable_nereids_rules) has a single collect, materialize, replace path, no longer appends a reused virtual column twice, and refuses to inline a volatile alias producer. 6. Responsibilities are explicit: CheckSearchUsage checks placement on the analyzed plan (a necessary condition only), materialize() alone decides where an index search may be computed, CheckAfterRewrite verifies the final plan. The qualifier-set "one table" check is gone; a SEARCH over two relations can never reach one scan and is rejected by the final check with the constraints spelled out. 7. Field names resolve like SQL column references by reusing ExpressionAnalyzer.bindSlotByScope, so `alias.field` selects one side of a self join before `column.subcolumn` is tried, and an ambiguous name is an error. 8. "One analyzer per field" compares the inverted indexes selected by OlapTable.getInvertedIndex, the lookup the translator sends to BE, instead of analyzer strings, so `name@Exact` and `name@exact` agree. Variant subcolumn paths keep their case when the plan fields are normalized. 9. Messages and comments no longer say "single-table scans"; the dead Rewriter registration of RewriteSearchToSlots is removed because binding happens in the Analyzer. Tests The pipeline randomizes the VARIANT defaults. With a small default_variant_max_subcolumns_count a subcolumn is stored in the sparse column without an inverted index, which made test_crm_search_join_document fail with "match_all not support execute_match" and test_crm_search_analyzers return an empty result (both reproduced by setting the variable by hand). The three CRM suites now pin those defaults like the other search suites. ### Release note In a SEARCH field reference an unquoted `@` always selects an analyzer; quote the segment or write `\@` for a literal `@` that follows a field name. SEARCH fields may be qualified with a table alias (`search('a.title:x')`). ### Check List (For Author) - Test: Unit Test (SearchJoinDocumentTest, RewriteSearchToSlotsTest, PushDownIndexSearchAsVirtualColumnTest, SearchDslParserTest, SearchExpressionTest, CheckSearchUsageTest and the FE tests that reference the changed rules) and Regression test (search/, inverted_index_p0/test_match_projection_virtual_column) - Behavior changed: Yes (see release note; error messages for unsupported SEARCH placement are more specific) - Does this need documentation: Yes (analyzer selector and table alias syntax of SEARCH fields) Co-Authored-By: Claude Fable 5.1 --- .../doris/analysis/SearchDslParser.java | 54 +++- .../glue/translator/ExpressionTranslator.java | 2 +- .../doris/nereids/jobs/executor/Rewriter.java | 11 +- .../rules/analysis/CheckAfterRewrite.java | 39 ++- .../rules/analysis/CheckSearchUsage.java | 18 +- .../rewrite/InferPredicateByReplace.java | 6 + .../rules/rewrite/InferPredicates.java | 10 + ...> PushDownIndexSearchAsVirtualColumn.java} | 209 +++++------- .../rules/rewrite/RewriteSearchToSlots.java | 304 +++++++++--------- .../trees/expressions/SearchExpression.java | 27 +- .../rules/analysis/CheckSearchUsageTest.java | 3 +- ...shDownIndexSearchAsVirtualColumnTest.java} | 16 +- .../rewrite/RewriteSearchToSlotsTest.java | 21 +- .../rules/rewrite/SearchJoinDocumentTest.java | 288 ++++++++++++++++- .../expressions/SearchExpressionTest.java | 23 +- .../functions/scalar/SearchDslParserTest.java | 39 +++ .../data/search/test_crm_search_analyzers.out | 4 + .../search/test_crm_search_join_document.out | 82 +++++ .../search/test_crm_search_analyzers.groovy | 15 + .../test_crm_search_join_document.groovy | 98 ++++++ .../test_crm_search_variant_topn.groovy | 5 + .../test_search_usage_restrictions.groovy | 4 +- 22 files changed, 933 insertions(+), 345 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/{PushDownMatchProjectionAsVirtualColumn.java => PushDownIndexSearchAsVirtualColumn.java} (55%) rename fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/{PushDownMatchProjectionAsVirtualColumnTest.java => PushDownIndexSearchAsVirtualColumnTest.java} (95%) diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java index 82ebceda51b394..fd29b0f2dad1f1 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchDslParser.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.common.Pair; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -35,6 +37,8 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -276,6 +280,9 @@ private static void validateFieldsList(List fields) { } } + // an escaped character, or an unescaped @ + private static final Pattern ESCAPED_OR_AT = Pattern.compile("\\\\.|@"); + private static String buildFieldPath(SearchParser.FieldPathContext ctx) { if (ctx == null) { throw new RuntimeException("Invalid field query: missing field path"); @@ -289,15 +296,49 @@ private static String buildFieldPath(SearchParser.FieldPathContext ctx) { } String segment = segments.get(i).getText(); if (segment.startsWith("\"") && segment.endsWith("\"")) { - // Preserve a literal @ in quoted field names until slot binding, - // where an unquoted @ selects the field's analyzer. - segment = segment.substring(1, segment.length() - 1).replace("@", "\\@"); + // An @ inside quotes is part of the field name; see splitAnalyzerSelector. + segment = ESCAPED_OR_AT.matcher(segment.substring(1, segment.length() - 1)) + .replaceAll(m -> Matcher.quoteReplacement(m.group().equals("@") ? "\\@" : m.group())); } fullPath.append(segment); } return fullPath.toString(); } + /** + * Splits a DSL field reference {@code path[@analyzer]} into the field path and the analyzer name (null when + * no analyzer is selected). The split is purely syntactic and never depends on the table schema: the last + * unescaped {@code @} that follows a non-empty path segment selects the analyzer, while {@code \@}, an + * {@code @} inside a quoted segment (escaped by buildFieldPath) and an {@code @} that starts a segment + * ({@code v.@timestamp}) belong to the field name. + */ + public static Pair splitAnalyzerSelector(String fieldReference) { + StringBuilder path = new StringBuilder(); + int selector = -1; + for (int i = 0; i < fieldReference.length(); i++) { + char c = fieldReference.charAt(i); + if (c == '\\' && i + 1 < fieldReference.length()) { + char escaped = fieldReference.charAt(++i); + if (escaped != '@') { + path.append(c); + } + path.append(escaped); + continue; + } + if (c == '@' && i > 0 && fieldReference.charAt(i - 1) != '.') { + selector = path.length(); + } + path.append(c); + } + if (selector < 0) { + return Pair.of(path.toString(), null); + } + if (selector == path.length() - 1) { + throw new SearchDslSyntaxException("SEARCH analyzer selector must be field@analyzer: " + fieldReference); + } + return Pair.of(path.substring(0, selector), path.substring(selector + 1)); + } + private static String normalizeNestedFieldPath(String fieldPath, @Nullable String nestedPath) { if (nestedPath == null || nestedPath.isEmpty()) { return fieldPath; @@ -1305,6 +1346,13 @@ public String getNestedPath() { return nestedPath; } + /** + * Sets the nested path (used for field name normalization). + */ + public void setNestedPath(String nestedPath) { + this.nestedPath = nestedPath; + } + /** * Returns whether the field was explicitly specified in the DSL syntax. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 7bded19895a282..ea0cebcef5f9df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -250,7 +250,7 @@ public Expr visitMatch(Match match, PlanTranslatorContext context) { // column/table reference (e.g., after CTE inlining or join projection remapping), // we gracefully fall back to invertedIndex = null. The BE can still evaluate MATCH // correctly without inverted index (slow path), or the PushDownProject / - // PushDownMatchProjectionAsVirtualColumn rules may have already pushed the expression + // PushDownIndexSearchAsVirtualColumn rules may have already pushed the expression // down for storage-level index evaluation (fast path). Index invertedIndex = null; String analyzer = match.getAnalyzer().orElse(null); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 7a3d6495204297..398bf06eb3fd5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -132,11 +132,11 @@ import org.apache.doris.nereids.rules.rewrite.PushDownEncodeSlot; import org.apache.doris.nereids.rules.rewrite.PushDownFilterIntoSchemaScan; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject; +import org.apache.doris.nereids.rules.rewrite.PushDownIndexSearchAsVirtualColumn; import org.apache.doris.nereids.rules.rewrite.PushDownJoinOnAssertNumRows; import org.apache.doris.nereids.rules.rewrite.PushDownLimit; import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughUnion; -import org.apache.doris.nereids.rules.rewrite.PushDownMatchProjectionAsVirtualColumn; import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit; import org.apache.doris.nereids.rules.rewrite.PushDownScoreTopNIntoOlapScan; import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin; @@ -154,7 +154,6 @@ import org.apache.doris.nereids.rules.rewrite.ReduceAggregateChildOutputRows; import org.apache.doris.nereids.rules.rewrite.ReorderJoin; import org.apache.doris.nereids.rules.rewrite.RewriteCteChildren; -import org.apache.doris.nereids.rules.rewrite.RewriteSearchToSlots; import org.apache.doris.nereids.rules.rewrite.RewriteSimpleAggToConstantRule; import org.apache.doris.nereids.rules.rewrite.SaltJoin; import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; @@ -797,7 +796,7 @@ public class Rewriter extends AbstractBatchJobExecutor { custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, EliminateUnnecessaryProject::new), topDown(new PushDownVectorTopNIntoOlapScan()), topDown(new PushDownVirtualColumnsIntoOlapScan()), - topDown(new PushDownMatchProjectionAsVirtualColumn()), + topDown(new PushDownIndexSearchAsVirtualColumn()), topic("score optimize", topDown(new PushDownScoreTopNIntoOlapScan(), new CheckScoreUsage()) @@ -929,12 +928,6 @@ private static List getWholeTreeRewriteJobs( custom(RuleType.DISTINCT_AGG_STRATEGY_SELECTOR, () -> DistinctAggStrategySelector.INSTANCE)))); - // Rewrite search function before VariantSubPathPruning - // so that ElementAt expressions from search can be processed - rewriteJobs.addAll(jobs( - bottomUp(new RewriteSearchToSlots()) - )); - if (needSubPathPushDown) { rewriteJobs.addAll(jobs( topic("variant element_at push down", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 957204f1c8754e..0e1da87971965d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -51,6 +51,7 @@ import org.apache.commons.lang3.StringUtils; import org.roaringbitmap.RoaringBitmap; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -66,17 +67,41 @@ public Rule build() { checkUnexpectedExpression(plan); checkMetricTypeIsUsedCorrectly(plan); checkMatchIsUsedCorrectly(plan); - if (!(plan instanceof LogicalOlapScan) - && !(plan instanceof LogicalFilter && plan.child(0) instanceof LogicalOlapScan) - && plan.getExpressions().stream().anyMatch(expression -> - expression.anyMatch(e -> e instanceof SearchExpression))) { - throw new AnalysisException("SEARCH must be evaluated by an OLAP scan; " - + "unsupported expression placement in " + plan.getType()); - } + checkSearchIsUsedCorrectly(plan); return null; }).toRule(RuleType.CHECK_ANALYSIS); } + /** + * The execution invariant of SEARCH, checked in this one place for every way a rewrite can move it: BE + * evaluates a SearchExpression only with inverted indexes inside an OLAP scan, so it must be a conjunct of + * the filter on the scan or a scan virtual column, and every child must still bind an index field. + */ + private void checkSearchIsUsedCorrectly(Plan plan) { + // Scan virtual columns are not part of LogicalOlapScan.getExpressions(). + List expressions = plan instanceof LogicalOlapScan + ? ((LogicalOlapScan) plan).getVirtualColumns() : plan.getExpressions(); + boolean evaluatedByScan = plan instanceof LogicalOlapScan + || (plan instanceof LogicalFilter && plan.child(0) instanceof LogicalOlapScan); + for (Expression expression : expressions) { + for (SearchExpression search : expression.collect(SearchExpression.class::isInstance)) { + if (!search.bindsOnlyFields()) { + // e.g. LEFT JOIN ... ON false, or an outer join converted to an anti join + throw new AnalysisException("SEARCH on the null-generating side of an outer join is not " + + "supported: its field is always NULL in " + search.toSql()); + } + if (!evaluatedByScan) { + throw new AnalysisException("SEARCH must be evaluated by an OLAP scan, but " + search.toSql() + + " remains in " + plan.getType() + ". Unless it is a WHERE conjunct that reaches the " + + "scan, all its fields must come from one DUP_KEYS or merge-on-write UNIQUE_KEYS " + + "table reference that is neither on the null-generating side of an outer join nor " + + "below a LIMIT, TopN, aggregate or window; combine separate SEARCH expressions with " + + "SQL AND/OR to search several tables"); + } + } + } + } + private void checkUnexpectedExpression(Plan plan) { boolean isGenerate = plan instanceof Generate; boolean isAgg = plan instanceof LogicalAggregate; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java index e65f912fa468fa..3066c79e5583fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsage.java @@ -37,9 +37,13 @@ import java.util.List; /** - * Check that search() is used in WHERE filters over OLAP tables. + * Check that search() is used in WHERE filters over OLAP tables, including joins of them. * This rule validates that search() expressions only appear in supported contexts. * Must run in analysis phase before search() gets optimized away. + * + *

This is a placement check on the analyzed plan and only a necessary condition: predicate push down may + * still move the filter anywhere below. Whether the search can be evaluated by a scan is decided by + * PushDownIndexSearchAsVirtualColumn and verified on the final plan by CheckAfterRewrite. */ public class CheckSearchUsage implements AnalysisRuleFactory { private static final Logger LOG = LogManager.getLogger(CheckSearchUsage.class); @@ -67,13 +71,13 @@ private void checkPlanRecursively(Plan plan) { for (Expression expr : agg.getGroupByExpressions()) { if (containsSearchExpression(expr)) { throw new AnalysisException("search() cannot appear in GROUP BY expressions; " - + "search predicates are only supported in WHERE filters on single-table scans"); + + "search predicates are only supported in WHERE filters over OLAP tables"); } } for (Expression expr : agg.getOutputExpressions()) { if (containsSearchExpression(expr)) { throw new AnalysisException("search() cannot appear in aggregate output expressions; " - + "search predicates are only supported in WHERE filters on single-table scans"); + + "search predicates are only supported in WHERE filters over OLAP tables"); } } } @@ -83,7 +87,6 @@ private void checkPlanRecursively(Plan plan) { LogicalProject project = (LogicalProject) plan; for (Expression expr : project.getProjects()) { if (containsSearchExpression(expr)) { - // Only allow if it's the project directly above a filter->scan pattern throw new AnalysisException("search() can only appear in WHERE filters on OLAP scans; " + "projection of search() is not supported"); } @@ -104,9 +107,9 @@ private void validateSearchUsage(Plan plan) { throw new AnalysisException("search() predicates require an OLAP scan pipeline"); } } else if (!(plan instanceof LogicalProject)) { - // search() can only appear in LogicalFilter or specific LogicalProject nodes - throw new AnalysisException("search() predicates are only supported inside WHERE filters on " - + "single-table scans"); + // a projection gets its own message in checkPlanRecursively + throw new AnalysisException("search() predicates are only supported inside WHERE filters over " + + "OLAP tables"); } } @@ -131,6 +134,7 @@ private boolean containsSearchExpression(Expression expression) { return false; } + // Every relation below the filter must be an OLAP scan; unary nodes in between are not restricted here. private boolean isOlapScanPipeline(Plan plan) { Plan current = plan; while (true) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java index c2ca99f0b61b9c..d8cd37ca999176 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Like; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -214,6 +215,11 @@ public static Set infer(Set inputs) { || input.getInputSlots().size() != 1) { continue; } + // A SEARCH is evaluated with the inverted indexes of the columns it binds, not from their values, + // so it does not hold for another column that is merely equal. + if (input.containsType(SearchExpression.class)) { + continue; + } input.accept(PredicatesCollector.INSTANCE, exprPredicates); } Set inferPredicates = new LinkedHashSet<>(inputs); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java index 8688db6a7d1486..b7dbc00c2895d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -226,6 +227,11 @@ private Plan inferNewPredicate(Plan plan, Set expressions) { // query semantics (see EXCEPT/INTERSECT regression cases). continue; } + if (expr.containsType(SearchExpression.class)) { + // A SEARCH is bound to the inverted indexes of its own scan. The slot substitution of the + // SetOp visitors would rebind it to the column of a sibling branch, where BE finds no index. + continue; + } Set slots = expr.getInputSlots(); if (!slots.isEmpty() && planOutputs.containsAll(slots)) { predicates.add(expr); @@ -255,6 +261,10 @@ private Plan inferNewPredicateRemoveUselessIsNull(Plan plan, Set exp // predicates into a subtree that did not already evaluate them. continue; } + if (expr.containsType(SearchExpression.class)) { + // See inferNewPredicate: a SEARCH stays on the scan whose indexes it binds. + continue; + } Set slots = expr.getInputSlots(); if (slots.isEmpty() || !planOutputs.containsAll(slots)) { continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumn.java similarity index 55% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumn.java index 1ada75da11c18c..26cee48e81e571 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumn.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumn.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; @@ -37,18 +38,26 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; /** - * Materialize MATCH and SEARCH expressions as virtual columns on OlapScan. - * Projections and residual Filter/Join conditions consume the resulting booleans, - * while BE evaluates the search expressions using inverted indexes during scan. + * Materialize index searches (MATCH and SEARCH expressions) as virtual columns on OlapScan, so that BE evaluates + * them with inverted indexes during the scan and the consumer reads the resulting boolean. + * + *

Consumers are projections, residual filters (a filter directly on a scan is evaluated by the scan itself) + * and join conditions. All of them go through the same steps: collect every index search in the consumer's + * expressions, however deeply nested, {@link #materialize} it in the child that produces its inputs, and + * replace it with the virtual column slot. + * + *

{@link #materialize} is the only place that decides where an index search may be computed. It descends + * through Project, Filter and Join down to one OlapScan that outputs all inputs, and refuses to cross the + * NULL-extended side of an outer join unless the expression is NULL for NULL input. A MATCH that cannot be + * materialized stays where it is; a SEARCH that is left outside a scan is rejected by CheckAfterRewrite. + * + *

The rule type keeps its original name because it is referenced by disable_nereids_rules. * * Example transformation: * Before: @@ -59,7 +68,7 @@ * Project[a, b, virtual_slot_ref] * └── OlapScan[table, virtual_columns=[(col MATCH_ANY 'hello') as alias]] */ -public class PushDownMatchProjectionAsVirtualColumn implements RewriteRuleFactory { +public class PushDownIndexSearchAsVirtualColumn implements RewriteRuleFactory { private boolean canPushDown(LogicalOlapScan scan) { boolean dupTblOrMOW = scan.getTable().getKeysType() == KeysType.DUP_KEYS @@ -71,135 +80,76 @@ private boolean canPushDown(LogicalOlapScan scan) { @Override public List buildRules() { return ImmutableList.of( - // Pattern 1: Project -> OlapScan - logicalProject(logicalOlapScan().when(this::canPushDown)) - .then(project -> { - LogicalOlapScan scan = project.child(); - return pushDown(project, scan, newScan -> newScan); - }).toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), - // Pattern 2: Project -> Filter -> OlapScan - logicalProject(logicalFilter(logicalOlapScan().when(this::canPushDown))) - .then(project -> { - LogicalFilter filter = project.child(); - LogicalOlapScan scan = filter.child(); - return pushDown(project, scan, - newScan -> filter.withChildren(newScan)); - }).toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), - logicalJoin().then(this::pushDownJoin) + logicalProject().thenApply(ctx -> pushDownFromProject(ctx.root, ctx.cascadesContext)) .toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), + // A filter directly on the scan is pushed into the scan and evaluated there. logicalFilter().when(filter -> !(filter.child() instanceof LogicalOlapScan)) - .then(this::pushDownResidual) + .thenApply(ctx -> pushDownFromFilter(ctx.root, ctx.cascadesContext)) .toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN), - logicalProject().when(project -> !(project.child() instanceof LogicalOlapScan)) - .then(this::pushDownResidual) + logicalJoin().thenApply(ctx -> pushDownFromJoin(ctx.root, ctx.cascadesContext)) .toRule(RuleType.PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN) ); } - /** - * Extract MATCH projections and push them as virtual columns on the scan. - * @param childRebuilder rebuilds the project's child tree with the new scan - */ - private LogicalProject pushDown( - LogicalProject project, LogicalOlapScan scan, - Function childRebuilder) { - List projections = project.getProjects(); - List virtualColumns = new ArrayList<>(); - Map replaceMap = new HashMap<>(); - - for (NamedExpression projection : projections) { - Expression matchExpr = unwrapMatch(projection); - if (matchExpr != null && !replaceMap.containsKey(matchExpr)) { - Alias alias = new Alias(matchExpr); - replaceMap.put(matchExpr, alias.toSlot()); - virtualColumns.add(alias); - } - } - - if (virtualColumns.isEmpty()) { - return null; - } - - ImmutableList.Builder newProjections = ImmutableList.builder(); - for (NamedExpression projection : projections) { - Expression matchExpr = unwrapMatch(projection); - if (matchExpr != null && replaceMap.containsKey(matchExpr)) { - Expression slot = replaceMap.get(matchExpr); - if (projection instanceof Alias) { - newProjections.add(new Alias(((Alias) projection).getExprId(), - slot, ((Alias) projection).getName())); - } else { - newProjections.add((NamedExpression) slot); - } - } else { - newProjections.add(projection); - } - } - - LogicalOlapScan newScan = scan.appendVirtualColumns(virtualColumns); - return (LogicalProject) project.withProjectsAndChild( - newProjections.build(), childRebuilder.apply(newScan)); - } - private boolean isIndexSearch(Expression expression) { return expression instanceof Match || expression instanceof SearchExpression; } - private Plan pushDownResidual(Plan plan) { - Plan child = plan.child(0); + /** + * Materialize every index search of the consumer's expressions in one of the children, which are updated in + * place. Returns the replacement of each materialized search. + */ + private Map materializeAll(List expressions, + List children, CascadesContext context) { Map replacements = new LinkedHashMap<>(); - for (Expression expression : plan.getExpressions()) { + for (Expression expression : expressions) { for (Expression search : expression.collect(e -> isIndexSearch((Expression) e))) { if (replacements.containsKey(search)) { continue; } - Pair result = materialize(search, child); - if (result != null) { - child = result.first; - replacements.put(search, result.second); + for (int i = 0; i < children.size(); i++) { + Pair result = materialize(search, children.get(i), context); + if (result != null) { + children.set(i, result.first); + replacements.put(search, result.second); + break; + } } } } + return replacements; + } + + private Plan pushDownFromProject(LogicalProject project, CascadesContext context) { + List children = new ArrayList<>(project.children()); + Map replacements = materializeAll(project.getProjects(), children, context); if (replacements.isEmpty()) { return null; } - if (plan instanceof LogicalFilter) { - Set conjuncts = new LinkedHashSet<>(); - for (Expression expression : ((LogicalFilter) plan).getConjuncts()) { - conjuncts.add(ExpressionUtils.replace(expression, replacements)); - } - // Hide additional scan values from the original filter's consumers. - return new LogicalProject<>(ImmutableList.copyOf(plan.getOutput()), - new LogicalFilter<>(conjuncts, child)); - } - LogicalProject project = (LogicalProject) plan; List projects = new ArrayList<>(); for (NamedExpression expression : project.getProjects()) { projects.add((NamedExpression) ExpressionUtils.replace(expression, replacements)); } - return project.withProjectsAndChild(projects, child); + return project.withProjectsAndChild(projects, children.get(0)); } - private Plan pushDownJoin(LogicalJoin join) { - List children = new ArrayList<>(join.children()); - Map replacements = new LinkedHashMap<>(); - for (Expression expression : join.getExpressions()) { - for (Expression search : expression.collect(e -> isIndexSearch((Expression) e))) { - if (replacements.containsKey(search)) { - continue; - } - for (int side = 0; side < children.size(); side++) { - // Join conditions consume child values before this join's NULL extension. - // This also handles WHERE predicates moved into an inner join by rewriting. - Pair result = materialize(search, children.get(side)); - if (result != null) { - children.set(side, result.first); - replacements.put(search, result.second); - break; - } - } - } + private Plan pushDownFromFilter(LogicalFilter filter, CascadesContext context) { + List children = new ArrayList<>(filter.children()); + Map replacements = materializeAll(filter.getExpressions(), children, context); + if (replacements.isEmpty()) { + return null; } + // Hide additional scan values from the original filter's consumers. + return new LogicalProject<>(ImmutableList.copyOf(filter.getOutput()), + new LogicalFilter<>(ExpressionUtils.replace(filter.getConjuncts(), replacements), + children.get(0))); + } + + private Plan pushDownFromJoin(LogicalJoin join, CascadesContext context) { + // Join conditions consume child values before this join's NULL extension, so they are materialized + // directly in the children. This also handles WHERE predicates moved into an inner join by rewriting. + List children = new ArrayList<>(join.children()); + Map replacements = materializeAll(join.getExpressions(), children, context); if (replacements.isEmpty()) { return null; } @@ -211,7 +161,12 @@ private Plan pushDownJoin(LogicalJoin join) { return new LogicalProject<>(ImmutableList.copyOf(join.getOutput()), rewritten); } - private Pair materialize(Expression expression, Plan plan) { + /** + * Compute the index search in the OlapScan below the plan that produces all its inputs, reusing an equal + * virtual column of that scan. Returns the rewritten plan and the slot, part of its output, that carries the + * value; or null when the search cannot be computed below the plan. + */ + private Pair materialize(Expression expression, Plan plan, CascadesContext context) { Set inputs = expression.getInputSlots(); if (inputs.isEmpty() || !plan.getOutputSet().containsAll(inputs)) { return null; @@ -234,17 +189,26 @@ private Pair materialize(Expression expression, Plan plan) { if (project.containsNoneMovableFunction()) { return null; } - Expression rewritten = ExpressionUtils.replace(expression, project.getAliasToProducer()); - Pair result = materialize(rewritten, project.child()); + // Like PushDownFilterThroughProject: inlining a volatile producer would evaluate it a second time. + Map aliasToProducer = project.getAliasToProducer(); + if (inputs.stream().map(aliasToProducer::get) + .anyMatch(producer -> producer != null && producer.containsVolatileExpression())) { + return null; + } + Expression rewritten = ExpressionUtils.replace(expression, aliasToProducer); + Pair result = materialize(rewritten, project.child(), context); if (result == null) { return null; } List projects = new ArrayList<>(project.getProjects()); - projects.add(result.second); + // A reused virtual column may already pass through this project. + if (!project.getOutputSet().contains(result.second)) { + projects.add(result.second); + } return Pair.of(project.withProjectsAndChild(projects, result.first), result.second); } if (plan instanceof LogicalFilter) { - Pair result = materialize(expression, plan.child(0)); + Pair result = materialize(expression, plan.child(0), context); return result == null ? null : Pair.of(plan.withChildren(result.first), result.second); } if (plan instanceof LogicalJoin) { @@ -255,12 +219,13 @@ private Pair materialize(Expression expression, Plan plan) { } boolean nullExtended = side == 0 ? join.getJoinType().isLeftSideNullable() : join.getJoinType().isRightSideNullable(); - // SEARCH has DSL-level existence and negation semantics. Do not assume - // every DSL node is NULL-propagating across an outer join. - if (nullExtended && expression instanceof SearchExpression) { + // The join turns a value computed below it into NULL for NULL-extended rows, so only + // an expression that is itself NULL for NULL input may be computed there. SEARCH has + // DSL-level existence and negation semantics and is not NULL-propagating. + if (nullExtended && !ExpressionUtils.isNullPropagating(expression, context)) { return null; } - Pair result = materialize(expression, join.child(side)); + Pair result = materialize(expression, join.child(side), context); if (result == null) { return null; } @@ -277,16 +242,4 @@ private Pair materialize(Expression expression, Plan plan) { } return null; } - - /** - * Unwrap a Match expression from a projection. - * Returns the Match expression if the projection is a Match directly or an Alias wrapping a Match. - * Returns null otherwise. - */ - private Expression unwrapMatch(NamedExpression projection) { - if (projection instanceof Alias && isIndexSearch(((Alias) projection).child())) { - return ((Alias) projection).child(); - } - return null; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java index c31a77bf8253a5..87b0c6f21faa14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlots.java @@ -22,9 +22,15 @@ import org.apache.doris.catalog.Index; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.info.IndexType; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; +import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SearchExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -34,26 +40,31 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; +import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** - * Rewrite search function to add proper slot reference children. + * Bind the fields of a search() function: resolve each DSL field reference to a slot of the filter's child and + * produce a SearchExpression whose children are those slots (or variant subcolumns of them). * This is crucial for BE's "action on slot" detection in normalize conjuncts. + * + *

The steps are kept apart on purpose. SearchDslParser owns the DSL syntax (field path, {@code @analyzer} + * selector, escapes). Field names resolve exactly like SQL column references, so {@code alias.field} selects a + * relation of a join before {@code column.subcolumn} is tried. Index validation and the field names sent to BE + * use the physical column behind the slot, never its output alias. Where a SEARCH may be evaluated is not + * decided here: PushDownIndexSearchAsVirtualColumn places it and CheckAfterRewrite verifies the final plan. */ public class RewriteSearchToSlots extends OneRewriteRuleFactory { private static final Logger LOG = LogManager.getLogger(RewriteSearchToSlots.class); @@ -62,15 +73,15 @@ public class RewriteSearchToSlots extends OneRewriteRuleFactory { public Rule build() { return logicalFilter() .when(filter -> ExpressionUtils.containsTypes(filter.getExpressions(), Search.class)) - .then(this::rewriteSearchExpressions) + .thenApply(ctx -> rewriteSearchExpressions(ctx.root, ctx.cascadesContext)) .toRule(RuleType.REWRITE_SEARCH_TO_SLOTS); } - private Plan rewriteSearchExpressions(LogicalFilter filter) { + private Plan rewriteSearchExpressions(LogicalFilter filter, CascadesContext cascadesContext) { List newExpressions = new ArrayList<>(); for (Expression expr : filter.getExpressions()) { - Expression rewritten = rewriteExpression(expr, filter.child()); + Expression rewritten = rewriteExpression(expr, filter.child(), cascadesContext); newExpressions.add(rewritten); } @@ -81,14 +92,14 @@ private Plan rewriteSearchExpressions(LogicalFilter filter) { return filter; } - private Expression rewriteExpression(Expression expr, Plan scan) { + private Expression rewriteExpression(Expression expr, Plan child, CascadesContext cascadesContext) { if (expr instanceof Search) { - return rewriteSearch((Search) expr, scan); + return rewriteSearch((Search) expr, child, cascadesContext); } // Recursively process children List newChildren = expr.children().stream() - .map(child -> rewriteExpression(child, scan)) + .map(c -> rewriteExpression(c, child, cascadesContext)) .collect(Collectors.toList()); if (!newChildren.equals(expr.children())) { @@ -98,7 +109,7 @@ private Expression rewriteExpression(Expression expr, Plan scan) { return expr; } - private Expression rewriteSearch(Search search, Plan scan) { + private Expression rewriteSearch(Search search, Plan child, CascadesContext cascadesContext) { try { // Parse DSL to get field bindings SearchDslParser.QsPlan qsPlan = search.getQsPlan(); @@ -108,95 +119,41 @@ private Expression rewriteSearch(Search search, Plan scan) { } Map normalizedFields = new HashMap<>(); - Map fieldAnalyzers = new HashMap<>(); - Set> qualifiers = new HashSet<>(); + // physical field -> the inverted index its analyzer selects + Map fieldIndexes = new HashMap<>(); + Scope scope = new Scope(child.getOutput()); + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(child, scope, cascadesContext, false, false); // Create slot reference children from field bindings List slotChildren = new ArrayList<>(); for (SearchDslParser.QsFieldBinding binding : qsPlan.getFieldBindings()) { - String bindingName = binding.getFieldName(); - String originalFieldName = bindingName; - int analyzerSeparator = bindingName.lastIndexOf('@'); - while (analyzerSeparator > 0 && bindingName.charAt(analyzerSeparator - 1) == '\\') { - analyzerSeparator = bindingName.lastIndexOf('@', analyzerSeparator - 1); - } - if (analyzerSeparator >= 0 && findSlotByName(bindingName, scan) == null) { - originalFieldName = bindingName.substring(0, analyzerSeparator); - String analyzer = bindingName.substring(analyzerSeparator + 1); - if (originalFieldName.isEmpty() || analyzer.isEmpty()) { - throw new AnalysisException("SEARCH analyzer selector must be field@analyzer: " + bindingName); - } - binding.setAnalyzerName(analyzer); - } - originalFieldName = originalFieldName.replace("\\@", "@"); - Expression childExpr; - String normalizedFieldName; - - if (originalFieldName.contains(".")) { - int firstDotPos = originalFieldName.indexOf('.'); - String parentFieldName = originalFieldName.substring(0, firstDotPos); - String subcolumnPath = originalFieldName.substring(firstDotPos + 1); - - // Find parent slot - Slot parentSlot = findSlotByName(parentFieldName, scan); - if (parentSlot == null) { - throw new AnalysisException(String.format( - "Parent field '%s' not found in table for search: %s", - parentFieldName, search.getDslString())); - } - - // Verify it's a variant type - if (!parentSlot.getDataType().isVariantType()) { - throw new AnalysisException(String.format( - "Field '%s' is not VARIANT type for subcolumn access: %s", - parentFieldName, search.getDslString())); - } - String normalizedParentFieldName = parentSlot.getName(); - - // Check the parent variant column has at least one INVERTED index. The concrete - // subcolumn binding is resolved per-segment in BE, so we only enforce the parent - // level here. See function_search.cpp is_variant_sub branch. - checkInvertedIndexExists(tableForSlot(parentSlot, scan), normalizedParentFieldName, - search.getDslString(), true); - - // Create ElementAt expression for variant subcolumn + String fieldReference = binding.getFieldName(); + Pair pathAndAnalyzer = SearchDslParser.splitAnalyzerSelector(fieldReference); + binding.setAnalyzerName(pathAndAnalyzer.second); + + Pair> field = resolveField(pathAndAnalyzer.first, child, analyzer, + scope, search.getDslString()); + SlotReference slot = field.first; + List subPath = field.second; + Index index = checkInvertedIndex(slot, subPath, binding.getAnalyzerName(), search.getDslString()); + + String normalizedFieldName = physicalFieldName(slot, subPath); + Expression childExpr = slot; + if (!subPath.isEmpty()) { // This will be converted to an extracted column slot by VariantSubPathPruning rule // If the subcolumn doesn't exist, ElementAt will remain and BE will handle it gracefully - childExpr = new ElementAt(parentSlot, new StringLiteral(subcolumnPath)); - normalizedFieldName = normalizedParentFieldName + "." + subcolumnPath; - - LOG.info( - "Created ElementAt expression for variant subcolumn: parent='{}', " - + "subcolumn='{}', field_name='{}'", - normalizedParentFieldName, subcolumnPath, normalizedFieldName); - } else { - // Normal field - find slot directly - Slot slot = findSlotByName(originalFieldName, scan); - if (slot == null) { - throw new AnalysisException(String.format( - "Field '%s' not found in table for search: %s", - originalFieldName, search.getDslString())); - } - checkInvertedIndexExists(tableForSlot(slot, scan), slot.getName(), search.getDslString(), false); - childExpr = slot; - normalizedFieldName = slot.getName(); + childExpr = new ElementAt(slot, new StringLiteral(String.join(".", subPath))); } - for (Slot input : childExpr.getInputSlots()) { - qualifiers.add(input.getQualifier()); - } - if (qualifiers.size() > 1) { - throw new AnalysisException("Each SEARCH expression must reference fields from one table; " - + "combine separate SEARCH expressions with SQL AND/OR"); - } - String fieldKey = normalizedFieldName.toLowerCase(Locale.ROOT); - if (fieldAnalyzers.containsKey(fieldKey) - && !Objects.equals(fieldAnalyzers.get(fieldKey), binding.getAnalyzerName())) { + // BE keeps one index per field, so two references to a field must select the same index. + // Comparing the selected indexes reuses the analyzer identity of the index lookup itself. + if (fieldIndexes.containsKey(normalizedFieldName) + && fieldIndexes.get(normalizedFieldName) != index) { throw new AnalysisException("SEARCH supports one analyzer per field; use separate SEARCH " + "expressions for different analyzers on " + normalizedFieldName); } - fieldAnalyzers.put(fieldKey, binding.getAnalyzerName()); - normalizedFields.put(bindingName, normalizedFieldName); + fieldIndexes.put(normalizedFieldName, index); + normalizedFields.put(fieldReference, normalizedFieldName); binding.setFieldName(normalizedFieldName); slotChildren.add(childExpr); } @@ -204,7 +161,12 @@ private Expression rewriteSearch(Search search, Plan scan) { LOG.info("Rewriting search function: dsl='{}' with {} slot children", search.getDslString(), slotChildren.size()); - normalizePlanFields(qsPlan.getRoot(), normalizedFields); + normalizePlanFields(qsPlan.getRoot(), normalizedFields, + nestedPath -> { + Pair> nested = resolveField(nestedPath, child, analyzer, scope, + search.getDslString()); + return physicalFieldName(nested.first, nested.second); + }); // Create SearchExpression with slot children return new SearchExpression(search.getDslString(), qsPlan, slotChildren); @@ -215,91 +177,117 @@ private Expression rewriteSearch(Search search, Plan scan) { } /** - * Ensure the column referenced by a Lucene-syntax SEARCH predicate has an inverted index. - * Without this check the BE path would silently fall back to an empty bitmap (i.e. all FALSE), - * which is indistinguishable from "no rows matched" to the user. Throw at planning time so the - * behavior is consistent with referencing a non-existent column. - * - * @param table table backing the LogicalOlapScan - * @param columnName column name (parent column name when isVariantParent) - * @param dsl original DSL, used in the error message - * @param isVariantParent true when {@code columnName} is the parent of a variant subcolumn - * access (e.g. {@code msg.body}); for that case any INVERTED index on - * the parent column is accepted because the concrete subcolumn binding - * is resolved per-segment in BE. + * Resolve a DSL field path against the child's output with the SQL name resolution rules + * (ExpressionAnalyzer#bindSlotByScope): {@code field}, {@code alias.field}, {@code db.tbl.field}, each + * optionally followed by a variant subcolumn path. Returns the slot and the subcolumn path (empty for a + * plain column). */ - private void checkInvertedIndexExists(OlapTable table, String columnName, String dsl, - boolean isVariantParent) { - Column column = table.getColumn(columnName); - if (column == null) { - // Field existence is already validated by findSlotByName; if we reach here the schema - // changed concurrently. Surface a clear error rather than fall through. - throw new AnalysisException(String.format( - "Column '%s' not found in table '%s' for search: %s", - columnName, table.getName(), dsl)); + private Pair> resolveField(String fieldPath, Plan child, + ExpressionAnalyzer analyzer, Scope scope, String dsl) { + List nameParts = Arrays.asList(fieldPath.split("\\.", -1)); + if (nameParts.stream().anyMatch(String::isEmpty)) { + throw new AnalysisException(String.format("Invalid field '%s' for search: %s", fieldPath, dsl)); + } + List candidates = analyzer.bindSlotByScope(new UnboundSlot(nameParts), scope) + .stream().distinct().collect(Collectors.toList()); + if (candidates.isEmpty()) { + throw new AnalysisException(String.format("Field '%s' not found in table for search: %s", + fieldPath, dsl)); + } + if (candidates.size() > 1) { + throw new AnalysisException(String.format("Ambiguous field '%s' in search(); qualify it with its " + + "table alias, as in a SQL column reference: %s", fieldPath, dsl)); } - if (isVariantParent) { - for (Index index : table.getIndexes()) { - if (index.getIndexType() != IndexType.INVERTED) { - continue; - } - List columns = index.getColumns(); - if (columns != null && !columns.isEmpty() - && columnName.equalsIgnoreCase(columns.get(0))) { - return; - } + // A nested reference is bound as Alias(element_at(element_at(slot, 'a'), 'b')). + Expression bound = candidates.get(0) instanceof Alias ? candidates.get(0).child(0) : candidates.get(0); + List subPath = new ArrayList<>(); + while (bound instanceof ElementAt) { + subPath.add(0, ((StringLiteral) bound.child(1)).getStringValue()); + bound = bound.child(0); + } + SlotReference boundSlot = (SlotReference) bound; + if (!subPath.isEmpty() && !boundSlot.getDataType().isVariantType()) { + throw new AnalysisException(String.format( + "Field '%s' is not VARIANT type for subcolumn access: %s", boundSlot.getName(), dsl)); + } + // bindSlotByScope renames the slot to the spelling of the reference; keep the child's own slot. + for (Slot output : child.getOutput()) { + if (output.getExprId().equals(boundSlot.getExprId())) { + return Pair.of((SlotReference) output, subPath); } - } else if (table.getInvertedIndex(column, null) != null) { - return; } - - throw new AnalysisException(String.format( - "Field '%s' has no inverted index, cannot be used in search: %s. " - + "Create an inverted index on the column first " - + "(ALTER TABLE ... ADD INDEX ... USING INVERTED).", - columnName, dsl)); + throw new AnalysisException(String.format("Field '%s' not found in table for search: %s", fieldPath, dsl)); } - private Slot findSlotByName(String fieldName, Plan scan) { - Slot result = null; - for (Slot slot : scan.getOutput()) { - if (slot.getName().equalsIgnoreCase(fieldName)) { - if (result != null) { - throw new AnalysisException("Ambiguous field '" + fieldName + "' in search()"); - } - result = slot; - } - } - return result; + // BE looks fields up in the tablet schema, so it gets the physical column name, never an output alias. + private String physicalFieldName(SlotReference slot, List subPath) { + String columnName = slot.getOriginalColumn().map(Column::getName).orElse(slot.getName()); + return subPath.isEmpty() ? columnName : columnName + "." + String.join(".", subPath); } - private OlapTable tableForSlot(Slot slot, Plan plan) { - if (plan instanceof LogicalOlapScan) { - return ((LogicalOlapScan) plan).getTable(); + /** + * Ensure the physical column referenced by a Lucene-syntax SEARCH predicate has an inverted index, and return + * the index the analyzer selects (OlapTable#getInvertedIndex, the lookup the translator sends to BE; null + * when a variant subcolumn gets its index per segment in BE). + * Without this check the BE path would silently fall back to an empty bitmap (i.e. all FALSE), + * which is indistinguishable from "no rows matched" to the user. Throw at planning time so the + * behavior is consistent with referencing a non-existent column. + * + * @param slot resolved field; its original table and column identify the physical column, whatever + * alias the slot carries + * @param subPath variant subcolumn path, empty for a plain column. For a subcolumn any INVERTED index on + * the parent column is accepted because the concrete subcolumn binding is resolved + * per-segment in BE. See function_search.cpp is_variant_sub branch. + * @param analyzer analyzer selected by {@code field@analyzer}, or null + * @param dsl original DSL, used in the error message + */ + private Index checkInvertedIndex(SlotReference slot, List subPath, String analyzer, String dsl) { + if (!(slot.getOriginalTable().orElse(null) instanceof OlapTable) || !slot.getOriginalColumn().isPresent()) { + throw new AnalysisException("search() requires a field from an OLAP table: " + slot.toSql()); + } + OlapTable table = (OlapTable) slot.getOriginalTable().get(); + Column column = slot.getOriginalColumn().get(); + Index index = table.getInvertedIndex(column, subPath, analyzer); + if (index == null && analyzer != null) { + throw new AnalysisException(String.format( + "No inverted index found for SEARCH analyzer '%s' on field '%s': %s", + analyzer, column.getName(), dsl)); } - if (slot instanceof SlotReference - && ((SlotReference) slot).getOriginalTable().orElse(null) instanceof OlapTable) { - return (OlapTable) ((SlotReference) slot).getOriginalTable().get(); + boolean hasIndex = index != null; + if (!subPath.isEmpty()) { + hasIndex = table.getIndexes().stream().anyMatch(i -> i.getIndexType() == IndexType.INVERTED + && i.getColumns() != null && !i.getColumns().isEmpty() + && column.getName().equalsIgnoreCase(i.getColumns().get(0))); } - throw new AnalysisException("search() requires a field from an OLAP table: " + slot.toSql()); + if (!hasIndex) { + throw new AnalysisException(String.format( + "Field '%s' has no inverted index, cannot be used in search: %s. " + + "Create an inverted index on the column first " + + "(ALTER TABLE ... ADD INDEX ... USING INVERTED).", + column.getName(), dsl)); + } + return index; } - private void normalizePlanFields(SearchDslParser.QsNode node, Map normalized) { + private void normalizePlanFields(SearchDslParser.QsNode node, Map normalized, + Function nestedPathNormalizer) { if (node == null) { return; } - if (node.getField() != null) { - for (Map.Entry entry : normalized.entrySet()) { - if (entry.getKey().equalsIgnoreCase(node.getField())) { - node.setField(entry.getValue()); - break; - } - } + // Variant subcolumn paths are case sensitive, so match the reference exactly as the binding was named. + if (node.getField() != null && !node.getField().isEmpty()) { + // The parser names every binding after a node's field. + Preconditions.checkState(normalized.containsKey(node.getField()), + "SEARCH field %s has no binding", node.getField()); + node.setField(normalized.get(node.getField())); + } + if (node.getNestedPath() != null) { + node.setNestedPath(nestedPathNormalizer.apply(node.getNestedPath())); } if (node.getChildren() != null) { for (SearchDslParser.QsNode child : node.getChildren()) { - normalizePlanFields(child, normalized); + normalizePlanFields(child, normalized, nestedPathNormalizer); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java index 1f7bba1e3b932e..b4011a9539e4d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java @@ -32,6 +32,11 @@ /** * SearchExpression represents a search query with bound slot references. * This is created by RewriteSearchToSlots rule from Search scalar function. + * + *

Each child binds one DSL field, in the order of the QsPlan field bindings: a slot, or element_at on a slot + * for a variant subcolumn. BE evaluates the expression only with the inverted indexes of those fields inside an + * OLAP scan, never row by row. Rewrites are free to move the expression; CheckAfterRewrite verifies on the final + * plan that it sits in a scan (filter conjunct or virtual column) and that {@link #bindsOnlyFields()} holds. */ public class SearchExpression extends Expression { private final String dslString; @@ -74,19 +79,35 @@ public boolean foldable() { @Override public SearchExpression withChildren(List children) { - // Null-rejection inference temporarily replaces input slots with NULL. - // Such symbolic expressions are not execution-time field bindings. + // Rewrites may replace a field with NULL: null-rejection inference does so on a temporary copy, and + // NULL padding of an outer join side (e.g. ON false, outer-to-anti join) does so in the plan. + // Such a SEARCH no longer binds an index field; CheckAfterRewrite rejects it via bindsOnlyFields. for (Expression child : children) { if (!(child instanceof SlotReference || child instanceof ElementAt || child instanceof NullLiteral)) { throw new IllegalArgumentException( - "SEARCH field binding must be a slot, subcolumn, or inference NULL, found " + "SEARCH field binding must be a slot, subcolumn, or NULL, found " + child.getClass().getSimpleName()); } } return new SearchExpression(dslString, qsPlan, children); } + /** + * Whether every child still binds an index field: a slot, or a variant subcolumn of a slot. + */ + public boolean bindsOnlyFields() { + return children().stream().allMatch(SearchExpression::isFieldBinding); + } + + private static boolean isFieldBinding(Expression expression) { + Expression current = expression; + while (current instanceof ElementAt) { + current = current.child(0); + } + return current instanceof SlotReference; + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitSearchExpression(this, context); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java index 40ce885224140b..5cd52f5f3b4a87 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java @@ -89,8 +89,7 @@ public void testSearchInGroupByRejected() { Assertions.assertTrue( exception.getMessage().contains("search()") && (exception.getMessage().contains("GROUP BY") - || exception.getMessage().contains("WHERE filters") - || exception.getMessage().contains("single-table")), + || exception.getMessage().contains("WHERE filters")), "Expected error about search() usage restrictions, got: " + exception.getMessage()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumnTest.java similarity index 95% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumnTest.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumnTest.java index 55735021e78480..a91a5cf395279c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMatchProjectionAsVirtualColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownIndexSearchAsVirtualColumnTest.java @@ -46,9 +46,9 @@ import java.util.stream.Collectors; /** - * Test for PushDownMatchProjectionAsVirtualColumn rule. + * Test for PushDownIndexSearchAsVirtualColumn rule. */ -public class PushDownMatchProjectionAsVirtualColumnTest implements MemoPatternMatchSupported { +public class PushDownIndexSearchAsVirtualColumnTest implements MemoPatternMatchSupported { @Test void testPushDownMatchProjection() { @@ -62,7 +62,7 @@ void testPushDownMatchProjection() { ImmutableList.of(idSlot, new Alias(matchExpr, "m")), scan); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); // Verify plan structure @@ -105,7 +105,7 @@ void testPushDownMatchProjectionWithFilter() { new LogicalFilter<>(ImmutableSet.of(filterPred), scan)); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); // Verify plan structure: Project -> Filter -> OlapScan @@ -141,7 +141,7 @@ void testNoMatchExpressionNoChange() { ImmutableList.of(idSlot), scan); PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .matches( logicalProject( logicalOlapScan().when(s -> s.getVirtualColumns().isEmpty()) @@ -163,7 +163,7 @@ void testDuplicateMatchDedup() { scan); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); Assertions.assertInstanceOf(LogicalProject.class, root); @@ -201,7 +201,7 @@ void testMultipleDistinctMatchExpressions() { scan); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); Assertions.assertInstanceOf(LogicalProject.class, root); @@ -245,7 +245,7 @@ void testAppendToExistingVirtualColumns() { ImmutableList.of(idSlot, new Alias(matchExpr, "m")), scanWithVc); Plan root = PlanChecker.from(MemoTestUtils.createConnectContext(), project) - .applyTopDown(new PushDownMatchProjectionAsVirtualColumn()) + .applyTopDown(new PushDownIndexSearchAsVirtualColumn()) .getPlan(); Assertions.assertInstanceOf(LogicalProject.class, root); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java index 46cb3c8b036fba..9883eaf9f9e8b8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/RewriteSearchToSlotsTest.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.TableIndexes; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.IndexType; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.trees.expressions.Expression; @@ -245,10 +246,10 @@ public void testRewriteSearchHandlesCaseInsensitiveField() throws Exception { Search searchFunc = new Search(new StringLiteral("NAME:alice")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, Plan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); - Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan); + Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan, null); Assertions.assertInstanceOf(SearchExpression.class, rewritten); SearchExpression searchExpression = (SearchExpression) rewritten; @@ -269,10 +270,10 @@ public void testRewriteSearchHandlesCaseInsensitiveVariantParentField() throws E Search searchFunc = new Search(new StringLiteral("V.foo:bar")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, Plan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); - Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan); + Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan, null); Assertions.assertInstanceOf(SearchExpression.class, rewritten); SearchExpression searchExpression = (SearchExpression) rewritten; @@ -294,11 +295,11 @@ public void testRewriteSearchThrowsWhenFieldMissing() throws Exception { Search searchFunc = new Search(new StringLiteral("unknown_field:value")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, Plan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); InvocationTargetException thrown = Assertions.assertThrows(InvocationTargetException.class, - () -> rewriteMethod.invoke(rewriteRule, searchFunc, scan)); + () -> rewriteMethod.invoke(rewriteRule, searchFunc, scan, null)); Assertions.assertNotNull(thrown.getCause()); Assertions.assertInstanceOf(AnalysisException.class, thrown.getCause()); Assertions.assertTrue(thrown.getCause().getMessage().contains("unknown_field")); @@ -313,11 +314,11 @@ public void testRewriteSearchThrowsWhenColumnHasNoInvertedIndex() throws Excepti Search searchFunc = new Search(new StringLiteral("name:alice")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, Plan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); InvocationTargetException thrown = Assertions.assertThrows(InvocationTargetException.class, - () -> rewriteMethod.invoke(rewriteRule, searchFunc, scan)); + () -> rewriteMethod.invoke(rewriteRule, searchFunc, scan, null)); Assertions.assertNotNull(thrown.getCause()); Assertions.assertInstanceOf(AnalysisException.class, thrown.getCause()); Assertions.assertTrue(thrown.getCause().getMessage().contains("inverted index"), @@ -332,10 +333,10 @@ public void testRewriteSearchSucceedsWhenColumnHasInvertedIndex() throws Excepti Search searchFunc = new Search(new StringLiteral("name:alice")); Method rewriteMethod = RewriteSearchToSlots.class.getDeclaredMethod( - "rewriteSearch", Search.class, Plan.class); + "rewriteSearch", Search.class, Plan.class, CascadesContext.class); rewriteMethod.setAccessible(true); - Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan); + Object rewritten = rewriteMethod.invoke(rewriteRule, searchFunc, scan, null); Assertions.assertInstanceOf(SearchExpression.class, rewritten); SearchExpression searchExpression = (SearchExpression) rewritten; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java index f56957ddedd49c..d22b03bffdaa15 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SearchJoinDocumentTest.java @@ -17,16 +17,27 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.analysis.SearchDslParser.QsFieldBinding; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + public class SearchJoinDocumentTest extends TestWithFeService { @Override protected void runBeforeAll() throws Exception { @@ -45,6 +56,15 @@ protected void runBeforeAll() throws Exception { createTable("CREATE TABLE associations (to_id BIGINT, from_id BIGINT) " + "DUPLICATE KEY(to_id) DISTRIBUTED BY HASH(to_id) BUCKETS 1 " + "PROPERTIES('replication_num'='1')"); + createTable("CREATE TABLE notes (id BIGINT, title TEXT, " + + "INDEX idx_title(title) USING INVERTED PROPERTIES('parser'='english')) " + + "DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); + createTable("CREATE TABLE docs (id BIGINT, title TEXT, " + + "INDEX idx_title_en(title) USING INVERTED PROPERTIES('parser'='english'), " + + "INDEX idx_title_uni(title) USING INVERTED PROPERTIES('parser'='unicode')) " + + "DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1')"); } @Test @@ -93,8 +113,170 @@ public void testMatchResidualUsesVirtualColumn() { @Test public void testAmbiguousSearchFieldRejected() { - Assertions.assertThrows(AnalysisException.class, () -> PlanChecker.from(connectContext).analyze( + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze( "SELECT a.id FROM objects a JOIN objects b ON a.id=b.id WHERE search('v.string_8:john')")); + Assertions.assertTrue(exception.getMessage().contains("Ambiguous field 'v.string_8'"), + exception.getMessage()); + } + + @Test + public void testSelfJoinFieldQualifiedByTableAlias() { + // alias.field resolves like a SQL column reference, so each side of a self join can be searched. + for (String side : new String[] {"a", "b"}) { + String otherSide = side.equals("a") ? "b" : "a"; + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT a.id FROM notes a JOIN notes b ON a.id=b.id " + + "WHERE search('" + side + ".title:john') OR " + otherSide + ".id=1", planner -> { + List scans = scansWithSearchVirtualColumn(planner.getPhysicalPlan()); + Assertions.assertEquals(1, scans.size(), planner.getPhysicalPlan().treeString()); + SearchExpression search = searchExpressions(planner.getPhysicalPlan()).get(0); + Assertions.assertTrue(((PhysicalOlapScan) scans.get(0)).getOutputSet() + .containsAll(search.getInputSlots())); + Assertions.assertEquals(ImmutableList.of(side), + search.getInputSlots().stream().map(slot -> slot.getQualifier() + .get(slot.getQualifier().size() - 1)).collect(Collectors.toList())); + }); + } + // alias.variant.subcolumn + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT a.id FROM objects a JOIN objects b ON a.id=b.id WHERE search('b.v.string_8:john')", + planner -> Assertions.assertEquals("v.string_8", searchExpressions(planner.getPhysicalPlan()) + .get(0).getQsPlan().getFieldBindings().get(0).getFieldName())); + } + + @Test + public void testFieldAliasBindsPhysicalColumn() { + // The DSL names the visible column; index validation and the field sent to BE use the physical column. + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT n.id FROM (SELECT id, title AS headline FROM notes) n " + + "LEFT JOIN lists l ON n.id=l.object_id WHERE search('headline:john') OR l.list_id=12", + planner -> assertSingleBinding(planner.getPhysicalPlan(), "title")); + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM (SELECT id, v AS props FROM objects) o " + + "JOIN lists l ON o.id=l.object_id WHERE search('props.string_8:john')", + planner -> assertSingleBinding(planner.getPhysicalPlan(), "v.string_8")); + // An alias that hides a column without an inverted index must not pass as the indexed column. + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze( + "SELECT id FROM (SELECT o.id, l.list_id AS title FROM notes o JOIN lists l ON o.id=l.object_id) t " + + "WHERE search('title:john')")); + Assertions.assertTrue(exception.getMessage().contains("'list_id' has no inverted index"), + exception.getMessage()); + } + + @Test + public void testNestedPathBindsPhysicalColumn() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT o.id FROM (SELECT id, v AS props FROM objects) o JOIN lists l ON o.id=l.object_id " + + "WHERE search('NESTED(props.items, name:john)')", planner -> { + SearchExpression search = searchExpressions(planner.getPhysicalPlan()).get(0); + Assertions.assertEquals("v.items", search.getQsPlan().getRoot().getNestedPath()); + Assertions.assertEquals("v.items.name", + search.getQsPlan().getFieldBindings().get(0).getFieldName()); + }); + } + + @Test + public void testSearchIsNotInferredForEqualColumn() { + // a.title = b.title lets predicates on a.title be inferred for b.title, but a SEARCH is bound to the + // inverted index of a.title and must stay there. + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT a.id FROM notes a JOIN docs b ON a.title=b.title " + + "WHERE search('a.title:john') OR a.title='x'", planner -> { + List searches = searchExpressions(planner.getPhysicalPlan()); + Assertions.assertEquals(1, searches.size(), planner.getPhysicalPlan().treeString()); + Slot field = searches.get(0).getInputSlots().iterator().next(); + Assertions.assertEquals("a", field.getQualifier().get(field.getQualifier().size() - 1)); + }); + } + + @Test + public void testSearchIsNotInferredThroughSetOperation() { + PlanChecker.from(connectContext).checkPlannerResult( + "(SELECT title FROM notes WHERE search('title:john')) INTERSECT (SELECT title FROM docs)", + planner -> Assertions.assertEquals(1, searchExpressions(planner.getPhysicalPlan()).size(), + planner.getPhysicalPlan().treeString())); + } + + @Test + public void testAnalyzerSelectorUsesIndexIdentity() { + // The selector is matched like the index lookup: case-insensitively, and the default analyzer of a + // field is the index it resolves to. + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT id FROM docs WHERE search('title@English:john AND title@english:smith')"); + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT id FROM notes WHERE search('title:john AND title@english:smith')"); + AnalysisException conflict = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze( + "SELECT id FROM docs WHERE search('title@english:john AND title@unicode:smith')")); + Assertions.assertTrue(conflict.getMessage().contains("one analyzer per field"), conflict.getMessage()); + AnalysisException missing = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze( + "SELECT id FROM docs WHERE search('title@chinese:john')")); + Assertions.assertTrue(missing.getMessage().contains("No inverted index found for SEARCH analyzer"), + missing.getMessage()); + } + + @Test + public void testNestedIndexSearchInProjectionUsesVirtualColumn() { + // Direct scans and joins share one materialization path, so a wrapped MATCH is handled by both. + for (String from : new String[] {"notes n", "notes n LEFT JOIN lists l ON n.id=l.object_id"}) { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT n.id, CASE WHEN n.title MATCH_ANY 'john' THEN 1 ELSE 0 END FROM " + from, + planner -> { + Assertions.assertTrue(hasMatchVirtualColumn(planner.getPhysicalPlan()), + planner.getPhysicalPlan().treeString()); + assertNoIndexSearchOutsideScan(planner.getPhysicalPlan()); + }); + } + } + + @Test + public void testVirtualColumnReusedBySelectAndWhere() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT n.id, n.title MATCH_ANY 'john' FROM notes n LEFT JOIN lists l ON n.id=l.object_id " + + "WHERE n.title MATCH_ANY 'john' OR l.list_id=12", planner -> { + Plan plan = planner.getPhysicalPlan(); + List scans = new ArrayList<>(); + collectPlans(plan, scans); + scans.removeIf(node -> !(node instanceof PhysicalOlapScan) + || ((PhysicalOlapScan) node).getVirtualColumns().isEmpty()); + Assertions.assertEquals(1, scans.size(), plan.treeString()); + Assertions.assertEquals(1, ((PhysicalOlapScan) scans.get(0)).getVirtualColumns().size(), + plan.treeString()); + assertNoIndexSearchOutsideScan(plan); + List nodes = new ArrayList<>(); + collectPlans(plan, nodes); + for (Plan node : nodes) { + List output = node.getOutput(); + Assertions.assertEquals(new HashSet<>(output).size(), output.size(), + "duplicate output slot in " + node + "\n" + plan.treeString()); + } + }); + } + + @Test + public void testIndexSearchNotReachingScanIsRejected() { + // The filter cannot move below the TopN and materialization does not cross it. + String from = "FROM (SELECT id, title FROM notes ORDER BY id LIMIT 10) t WHERE "; + assertRejected("SELECT id " + from + "search('title:john')", "SEARCH must be evaluated by an OLAP scan"); + assertRejected("SELECT id " + from + "title MATCH_ANY 'john'", "only support in olapScan filter"); + // Fields of two tables can never be evaluated by one scan. + assertRejected("SELECT n.id FROM notes n JOIN objects o ON n.id=o.id " + + "WHERE search('title:john OR v.string_8:john')", "SEARCH must be evaluated by an OLAP scan"); + } + + @Test + public void testPushDownIsIdempotent() { + PlanChecker checker = PlanChecker.from(connectContext).analyze( + "SELECT n.id, CASE WHEN n.title MATCH_ANY 'john' THEN 1 ELSE 0 END FROM notes n " + + "LEFT JOIN lists l ON n.id=l.object_id WHERE search('title:smith') OR l.list_id=12").rewrite(); + String rewritten = checker.getPlan().treeString(); + Assertions.assertTrue(checker.getPlan().anyMatch(node -> node instanceof LogicalOlapScan + && ((LogicalOlapScan) node).getVirtualColumns().size() == 2), rewritten); + Assertions.assertEquals(rewritten, + checker.applyTopDown(new PushDownIndexSearchAsVirtualColumn()).getPlan().treeString()); } @Test @@ -129,4 +311,108 @@ public void testSearchOrMovedIntoInnerJoin() { planner.getPhysicalPlan().treeString()); }); } + + @Test + public void testNullPropagatingMatchOnNullExtendedSideUsesVirtualColumn() { + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT l.object_id FROM lists l LEFT JOIN objects o ON l.object_id=o.id " + + "WHERE CAST(o.v['string_8'] AS VARCHAR) MATCH_ANY 'john' OR l.list_id=12", planner -> { + Assertions.assertTrue(hasMatchVirtualColumn(planner.getPhysicalPlan()), + planner.getPhysicalPlan().treeString()); + }); + } + + @Test + public void testNonNullPropagatingMatchStaysAboveOuterJoin() { + // nvl(NULL, 'john') matches, so the MATCH cannot be computed below the NULL-extended side. + PlanChecker.from(connectContext).checkPlannerResult( + "SELECT l.object_id FROM lists l LEFT JOIN objects o ON l.object_id=o.id " + + "WHERE nvl(CAST(o.v['string_8'] AS VARCHAR), 'john') MATCH_ANY 'john' OR l.list_id=12", + planner -> { + Assertions.assertFalse(hasMatchVirtualColumn(planner.getPhysicalPlan()), + planner.getPhysicalPlan().treeString()); + }); + } + + @Test + public void testSearchOnNullPaddedSideRejected() { + // LEFT JOIN ... ON false becomes Project(l.*, NULL AS o.*). + assertNullSideSearchRejected("SELECT l.object_id FROM lists l LEFT JOIN notes n ON false " + + "WHERE NOT search('title:john') OR l.list_id=8"); + // LEFT JOIN ... WHERE n.id IS NULL becomes a left anti join with NULL aliases for n.*. + assertNullSideSearchRejected("SELECT l.object_id FROM lists l LEFT JOIN notes n ON l.object_id=n.id " + + "WHERE n.id IS NULL AND (NOT search('title:john') OR l.list_id=8)"); + // A derived table exposing both sides lets materialize() inline NULL into a scan virtual column. + assertNullSideSearchRejected("SELECT s.id FROM (SELECT o.id, o.v, n.title FROM objects o " + + "LEFT JOIN notes n ON false) s JOIN lists l ON s.id=l.object_id " + + "WHERE NOT search('title:john OR v.string_8:hello') OR l.list_id=1"); + } + + private void collectPlans(Plan plan, List plans) { + plans.add(plan); + plan.children().forEach(child -> collectPlans(child, plans)); + } + + private List scanEvaluatedExpressions(Plan node) { + return node instanceof PhysicalOlapScan + ? new ArrayList<>(((PhysicalOlapScan) node).getVirtualColumns()) : new ArrayList<>(); + } + + private List scansWithSearchVirtualColumn(Plan plan) { + List plans = new ArrayList<>(); + collectPlans(plan, plans); + return plans.stream().filter(node -> scanEvaluatedExpressions(node).stream() + .anyMatch(column -> column.anyMatch(e -> e instanceof SearchExpression))) + .collect(Collectors.toList()); + } + + private List searchExpressions(Plan plan) { + List plans = new ArrayList<>(); + collectPlans(plan, plans); + List result = new ArrayList<>(); + for (Plan node : plans) { + List expressions = scanEvaluatedExpressions(node); + expressions.addAll(node.getExpressions()); + expressions.forEach(e -> result.addAll(e.collect(SearchExpression.class::isInstance))); + } + return result; + } + + private void assertSingleBinding(Plan plan, String physicalField) { + List searches = searchExpressions(plan); + Assertions.assertEquals(1, searches.size(), plan.treeString()); + List bindings = searches.get(0).getQsPlan().getFieldBindings(); + Assertions.assertEquals(1, bindings.size()); + Assertions.assertEquals(physicalField, bindings.get(0).getFieldName()); + Assertions.assertEquals(physicalField, searches.get(0).getQsPlan().getRoot().getField()); + } + + // Every MATCH/SEARCH of these plans can be materialized, so none may be left for row evaluation. + private void assertNoIndexSearchOutsideScan(Plan plan) { + List plans = new ArrayList<>(); + collectPlans(plan, plans); + for (Plan node : plans) { + Assertions.assertFalse(node.getExpressions().stream().anyMatch(expression -> expression.anyMatch( + e -> e instanceof Match || e instanceof SearchExpression)), plan.treeString()); + } + } + + private void assertRejected(String sql, String message) { + Throwable thrown = Assertions.assertThrows(Throwable.class, + () -> PlanChecker.from(connectContext).checkPlannerResult(sql)); + Throwable cause = thrown; + while (cause != null && (cause.getMessage() == null || !cause.getMessage().contains(message))) { + cause = cause.getCause(); + } + Assertions.assertNotNull(cause, sql + " failed with: " + thrown); + } + + private boolean hasMatchVirtualColumn(Plan plan) { + return plan.anyMatch(node -> node instanceof PhysicalOlapScan && ((PhysicalOlapScan) node).getVirtualColumns() + .stream().anyMatch(column -> column.anyMatch(e -> e instanceof Match))); + } + + private void assertNullSideSearchRejected(String sql) { + assertRejected(sql, "null-generating side of an outer join"); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java index 90623a98a3e1db..96111f32873360 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java @@ -18,7 +18,9 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.analysis.SearchDslParser; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.StringType; @@ -141,13 +143,22 @@ public void testToString() { } @Test - public void testSymbolicNullChildForNullRejectionInference() { + public void testNullFieldIsNotAFieldBinding() { + SlotReference title = createTestSlot("title"); SearchExpression search = new SearchExpression("title:hello", createTestPlan(), - Collections.singletonList(createTestSlot("title"))); - SearchExpression symbolic = search.withChildren(Collections.singletonList(NullLiteral.INSTANCE)); - Assertions.assertEquals(NullLiteral.INSTANCE, symbolic.child(0)); - Assertions.assertFalse(symbolic.foldable()); - Assertions.assertEquals(search.getQsPlan(), symbolic.getQsPlan()); + Collections.singletonList(title)); + Assertions.assertTrue(search.bindsOnlyFields()); + + // Null-rejection inference and outer join NULL padding substitute NULL for the field. + SearchExpression nullField = search.withChildren(Collections.singletonList(NullLiteral.INSTANCE)); + Assertions.assertFalse(nullField.foldable()); + Assertions.assertFalse(nullField.bindsOnlyFields()); + + StringLiteral key = new StringLiteral("name"); + Assertions.assertTrue(search.withChildren( + Collections.singletonList(new ElementAt(title, key))).bindsOnlyFields()); + Assertions.assertFalse(search.withChildren( + Collections.singletonList(new ElementAt(NullLiteral.INSTANCE, key))).bindsOnlyFields()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java index c078e56912118e..3575d454970bb4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.SearchDslParser.QsNode; import org.apache.doris.analysis.SearchDslParser.QsOccur; import org.apache.doris.analysis.SearchDslParser.QsPlan; +import org.apache.doris.common.Pair; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -2814,4 +2815,42 @@ public void testMultiFieldMatchAllDocsPreservesOccurWithAndOperator() { Assertions.assertTrue(foundMatchAllWithMust, "Should contain MATCH_ALL_DOCS node with MUST occur"); } + + private void assertFieldReference(String reference, String path, String analyzer) { + Pair result = SearchDslParser.splitAnalyzerSelector(reference); + Assertions.assertEquals(path, result.first, reference); + Assertions.assertEquals(analyzer, result.second, reference); + } + + @Test + public void testAnalyzerSelectorIsSyntactic() { + assertFieldReference("title", "title", null); + assertFieldReference("title@exact", "title", "exact"); + assertFieldReference("v.name@exact", "v.name", "exact"); + // Only the last unescaped @ is the selector. + assertFieldReference("a@b@exact", "a@b", "exact"); + // Escaped @ (also produced for quoted segments) is part of the name. + assertFieldReference("name\\@literal", "name@literal", null); + assertFieldReference("name\\@literal@exact", "name@literal", "exact"); + assertFieldReference("name\\\\@exact", "name\\\\", "exact"); + // An @ that starts a path segment cannot follow a field, so it is part of the name. + assertFieldReference("@timestamp", "@timestamp", null); + assertFieldReference("v.@timestamp", "v.@timestamp", null); + assertFieldReference("v.@timestamp@exact", "v.@timestamp", "exact"); + Assertions.assertThrows(SearchDslParser.SearchDslSyntaxException.class, + () -> SearchDslParser.splitAnalyzerSelector("title@")); + } + + @Test + public void testQuotedAtSignIsNotAnalyzerSelector() { + QsPlan quoted = SearchDslParser.parseDsl("\"name@literal\":john"); + assertFieldReference(quoted.getFieldBindings().get(0).getFieldName(), "name@literal", null); + + // An @ that is already escaped inside the quotes is not escaped twice. + QsPlan escaped = SearchDslParser.parseDsl("\"name\\@literal\":john"); + assertFieldReference(escaped.getFieldBindings().get(0).getFieldName(), "name@literal", null); + + QsPlan unquoted = SearchDslParser.parseDsl("name@literal:john"); + assertFieldReference(unquoted.getFieldBindings().get(0).getFieldName(), "name", "literal"); + } } diff --git a/regression-test/data/search/test_crm_search_analyzers.out b/regression-test/data/search/test_crm_search_analyzers.out index e0e3b89f4a67e8..3f3e7d67787a90 100644 --- a/regression-test/data/search/test_crm_search_analyzers.out +++ b/regression-test/data/search/test_crm_search_analyzers.out @@ -16,6 +16,10 @@ 1 2 +-- !document_11_selector_case -- +1 +2 + -- !document_13_ordinary_in -- 1 diff --git a/regression-test/data/search/test_crm_search_join_document.out b/regression-test/data/search/test_crm_search_join_document.out index 48ec32c04e51d1..fcec5f1bdf09fa 100644 --- a/regression-test/data/search/test_crm_search_join_document.out +++ b/regression-test/data/search/test_crm_search_join_document.out @@ -169,3 +169,85 @@ 2 2 3 3 +-- !search_self_join_left -- +1 1 +2 2 +3 3 +6 6 + +-- !search_self_join_right -- +1 1 +2 2 +3 3 +7 7 + +-- !search_renamed_column -- +\N \N +1 1 +1 1 +2 \N +4 4 +6 6 + +-- !search_renamed_variant -- +1 1 +1 1 +2 \N + +-- !match_case_scan -- +\N hit +1 hit +2 hit +3 miss +4 miss +5 miss +6 hit +7 miss + +-- !match_case_join -- +\N \N hit +1 1 hit +1 1 hit +2 \N hit +3 \N miss +4 4 miss +5 5 miss +6 6 hit +7 \N miss + +-- !match_case_null_side -- +\N null +1 hit +1 hit +4 miss +5 null +6 hit +8 null + +-- !search_not_inferred_for_equal_column -- +\N 10 +1 10 +2 20 +6 20 + +-- !search_not_inferred_through_intersect -- +hello +hello world + +-- !nullside_nonstrict_match_where -- +\N +1 +1 +5 +6 +8 + +-- !nullside_nonstrict_match_select -- +\N true +1 true +1 true +4 false +5 true +6 true +8 true + diff --git a/regression-test/suites/search/test_crm_search_analyzers.groovy b/regression-test/suites/search/test_crm_search_analyzers.groovy index cd60d1928521e2..aeb841b7b1f175 100644 --- a/regression-test/suites/search/test_crm_search_analyzers.groovy +++ b/regression-test/suites/search/test_crm_search_analyzers.groovy @@ -16,6 +16,11 @@ // under the License. suite("test_crm_search_analyzers") { + // The pipeline randomizes these defaults. A VARIANT subcolumn stored in the sparse or doc column has no + // inverted index, so SEARCH finds nothing in it and MATCH fails without enable_match_without_inverted_index. + sql "set default_variant_enable_doc_mode = false" + sql "set default_variant_enable_typed_paths_to_sparse = false" + sql "set default_variant_max_subcolumns_count = 0" // Chapters X and XI: a normalized keyword index and a full-text index. sql "DROP TABLE IF EXISTS crm_search_analyzers" sql "DROP INVERTED INDEX ANALYZER IF EXISTS crm_doc_text" @@ -74,6 +79,11 @@ suite("test_crm_search_analyzers") { sql """SELECT id FROM crm_search_analyzers WHERE search('name@does_not_exist:John')""" exception "No inverted index found for SEARCH analyzer" } + // The selector names an analyzer the way the index lookup does, ignoring case. + order_qt_document_11_selector_case """ + SELECT id FROM crm_search_analyzers + WHERE search('name@CRM_DOC_EXACT:"John Smith" AND name@crm_doc_exact:"john smith"') + """ // Chapter XIII is outside P0-P2. Ordinary IN must keep SQL equality semantics. order_qt_document_13_ordinary_in """ SELECT id FROM crm_search_analyzers WHERE name IN ('John Smith','Mason Jackson') @@ -102,5 +112,10 @@ suite("test_crm_search_analyzers") { order_qt_literal_variant_at """ SELECT id FROM crm_search_literal_fields WHERE search('"v.email@work":john') """ + // An unquoted @ always selects an analyzer, whatever columns the table has. + test { + sql """SELECT id FROM crm_search_literal_fields WHERE search('name@literal:john')""" + exception "Field 'name' not found" + } } diff --git a/regression-test/suites/search/test_crm_search_join_document.groovy b/regression-test/suites/search/test_crm_search_join_document.groovy index fad0826c2f5598..c3bbc8c8275ac2 100644 --- a/regression-test/suites/search/test_crm_search_join_document.groovy +++ b/regression-test/suites/search/test_crm_search_join_document.groovy @@ -18,6 +18,11 @@ // Source: query string and search+join examples, revision 23. suite("test_crm_search_join_document") { sql "set enable_match_without_inverted_index = false" + // The pipeline randomizes these defaults. A VARIANT subcolumn stored in the sparse or doc column has no + // inverted index, so SEARCH finds nothing in it and MATCH fails without enable_match_without_inverted_index. + sql "set default_variant_enable_doc_mode = false" + sql "set default_variant_enable_typed_paths_to_sparse = false" + sql "set default_variant_max_subcolumns_count = 0" sql "DROP TABLE IF EXISTS crm_search_objects" sql """CREATE TABLE crm_search_objects ( @@ -931,11 +936,104 @@ select objects_0_1___OBJECTID from `results` WHERE search('OVERFLOWPROPERTIES.string_8:john') OR search('v.name:john') """ + // A SEARCH field resolves like a SQL column reference: a table alias picks one side of a self join, + // and an output alias still searches the physical column and its index. + order_qt_search_self_join_left """ + SELECT a.k1, b.k1 FROM crm_search_full_a a JOIN crm_search_full_a b ON a.k1=b.k1 + WHERE search('a.content:hello') OR b.k1=3 + """ + order_qt_search_self_join_right """ + SELECT a.k1, b.k1 FROM crm_search_full_a a JOIN crm_search_full_a b ON a.k1=b.k1 + WHERE search('b.content:world') OR a.k1=2 + """ + test { + sql """SELECT a.k1 FROM crm_search_full_a a JOIN crm_search_full_a b ON a.k1=b.k1 + WHERE search('content:hello')""" + exception "Ambiguous field 'content'" + } + // b has k1=1 twice: the join's duplicate rows must survive the virtual column. + order_qt_search_renamed_column """ + SELECT t.k1, b.k1 FROM (SELECT k1, content AS body FROM crm_search_full_a) t + LEFT JOIN crm_search_full_b b ON t.k1=b.k1 + WHERE search('body:hello') OR b.k1=4 + """ + order_qt_search_renamed_variant """ + SELECT o.id, l.k1 FROM (SELECT id, v AS props FROM crm_search_mow) o + LEFT JOIN crm_search_full_b l ON o.id=l.k1 + WHERE search('props.name:john') OR l.k1=1 + """ + // A MATCH wrapped in another expression is materialized the same way above a scan and above a join; + // on the null-generating side it stays NULL for rows without a join partner. + order_qt_match_case_scan """ + SELECT k1, CASE WHEN content MATCH_ANY 'hello' THEN 'hit' ELSE 'miss' END + FROM crm_search_full_a + """ + order_qt_match_case_join """ + SELECT a.k1, b.k1, CASE WHEN a.content MATCH_ANY 'hello' THEN 'hit' ELSE 'miss' END + FROM crm_search_full_a a LEFT JOIN crm_search_full_b b ON a.k1=b.k1 + """ + order_qt_match_case_null_side """ + SELECT b.k1, CASE WHEN a.content MATCH_ANY 'hello' THEN 'hit' + WHEN (a.content MATCH_ANY 'hello') IS NULL THEN 'null' ELSE 'miss' END + FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON b.k1=a.k1 + """ + // a.content = n.name lets predicates on a.content be inferred for n.name, but a SEARCH is bound to the + // inverted index of a.content: copied to n.name, which has no index, it would drop every joined row. + sql "DROP TABLE IF EXISTS crm_search_names" + sql """CREATE TABLE crm_search_names (id INT, name TEXT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO crm_search_names VALUES (10,'hello world'),(20,'hello'),(30,'world')" + order_qt_search_not_inferred_for_equal_column """ + SELECT a.k1, n.id FROM crm_search_full_a a JOIN crm_search_names n ON a.content=n.name + WHERE search('a.content:hello') OR a.content='zzz' + """ + // The same holds for the positional inference of INTERSECT: the SEARCH stays in its own branch. + order_qt_search_not_inferred_through_intersect """ + (SELECT content FROM crm_search_full_a WHERE search('content:hello')) + INTERSECT (SELECT name FROM crm_search_names) + """ + // A SEARCH that cannot reach one scan is rejected instead of being evaluated without an index. + test { + sql """SELECT k1 FROM (SELECT k1, content FROM crm_search_full_a ORDER BY k1 LIMIT 3) t + WHERE search('content:hello')""" + exception "SEARCH must be evaluated by an OLAP scan" + } + test { + sql """SELECT a.k1 FROM crm_search_full_a a JOIN crm_search_mow m ON a.k1=m.id + WHERE search('content:hello OR v.name:john')""" + exception "SEARCH must be evaluated by an OLAP scan" + } + // Keep SEARCH on the null-generating side gated until its full DSL NULL contract is established. test { sql """SELECT b.k1 FROM crm_search_full_b b LEFT JOIN crm_search_mow m ON b.k1=m.id WHERE search('NOT v.name:john') OR b.k1=8""" exception "SEARCH must be evaluated by an OLAP scan" } + // Rewrites that NULL-pad that side (ON false, outer join to anti join) must not bypass the gate. + test { + sql """SELECT b.k1 FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON false + WHERE NOT search('content:hello') OR b.k1=8""" + exception "null-generating side of an outer join" + } + test { + sql """SELECT b.k1 FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON b.k1=a.k1 + WHERE a.k1 IS NULL AND (NOT search('content:hello') OR b.k1=8)""" + exception "null-generating side of an outer join" + } + + // nvl(NULL, 'hello') matches: a MATCH with such an operand must stay above the outer join, + // so rows without a join partner (b.k1 = 8 and NULL) are kept and project TRUE. + sql "set enable_match_without_inverted_index = true" + order_qt_nullside_nonstrict_match_where """ + SELECT b.k1 FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON b.k1=a.k1 + WHERE nvl(a.content, 'hello') MATCH_ANY 'hello' OR b.k1=100 + """ + order_qt_nullside_nonstrict_match_select """ + SELECT b.k1, nvl(a.content, 'hello') MATCH_ANY 'hello' + FROM crm_search_full_b b LEFT JOIN crm_search_full_a a ON b.k1=a.k1 + """ + sql "set enable_match_without_inverted_index = false" } diff --git a/regression-test/suites/search/test_crm_search_variant_topn.groovy b/regression-test/suites/search/test_crm_search_variant_topn.groovy index c900e01b83c06d..89ba1079ad4515 100644 --- a/regression-test/suites/search/test_crm_search_variant_topn.groovy +++ b/regression-test/suites/search/test_crm_search_variant_topn.groovy @@ -16,6 +16,11 @@ // under the License. suite("test_crm_search_variant_topn") { + // The pipeline randomizes these defaults. A VARIANT subcolumn stored in the sparse or doc column has no + // inverted index, so SEARCH finds nothing in it and MATCH fails without enable_match_without_inverted_index. + sql "set default_variant_enable_doc_mode = false" + sql "set default_variant_enable_typed_paths_to_sparse = false" + sql "set default_variant_max_subcolumns_count = 0" // Chapter XII. More than LIMIT matching rows and absent payload paths. sql "DROP TABLE IF EXISTS crm_search_products" sql """CREATE TABLE crm_search_products (id BIGINT, v VARIANT, diff --git a/regression-test/suites/search/test_search_usage_restrictions.groovy b/regression-test/suites/search/test_search_usage_restrictions.groovy index 5f2c13c6a6b140..3f2cb513822fa1 100644 --- a/regression-test/suites/search/test_search_usage_restrictions.groovy +++ b/regression-test/suites/search/test_search_usage_restrictions.groovy @@ -86,7 +86,7 @@ suite("test_search_usage_restrictions", "p0") { // Test 5: search() in GROUP BY should fail test { sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ count(*) FROM ${tableName} GROUP BY search('title:Learning')" - exception "predicates are only supported inside WHERE filters on single-table scans" + exception "predicates are only supported inside WHERE filters over OLAP tables" } // Test 6: search() in SELECT then GROUP BY alias should fail @@ -110,7 +110,7 @@ suite("test_search_usage_restrictions", "p0") { // Test 9: search() in HAVING clause should fail test { sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ category, count(*) FROM ${tableName} GROUP BY category HAVING search('title:Learning')" - exception "predicates are only supported inside WHERE filters on single-table scans" + exception "predicates are only supported inside WHERE filters over OLAP tables" } // Test 10: SEARCH fields belong to t1 even though the WHERE is above a join. From b0e650e6d381e843fff6885cd4912681a2236e69 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Mon, 21 Sep 2026 15:42:51 +0800 Subject: [PATCH 5/5] [fix](nereids) Declare SEARCH nullable so an UNKNOWN index result stays UNKNOWN ### What problem does this PR solve? Issue Number: None Related PR: #67932 Problem Summary: A SEARCH materialized as a scan virtual column returned FALSE instead of UNKNOWN whenever its fields were NOT NULL, so `NOT search(...)` selected every row. With `age INT NOT NULL` carrying an inverted index and rows 20, 25, 40, 50: SELECT a.id FROM t a LEFT JOIN j ON a.id = j.k1 WHERE (NOT search('age:[18 TO 30]')) OR j.k1 = 100 returned all four rows, while the same predicate as a filter on the scan returned none, and `search('age:[18 TO 30]') IS NULL` returned no row at all. Declaring the column nullable, with the same data, produced the opposite (and correct) result for both. A BKD index answers only TERM and EXACT clauses (direct_index_query_type_for_clause), so BE returns make_unknown_query - a null bitmap covering every row - for a range, an unparseable value or a missing iterator, independently of whether the field can be NULL. The filter path keeps three-valued semantics in the bitmap operators, but the virtual column path fills the boolean column in SegmentIterator::_output_index_result_column, which builds a null map only `if (has_null_bitmap && expr_returns_nullable)` and otherwise writes UNKNOWN rows as 0. `expr_returns_nullable` comes from SearchExpression.nullable(), which followed its children. SearchExpression is therefore always nullable: the UNKNOWN of a SEARCH is a property of what its indexes can answer, not of its fields' nullability. Text (Lucene) fields were unaffected because their reader implements the other clause types. ### Release note Fix `NOT search(...)` returning every row when a SEARCH over NOT NULL fields is evaluated above a join and its inverted index cannot answer the DSL clause. ### Check List (For Author) - Test: Unit Test (SearchExpressionTest.testAlwaysNullable) and Regression test (search/test_crm_search_join_document: search_bkd_term_{scan,join}, search_bkd_range_{scan,join} - each pair asserts that the filter-on-scan and the virtual-column shapes agree) - Behavior changed: No (a wrong result is corrected) - Does this need documentation: No Co-Authored-By: Claude Opus 5 --- .../trees/expressions/SearchExpression.java | 8 +++-- .../expressions/SearchExpressionTest.java | 13 ++++++++ .../search/test_crm_search_join_document.out | 14 +++++++++ .../test_crm_search_join_document.groovy | 31 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java index b4011a9539e4d2..e11c159527f1e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java @@ -62,8 +62,12 @@ public List getSlotChildren() { @Override public boolean nullable() throws UnboundException { - // Search expressions can be null if any child slot is null - return children().stream().anyMatch(Expression::nullable); + // A SEARCH is UNKNOWN wherever its inverted indexes cannot answer the DSL, not only where its fields are + // NULL: BE returns an all-rows null bitmap for a clause type an index does not implement (a range on a BKD + // index), an unparseable value, or a missing iterator. A scan writes that bitmap into a virtual column only + // when this expression is nullable (segment_iterator.cpp, _output_index_result_column), so declaring it + // non-nullable over NOT NULL fields would turn UNKNOWN into FALSE and make NOT search(...) true everywhere. + return true; } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java index 96111f32873360..84732544c3fec0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SearchExpressionTest.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.junit.jupiter.api.Assertions; @@ -66,6 +67,18 @@ public void testSearchExpressionCreation() { Assertions.assertEquals(titleSlot, searchExpr.children().get(0)); } + @Test + public void testAlwaysNullable() { + // BE answers UNKNOWN for a clause its index cannot evaluate, whatever the field's nullability, and a scan + // only carries that UNKNOWN into a virtual column while this expression is nullable. + SlotReference notNullSlot = new SlotReference("age", IntegerType.INSTANCE, false, Arrays.asList()); + SearchExpression searchExpr = new SearchExpression("age:[18 TO 30]", createTestPlan(), + Arrays.asList(notNullSlot)); + + Assertions.assertFalse(notNullSlot.nullable()); + Assertions.assertTrue(searchExpr.nullable()); + } + @Test public void testDataType() { String dsl = "title:hello"; diff --git a/regression-test/data/search/test_crm_search_join_document.out b/regression-test/data/search/test_crm_search_join_document.out index fcec5f1bdf09fa..47e4454dfb69f6 100644 --- a/regression-test/data/search/test_crm_search_join_document.out +++ b/regression-test/data/search/test_crm_search_join_document.out @@ -234,6 +234,20 @@ hello hello world +-- !search_bkd_term_scan -- +2 +3 +4 + +-- !search_bkd_term_join -- +2 +3 +4 + +-- !search_bkd_range_scan -- + +-- !search_bkd_range_join -- + -- !nullside_nonstrict_match_where -- \N 1 diff --git a/regression-test/suites/search/test_crm_search_join_document.groovy b/regression-test/suites/search/test_crm_search_join_document.groovy index c3bbc8c8275ac2..b669527b11f9d6 100644 --- a/regression-test/suites/search/test_crm_search_join_document.groovy +++ b/regression-test/suites/search/test_crm_search_join_document.groovy @@ -993,6 +993,37 @@ select objects_0_1___OBJECTID from `results` (SELECT content FROM crm_search_full_a WHERE search('content:hello')) INTERSECT (SELECT name FROM crm_search_names) """ + // A BKD (numeric) index answers only TERM/EXACT clauses, so BE returns UNKNOWN for every row of any other + // clause. A residual SEARCH materialized as a scan virtual column must carry that UNKNOWN exactly as the + // filter on the scan does, whatever the field's nullability: when it does not, UNKNOWN becomes FALSE and + // NOT search(...) selects every row. Each pair below must keep matching, whichever clauses the index grows. + sql "DROP TABLE IF EXISTS crm_search_ages" + sql """CREATE TABLE crm_search_ages (id INT NOT NULL, age INT NOT NULL, + INDEX idx_age(age) USING INVERTED) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO crm_search_ages VALUES (1,20),(2,25),(3,40),(4,50)" + sql "DROP TABLE IF EXISTS crm_search_age_join" + sql """CREATE TABLE crm_search_age_join (k1 INT) + DUPLICATE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO crm_search_age_join VALUES (1),(2),(3),(4)" + order_qt_search_bkd_term_scan """ + SELECT id FROM crm_search_ages WHERE NOT search('age:20') + """ + order_qt_search_bkd_term_join """ + SELECT a.id FROM crm_search_ages a LEFT JOIN crm_search_age_join j ON a.id = j.k1 + WHERE (NOT search('age:20')) OR j.k1 = 100 + """ + // A range clause the BKD index cannot answer: UNKNOWN, so neither shape selects a row. + order_qt_search_bkd_range_scan """ + SELECT id FROM crm_search_ages WHERE NOT search('age:[18 TO 30]') + """ + order_qt_search_bkd_range_join """ + SELECT a.id FROM crm_search_ages a LEFT JOIN crm_search_age_join j ON a.id = j.k1 + WHERE (NOT search('age:[18 TO 30]')) OR j.k1 = 100 + """ + // A SEARCH that cannot reach one scan is rejected instead of being evaluated without an index. test { sql """SELECT k1 FROM (SELECT k1, content FROM crm_search_full_a ORDER BY k1 LIMIT 3) t