diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectFilterAboveConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectFilterAboveConsumer.java index 25fa5d2ccc8a68..91b003e06470ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectFilterAboveConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectFilterAboveConsumer.java @@ -38,7 +38,7 @@ public Rule build() { LogicalCTEConsumer cteConsumer = filter.child(); Set exprs = filter.getConjuncts(); for (Expression expr : exprs) { - if (expr.containsVolatileExpression()) { + if (expr.containsVolatileOrNoneMovableExpression()) { continue; } Expression rewrittenExpr = expr.rewriteUp(e -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java index c2ca99f0b61b9c..9003b802476b62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java @@ -32,9 +32,11 @@ import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.util.ExpressionUtils; @@ -153,12 +155,15 @@ private static Set getEqualSetAndDoReplace(T ExpressionAnalyzer analyzer = new ReplaceAnalyzer(null, new Scope(ImmutableList.of()), null, false, false); Set res = new LinkedHashSet<>(); for (T equals : equalSet) { - Map replaceMap = new HashMap<>(); - replaceMap.put(equals, replaceToThis); if (!exprPredicates.containsKey(equals)) { continue; } + Map replaceMap = new HashMap<>(); + replaceMap.put(equals, replaceToThis); for (Expression predicate : exprPredicates.get(equals)) { + if (!canReplace(equals, replaceToThis, predicate)) { + continue; + } Expression newPredicates = ExpressionUtils.replace(predicate, replaceMap); try { Expression analyzed = analyzer.analyze(newPredicates); @@ -171,6 +176,27 @@ private static Set getEqualSetAndDoReplace(T return res; } + private static boolean canReplace(Expression source, Expression target, Expression predicate) { + Expression comparison = predicate instanceof Not ? predicate.child(0) : predicate; + // Direct comparisons observe comparison equality rather than a value's type or representation. + // Do not descend through functions, casts or OR to apply this exception. + if ((comparison instanceof ComparisonPredicate || comparison instanceof InPredicate) + && comparison.child(0).equals(source)) { + return true; + } + DataType type = source.getDataType(); + // Comparison equality across types does not preserve type-sensitive expressions such as CAST to STRING. + if (!type.equals(target.getDataType())) { + return false; + } + // Only substitute types whose equality preserves the value observed by enclosing expressions. + // In particular, FLOAT/DOUBLE equality cannot distinguish signed zero, but SIGNBIT can. + // Comparisons can still be propagated separately by UnequalPredicateInfer. + return type.isBooleanType() || type.isIntegralType() || type.isDecimalLikeType() + || type.isStringLikeType() || type.isIPType() + || (type.isDateLikeType() && !type.isTimeStampTzType()); + } + /* Extract the equivalence relationship a=b, and when case (d_tinyint as int)=d_int is encountered, remove the cast and extract d_tinyint=d_int EqualPairs is the output parameter and the equivalent pair of predicate derivation input, @@ -210,7 +236,9 @@ public static Set infer(Set inputs) { } Map> exprPredicates = new HashMap<>(); for (Expression input : inputs) { - if (input.anyMatch(expr -> !((ExpressionTrait) expr).isDeterministic()) + // Inference can evaluate a predicate on rows that never reach its original filter. + if (input.anyMatch(expr -> expr instanceof NoneMovableFunction + || !((ExpressionTrait) expr).isDeterministic()) || input.getInputSlots().size() != 1) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java index 8688db6a7d1486..4ac9da39a7e133 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java @@ -218,7 +218,7 @@ private Plan inferNewPredicate(Plan plan, Set expressions) { Set predicates = new LinkedHashSet<>(); Set planOutputs = plan.getOutputSet(); for (Expression expr : expressions) { - if (expr.containsVolatileExpression()) { + if (expr.containsVolatileOrNoneMovableExpression()) { // Volatile expressions (e.g. rand(), uuid()) must not be cloned into // subtrees that did not already evaluate them. Otherwise, callers that perform // slot substitution (e.g. SetOp visitors below) would introduce a fresh @@ -250,7 +250,7 @@ private Plan inferNewPredicateRemoveUselessIsNull(Plan plan, Set exp Set predicates = new LinkedHashSet<>(); Set planOutputs = plan.getOutputSet(); for (Expression expr : expressions) { - if (expr.containsVolatileExpression()) { + if (expr.containsVolatileOrNoneMovableExpression()) { // See inferNewPredicate for rationale: never clone volatile // predicates into a subtree that did not already evaluate them. continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java index 72f1752c375c36..98fb14db1c655a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java @@ -114,7 +114,7 @@ public Expression visit(Expression expression, ReplacerContext ctx) { // pair" to "per row of that child", which silently changes results. Keep such // expressions inline in otherJoinConjuncts, but still recurse to extract deterministic // child expressions. - if (expression.containsVolatileExpression()) { + if (expression.containsVolatileOrNoneMovableExpression()) { return super.visit(expression, ctx); } if (ctx.leftSlots.containsAll(input)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregation.java index 0945162f6d0d1d..c61f5b28d42b97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregation.java @@ -69,7 +69,7 @@ public Rule build() { // 2. if the conjunct contains unique function, it should not be pushed down; // e.g. 'select a, sum(a) from t group by a having a + random() > 10' // not equals 'select a, sum(a) from t where a + random() > 10 group by a' - if (!conjunct.containsVolatileExpression() + if (!conjunct.containsVolatileOrNoneMovableExpression() && !conjunctSlots.isEmpty() && canPushDownSlots.containsAll(conjunctSlots)) { pushDownPredicates.add(conjunct); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughGenerate.java index 85de47ab1274b4..a7e885a501fb0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughGenerate.java @@ -50,7 +50,7 @@ public Rule build() { filter.getConjuncts().forEach(conjunct -> { Set conjunctSlots = conjunct.getInputSlots(); if (!conjunctSlots.isEmpty() && childOutputs.containsAll(conjunctSlots) - && !conjunct.containsVolatileExpression()) { + && !conjunct.containsVolatileOrNoneMovableExpression()) { pushDownPredicates.add(conjunct); } else { remainPredicates.add(conjunct); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java index ddcff70759fb7e..7e3532c1f103d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java @@ -120,7 +120,7 @@ public Rule build() { Set rightPredicates = Sets.newLinkedHashSet(); Set remainingPredicates = Sets.newLinkedHashSet(); for (Expression p : filterPredicates) { - if (p.containsVolatileExpression()) { + if (p.containsVolatileOrNoneMovableExpression()) { remainingPredicates.add(p); continue; } @@ -162,7 +162,7 @@ private boolean convertJoinCondition(Expression predicate, Set leftOutputs if (!(predicate instanceof EqualTo)) { return false; } - if (predicate.containsVolatileExpression()) { + if (predicate.containsVolatileOrNoneMovableExpression()) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughPartitionTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughPartitionTopN.java index 5c5275730a1499..62d3e139b9eca4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughPartitionTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughPartitionTopN.java @@ -77,7 +77,7 @@ public Rule build() { // top-N", and the surviving rows of every partition would no longer be the true // top-N. Empty-input-slot predicates like `rand() > 0.5` would also bypass the // `containsAll` check otherwise. - if (!expr.containsVolatileExpression() && partitionKeySlots.containsAll(exprInputSlots)) { + if (!expr.containsVolatileOrNoneMovableExpression() && partitionKeySlots.containsAll(exprInputSlots)) { bottomConjunctsBuilder.add(expr); } else { upperConjunctsBuilder.add(expr); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java index 1a46b51a2468b7..0f3478e0642a62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java @@ -125,7 +125,7 @@ private static Pair, Set> splitConjunctsByChildOutpu // `project(b + random(1, 10) as a) -> filter(b + random(1, 10) > 1)`, it contains two distinct RANDOM. if (childOutputs.containsAll(conjunctSlots) && conjunctSlots.stream().map(childAlias::get).filter(Objects::nonNull) - .noneMatch(Expression::containsVolatileExpression)) { + .noneMatch(Expression::containsVolatileOrNoneMovableExpression)) { pushDownPredicates.add(conjunct); } else { remainPredicates.add(conjunct); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughSetOperation.java index 85d78be1aef2c0..b5f0c243d4fca5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughSetOperation.java @@ -87,7 +87,7 @@ public Rule build() { pushableConjuncts = new LinkedHashSet<>(); Set kept = new LinkedHashSet<>(); for (Expression c : origFilter.getConjuncts()) { - if (c.containsVolatileExpression()) { + if (c.containsVolatileOrNoneMovableExpression()) { kept.add(c); } else { pushableConjuncts.add(c); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughWindow.java index 3fc7c0b8dfa823..cc64a3386662c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughWindow.java @@ -97,7 +97,7 @@ public static boolean canPushDown(Expression conjunct, Set common // changes the value of every window function (row_number, rank, sum, ...). In addition, // a predicate like `rand() > 0.5` has empty input slots, so `containsAll(emptySet)` // would otherwise wrongly return true. - return !conjunct.containsVolatileExpression() + return !conjunct.containsVolatileOrNoneMovableExpression() && commonPartitionKeys.containsAll(conjunct.getInputSlots()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java index 098175ff462e43..e5e89742e1b461 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java @@ -78,7 +78,7 @@ public Rule build() { // child changes their evaluation granularity from per joined row to per // input row. Repeated volatile occurrences are materialized later by // AddProjectForVolatileExpression. - if (otherConjunct.containsVolatileExpression()) { + if (otherConjunct.containsVolatileOrNoneMovableExpression()) { remainingOther.add(otherConjunct); } else if (PUSH_DOWN_LEFT_VALID_TYPE.contains(join.getJoinType()) && allCoveredBy(otherConjunct, join.left().getOutputSet())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoin.java index 7c529ee6669acb..0fcc24bae2120f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoin.java @@ -59,7 +59,7 @@ public Rule build() { List otherConditions = Lists.newArrayListWithExpectedSize( filter.getConjuncts().size() + join.getOtherJoinConjuncts().size()); for (Expression expr : filter.getConjuncts()) { - if (expr.containsVolatileExpression()) { + if (expr.containsVolatileOrNoneMovableExpression()) { remainConditions.add(expr); } else { otherConditions.add(expr); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java index 3f5c520b27fa3d..d71af183080397 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java @@ -100,7 +100,7 @@ public Rule build() { for (Expression conjunct : filter.getConjuncts()) { // after reorder and push down the random() down to lower join, // the rewritten sql may have less rows() than the origin sql - if (conjunct.containsVolatileExpression()) { + if (conjunct.containsVolatileOrNoneMovableExpression()) { uniqueExprConjuncts.add(conjunct); } else { nonUniqueExprConjuncts.add(conjunct); @@ -153,7 +153,7 @@ public Plan joinToMultiJoin(Plan plan, Map planToHintType) // (t1 join t2) join t3 where t1.a = t3.x + random() // if reorder, then may have ((t1 join t3) on t1.a = t3.x + random()) join t2, // then the reorder result will less rows than origin. - if (conjunct.containsVolatileExpression()) { + if (conjunct.containsVolatileOrNoneMovableExpression()) { return plan; } } @@ -163,7 +163,8 @@ public Plan joinToMultiJoin(Plan plan, Map planToHintType) join = (LogicalJoin) plan; } - if (join.isMarkJoin() || join.getJoinType().isAsofJoin()) { + if (join.isMarkJoin() || join.getJoinType().isAsofJoin() + || join.getExpressions().stream().anyMatch(Expression::containsVolatileExpression)) { return plan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnequalPredicateInfer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnequalPredicateInfer.java index adf369221e1ab1..1af8f59ca91543 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnequalPredicateInfer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnequalPredicateInfer.java @@ -396,7 +396,8 @@ private Relation getType(ComparisonPredicate comparisonPredicate) { private void clear(Relation[][] graph, int left, int right, Relation type) { graph[left][right] = Relation.UNDEFINED; - if (type == Relation.EQ) { + // A reverse inequality is a separate constraint, not the duplicate of this equality. + if (type == Relation.EQ && graph[right][left] == Relation.EQ) { graph[right][left] = Relation.UNDEFINED; } } @@ -448,7 +449,8 @@ public Set chooseInputPredicates(Relation[][] chosen) { clear(chosen, left, right, type); } else if (deduced[left][right] != type) { keep[i] = true; - set(deduced, left, right, Relation.EQ); + // Preserve the relation of the retained predicate; an inequality is not an equality. + set(deduced, left, right, type); expandGraph(deduced, left, right); if (type == Relation.EQ) { expandGraph(deduced, right, left); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java index 8740a200cd70ab..64463f1bb82214 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java @@ -114,4 +114,9 @@ default boolean isVolatile() { default boolean containsVolatileExpression() { return containsType(VolatileExpression.class) && anyMatch(expr -> ((ExpressionTrait) expr).isVolatile()); } + + default boolean containsVolatileOrNoneMovableExpression() { + return containsType(VolatileExpression.class, NoneMovableFunction.class) + && anyMatch(expr -> ((ExpressionTrait) expr).isVolatile() || expr instanceof NoneMovableFunction); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToBitmapWithCheck.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToBitmapWithCheck.java index d238ad782e6734..8805e5c55dfcfd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToBitmapWithCheck.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToBitmapWithCheck.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -37,7 +38,7 @@ * ScalarFunction 'to_bitmap_with_check'. This class is generated by GenerateFunction. */ public class ToBitmapWithCheck extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, NoneMovableFunction { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BitmapType.INSTANCE).args(BigIntType.INSTANCE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java index c6ce702ebe75f5..c9991e7ee86537 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java @@ -48,7 +48,9 @@ private DateType(boolean shouldConversion) { @Override public boolean isInjectiveCastTo(DataType target) { - return target instanceof DateType || target instanceof DateV2Type || target instanceof CharacterType; + return target instanceof DateType || target instanceof DateV2Type + || target instanceof DateTimeType || target instanceof DateTimeV2Type + || target instanceof CharacterType; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateV2Type.java index 2acac3343048db..ceeb3783a8ca79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateV2Type.java @@ -36,6 +36,12 @@ public class DateV2Type extends DateLikeType { private DateV2Type() { } + @Override + public boolean isInjectiveCastTo(DataType target) { + return target instanceof DateType || target instanceof DateV2Type + || target instanceof DateTimeType || target instanceof DateTimeV2Type; + } + @Override public Type toCatalogDataType() { return Type.DATEV2; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PredicateInferUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PredicateInferUtils.java index 2a3ac016c49819..96b20606cdb579 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PredicateInferUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PredicateInferUtils.java @@ -26,10 +26,6 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.DateTimeType; -import org.apache.doris.nereids.types.DateTimeV2Type; -import org.apache.doris.nereids.types.DateType; -import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.coercion.CharacterType; import org.apache.doris.nereids.types.coercion.DateLikeType; import org.apache.doris.nereids.types.coercion.IntegralType; @@ -128,37 +124,9 @@ private static Optional validForInfer(Expression expression, InferTy Expression child = cast.child(); DataType dataType = cast.getDataType(); DataType childType = child.getDataType(); - if (inferType == InferType.INTEGRAL) { - if (dataType instanceof IntegralType) { - IntegralType integralType = (IntegralType) dataType; - if (childType instanceof IntegralType && integralType.widerThan((IntegralType) childType)) { - return validForInfer(((Cast) expression).child(), inferType); - } - } - } else if (inferType == InferType.DATE) { - // avoid lost precision - if (dataType instanceof DateType) { - if (childType instanceof DateV2Type || childType instanceof DateType) { - return validForInfer(child, inferType); - } - } else if (dataType instanceof DateV2Type) { - if (childType instanceof DateType || childType instanceof DateV2Type) { - return validForInfer(child, inferType); - } - } else if (dataType instanceof DateTimeType) { - if (childType.isTimeStampNsType()) { - return Optional.empty(); - } - if (!(childType instanceof DateTimeV2Type)) { - return validForInfer(child, inferType); - } - } else if (dataType instanceof DateTimeV2Type) { - if (childType.isTimeStampNsType()) { - return Optional.empty(); - } - if (!(childType instanceof DateTimeV2Type) || childType.isInjectiveCastTo(dataType)) { - return validForInfer(child, inferType); - } + if (inferType == InferType.INTEGRAL || inferType == InferType.DATE) { + if (childType.isInjectiveCastTo(dataType)) { + return validForInfer(child, inferType); } } else if (inferType == InferType.STRING) { // avoid substring cast such as cast(char(3) as char(2)) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java index 5c174cb6348f10..f9bd8835bdabb5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java @@ -28,30 +28,68 @@ import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs; +import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; +import org.apache.doris.nereids.trees.expressions.functions.scalar.SignBit; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.CharType; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateType; +import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.DecimalV3Type; +import org.apache.doris.nereids.types.DoubleType; +import org.apache.doris.nereids.types.FloatType; +import org.apache.doris.nereids.types.IPv4Type; +import org.apache.doris.nereids.types.IPv6Type; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TimeStampNsType; +import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.util.PredicateInferUtils; +import org.apache.doris.nereids.util.TypeCoercionUtils; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; +import java.util.stream.Stream; public class InferPredicateByReplaceTest { + @Test + public void testDoNotInferNoneMovablePredicateInsideOr() { + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + Expression predicate = new Or( + new AssertTrue(new GreaterThan(a, new IntegerLiteral(0)), new StringLiteral("bad")), + new GreaterThan(a, new IntegerLiteral(10))); + Set inputs = new LinkedHashSet<>(ImmutableList.of(new EqualTo(a, b), predicate)); + + Assertions.assertEquals(inputs, InferPredicateByReplace.infer(inputs)); + Assertions.assertEquals(inputs, PredicateInferUtils.inferAllPredicate(inputs)); + } + @Test public void testInferWithEqualTo() { SlotReference a = new SlotReference("a", IntegerType.INSTANCE); @@ -231,6 +269,23 @@ public void testTimestampNsCastIsNotRemovedForPredicateInference() { Assertions.assertFalse(PredicateInferUtils.getPairFromCast(legacyEqualTo).isPresent()); } + @Test + public void testTimestampTzCastIsNotRemovedForPredicateInference() { + for (int sourceScale : new int[] {0, 3, 6}) { + SlotReference timestampTz = new SlotReference("tz", TimeStampTzType.of(sourceScale)); + for (DataType target : ImmutableList.of(DateTimeType.INSTANCE, + DateTimeV2Type.of(0), DateTimeV2Type.of(3), DateTimeV2Type.of(6))) { + SlotReference localTime = new SlotReference("dt", target); + Cast cast = new Cast(timestampTz, target); + Assertions.assertFalse(PredicateInferUtils.getPairFromCast(new EqualTo(cast, localTime)).isPresent()); + Assertions.assertFalse(PredicateInferUtils.getPairFromCast(new GreaterThan(cast, localTime)).isPresent()); + Assertions.assertFalse(PredicateInferUtils.getPairFromCast( + new EqualTo(new Cast(cast, DateTimeV2Type.of(6)), + new Cast(localTime, DateTimeV2Type.of(6)))).isPresent()); + } + } + } + @Test public void testNotInferWithTransitiveEqualitySameTable() { // a = b, b = c @@ -245,4 +300,106 @@ public void testNotInferWithTransitiveEqualitySameTable() { Set result = InferPredicateByReplace.infer(inputs); Assertions.assertEquals(2, result.size()); } + + static Stream replacementTypes() { + List cases = new ArrayList<>(); + for (DataType type : ImmutableList.of(BooleanType.INSTANCE, IntegerType.INSTANCE, BigIntType.INSTANCE, + StringType.INSTANCE, DateV2Type.INSTANCE, DateTimeV2Type.of(0), DateTimeV2Type.of(6), + DecimalV3Type.createDecimalV3Type(9, 2), TimeStampNsType.INSTANCE, + CharType.createCharType(10), VarcharType.createVarcharType(10), + IPv4Type.INSTANCE, IPv6Type.INSTANCE)) { + cases.add(Arguments.of(type, type, true)); + } + for (List pair : ImmutableList.>of( + ImmutableList.of(DateV2Type.INSTANCE, DateTimeV2Type.of(0)), + ImmutableList.of(DateTimeV2Type.of(0), DateTimeV2Type.of(6)), + ImmutableList.of(IntegerType.INSTANCE, BigIntType.INSTANCE), + ImmutableList.of(DecimalV3Type.createDecimalV3Type(9, 2), + DecimalV3Type.createDecimalV3Type(9, 3)))) { + cases.add(Arguments.of(pair.get(0), pair.get(1), false)); + cases.add(Arguments.of(pair.get(1), pair.get(0), false)); + } + cases.add(Arguments.of(FloatType.INSTANCE, FloatType.INSTANCE, false)); + cases.add(Arguments.of(DoubleType.INSTANCE, DoubleType.INSTANCE, false)); + cases.add(Arguments.of(ArrayType.of(DoubleType.INSTANCE), ArrayType.of(DoubleType.INSTANCE), false)); + return cases.stream(); + } + + @ParameterizedTest(name = "{0} -> {1}, replace={2}") + @MethodSource("replacementTypes") + public void testTypeSensitiveReplacement(DataType sourceType, DataType targetType, boolean canReplace) { + SlotReference a = new SlotReference("a", sourceType); + SlotReference b = new SlotReference("b", targetType); + Expression equality = TypeCoercionUtils.processComparisonPredicate(new EqualTo(a, b)); + Expression predicate = new EqualTo(new Length(new Cast(a, StringType.INSTANCE)), new IntegerLiteral(10)); + Set inputs = new LinkedHashSet<>(ImmutableList.of(equality, predicate)); + Set result = InferPredicateByReplace.infer(inputs); + if (canReplace) { + Expression expected = new EqualTo(new Length(new Cast(b, StringType.INSTANCE)), new IntegerLiteral(10)); + Assertions.assertTrue(result.contains(expected), () -> "Missing " + expected + " in " + result); + } else { + Assertions.assertEquals(inputs, result); + } + } + + @Test + public void testSignedZeroInOr() { + SlotReference x = new SlotReference("x", DoubleType.INSTANCE); + SlotReference y = new SlotReference("y", DoubleType.INSTANCE); + Expression predicate = new Or(new SignBit(x), new GreaterThan(x, new DoubleLiteral(1.0))); + Set inputs = new LinkedHashSet<>(ImmutableList.of(new EqualTo(x, y), predicate)); + // x = -0.0 and y = +0.0 satisfy the input, but not the predicate with x replaced by y. + Assertions.assertEquals(inputs, InferPredicateByReplace.infer(inputs)); + Assertions.assertEquals(inputs, PredicateInferUtils.inferPredicate(inputs)); + } + + @Test + public void testSafeComparisonPropagation() { + SlotReference x = new SlotReference("x", DoubleType.INSTANCE, true, ImmutableList.of("left")); + SlotReference y = new SlotReference("y", DoubleType.INSTANCE, true, ImmutableList.of("right")); + Set floating = new LinkedHashSet<>(ImmutableList.of(new EqualTo(x, y), + new GreaterThan(x, new DoubleLiteral(1.0)))); + Assertions.assertTrue(PredicateInferUtils.inferPredicate(floating) + .contains(new GreaterThan(y, new DoubleLiteral(1.0)))); + + SlotReference small = new SlotReference("small", IntegerType.INSTANCE, true, ImmutableList.of("left")); + SlotReference wide = new SlotReference("wide", BigIntType.INSTANCE, true, ImmutableList.of("right")); + Set integers = new LinkedHashSet<>(ImmutableList.of( + new EqualTo(new Cast(small, BigIntType.INSTANCE), wide), + new GreaterThan(small, new IntegerLiteral(1)))); + Assertions.assertTrue(PredicateInferUtils.inferPredicate(integers).stream() + .anyMatch(p -> p instanceof GreaterThan && p.child(0).equals(wide) + && p.child(1).equals(new BigIntLiteral(1)))); + } + + @Test + public void testDirectComparisonReplacement() { + SlotReference small = new SlotReference("small", IntegerType.INSTANCE); + SlotReference wide = new SlotReference("wide", BigIntType.INSTANCE); + Expression equality = new EqualTo(new Cast(small, BigIntType.INSTANCE), wide); + List predicates = ImmutableList.of( + new Not(new EqualTo(small, new IntegerLiteral(10))), + new InPredicate(small, ImmutableList.of(new IntegerLiteral(10), new IntegerLiteral(20))), + new Not(new InPredicate(small, + ImmutableList.of(new IntegerLiteral(10), new IntegerLiteral(20))))); + List expected = ImmutableList.of( + new Not(new EqualTo(wide, new BigIntLiteral(10))), + new InPredicate(wide, ImmutableList.of(new BigIntLiteral(10), new BigIntLiteral(20))), + new Not(new InPredicate(wide, ImmutableList.of(new BigIntLiteral(10), new BigIntLiteral(20))))); + for (int i = 0; i < predicates.size(); i++) { + Set inputs = new LinkedHashSet<>(ImmutableList.of(equality, predicates.get(i))); + Assertions.assertTrue(InferPredicateByReplace.infer(inputs).contains(expected.get(i))); + } + } + + @Test + public void testSameTypeBehindWideningCasts() { + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + Expression equality = new EqualTo(new Cast(a, BigIntType.INSTANCE), new Cast(b, BigIntType.INSTANCE)); + Expression predicate = new EqualTo(new Length(new Cast(a, StringType.INSTANCE)), new IntegerLiteral(1)); + Set inputs = new LinkedHashSet<>(ImmutableList.of(equality, predicate)); + Expression expected = new EqualTo(new Length(new Cast(b, StringType.INSTANCE)), new IntegerLiteral(1)); + Assertions.assertTrue(InferPredicateByReplace.infer(inputs).contains(expected)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/UnequalPredicateInferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/UnequalPredicateInferTest.java index 7bd43c98929bc2..7115af40297162 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/UnequalPredicateInferTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/UnequalPredicateInferTest.java @@ -36,14 +36,18 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.PredicateInferUtils; +import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; public class UnequalPredicateInferTest { @@ -685,4 +689,99 @@ public void testInferWithTransitiveEqualityWithCastDatev2andDate() { EqualTo expected = new EqualTo(a, b); Assertions.assertTrue(result.contains(expected) || result.contains(expected.commute()), "Expected to find a = b in the result."); } + + @Test + public void testInputPredicateSemantics() { + SlotReference a = new SlotReference("a", IntegerType.INSTANCE, false, ImmutableList.of("t")); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE, false, ImmutableList.of("t")); + List> relations = new ArrayList<>(); + for (Relation first : ImmutableList.of(Relation.GT, Relation.GTE)) { + for (Relation second : ImmutableList.of(Relation.GT, Relation.GTE, Relation.EQ)) { + for (Relation third : ImmutableList.of(Relation.GT, Relation.GTE)) { + relations.add(ImmutableList.of(first, second, third)); + } + } + } + relations.add(ImmutableList.of(Relation.EQ, Relation.EQ, Relation.EQ)); + // Window outputs have no table qualifier. Also cover same-table and cross-table slots. + for (List qualifier : ImmutableList.of(ImmutableList.of(), + ImmutableList.of("t"), ImmutableList.of("other"))) { + SlotReference rn = new SlotReference("rn", IntegerType.INSTANCE, false, qualifier); + for (List types : relations) { + List predicates = ImmutableList.of(comparison(a, b, types.get(0)), + comparison(rn, b, types.get(1)), comparison(a, rn, types.get(2))); + for (List permutation : Collections2.permutations(predicates)) { + Set inputs = new LinkedHashSet<>(permutation); + Set inferred = UnequalPredicateInfer.inferUnequalPredicates(inputs); + assertPredicateSemantics(inputs, inferred, a, b, rn); + } + } + } + } + + @Test + public void testStrictReverseRelationWithEquality() { + SlotReference a = new SlotReference("a", IntegerType.INSTANCE, false, ImmutableList.of("t")); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE, false, ImmutableList.of("t")); + for (List qualifier : ImmutableList.of(ImmutableList.of(), + ImmutableList.of("t"), ImmutableList.of("other"))) { + SlotReference c = new SlotReference("c", IntegerType.INSTANCE, false, qualifier); + List predicates = ImmutableList.of(new EqualTo(a, c), new GreaterThan(c, a), + new GreaterThanEqual(c, b), new GreaterThan(b, c), new EqualTo(b, c)); + // Check the reported order first, then all permutations. Clearing a chosen equality must + // not discard a distinct reverse inequality needed to keep the contradiction. + Set inputs = new LinkedHashSet<>(predicates); + assertPredicateSemantics(inputs, UnequalPredicateInfer.inferUnequalPredicates(inputs), a, b, c); + for (List permutation : Collections2.permutations(predicates)) { + inputs = new LinkedHashSet<>(permutation); + assertPredicateSemantics(inputs, UnequalPredicateInfer.inferUnequalPredicates(inputs), a, b, c); + assertPredicateSemantics(inputs, UnequalPredicateInfer.inferAllPredicates(inputs), a, b, c); + } + } + } + + private static void assertPredicateSemantics(Set inputs, Set inferred, + SlotReference a, SlotReference b, SlotReference c) { + for (int av = 0; av <= 3; av++) { + for (int bv = 0; bv <= 3; bv++) { + for (int cv = 0; cv <= 3; cv++) { + Map values = ImmutableMap.of(a, av, b, bv, c, cv); + boolean expected = inputs.stream().allMatch(p -> evaluateComparison(p, values)); + boolean actual = inferred.stream().allMatch(p -> evaluateComparison(p, values)); + Assertions.assertEquals(expected, actual, + () -> "inputs=" + inputs + ", inferred=" + inferred + ", values=" + values); + } + } + } + } + + private static Expression comparison(Expression left, Expression right, Relation relation) { + switch (relation) { + case GT: + return new GreaterThan(left, right); + case GTE: + return new GreaterThanEqual(left, right); + case EQ: + return new EqualTo(left, right); + default: + throw new AssertionError("Unexpected relation: " + relation); + } + } + + private static boolean evaluateComparison(Expression expression, Map values) { + int left = values.get(expression.child(0)); + int right = values.get(expression.child(1)); + if (expression instanceof GreaterThan) { + return left > right; + } else if (expression instanceof GreaterThanEqual) { + return left >= right; + } else if (expression instanceof LessThan) { + return left < right; + } else if (expression instanceof LessThanEqual) { + return left <= right; + } else if (expression instanceof EqualTo) { + return left == right; + } + throw new AssertionError("Unexpected comparison: " + expression); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java index 6c3ccf45fb4c32..55794e7af6362a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java @@ -253,6 +253,21 @@ public void testIsInjectiveCastToForPrimitiveTypes() { assertSafeCast(v1, anotherV1); } + @Test + public void testIsInjectiveCastToForDateTypes() { + for (DataType source : ImmutableList.of(DateType.INSTANCE, DateV2Type.INSTANCE)) { + assertSafeCast(source, DateType.INSTANCE); + assertSafeCast(source, DateV2Type.INSTANCE); + assertSafeCast(source, DateTimeType.INSTANCE); + for (int scale = 0; scale <= DateTimeV2Type.MAX_SCALE; scale++) { + assertSafeCast(source, DateTimeV2Type.of(scale)); + } + assertUnsafeCast(source, TimeStampTzType.MAX); + assertUnsafeCast(source, TimeStampNsType.INSTANCE); + assertUnsafeCast(DateTimeV2Type.MAX, source); + } + } + @Test public void testIsInjectiveCastToForComplexTypes() { assertSafeCast(ArrayType.of(IntegerType.INSTANCE), ArrayType.of(BigIntType.INSTANCE)); diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/infer_none_movable_predicate.out b/regression-test/data/nereids_rules_p0/infer_predicate/infer_none_movable_predicate.out new file mode 100644 index 00000000000000..6a145a72a41c3d --- /dev/null +++ b/regression-test/data/nereids_rules_p0/infer_predicate/infer_none_movable_predicate.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !assert_in_or -- +1 1 +11 11 + diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/infer_predicate_reverse_relation.out b/regression-test/data/nereids_rules_p0/infer_predicate/infer_predicate_reverse_relation.out new file mode 100644 index 00000000000000..35ba57417f0075 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/infer_predicate/infer_predicate_reverse_relation.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !strict -- + +-- !reordered -- + +-- !commuted -- + +-- !non_strict -- +1 1 1 1 +2 2 2 2 + diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/infer_timestamptz_cast.out b/regression-test/data/nereids_rules_p0/infer_predicate/infer_timestamptz_cast.out new file mode 100644 index 00000000000000..9469cb5a353564 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/infer_predicate/infer_timestamptz_cast.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dst_not_equal -- +1 10 + +-- !dst_greater_than -- +1 10 + +-- !scale_not_equal -- +2 20 + +-- !scale_less_than -- +2 20 + diff --git a/regression-test/data/nereids_rules_p0/infer_predicate_qualify.out b/regression-test/data/nereids_rules_p0/infer_predicate_qualify.out new file mode 100644 index 00000000000000..141f1db1e8d237 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/infer_predicate_qualify.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !strict -- + +-- !non_strict -- +1 2 1 1 +2 2 1 2 + +-- !mixed -- +1 2 1 1 + +-- !reordered -- + +-- !matching_row -- +3 4 1 3 + diff --git a/regression-test/data/nereids_rules_p0/infer_predicate_replace_type.out b/regression-test/data/nereids_rules_p0/infer_predicate_replace_type.out new file mode 100644 index 00000000000000..18d06474878861 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/infer_predicate_replace_type.out @@ -0,0 +1,36 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !date_length -- +1 10 + +-- !datetime_length -- +1 10 + +-- !date_same_type -- +1 1 + +-- !date_comparison -- +1 10 + +-- !signed_zero_facts -- +1 true true false +2 true false true +3 true false false +4 true false false +5 true true true + +-- !signed_zero_or -- +1 +3 +5 + +-- !signed_zero_or_reversed -- +2 +3 +5 + +-- !float_comparison -- +3 + +-- !decimal_scale -- +1 10 + diff --git a/regression-test/suites/nereids_rules_p0/infer_predicate/infer_none_movable_predicate.groovy b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_none_movable_predicate.groovy new file mode 100644 index 00000000000000..0c6d53c96b10ff --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_none_movable_predicate.groovy @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("infer_none_movable_predicate", "p0") { + sql "DROP TABLE IF EXISTS infer_none_movable_l" + sql "DROP TABLE IF EXISTS infer_none_movable_r" + sql """ + CREATE TABLE infer_none_movable_l (a INT NOT NULL) + DUPLICATE KEY(a) DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """ + CREATE TABLE infer_none_movable_r (b INT NOT NULL) + DUPLICATE KEY(b) DISTRIBUTED BY HASH(b) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql "INSERT INTO infer_none_movable_l VALUES (1), (11)" + sql "INSERT INTO infer_none_movable_r VALUES (-1), (1), (11)" + // The unmatched negative row must never evaluate the left-side assertion. + order_qt_assert_in_or """ + SELECT l.a, r.b + FROM (SELECT a FROM infer_none_movable_l + WHERE assert_true(a > 0, 'bad') OR a > 10) l + JOIN infer_none_movable_r r ON l.a = r.b + """ +} diff --git a/regression-test/suites/nereids_rules_p0/infer_predicate/infer_predicate_reverse_relation.groovy b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_predicate_reverse_relation.groovy new file mode 100644 index 00000000000000..459dd6dc2aae63 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_predicate_reverse_relation.groovy @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("infer_predicate_reverse_relation") { + sql "drop table if exists infer_predicate_reverse_relation_input" + sql """ + create table infer_predicate_reverse_relation_input ( + k int not null, a int null, b int null, c int null + ) duplicate key(k) distributed by hash(k) buckets 1 + properties("replication_num"="1") + """ + sql """ + insert into infer_predicate_reverse_relation_input values + (1,1,1,1),(2,2,2,2),(3,3,2,1),(4,1,2,3), + (5,null,1,1),(6,1,null,1),(7,1,1,null),(8,null,null,null) + """ + + // Both strict comparisons contradict the equalities. Inference must not admit a = b = c. + order_qt_strict """ + select * from infer_predicate_reverse_relation_input + where a = c and c > a and c >= b and b > c and b = c + """ + order_qt_reordered """ + select * from infer_predicate_reverse_relation_input + where b = c and b > c and c >= b and c > a and a = c + """ + order_qt_commuted """ + select * from infer_predicate_reverse_relation_input + where c = a and a < c and b <= c and c < b and c = b + """ + // Equal non-null rows do satisfy the non-strict variant. + order_qt_non_strict """ + select * from infer_predicate_reverse_relation_input + where a = c and c >= a and c >= b and b >= c and b = c + """ +} diff --git a/regression-test/suites/nereids_rules_p0/infer_predicate/infer_timestamptz_cast.groovy b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_timestamptz_cast.groovy new file mode 100644 index 00000000000000..6325686623b328 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_timestamptz_cast.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("infer_timestamptz_cast", "p0") { + sql "DROP TABLE IF EXISTS infer_timestamptz_l" + sql "DROP TABLE IF EXISTS infer_timestamptz_r" + sql """ + CREATE TABLE infer_timestamptz_l (id INT, tz TIMESTAMPTZ(6)) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """ + CREATE TABLE infer_timestamptz_r (id INT, dt DATETIMEV2(3)) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """ + INSERT INTO infer_timestamptz_l VALUES + (1, CAST('2024-11-03 06:30:00 +00:00' AS TIMESTAMPTZ(6))), + (2, CAST('2024-01-01 00:00:00.123600 +00:00' AS TIMESTAMPTZ(6))) + """ + sql """ + INSERT INTO infer_timestamptz_r VALUES + (10, '2024-11-03 01:30:00'), (20, '2024-01-01 00:00:00.124') + """ + def originalTimeZone = sql "SELECT @@time_zone" + try { + sql "SET time_zone = 'America/New_York'" + // 05:30Z and 06:30Z both map to 01:30 during the fall-back overlap. + order_qt_dst_not_equal """ + SELECT l.id, r.id FROM infer_timestamptz_l l JOIN infer_timestamptz_r r + ON CAST(l.tz AS DATETIMEV2(0)) = r.dt + WHERE NOT (l.tz = CAST('2024-11-03 05:30:00 +00:00' AS TIMESTAMPTZ(6))) + """ + order_qt_dst_greater_than """ + SELECT l.id, r.id FROM infer_timestamptz_l l JOIN infer_timestamptz_r r + ON CAST(l.tz AS DATETIMEV2(0)) = r.dt + WHERE l.tz > CAST('2024-11-03 05:30:00 +00:00' AS TIMESTAMPTZ(6)) + """ + sql "SET time_zone = '+00:00'" + // .123600 and .124000 become equal after rounding to milliseconds. + order_qt_scale_not_equal """ + SELECT l.id, r.id FROM infer_timestamptz_l l JOIN infer_timestamptz_r r + ON CAST(l.tz AS DATETIMEV2(3)) = r.dt + WHERE NOT (l.tz = CAST('2024-01-01 00:00:00.124000 +00:00' AS TIMESTAMPTZ(6))) + """ + order_qt_scale_less_than """ + SELECT l.id, r.id FROM infer_timestamptz_l l JOIN infer_timestamptz_r r + ON CAST(l.tz AS DATETIMEV2(3)) = r.dt + WHERE l.tz < CAST('2024-01-01 00:00:00.124000 +00:00' AS TIMESTAMPTZ(6)) + """ + } finally { + sql "SET time_zone = '${originalTimeZone[0][0]}'" + } +} diff --git a/regression-test/suites/nereids_rules_p0/infer_predicate_qualify.groovy b/regression-test/suites/nereids_rules_p0/infer_predicate_qualify.groovy new file mode 100644 index 00000000000000..e29879738d8e17 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/infer_predicate_qualify.groovy @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("infer_predicate_qualify") { + sql "drop table if exists infer_predicate_qualify_input" + sql """ + create table infer_predicate_qualify_input (k bigint not null, a bigint not null, b bigint not null) + duplicate key(k) distributed by hash(k) buckets 1 + properties("replication_num"="1") + """ + sql "insert into infer_predicate_qualify_input values (1,2,1),(2,2,1)" + + order_qt_strict """ + select t.k, t.a, t.b, row_number() over (order by t.k) as rn + from infer_predicate_qualify_input t + qualify t.a > t.b and rn > t.b and t.a > rn + """ + order_qt_non_strict """ + select t.k, t.a, t.b, row_number() over (order by t.k) as rn + from infer_predicate_qualify_input t + qualify t.a >= t.b and rn >= t.b and t.a >= rn + """ + order_qt_mixed """ + select t.k, t.a, t.b, row_number() over (order by t.k) as rn + from infer_predicate_qualify_input t + qualify t.a >= t.b and rn >= t.b and t.a > rn + """ + order_qt_reordered """ + select t.k, t.a, t.b, row_number() over (order by t.k) as rn + from infer_predicate_qualify_input t + qualify t.a > rn and rn > t.b and t.a > t.b + """ + // Include a row satisfying all strict comparisons, so preserving every row is also detected. + sql "insert into infer_predicate_qualify_input values (3,4,1)" + order_qt_matching_row """ + select t.k, t.a, t.b, row_number() over (order by t.k) as rn + from infer_predicate_qualify_input t + qualify t.a > t.b and rn > t.b and t.a > rn + """ +} diff --git a/regression-test/suites/nereids_rules_p0/infer_predicate_replace_type.groovy b/regression-test/suites/nereids_rules_p0/infer_predicate_replace_type.groovy new file mode 100644 index 00000000000000..ada35647762caa --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/infer_predicate_replace_type.groovy @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("infer_predicate_replace_type") { + sql "drop table if exists infer_replace_date_l" + sql """ + create table infer_replace_date_l (id int not null, d date not null) + duplicate key(id) distributed by hash(id) buckets 1 properties("replication_num"="1") + """ + sql "drop table if exists infer_replace_date_r" + sql """ + create table infer_replace_date_r (id int not null, ts datetime(0) not null) + duplicate key(id) distributed by hash(id) buckets 1 properties("replication_num"="1") + """ + sql "insert into infer_replace_date_l values (1,'2024-01-01')" + sql "insert into infer_replace_date_r values (10,'2024-01-01 00:00:00')" + + order_qt_date_length """ + select l.id, r.id from infer_replace_date_l l join infer_replace_date_r r on l.d = r.ts + where length(cast(l.d as string)) = 10 + """ + order_qt_datetime_length """ + select l.id, r.id from infer_replace_date_l l join infer_replace_date_r r on l.d = r.ts + where length(cast(r.ts as string)) = 19 + """ + order_qt_date_same_type """ + select l.id, r.id from infer_replace_date_l l join infer_replace_date_l r on l.d = r.d + where length(cast(l.d as string)) = 10 + """ + order_qt_date_comparison """ + select l.id, r.id from infer_replace_date_l l join infer_replace_date_r r on l.d = r.ts + where l.d > cast('2023-12-31' as date) + """ + + sql "drop table if exists infer_replace_fp" + sql """ + create table infer_replace_fp (id bigint not null, x double not null, y double not null) + duplicate key(id) distributed by hash(id) buckets 1 properties("replication_num"="1") + """ + sql """ + insert into infer_replace_fp values + (1,cast('-0.0' as double),cast('0.0' as double)), + (2,cast('0.0' as double),cast('-0.0' as double)), + (3,2,2),(4,0,0),(5,-2,-2) + """ + order_qt_signed_zero_facts """ + select id, x = y, signbit(x), signbit(y) from infer_replace_fp + """ + order_qt_signed_zero_or """ + select id from infer_replace_fp where x = y and (signbit(x) or x > cast(1 as double)) + """ + order_qt_signed_zero_or_reversed """ + select id from infer_replace_fp where x = y and (signbit(y) or y > cast(1 as double)) + """ + order_qt_float_comparison """ + select id from infer_replace_fp where x = y and x > cast(1 as double) + """ + + sql "drop table if exists infer_replace_decimal_l" + sql """ + create table infer_replace_decimal_l (id int not null, d decimal(9,2) not null) + duplicate key(id) distributed by hash(id) buckets 1 properties("replication_num"="1") + """ + sql "drop table if exists infer_replace_decimal_r" + sql """ + create table infer_replace_decimal_r (id int not null, d decimal(9,3) not null) + duplicate key(id) distributed by hash(id) buckets 1 properties("replication_num"="1") + """ + sql "insert into infer_replace_decimal_l values (1,1.20)" + sql "insert into infer_replace_decimal_r values (10,1.200)" + order_qt_decimal_scale """ + select l.id, r.id from infer_replace_decimal_l l join infer_replace_decimal_r r on l.d = r.d + where length(cast(l.d as string)) = 4 + """ +}