From 9d333ed633c53b2f971615f1145ae7d6a2296677 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 14 Sep 2026 13:15:05 +0800 Subject: [PATCH] [fix](fe) Preserve Paimon CAST predicate semantics on branch-4.1 ### What problem does this PR solve? Problem Summary: Paimon predicate conversion on branch-4.1 strips CAST from column references before building source filters. For a STRING column containing '05', ' 5', and '5', `CAST(code AS INT) = 5` can become the source predicate `code = '5'`, incorrectly pruning matching rows before Doris evaluates the original predicate. Decimal casts that reduce scale have the same risk. Only convert bare column references. Keep CAST predicates in Doris, preserving their value and null semantics. Ordinary column predicates can still be pushed down independently. This follows the conservative CAST policy already used by the connector on master. ### Release note Fix missing rows in Paimon queries with predicates on casted columns on branch-4.1. ### Check List (For Author) - Test: FE planner regression tests for CAST comparisons, IN/NOT IN, null checks, OR, independent AND conjuncts, and ordinary column pushdown. The 11 predicate-converter tests pass after reproducing 8 failures on the original code. FE Checkstyle passes with 0 violations. All 59 tests across PaimonPredicateConverterTest, PaimonScanNodeTest, and PaimonSourceTest pass. No external Paimon cluster was required for these FE tests. - Behavior changed: Yes. CAST column predicates remain in Doris instead of being rewritten into potentially stricter Paimon filters. This may reduce pushdown for otherwise safe casts. - Does this need documentation: No. --- .../source/PaimonPredicateConverter.java | 11 ++-- .../planner/PaimonPredicateConverterTest.java | 56 +++++++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java index ae45c2427184e4..45b351aa618388 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java @@ -185,15 +185,12 @@ private int getFieldIndex(String colName) { public static SlotRef convertDorisExprToSlotRef(Expr expr) { - SlotRef slotRef = null; + // Stripping CAST can prune matching rows, e.g. CAST('05' AS INT) = 5 is not '05' = '5'. + // Keep casted columns in the original Doris conjuncts to preserve value and null semantics. if (expr instanceof SlotRef) { - slotRef = (SlotRef) expr; - } else if (expr instanceof CastExpr) { - if (expr.getChild(0) instanceof SlotRef) { - slotRef = (SlotRef) expr.getChild(0); - } + return (SlotRef) expr; } - return slotRef; + return null; } public LiteralExpr convertDorisExprToLiteralExpr(Expr expr) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java index fde1b6f74c244c..85ff507ddb60c0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java @@ -17,6 +17,7 @@ package org.apache.doris.planner; +import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.Expr; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.paimon.source.PaimonPredicateConverter; @@ -29,10 +30,14 @@ import org.apache.paimon.predicate.Or; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.List; @@ -46,6 +51,57 @@ protected void runBeforeAll() throws Exception { String tbl1 = "create table db1.tbl1(" + "k1 int," + " k2 int," + " v1 int)" + " distributed by hash(k1)" + " properties('replication_num' = '1');"; createTables(tbl1); + createTables("create table db1.cast_predicates (id int, code string, amount decimal(10, 2))" + + " distributed by hash(id) properties('replication_num' = '1')"); + } + + @ParameterizedTest + @ValueSource(strings = {"cast(code as int) = 5", "5 = cast(code as int)", "cast(code as int) > 5", + "cast(code as int) in (5, 6)", "cast(code as int) not in (5, 6)", + "cast(code as int) is null", "cast(code as int) is not null", + "cast(code as int) = 5 or id = 1"}) + public void rejectStringToIntegerCast(String predicate) throws Exception { + List conjuncts = planCastPredicate(predicate); + Assertions.assertTrue(conjuncts.stream().anyMatch(expr -> expr.contains(CastExpr.class)), predicate); + List remaining = Expr.cloneList(conjuncts); + Assertions.assertTrue(castPredicateConverter().convertToPaimonExpr(conjuncts).isEmpty(), predicate); + Assertions.assertEquals(remaining, conjuncts); + } + + @Test + public void rejectDecimalScaleCast() throws Exception { + List conjuncts = planCastPredicate("cast(amount as decimal(10, 1)) = 1.2"); + Assertions.assertTrue(conjuncts.stream().anyMatch(expr -> expr.contains(CastExpr.class))); + Assertions.assertTrue(castPredicateConverter().convertToPaimonExpr(conjuncts).isEmpty()); + } + + @Test + public void retainIndependentUncastPredicates() throws Exception { + List conjuncts = planCastPredicate("cast(code as int) = 5 and id = 1"); + List remaining = Expr.cloneList(conjuncts); + List predicates = castPredicateConverter().convertToPaimonExpr(conjuncts); + Assertions.assertEquals(1, predicates.size()); + Assertions.assertEquals("id", ((LeafPredicate) predicates.get(0)).fieldName()); + Assertions.assertEquals(remaining, conjuncts); + + for (String predicate : Lists.newArrayList("code = '5'", "id > 1", "id in (1, 2)")) { + Assertions.assertEquals(1, castPredicateConverter().convertToPaimonExpr(planCastPredicate(predicate)).size(), + predicate); + } + } + + private List planCastPredicate(String predicate) throws Exception { + StmtExecutor executor = new StmtExecutor(connectContext, + "select * from db1.cast_predicates where " + predicate); + executor.execute(); + return executor.planner().getScanNodes().get(0).getConjuncts(); + } + + private PaimonPredicateConverter castPredicateConverter() { + return new PaimonPredicateConverter(new RowType(Lists.newArrayList( + new DataField(0, "id", new IntType()), + new DataField(1, "code", new VarCharType()), + new DataField(2, "amount", new DecimalType(10, 2))))); } @Test