diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index f30b1187b7bca..619d53fb045fe 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -576,6 +576,15 @@ fn infer_join_predicates( predicates: &[Expr], on_filters: &[Expr], ) -> Result> { + let join_type = join.join_type; + + // Null-aware anti joins implement NOT IN semantics, where NULLs on the + // right side can affect the result. Inferring predicates across the join can + // incorrectly filter those NULLs. + if join_type == JoinType::LeftAnti && join.null_aware { + return Ok(vec![]); + } + // Only allow both side key is column. let join_col_keys = join .on @@ -587,8 +596,6 @@ fn infer_join_predicates( }) .collect::>(); - let join_type = join.join_type; - let mut inferred_predicates = InferredPredicates::new(join_type); infer_join_predicates_from_predicates( diff --git a/datafusion/sqllogictest/test_files/not_in_null_pushdown.slt b/datafusion/sqllogictest/test_files/not_in_null_pushdown.slt new file mode 100644 index 0000000000000..d2ef490d6813f --- /dev/null +++ b/datafusion/sqllogictest/test_files/not_in_null_pushdown.slt @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Regression test for NOT IN null-aware semantics. The right-side NULL must not +# be removed by inferred predicate pushdown through the null-aware anti join. + +statement ok +CREATE TABLE pushdown_t(id INT, a INT) + +statement ok +INSERT INTO pushdown_t VALUES (1, 1), (2, 2) + +statement ok +CREATE TABLE pushdown_u(k INT) + +statement ok +INSERT INTO pushdown_u VALUES (NULL) + +# Expected result is empty: +# - id=1: a <> 1 is false +# - id=2: a <> 1 is true, but 2 NOT IN (NULL) is UNKNOWN +# WHERE only keeps TRUE, so neither row should be returned. +query I +SELECT id +FROM pushdown_t +WHERE a != 1 + AND a NOT IN (SELECT k FROM pushdown_u); +---- + +statement ok +DROP TABLE pushdown_t + +statement ok +DROP TABLE pushdown_u