diff --git a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala index fa46e3d02aa1..2397b0e060cb 100644 --- a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala +++ b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala @@ -272,7 +272,7 @@ object ScalarOperatorGens { | } | | ${elementEqualsExpr.code} - | if (!${elementEqualsExpr.resultTerm}) { + | if (!${nullSafeEquals(leftElementNullTerm, rightElementNullTerm, elementEqualsExpr.resultTerm)}) { | $resultTerm = false; | break; | } @@ -396,7 +396,7 @@ object ScalarOperatorGens { | } | | ${valueEqualsExpr.code} - | if (${valueEqualsExpr.resultTerm}) { + | if (${nullSafeEquals(leftValueNullTerm, rightValueNullTerm, valueEqualsExpr.resultTerm)}) { | $matchedTerm[$rightIndexTerm] = true; | $foundTerm = true; | } @@ -432,7 +432,7 @@ object ScalarOperatorGens { | boolean $rightValueNullTerm = ($rightValueTerm == null); | | ${valueEqualsExpr.code} - | if (!${valueEqualsExpr.resultTerm}) { + | if (!${nullSafeEquals(leftValueNullTerm, rightValueNullTerm, valueEqualsExpr.resultTerm)}) { | $resultTerm = false; | break; | } @@ -464,6 +464,17 @@ object ScalarOperatorGens { } } + /** + * Null-safe equality on a nested element or value: two nulls are equal, a single null is not, + * otherwise the generated equality decides. `generateEquals` leaves its result flag `false` when + * either operand is null, so reading that flag alone treats two null elements as unequal. + */ + private def nullSafeEquals( + leftNullTerm: String, + rightNullTerm: String, + equalsTerm: String): String = + s"(($leftNullTerm && $rightNullTerm) || (!$leftNullTerm && !$rightNullTerm && $equalsTerm))" + private def containsFloatingPoint(t: DataType): Boolean = t.getTypeRoot match { case FLOAT | DOUBLE => true case DataTypeRoot.ARRAY | DataTypeRoot.MAP | DataTypeRoot.MULTISET | DataTypeRoot.ROW | diff --git a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java index ab4ad74af83f..fc0716903c32 100644 --- a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java +++ b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java @@ -571,6 +571,84 @@ public void testIntKeyMapEqualiserIgnoresEntryOrder() { false); } + @Test + public void testArrayEqualiserTreatsNullElementsAsEqual() { + DataType intArrayType = DataTypes.ARRAY(DataTypes.INT()); + RecordEqualiser intArrayEqualiser = + new EqualiserCodeGenerator(new DataType[] {intArrayType}) + .generateRecordEqualiser("nullElementIntArrayFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertFloatingPointEqualiser( + intArrayEqualiser, + intArrayType, + new GenericArray(new Object[] {1, null}), + new GenericArray(new Object[] {1, null}), + true); + assertFloatingPointEqualiser( + intArrayEqualiser, + intArrayType, + new GenericArray(new Object[] {1, null}), + new GenericArray(new Object[] {1, 2}), + false); + + DataType doubleArrayType = DataTypes.ARRAY(DataTypes.DOUBLE()); + RecordEqualiser doubleArrayEqualiser = + new EqualiserCodeGenerator(new DataType[] {doubleArrayType}) + .generateRecordEqualiser("nullElementDoubleArrayFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertFloatingPointEqualiser( + doubleArrayEqualiser, + doubleArrayType, + new GenericArray(new Object[] {1.0d, null}), + new GenericArray(new Object[] {1.0d, null}), + true); + + DataType rowArrayType = DataTypes.ARRAY(DataTypes.ROW(DataTypes.INT())); + RecordEqualiser rowArrayEqualiser = + new EqualiserCodeGenerator(new DataType[] {rowArrayType}) + .generateRecordEqualiser("nullElementRowArrayFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertFloatingPointEqualiser( + rowArrayEqualiser, + rowArrayType, + new GenericArray(new Object[] {null}), + new GenericArray(new Object[] {null}), + true); + } + + @Test + public void testMapEqualiserTreatsNullValuesAsEqual() { + DataType intKeyMapType = DataTypes.MAP(DataTypes.INT(), DataTypes.INT()); + RecordEqualiser intKeyMapEqualiser = + new EqualiserCodeGenerator(new DataType[] {intKeyMapType}) + .generateRecordEqualiser("nullValueIntKeyMapFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertMapEqualiser( + intKeyMapEqualiser, + intKeyMapType, + new GenericMap(singletonMap(1, null)), + new GenericMap(singletonMap(1, null)), + true); + assertMapEqualiser( + intKeyMapEqualiser, + intKeyMapType, + new GenericMap(singletonMap(1, null)), + new GenericMap(singletonMap(1, 2)), + false); + + DataType binaryKeyMapType = DataTypes.MAP(DataTypes.BYTES(), DataTypes.INT()); + RecordEqualiser binaryKeyMapEqualiser = + new EqualiserCodeGenerator(new DataType[] {binaryKeyMapType}) + .generateRecordEqualiser("nullValueBinaryKeyMapFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertMapEqualiser( + binaryKeyMapEqualiser, + binaryKeyMapType, + new GenericMap(singletonMap("k".getBytes(), null)), + new GenericMap(singletonMap("k".getBytes(), null)), + true); + } + private static Map singletonMap(Object key, Object value) { Map map = new HashMap<>(); map.put(key, value);