From 72ada736ef2cdd0ff299eba6c21bf497b6e4d1a8 Mon Sep 17 00:00:00 2001 From: Eunbin Son Date: Wed, 16 Sep 2026 09:23:26 +0900 Subject: [PATCH] [codegen] Fix equaliser treating null array elements and map values as unequal The array element check and both map value checks in ScalarOperatorGens only read the result flag of generateEquals, which stays false when either operand is null. Two null elements or two null map values were therefore reported as unequal, so ARRAY [1, null] != [1, null] and MAP {1 -> null} != {1 -> null} in the generated RecordEqualiser, while top-level fields and map keys already treat null == null as equal. Add a nullSafeEquals helper and use it at the three sites so that two nulls compare equal and a single null compares unequal. Generated-by: Claude Code --- .../paimon/codegen/ScalarOperatorGens.scala | 17 +++- .../codegen/EqualiserCodeGeneratorTest.java | 78 +++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala index fa46e3d02aa1..2397b0e060cb 100644 --- a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala +++ b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala @@ -272,7 +272,7 @@ object ScalarOperatorGens { | } | | ${elementEqualsExpr.code} - | if (!${elementEqualsExpr.resultTerm}) { + | if (!${nullSafeEquals(leftElementNullTerm, rightElementNullTerm, elementEqualsExpr.resultTerm)}) { | $resultTerm = false; | break; | } @@ -396,7 +396,7 @@ object ScalarOperatorGens { | } | | ${valueEqualsExpr.code} - | if (${valueEqualsExpr.resultTerm}) { + | if (${nullSafeEquals(leftValueNullTerm, rightValueNullTerm, valueEqualsExpr.resultTerm)}) { | $matchedTerm[$rightIndexTerm] = true; | $foundTerm = true; | } @@ -432,7 +432,7 @@ object ScalarOperatorGens { | boolean $rightValueNullTerm = ($rightValueTerm == null); | | ${valueEqualsExpr.code} - | if (!${valueEqualsExpr.resultTerm}) { + | if (!${nullSafeEquals(leftValueNullTerm, rightValueNullTerm, valueEqualsExpr.resultTerm)}) { | $resultTerm = false; | break; | } @@ -464,6 +464,17 @@ object ScalarOperatorGens { } } + /** + * Null-safe equality on a nested element or value: two nulls are equal, a single null is not, + * otherwise the generated equality decides. `generateEquals` leaves its result flag `false` when + * either operand is null, so reading that flag alone treats two null elements as unequal. + */ + private def nullSafeEquals( + leftNullTerm: String, + rightNullTerm: String, + equalsTerm: String): String = + s"(($leftNullTerm && $rightNullTerm) || (!$leftNullTerm && !$rightNullTerm && $equalsTerm))" + private def containsFloatingPoint(t: DataType): Boolean = t.getTypeRoot match { case FLOAT | DOUBLE => true case DataTypeRoot.ARRAY | DataTypeRoot.MAP | DataTypeRoot.MULTISET | DataTypeRoot.ROW | diff --git a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java index ab4ad74af83f..fc0716903c32 100644 --- a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java +++ b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java @@ -571,6 +571,84 @@ public void testIntKeyMapEqualiserIgnoresEntryOrder() { false); } + @Test + public void testArrayEqualiserTreatsNullElementsAsEqual() { + DataType intArrayType = DataTypes.ARRAY(DataTypes.INT()); + RecordEqualiser intArrayEqualiser = + new EqualiserCodeGenerator(new DataType[] {intArrayType}) + .generateRecordEqualiser("nullElementIntArrayFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertFloatingPointEqualiser( + intArrayEqualiser, + intArrayType, + new GenericArray(new Object[] {1, null}), + new GenericArray(new Object[] {1, null}), + true); + assertFloatingPointEqualiser( + intArrayEqualiser, + intArrayType, + new GenericArray(new Object[] {1, null}), + new GenericArray(new Object[] {1, 2}), + false); + + DataType doubleArrayType = DataTypes.ARRAY(DataTypes.DOUBLE()); + RecordEqualiser doubleArrayEqualiser = + new EqualiserCodeGenerator(new DataType[] {doubleArrayType}) + .generateRecordEqualiser("nullElementDoubleArrayFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertFloatingPointEqualiser( + doubleArrayEqualiser, + doubleArrayType, + new GenericArray(new Object[] {1.0d, null}), + new GenericArray(new Object[] {1.0d, null}), + true); + + DataType rowArrayType = DataTypes.ARRAY(DataTypes.ROW(DataTypes.INT())); + RecordEqualiser rowArrayEqualiser = + new EqualiserCodeGenerator(new DataType[] {rowArrayType}) + .generateRecordEqualiser("nullElementRowArrayFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertFloatingPointEqualiser( + rowArrayEqualiser, + rowArrayType, + new GenericArray(new Object[] {null}), + new GenericArray(new Object[] {null}), + true); + } + + @Test + public void testMapEqualiserTreatsNullValuesAsEqual() { + DataType intKeyMapType = DataTypes.MAP(DataTypes.INT(), DataTypes.INT()); + RecordEqualiser intKeyMapEqualiser = + new EqualiserCodeGenerator(new DataType[] {intKeyMapType}) + .generateRecordEqualiser("nullValueIntKeyMapFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertMapEqualiser( + intKeyMapEqualiser, + intKeyMapType, + new GenericMap(singletonMap(1, null)), + new GenericMap(singletonMap(1, null)), + true); + assertMapEqualiser( + intKeyMapEqualiser, + intKeyMapType, + new GenericMap(singletonMap(1, null)), + new GenericMap(singletonMap(1, 2)), + false); + + DataType binaryKeyMapType = DataTypes.MAP(DataTypes.BYTES(), DataTypes.INT()); + RecordEqualiser binaryKeyMapEqualiser = + new EqualiserCodeGenerator(new DataType[] {binaryKeyMapType}) + .generateRecordEqualiser("nullValueBinaryKeyMapFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertMapEqualiser( + binaryKeyMapEqualiser, + binaryKeyMapType, + new GenericMap(singletonMap("k".getBytes(), null)), + new GenericMap(singletonMap("k".getBytes(), null)), + true); + } + private static Map singletonMap(Object key, Object value) { Map map = new HashMap<>(); map.put(key, value);