From 02c4534cd64a81c9c627ed249db8ba6896ee6d20 Mon Sep 17 00:00:00 2001 From: LautaroPetaccio Date: Mon, 14 Sep 2026 10:31:34 -0300 Subject: [PATCH] Add tests for Mongo semantics the heuristics calculator does not reproduce Twenty queries where the calculator disagrees with the database, and three neighbouring cases that it already answers correctly, kept as guards so that a fix can be told apart from a regression in the path next to it. They extend the section at the end of MongoHeuristicsCalculatorTest, which was written for exactly this, and follow its convention: every expected value was taken from a MongoDB 7.0.41 server answering the same query against the same document, one test per defect, and @Disabled with the reason on the ones that fail today. Where the database refuses the query outright there is no answer to copy, so those tests assert that the calculator answers false without throwing, that being the only answer available to it. Four of them concern values compareNonNullValues has no branch for, being two sub-documents or two binary values, which reach the case for types that cannot be compared. That answers false for $eq, missing the plainest nested-object query there is, and true for $ne and $nin, which is a match the database does not have. On master these threw instead, so this is a step forward that stops short. Six of the queries make the calculator throw, and MongoHandler catches nothing around computeDistanceDocuments, so the exception costs the action its whole ExtraHeuristicsDto, the SQL heuristics included. Two of those six are queries the database answers normally: $regex against an array holding no strings, and $not holding a bare regex. --- .../mongo/MongoHeuristicsCalculatorTest.java | 451 ++++++++++++++++++ 1 file changed, 451 insertions(+) diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/mongo/MongoHeuristicsCalculatorTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/mongo/MongoHeuristicsCalculatorTest.java index 3efc888435..80373e8f8b 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/mongo/MongoHeuristicsCalculatorTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/mongo/MongoHeuristicsCalculatorTest.java @@ -2114,4 +2114,455 @@ public void testNotWithMoreThanOneInnerOperator() { assertTrue(calculator.computeHeuristicDocument(nested, doc).isFalse()); assertTrue(calculator.computeHeuristicDocument(nested, new Document("a", 5)).isTrue()); } + + /* + ================================================================================ + Further cases, in the same form: every expected value comes from a MongoDB + 7.0.41 server answering that query against that document, one test per defect, + and @Disabled on the ones that fail today. + + Where the database refuses the query outright there is no answer to copy, so + those tests assert that the calculator answers false without throwing, that + being the only answer available to it. + ================================================================================ + */ + + @Test + @Disabled("$regex against an array holding no strings throws IllegalArgumentException") + public void testRegexAgainstAnArrayWithoutStringElements() { + /* + mongo: no match, and no error. The elements are compared one by one and a + number never matches a regex. + + The array branch keeps only the strings and then aggregates what is left, + and the aggregation rejects an empty array. The scalar branch of the same + operator answers C_FALSE for a number, as this one used to. + */ + Document doc = new Document().append("a", new ArrayList<>(Arrays.asList(1, 2, 3))); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$regex", new BsonRegularExpression("x"))); + Truthness truthness = assertDoesNotThrow(() -> calculator.computeHeuristicDocument(query, doc)); + assertTrue(truthness.isFalse()); + } + + @Test + @Disabled("a bitmask held as org.bson.types.Binary is not parsed, so the calculator throws") + public void testBitmaskGivenAsBinaryData() { + /* + mongo: matches, bit 1 of the mask is set and so is bit 1 of the value. + + A bitmask given as binary data is only recognised when it is a byte[]. A + query that has been through a BSON decode holds a Binary instead, which + leaves the mask unparsed, and the whole query then parses to null. + */ + Document doc = new Document().append("a", 2); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$bitsAllSet", new Binary(new byte[]{(byte) 0x02}))); + Truthness truthness = assertDoesNotThrow(() -> calculator.computeHeuristicDocument(query, doc)); + assertTrue(truthness.isTrue()); + } + + @Test + public void testBitmaskGivenAsAByteArray() { + // the shape of binary bitmask that is handled today, next to the one above + Document doc = new Document().append("a", 2); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$bitsAllSet", new byte[]{(byte) 0x02})); + assertTrue(calculator.computeHeuristicDocument(query, doc).isTrue()); + } + + @Test + @Disabled("$not holding a bare regex is not parsed, so the calculator throws a NullPointerException") + public void testNotWithABareRegex() { + /* + mongo: matches, {"a": {"$not": /x/}} is a documented form of the operator. + Only a document is accepted as the value of $not today, so this parses to + null. + */ + Document doc = new Document().append("a", "aa"); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$not", new BsonRegularExpression("x"))); + Truthness truthness = assertDoesNotThrow(() -> calculator.computeHeuristicDocument(query, doc)); + assertTrue(truthness.isTrue()); + } + + @Test + @Disabled("$comment is removed from documents that are values rather than operators") + public void testCommentInsideALiteralValue() { + /* + mongo: matches. A field name that starts with "$" is storable since 5.0, and + under an explicit $eq the document is compared as a literal. The implicit + form, {"a": {"$comment": ...}}, is rejected instead. + + The comment operator is removed everywhere it appears in the query, so the + value being compared loses the key as well and no longer equals the stored + document. + */ + Document doc = new Document().append("a", + new Document().append("$comment", "note").append("x", 1)); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", new Document().append("$eq", + new Document().append("$comment", "note").append("x", 1))); + assertTrue(calculator.computeHeuristicDocument(query, doc).isTrue()); + } + + @Test + public void testCommentAtTopLevel() { + // the placement of $comment that the query is actually allowed to use + Document doc = new Document().append("a", 1); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", 1).append("$comment", "note"); + assertTrue(calculator.computeHeuristicDocument(query, doc).isTrue()); + } + + @Test + @Disabled("$all with a repeated element does not match a field that is not an array") + public void testAllWithARepeatedElement() { + // mongo: matches. $all is an $and of $eq, so a repeat holds against a scalar. + Document doc = new Document().append("a", 1); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$all", Arrays.asList(1, 1))); + assertTrue(calculator.computeHeuristicDocument(query, doc).isTrue()); + } + + @Test + @Disabled("$all holding more than one null does not match a missing field") + public void testAllWithRepeatedNullsAgainstAMissingField() { + // mongo: matches, null matches a missing field however many times it is asked for + Document doc = new Document().append("b", 1); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$all", Arrays.asList(null, null))); + assertTrue(calculator.computeHeuristicDocument(query, doc).isTrue()); + } + + @Test + public void testAllWithASingleNullAgainstAMissingField() { + // the single element case, which is special-cased today + Document doc = new Document().append("b", 1); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$all", Collections.singletonList(null))); + assertTrue(calculator.computeHeuristicDocument(query, doc).isTrue()); + } + + @Test + @Disabled("$in does not match an element of the list that is a regex") + public void testInWithARegexElement() { + // mongo: matches, $in accepts regexes among its elements + Document doc = new Document().append("a", "xy"); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", new Document().append("$in", + Collections.singletonList(new BsonRegularExpression("x")))); + assertTrue(calculator.computeHeuristicDocument(query, doc).isTrue()); + } + + @Test + @Disabled("two sub-documents are never equal to one another, so the query never matches") + public void testEqualityBetweenTwoSubDocuments() { + /* + mongo: matches every one of these. A sub-document is compared field by field + in order, and an array is searched for it as an element. + + compareNonNullValues has a branch for numbers, strings, booleans, lists, + dates, timestamps and object ids, but none for two documents, so they reach + the branch for values of types that cannot be compared and answer false. + testFieldsHoldingASubDocument above covers a sub-document against a scalar, + where answering false is right, which is why this one is not caught there. + */ + Document doc = new Document().append("a", new Document().append("x", 1)); + Document arrayOfSubDocuments = new Document().append("a", new ArrayList<>( + Arrays.asList(new Document().append("x", 1), new Document().append("x", 2)))); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document implicitEquality = new Document().append("a", new Document().append("x", 1)); + assertTrue(calculator.computeHeuristicDocument(implicitEquality, doc).isTrue()); + + Document explicitEquality = new Document().append("a", + new Document().append("$eq", new Document().append("x", 1))); + assertTrue(calculator.computeHeuristicDocument(explicitEquality, doc).isTrue()); + + // mongo: matches, the array is searched for the sub-document as one of its elements + assertTrue(calculator.computeHeuristicDocument(implicitEquality, arrayOfSubDocuments).isTrue()); + } + + @Test + @Disabled("$ne and $nin report a match between two sub-documents that are equal") + public void testInequalityBetweenTwoSubDocuments() { + /* + mongo: no match for either, the sub-document held by the field is the one the + query names. + + The same gap as the test above, in the direction that hurts: two documents + reach the branch for values of types that cannot be compared, which answers + true for $ne, and $nin reaches the same answer by inverting it once more. + */ + Document doc = new Document().append("a", new Document().append("x", 1)); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document notEquals = new Document().append("a", + new Document().append("$ne", new Document().append("x", 1))); + assertTrue(calculator.computeHeuristicDocument(notEquals, doc).isFalse()); + + Document notIn = new Document().append("a", new Document().append("$nin", + Collections.singletonList(new Document().append("x", 1)))); + assertTrue(calculator.computeHeuristicDocument(notIn, doc).isFalse()); + } + + @Test + @Disabled("$eq does not match an array holding the given array as one of its elements") + public void testEqualsMatchingAnArrayThatIsAnElement() { + /* + mongo: matches, the field holds [1,2] as one of its elements. The other half + of this operator, an array equal to the query value as a whole, is covered by + testEqualsMatchingAnArrayFieldAsAWhole above and already answers correctly. + + $in and $all both find a nested array in this position, so $eq is the one of + the three that does not. + */ + Document doc = new Document().append("a", new ArrayList<>( + Arrays.asList(new ArrayList<>(Arrays.asList(1, 2)), 3))); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$eq", Arrays.asList(1, 2))); + assertTrue(calculator.computeHeuristicDocument(query, doc).isTrue()); + } + + @Test + @Disabled("two binary values are never equal to one another, so the query never matches") + public void testComparisonBetweenTwoBinaryValues() { + /* + mongo: the first three match and the last two do not. + + The same gap as testEqualityBetweenTwoSubDocuments, reached by the other value + compareNonNullValues has no branch for. testFieldsHoldingASubDocument above + covers binary data against a number, where answering false is right. + */ + Document doc = new Document().append("a", new Binary(new byte[]{1, 2})); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + assertTrue(calculator.computeHeuristicDocument( + new Document().append("a", new Binary(new byte[]{1, 2})), doc).isTrue()); + assertTrue(calculator.computeHeuristicDocument( + new Document().append("a", new Document().append("$eq", new Binary(new byte[]{1, 2}))), + doc).isTrue()); + assertTrue(calculator.computeHeuristicDocument( + new Document().append("a", new Document().append("$in", + Collections.singletonList(new Binary(new byte[]{1, 2})))), doc).isTrue()); + + // mongo: no match, the value held by the field is the one the query names + assertTrue(calculator.computeHeuristicDocument( + new Document().append("a", new Document().append("$ne", new Binary(new byte[]{1, 2}))), + doc).isFalse()); + assertTrue(calculator.computeHeuristicDocument( + new Document().append("a", new Document().append("$nin", + Collections.singletonList(new Binary(new byte[]{1, 2})))), doc).isFalse()); + } + + @Test + @Disabled("$all holding an $elemMatch is compared as a value instead of applied as a condition") + public void testAllHoldingAnElemMatch() { + /* + mongo: matches both. $all accepts $elemMatch among its elements, and then every + one of them has to be satisfied by some element of the array. + + Each element of $all is compared for equality against the elements of the field + instead, so the document holding the operator never equals any of them. + */ + Document doc = new Document().append("a", new ArrayList<>(Arrays.asList(1, 5))); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document one = new Document().append("a", new Document().append("$all", + Collections.singletonList(new Document().append("$elemMatch", + new Document().append("$gt", 2))))); + assertTrue(calculator.computeHeuristicDocument(one, doc).isTrue()); + + // mongo: matches, 5 satisfies the first and 1 satisfies the second + Document two = new Document().append("a", new Document().append("$all", Arrays.asList( + new Document().append("$elemMatch", new Document().append("$gt", 4)), + new Document().append("$elemMatch", new Document().append("$lt", 2))))); + assertTrue(calculator.computeHeuristicDocument(two, doc).isTrue()); + } + + @Test + @Disabled("an ordering operator given null is not parsed, so the calculator throws") + public void testOrderingComparisonsGivenANullArgument() { + /* + mongo: runs all four and matches nothing, null orders against nothing. It is a + query the database accepts, unlike $mod with a zero divisor or a negative $size. + + $eq and $ne accept null and answer, so the four below are the odd ones out in + their own family. + */ + Document doc = new Document().append("a", 1); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + for (String operator : Arrays.asList("$gt", "$gte", "$lt", "$lte")) { + Document query = new Document().append("a", new Document().append(operator, null)); + Truthness truthness = assertDoesNotThrow( + () -> calculator.computeHeuristicDocument(query, doc), operator); + assertTrue(truthness.isFalse(), operator); + } + } + + @Test + @Disabled("$exists given anything but a boolean is not parsed, so the calculator throws") + public void testExistsGivenAValueThatIsNotABoolean() { + /* + mongo: takes any value for $exists and reads it as a truth value, so 1 and a + string ask for the field to be present, and 0 and null ask for it to be absent. + All four are queries the database accepts and answers. + */ + Document doc = new Document().append("a", 1); + Document without = new Document().append("b", 1); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document truthy = new Document().append("a", new Document().append("$exists", 1)); + assertTrue(assertDoesNotThrow(() -> calculator.computeHeuristicDocument(truthy, doc)).isTrue()); + + Document falsy = new Document().append("a", new Document().append("$exists", 0)); + assertTrue(assertDoesNotThrow(() -> calculator.computeHeuristicDocument(falsy, doc)).isFalse()); + assertTrue(assertDoesNotThrow(() -> calculator.computeHeuristicDocument(falsy, without)).isTrue()); + + Document nullValue = new Document().append("a", new Document().append("$exists", null)); + assertTrue(assertDoesNotThrow(() -> calculator.computeHeuristicDocument(nullValue, doc)).isFalse()); + } + + @Test + @Disabled("a negative bit position is read as bit 63 instead of being rejected") + public void testBitmaskWithANegativeBitPosition() { + /* + mongo: "Failed to parse bit position. Expected a non-negative number in: 0: -1". + + A shift count is masked to its low six bits in Java, so 1L << -1 sets bit 63 + and the mask becomes Long.MIN_VALUE, which the value below then matches. + Rejecting the mask on its own is not enough here: if the mask stops parsing, + the selector answers null and the query goes unparsed, which is the throwing + case rather than a fix. + */ + Document doc = new Document().append("a", Long.MIN_VALUE); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$bitsAllSet", Collections.singletonList(-1))); + Truthness truthness = assertDoesNotThrow(() -> calculator.computeHeuristicDocument(query, doc)); + assertTrue(truthness.isFalse()); + } + + @Test + @Disabled("a bitmask that is not an integer is truncated instead of being rejected") + public void testBitmaskThatIsNotAnInteger() { + // mongo: "Expected an integer: $bitsAllSet: 3.9". Here 3.9 is read as the mask 3. + Document doc = new Document().append("a", 3); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", new Document().append("$bitsAllSet", 3.9d)); + Truthness truthness = assertDoesNotThrow(() -> calculator.computeHeuristicDocument(query, doc)); + assertTrue(truthness.isFalse()); + } + + @Test + @Disabled("a negative bitmask is read as all ones instead of being rejected") + public void testBitmaskThatIsNegative() { + // mongo: "Expected a non-negative number in: $bitsAllSet: -1" + Document doc = new Document().append("a", -1); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", new Document().append("$bitsAllSet", -1)); + Truthness truthness = assertDoesNotThrow(() -> calculator.computeHeuristicDocument(query, doc)); + assertTrue(truthness.isFalse()); + } + + @Test + @Disabled("$mod with a divisor of 0 throws ArithmeticException") + public void testModByZero() { + // mongo: "divisor cannot be 0" + Document doc = new Document().append("a", 5); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", + new Document().append("$mod", Arrays.asList(0, 0))); + Truthness truthness = assertDoesNotThrow(() -> calculator.computeHeuristicDocument(query, doc)); + assertTrue(truthness.isFalse()); + } + + @Test + @Disabled("$size with a negative value is not parsed, so the calculator throws a NullPointerException") + public void testNegativeSize() { + // mongo: "Expected a non-negative number in: $size: -1" + Document doc = new Document().append("a", Arrays.asList(1, 2)); + MongoHeuristicsCalculator calculator = new MongoHeuristicsCalculator(); + + Document query = new Document().append("a", new Document().append("$size", -1)); + Truthness truthness = assertDoesNotThrow(() -> calculator.computeHeuristicDocument(query, doc)); + assertTrue(truthness.isFalse()); + } + + @Test + @Disabled("the documents are traversed more than once, so a one-shot Iterable is scored wrongly") + public void testDocumentsThatCanOnlyBeTraversedOnce() { + /* + Not a question of MongoDB semantics but of the contract of the calculator. + + computeDistanceDocuments counts the documents, computeHeuristicOnDocuments + counts them again, and the loop then walks them a third time. What is passed + in today is a FindIterable, which replays, so this costs a round trip to the + database per traversal rather than a wrong answer. The signature accepts any + Iterable though, and one that cannot be replayed is scored as if the + collection were empty while still reporting the documents counted first. + */ + List docs = Arrays.asList( + new Document().append("a", 1), + new Document().append("a", 2)); + Document query = convertToDocument(Filters.eq("a", 2)); + + // the second document matches the query, so the distance is 0 over either Iterable + MongoDistanceWithMetrics fromList = new MongoHeuristicsCalculator() + .computeDistanceDocuments(query, docs); + assertEquals(2, fromList.numberOfEvaluatedDocuments); + assertEquals(0d, fromList.mongoDistance, 0.000001d); + + MongoDistanceWithMetrics fromOneShot = new MongoHeuristicsCalculator() + .computeDistanceDocuments(query, new OneShotIterable<>(docs)); + assertEquals(2, fromOneShot.numberOfEvaluatedDocuments); + assertEquals(0d, fromOneShot.mongoDistance, 0.000001d); + } + + /** + * An Iterable that can be traversed only once. + */ + private static final class OneShotIterable implements Iterable { + + private final Iterable delegate; + private boolean consumed = false; + + private OneShotIterable(Iterable delegate) { + this.delegate = delegate; + } + + @Override + public Iterator iterator() { + if (consumed) { + return Collections.emptyIterator(); + } + consumed = true; + return delegate.iterator(); + } + } }