diff --git a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/app/CqlTypeAndFunctionChecker.java b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/app/CqlTypeAndFunctionChecker.java index a00aaa990..51a33d212 100644 --- a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/app/CqlTypeAndFunctionChecker.java +++ b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/app/CqlTypeAndFunctionChecker.java @@ -36,11 +36,13 @@ import de.ii.xtraplatform.cql.domain.ImmutableLte; import de.ii.xtraplatform.cql.domain.ImmutableNeq; import de.ii.xtraplatform.cql.domain.In; +import de.ii.xtraplatform.cql.domain.InResultSet; import de.ii.xtraplatform.cql.domain.Interval; import de.ii.xtraplatform.cql.domain.IsNull; import de.ii.xtraplatform.cql.domain.Like; import de.ii.xtraplatform.cql.domain.LogicalOperation; import de.ii.xtraplatform.cql.domain.Not; +import de.ii.xtraplatform.cql.domain.Parameter; import de.ii.xtraplatform.cql.domain.Property; import de.ii.xtraplatform.cql.domain.Scalar; import de.ii.xtraplatform.cql.domain.ScalarLiteral; @@ -149,6 +151,11 @@ public Type visit(IsNull isNull, List children) { @Override public Type visit(BinaryScalarOperation scalarOperation, List children) { + if (scalarOperation instanceof InResultSet) { + // the first argument may be of any queryable type, the second is always the + // name of a result set + return Type.Boolean; + } checkOperation(scalarOperation, children); return Type.Boolean; } @@ -249,6 +256,13 @@ public Type visit(Property property, List children) { return Type.UNKNOWN; } + @Override + public Type visit(Parameter parameter, List children) { + // an unbound parameter (e.g. in a stored query validated before invocation) has no known type; + // treat it as a wildcard so the concrete operands are still checked + return Type.UNKNOWN; + } + @Override public Type visit(ScalarLiteral scalarLiteral, List children) { return Type.valueOf(scalarLiteral.getType().getSimpleName()); diff --git a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlBuiltInFunctions.java b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlBuiltInFunctions.java index 4c0d97231..b3c3d099d 100644 --- a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlBuiltInFunctions.java +++ b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlBuiltInFunctions.java @@ -73,7 +73,21 @@ public final class CqlBuiltInFunctions { ImmutableList.of(TYPE_BOOLEAN), Map.of( "SQL/PGIS", "$values::varchar LIKE $pattern", - "SQL/GPKG", "cast($values as text) LIKE $pattern"))); + "SQL/GPKG", "cast($values as text) LIKE $pattern")), + CustomFunction.ofQueryExpressionOnly( + InResultSet.TYPE, + "Tests whether the value of a property, or the feature id, is contained in a named " + + "result set. Result sets are defined by other queries of the same query " + + "expression; this function can therefore only be used within a query " + + "expression, not in a standalone CQL2 filter.", + ImmutableList.of( + argument( + "value", + "Property to test: a feature reference, a value property holding feature " + + "ids, or the feature id.", + TYPE_STRING), + argument("resultSet", "Name of the result set.", TYPE_STRING)), + ImmutableList.of(TYPE_BOOLEAN))); private CqlBuiltInFunctions() {} diff --git a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlToText.java b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlToText.java index 9a1d5dd03..1a64645b4 100644 --- a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlToText.java +++ b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlToText.java @@ -203,6 +203,9 @@ public String visit(Not not, List children) { @Override public String visit(BinaryScalarOperation scalarOperation, List children) { + if (scalarOperation instanceof InResultSet) { + return String.format("INRESULTSET(%s, %s)", children.get(0), children.get(1)); + } String operator = SCALAR_OPERATORS.get(scalarOperation.getClass()); return String.format("%s %s %s", children.get(0), operator, children.get(1)); } diff --git a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlVisitorCopy.java b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlVisitorCopy.java index 8c40397c2..31fbd66b7 100644 --- a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlVisitorCopy.java +++ b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CqlVisitorCopy.java @@ -33,6 +33,18 @@ public CqlNode visit(Not not, List children) { @Override public CqlNode visit(BinaryScalarOperation scalarOperation, List children) { + if (scalarOperation instanceof InResultSet) { + // keep the resolved producer context, only the args are copied + return new ImmutableInResultSet.Builder() + .from((InResultSet) scalarOperation) + .args( + children.stream() + .filter(child -> child instanceof Scalar) + .map(child -> (Scalar) child) + .toList()) + .build(); + } + BinaryScalarOperation.Builder builder = null; if (scalarOperation instanceof Eq) { diff --git a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CustomFunction.java b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CustomFunction.java index 76845fa1e..5ccda2723 100644 --- a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CustomFunction.java +++ b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/CustomFunction.java @@ -173,11 +173,33 @@ default Map getExpressions() { return Map.of(); } + /** + * A built-in function that is only meaningful within a query expression (a semi-join against a + * result set defined by another query) and is encoded by a dedicated handler rather than a SQL + * template. Such a function therefore defines neither {@code expression} nor {@code expressions}. + * Not intended for user-defined functions. + */ + @Value.Default + default boolean getQueryExpressionOnly() { + return false; + } + @Value.Check default void checkExpressionDefinition() { boolean hasExpression = getExpression() != null && !getExpression().isBlank(); boolean hasExpressions = !getExpressions().isEmpty(); + if (getQueryExpressionOnly()) { + if (hasExpression || hasExpressions) { + throw new IllegalStateException( + String.format( + "Custom function '%s' is restricted to query expressions and must not define a SQL" + + " expression", + getName())); + } + return; + } + if (hasExpression == hasExpressions) { throw new IllegalStateException( String.format( @@ -224,4 +246,18 @@ static CustomFunction of( .expressions(expressions) .build(); } + + static CustomFunction ofQueryExpressionOnly( + String name, + @Nullable String description, + List arguments, + List returns) { + return new ImmutableCustomFunction.Builder() + .name(name) + .description(description) + .arguments(arguments) + .returns(returns) + .queryExpressionOnly(true) + .build(); + } } diff --git a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/InResultSet.java b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/InResultSet.java new file mode 100644 index 000000000..f1b04483c --- /dev/null +++ b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/InResultSet.java @@ -0,0 +1,100 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.cql.domain; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.Optional; +import org.immutables.value.Value; + +/** + * Predicate that tests whether a property value (or the feature id) is contained in a named result + * set that is defined by another query of the same query expression. + * + *

The two arguments are the property and the name of the result set (a string literal). The + * predicate behaves like an IN expression (single values) or an A_OVERLAPS expression (arrays) + * against the object ids in the result set. + * + *

The result-set reference is resolved by the service before the filter is encoded: the + * producing query's feature type and its effective filter are attached to this node. They are not + * part of the JSON or text encoding. + */ +@Value.Immutable +@JsonDeserialize(builder = ImmutableInResultSet.Builder.class) +public interface InResultSet extends BinaryScalarOperation { + + String TYPE = "inResultSet"; + + @Override + @Value.Derived + default String getOp() { + return TYPE; + } + + /** Feature type of the query that defines the result set. */ + @JsonIgnore + Optional getProducerType(); + + /** Effective filter of the query that defines the result set. */ + @JsonIgnore + Optional getProducerFilter(); + + /** + * Property of the producing feature type whose values form the result set (projected result set). + * If empty, the result set consists of the ids of the selected features. + */ + @JsonIgnore + Optional getProducerValues(); + + /** + * Values of the result set, materialized by the service before the filter is encoded. When + * present, the predicate is encoded as a literal {@code IN} list rather than a nested subquery; + * an empty list means the result set has no members. When absent, the result set is re-derived + * inline (subquery / CTE). + */ + @JsonIgnore + Optional> getMaterializedValues(); + + @JsonIgnore + @Value.Lazy + default String getSetName() { + return String.valueOf(((ScalarLiteral) getArgs().get(1)).getValue()); + } + + @Value.Check + default void checkArgs() { + Preconditions.checkState( + getArgs().size() == 2 && getArgs().get(0) instanceof Property, + "the first argument of %s must be a property, found: %s", + TYPE, + getArgs()); + Preconditions.checkState( + getArgs().get(1) instanceof ScalarLiteral + && ((ScalarLiteral) getArgs().get(1)).getType() == String.class, + "the second argument of %s must be the name of a result set, found: %s", + TYPE, + getArgs().get(1)); + } + + static InResultSet of(String property, String setName) { + return new ImmutableInResultSet.Builder() + .args(ImmutableList.of(Property.of(property), ScalarLiteral.of(setName))) + .build(); + } + + static InResultSet of(Property property, String setName) { + return new ImmutableInResultSet.Builder() + .args(ImmutableList.of(property, ScalarLiteral.of(setName))) + .build(); + } + + abstract class Builder extends BinaryScalarOperation.Builder {} +} diff --git a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/Operand.java b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/Operand.java index 3199b7b3f..31494ee7f 100644 --- a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/Operand.java +++ b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/Operand.java @@ -177,6 +177,11 @@ private Optional getFilterCrs(ObjectCodec oc) throws JsonMappingExcepti if (Objects.nonNull(iv)) { try { Object value = iv.findInjectableValue("filterCrs", null, null, null); + // the value is injected as an Optional (see CqlImpl#read), but a bare EpsgCrs is + // also accepted + if (value instanceof Optional) { + value = ((Optional) value).orElse(null); + } if (value instanceof EpsgCrs) { return Optional.of((EpsgCrs) value); } diff --git a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/Operation.java b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/Operation.java index 3e1fce04c..53d0b85e3 100644 --- a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/Operation.java +++ b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/domain/Operation.java @@ -29,6 +29,7 @@ @Type(value = Or.class, name = Or.TYPE), @Type(value = IsNull.class, name = IsNull.TYPE), @Type(value = In.class, name = In.TYPE), + @Type(value = InResultSet.class, name = InResultSet.TYPE), @Type(value = Between.class, name = Between.TYPE), @Type(value = TAfter.class, name = TAfter.TYPE), @Type(value = TBefore.class, name = TBefore.TYPE), diff --git a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/infra/CqlTextVisitor.java b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/infra/CqlTextVisitor.java index c4f682131..bbedd5d98 100644 --- a/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/infra/CqlTextVisitor.java +++ b/xtraplatform-cql/src/main/java/de/ii/xtraplatform/cql/infra/CqlTextVisitor.java @@ -38,6 +38,7 @@ import de.ii.xtraplatform.cql.domain.ImmutableLte; import de.ii.xtraplatform.cql.domain.ImmutableNeq; import de.ii.xtraplatform.cql.domain.In; +import de.ii.xtraplatform.cql.domain.InResultSet; import de.ii.xtraplatform.cql.domain.IsNull; import de.ii.xtraplatform.cql.domain.Like; import de.ii.xtraplatform.cql.domain.Not; @@ -602,6 +603,15 @@ public CqlNode visitFunction(CqlParser.FunctionContext ctx) { ctx.argumentList().positionalArgument().argument().stream() .map(arg -> (Operand) arg.accept(this)) .collect(Collectors.toList()); + + if ("INRESULTSET".equalsIgnoreCase(functionName) + && args.size() == 2 + && args.get(0) instanceof Property + && args.get(1) instanceof ScalarLiteral) { + return InResultSet.of( + (Property) args.get(0), String.valueOf(((ScalarLiteral) args.get(1)).getValue())); + } + return Function.of(functionName, args); } diff --git a/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/app/CqlTypeCheckerSpec.groovy b/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/app/CqlTypeCheckerSpec.groovy index 4fa192d21..e83ead580 100644 --- a/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/app/CqlTypeCheckerSpec.groovy +++ b/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/app/CqlTypeCheckerSpec.groovy @@ -29,6 +29,8 @@ import de.ii.xtraplatform.cql.domain.Lte import de.ii.xtraplatform.cql.domain.Neq import de.ii.xtraplatform.cql.domain.Not import de.ii.xtraplatform.cql.domain.Or +import de.ii.xtraplatform.cql.domain.Parameter +import de.ii.xtraplatform.jsonschema.domain.ImmutableJsonSchemaString import de.ii.xtraplatform.cql.domain.Lt import de.ii.xtraplatform.cql.domain.Gt import de.ii.xtraplatform.cql.domain.Property @@ -117,6 +119,26 @@ class CqlTypeCheckerSpec extends Specification { noExceptionThrown() } + def 'an unbound parameter is treated as a wildcard'() { + given: 'a parameter operand, e.g. in a stored query validated before invocation' + def param = Parameter.of("aoi", new ImmutableJsonSchemaString.Builder().build()) + + when: 'a parameter is an operand of a spatial or array operation' + BinarySpatialOperation.of(SpatialFunction.S_INTERSECTS, Property.of("location_geometry"), param).accept(visitor3) + BinaryArrayOperation.of(ArrayFunction.A_CONTAINS, Property.of("seats_per_class"), param).accept(visitor3) + + then: 'the concrete operand is checked, the parameter is not constrained' + noExceptionThrown() + } + + def 'a scalar comparison on an array property is rejected'() { + when: 'a scalar = is used on an array-valued property (the modellart case)' + Eq.of(Property.of("seats_per_class"), ScalarLiteral.of("DLKM")).accept(visitor3) + + then: + thrown CqlIncompatibleTypes + } + def 'Test the predicate-checking visitor'() { given: // run the test on 2 different queries to make sure that old reports are removed diff --git a/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/app/InResultSetSpec.groovy b/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/app/InResultSetSpec.groovy new file mode 100644 index 000000000..245cc5419 --- /dev/null +++ b/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/app/InResultSetSpec.groovy @@ -0,0 +1,126 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.cql.app + +import de.ii.xtraplatform.cql.domain.And +import de.ii.xtraplatform.cql.domain.Cql +import de.ii.xtraplatform.cql.domain.Cql2Expression +import de.ii.xtraplatform.cql.domain.Eq +import de.ii.xtraplatform.cql.domain.ImmutableInResultSet +import de.ii.xtraplatform.cql.domain.InResultSet +import de.ii.xtraplatform.cql.domain.Property +import de.ii.xtraplatform.cql.domain.ScalarLiteral +import org.skyscreamer.jsonassert.JSONAssert +import spock.lang.Shared +import spock.lang.Specification + +class InResultSetSpec extends Specification { + + @Shared + Cql cql + + def setupSpec() { + cql = new CqlImpl() + } + + def 'cql2-json round-trip'() { + + given: + String cqlJson = """ + { + "op": "inResultSet", + "args": [ { "property": "id" }, "flst" ] + } + """ + + when: 'reading json' + Cql2Expression actual = cql.read(cqlJson, Cql.Format.JSON) + + then: + actual == InResultSet.of("id", "flst") + ((InResultSet) actual).getSetName() == "flst" + + and: + + when: 'writing json' + String actual2 = cql.write(InResultSet.of("id", "flst"), Cql.Format.JSON) + + then: + JSONAssert.assertEquals(cqlJson, actual2, true) + } + + def 'cql2-json in a conjunction'() { + + given: + String cqlJson = """ + { + "op": "and", + "args": [ + { "op": "inResultSet", "args": [ { "property": "istBestandteilVon" }, "bb" ] }, + { "op": "=", "args": [ { "property": "name" }, "foo" ] } + ] + } + """ + + when: 'reading json' + Cql2Expression actual = cql.read(cqlJson, Cql.Format.JSON) + + then: + actual == And.of( + InResultSet.of("istBestandteilVon", "bb"), + Eq.of(Property.of("name"), ScalarLiteral.of("foo"))) + } + + def 'cql2-text round-trip'() { + + when: 'writing text' + String text = cql.write(InResultSet.of("id", "flst"), Cql.Format.TEXT) + + then: + text == "INRESULTSET(id, 'flst')" + + and: + + when: 'reading text' + Cql2Expression actual = cql.read(text, Cql.Format.TEXT) + + then: + actual == InResultSet.of("id", "flst") + } + + def 'resolved producer context is not part of the json encoding'() { + + given: + InResultSet resolved = new ImmutableInResultSet.Builder() + .from(InResultSet.of("id", "flst")) + .producerType("ax_flurstueck") + .producerFilter(Eq.of(Property.of("name"), ScalarLiteral.of("foo"))) + .build() + + when: + String json = cql.write(resolved, Cql.Format.JSON) + + then: + JSONAssert.assertEquals("""{ "op": "inResultSet", "args": [ { "property": "id" }, "flst" ] }""", json, true) + } + + def 'invalid arguments are rejected'() { + + when: 'the second argument is not a string' + cql.read("""{ "op": "inResultSet", "args": [ { "property": "id" }, 5 ] }""", Cql.Format.JSON) + + then: + thrown Exception + + when: 'the first argument is not a property' + cql.read("""{ "op": "inResultSet", "args": [ "id", "flst" ] }""", Cql.Format.JSON) + + then: + thrown Exception + } +} diff --git a/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/domain/CustomFunctionSpec.groovy b/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/domain/CustomFunctionSpec.groovy index 8e7f09edf..f808f3b93 100644 --- a/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/domain/CustomFunctionSpec.groovy +++ b/xtraplatform-cql/src/test/groovy/de/ii/xtraplatform/cql/domain/CustomFunctionSpec.groovy @@ -66,4 +66,48 @@ class CustomFunctionSpec extends Specification { function.getExpression() == null function.getExpressions().get('SQL/PGIS') == 'UPPER($value) LIKE $pattern' } + + def 'accepts a query-expression-only function without an expression'() { + when: + def function = new ImmutableCustomFunction.Builder() + .name('OK_QE') + .arguments([]) + .returns(['BOOLEAN']) + .queryExpressionOnly(true) + .build() + + then: + function.getQueryExpressionOnly() + function.getExpression() == null + function.getExpressions().isEmpty() + } + + def 'rejects a query-expression-only function that defines an expression'() { + when: + new ImmutableCustomFunction.Builder() + .name('BAD_QE') + .arguments([]) + .returns(['BOOLEAN']) + .queryExpressionOnly(true) + .expression('UPPER($value)') + .build() + + then: + def ex = thrown(IllegalStateException) + ex.message.contains('restricted to query expressions') + } + + def 'inResultSet is a registered built-in function restricted to query expressions'() { + when: + def function = CqlBuiltInFunctions.FUNCTIONS.find { it.name == InResultSet.TYPE } + + then: + function != null + function.getQueryExpressionOnly() + function.getExpression() == null + function.getExpressions().isEmpty() + function.getReturns() == ['BOOLEAN'] + // each argument declares a single value type, so /functions renders no duplicate types + function.getArguments().every { it.getType().size() == 1 } + } } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureDecoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureDecoderSql.java index 8fe4593bf..f246e7a7d 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureDecoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureDecoderSql.java @@ -57,6 +57,7 @@ public class FeatureDecoderSql private boolean started; private boolean featureStarted; private Object currentId; + private int currentQueryIndex; private boolean isAtLeastOneFeatureWritten; private ModifiableContext context; @@ -189,7 +190,9 @@ private void handleValueRow(SqlRow sqlRow) { schemaIndexes.clear(); } - if (!Objects.equals(currentId, featureId) || !Objects.equals(context.type(), featureType)) { + if (!Objects.equals(currentId, featureId) + || !Objects.equals(context.type(), featureType) + || currentQueryIndex != sqlRow.getQueryIndex()) { if (featureStarted) { getDownstream().onFeatureEnd(context); this.featureStarted = false; @@ -202,6 +205,7 @@ private void handleValueRow(SqlRow sqlRow) { getDownstream().onFeatureStart(context); this.featureStarted = true; this.currentId = featureId; + this.currentQueryIndex = sqlRow.getQueryIndex(); } List multiplicitiesForPath = diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java index 9d4aeb2b0..086cac6a3 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java @@ -113,7 +113,8 @@ private SqlQueryBatch encode(FeatureQuery query, Map additionalQ query, query, additionalQueryParameters, - query.returnsSingleFeature()))) + query.returnsSingleFeature(), + 0))) .flatMap(s -> s) .collect(Collectors.toList()); @@ -133,9 +134,11 @@ private SqlQueryBatch encode( int chunks = (query.getLimit() / chunkSize) + (query.getLimit() % chunkSize > 0 ? 1 : 0); List querySets = - query.getQueries().stream() + IntStream.range(0, query.getQueries().size()) + .boxed() .flatMap( - typeQuery -> { + queryIndex -> { + TypeQuery typeQuery = query.getQueries().get(queryIndex); List queryTemplates = allQueryTemplates.get(typeQuery.getType()); @@ -153,7 +156,8 @@ private SqlQueryBatch encode( typeQuery, query, additionalQueryParameters, - false))) + false, + queryIndex))) .flatMap(s -> s); }) .collect(Collectors.toList()); @@ -176,10 +180,15 @@ private SqlQuerySet createQuerySet( TypeQuery typeQuery, Query query, Map additionalQueryParameters, - boolean skipMetaQuery) { + boolean skipMetaQuery, + int queryIndex) { List sortKeys = transformSortKeys(typeQuery.getSortKeys(), queryTemplates.getMapping()); boolean useMinMaxKeys = queryTemplates.getMapping().getMainTable().isSortKeyUnique(); + // a multi-query may opt out of computing numberMatched to avoid a count query per sub-query + boolean computeNumberMatched = + !(query instanceof MultiFeatureQuery) + || ((MultiFeatureQuery) query).getComputeNumberMatched(); BiFunction> metaQuery = (maxLimit, skipped) -> @@ -200,7 +209,7 @@ private SqlQuerySet createQuerySet( query.hitsOnly(), // numberMatched is invariant across chunks, so compute it only on the // first chunk of each collection; later chunks reuse that value - chunk == 0)); + chunk == 0 && computeNumberMatched)); TriFunction> valueQueries = (metaResult, maxLimit, skipped) -> @@ -228,6 +237,7 @@ private SqlQuerySet createQuerySet( .metaQuery(metaQuery) .valueQueries(valueQueries) .options(getOptions(typeQuery, query)) + .queryIndex(queryIndex) .build() .withTableSchemas(queryTemplates.getMapping().getTables()); } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java index c0babff6b..016749522 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java @@ -37,6 +37,7 @@ import de.ii.xtraplatform.cql.domain.Function; import de.ii.xtraplatform.cql.domain.GeometryNode; import de.ii.xtraplatform.cql.domain.In; +import de.ii.xtraplatform.cql.domain.InResultSet; import de.ii.xtraplatform.cql.domain.IsNull; import de.ii.xtraplatform.cql.domain.Like; import de.ii.xtraplatform.cql.domain.LogicalOperation; @@ -54,6 +55,7 @@ import de.ii.xtraplatform.crs.domain.CrsTransformerFactory; import de.ii.xtraplatform.crs.domain.EpsgCrs; import de.ii.xtraplatform.features.domain.FeatureSchema; +import de.ii.xtraplatform.features.domain.SchemaConstraints; import de.ii.xtraplatform.features.domain.Tuple; import de.ii.xtraplatform.features.sql.domain.SchemaSql; import de.ii.xtraplatform.features.sql.domain.SchemaSql.PropertyTypeInfo; @@ -73,14 +75,19 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,6 +107,7 @@ public class FilterEncoderSql { private final Cql cql; private final String accentiCollation; private final Map customFunctions; + private final java.util.function.Function> mappingResolver; BiFunction, Optional, Geometry> coordinatesTransformer; public FilterEncoderSql( @@ -120,12 +128,33 @@ public FilterEncoderSql( Cql cql, List customFunctions, String accentiCollation) { + this( + nativeCrs, + sqlDialect, + crsTransformerFactory, + crsInfo, + cql, + customFunctions, + accentiCollation, + type -> Optional.empty()); + } + + public FilterEncoderSql( + EpsgCrs nativeCrs, + SqlDialect sqlDialect, + CrsTransformerFactory crsTransformerFactory, + CrsInfo crsInfo, + Cql cql, + List customFunctions, + String accentiCollation, + java.util.function.Function> mappingResolver) { this.nativeCrs = nativeCrs; this.sqlDialect = sqlDialect; this.crsTransformerFactory = crsTransformerFactory; this.crsInfo = crsInfo; this.cql = cql; this.accentiCollation = accentiCollation; + this.mappingResolver = mappingResolver; this.customFunctions = ImmutableMap.copyOf( CqlBuiltInFunctions.prependBuiltInFunctions(customFunctions).stream() @@ -237,6 +266,149 @@ private String reduceSelectToColumnForTemplate(String expression) { return expression; } + // output column alias of every result-set CTE; consumers reference it as `SELECT FROM + // ` + private static final String CTE_VALUE_COL = "rs_value"; + + /** + * Collects the result-set subqueries of one top-level {@code inResultSet} predicate as named, + * materialized CTEs so that each result set is evaluated exactly once instead of being + * re-embedded (and re-evaluated) at every nesting level. CTEs are stored in dependency order — a + * nested set is registered while building the body of its parent, so it is inserted first and may + * be referenced by the parent. + */ + private static final class CteCollector { + private final LinkedHashMap ctes = new LinkedHashMap<>(); + private final Map namesBySet = new java.util.HashMap<>(); + private int counter; + + String register(String setName, Supplier bodySupplier) { + String existing = namesBySet.get(setName); + if (existing != null) { + return existing; + } + String name = "_rs_" + (counter++) + "_" + setName.replaceAll("[^A-Za-z0-9_]", "_"); + namesBySet.put(setName, name); + // building the body registers any nested result sets first, preserving dependency order + String body = bodySupplier.get(); + ctes.put(name, body); + return name; + } + + String renderWith(SqlDialect dialect) { + return "WITH " + + ctes.entrySet().stream() + .map(e -> dialect.materializedCte(e.getKey(), e.getValue())) + .collect(Collectors.joining(", ")); + } + } + + private static String renderInlineLiteral(Object value) { + if (value instanceof String) { + return "'" + ((String) value).replace("'", "''") + "'"; + } + return String.valueOf(value); + } + + /** + * Build the producer SELECT of a result set: {@code SELECT FROM + * WHERE }. With {@code withValueAlias} the value column is aliased as + * the CTE value column (for use inside a CTE); without it the bare value column is selected (for + * the materializer, which reads that single column). The optional collector lets nested result + * sets in the producer filter hoist into the same WITH clause. + */ + String resultSetProducerSelect( + InResultSet inResultSet, boolean withValueAlias, CteCollector collector) { + String setName = inResultSet.getSetName(); + String producerType = + inResultSet + .getProducerType() + .orElseThrow( + () -> + new IllegalArgumentException( + String.format("Filter is invalid. Unknown result set: '%s'.", setName))); + SqlQueryMapping producerMapping = + mappingResolver + .apply(producerType) + .orElseThrow( + () -> + new IllegalArgumentException( + String.format( + "Filter is invalid. Result set '%s' cannot be resolved for feature type '%s'.", + setName, producerType))); + de.ii.xtraplatform.base.domain.util.Tuple setColumn = + inResultSet.getProducerValues().isPresent() + ? producerMapping + .getColumnForValue(inResultSet.getProducerValues().get()) + .orElseThrow( + () -> + new IllegalArgumentException( + String.format( + "Filter is invalid. Result set '%s' projects the property '%s', which is unknown for feature type '%s'.", + setName, inResultSet.getProducerValues().get(), producerType))) + : producerMapping + .getColumnForId() + .orElseThrow( + () -> + new IllegalArgumentException( + String.format( + "Filter is invalid. Feature type '%s' has no id property for result set '%s'.", + producerType, setName))); + + SqlQuerySchema valueTable = setColumn.first(); + List aliases = AliasGenerator.getAliases(valueTable); + SqlQueryTable producerMain = + valueTable.getRelations().isEmpty() ? valueTable : valueTable.getRelations().get(0); + String join = JoinGenerator.getJoins(valueTable, aliases, this); + String valueColumn = + String.format("%s.%s", aliases.get(aliases.size() - 1), setColumn.second().getName()); + + Optional tableFilter = + producerMapping.getMainTable().getFilter().map(filter -> (Cql2Expression) filter); + Optional producerFilter = inResultSet.getProducerFilter(); + Optional effectiveFilter = + tableFilter.isPresent() && producerFilter.isPresent() + ? Optional.of(And.of(tableFilter.get(), producerFilter.get())) + : tableFilter.isPresent() ? tableFilter : producerFilter; + String where = + effectiveFilter + .map( + filter -> + " WHERE " + + prepareExpression(filter) + .accept(new CqlToSql2(producerMapping, collector))) + .orElse(""); + + String valueExpr = + withValueAlias ? String.format("%s AS %s", valueColumn, CTE_VALUE_COL) : valueColumn; + return String.format( + "SELECT %2$s FROM %1$s %3$s%4$s%5$s%6$s", + producerMain.getName(), valueExpr, aliases.get(0), join.isEmpty() ? "" : " ", join, where); + } + + /** + * Producer SELECT of a result set for up-front materialization: returns the bare value column so + * the caller can run it once and collect the values. Any nested result sets referenced by the + * producer filter that already carry materialized values are inlined as literals. + */ + public String encodeResultSetProducer(InResultSet inResultSet) { + return resultSetProducerSelect(inResultSet, false, null); + } + + /** + * Type of the value column of a result set, used to coerce and render its materialized values. + */ + public de.ii.xtraplatform.features.domain.SchemaBase.Type resultSetValueType( + InResultSet inResultSet) { + String producerType = inResultSet.getProducerType().orElseThrow(); + SqlQueryMapping producerMapping = mappingResolver.apply(producerType).orElseThrow(); + return (inResultSet.getProducerValues().isPresent() + ? producerMapping.getColumnForValue(inResultSet.getProducerValues().get()) + : producerMapping.getColumnForId()) + .map(column -> column.second().getType()) + .orElse(de.ii.xtraplatform.features.domain.SchemaBase.Type.STRING); + } + public String encode(Cql2Expression cqlFilter, SchemaSql schema) { return prepareExpression(cqlFilter).accept(new CqlToSql(schema)); } @@ -361,6 +533,66 @@ public Optional encodeRelationFilter2( return Optional.of(encodeNested(null, mergedFilter, table.get(), true)); } + private static final String DYNAMIC_REF_TYPE = "DYNAMIC"; + + /** The valid target types of a property in a mapping, or empty if they are not constrained. */ + private Optional> targetTypes(SqlQueryMapping mapping, String propertyName) { + FeatureSchema schema = + mapping + .getSchemaForObject(propertyName) + .or(() -> mapping.getSchemaForValue(propertyName)) + .orElse(null); + return validTargetTypes(schema); + } + + /** + * The valid target types of a feature-reference property: from its {@code refType} (case 1), the + * union over its {@code concat}/{@code coalesce} members (case 2), or a constant or enum on its + * {@code type} sub-property (case 3). Empty when the target type is not constrained (case 4) or + * any branch is open. + */ + Optional> validTargetTypes(FeatureSchema schema) { + if (Objects.isNull(schema)) { + return Optional.empty(); + } + + List members = + !schema.getConcat().isEmpty() + ? schema.getConcat() + : !schema.getCoalesce().isEmpty() ? schema.getCoalesce() : List.of(); + if (!members.isEmpty()) { + Set types = new LinkedHashSet<>(); + for (FeatureSchema member : members) { + Optional> memberTypes = validTargetTypes(member); + if (memberTypes.isEmpty()) { + return Optional.empty(); // an open branch leaves the overall target type unconstrained + } + types.addAll(memberTypes.get()); + } + return types.isEmpty() ? Optional.empty() : Optional.of(types); + } + + if (schema.getRefType().filter(type -> !DYNAMIC_REF_TYPE.equals(type)).isPresent()) { + return Optional.of(Set.of(schema.getRefType().get())); + } + + Optional typeProperty = + schema.getProperties().stream().filter(FeatureSchema::isType).findFirst(); + if (typeProperty.isPresent()) { + FeatureSchema type = typeProperty.get(); + if (type.getConstantValue().isPresent()) { + return Optional.of(Set.of(type.getConstantValue().get())); + } + List enumValues = + type.getConstraints().map(SchemaConstraints::getEnumValues).orElse(List.of()); + if (!enumValues.isEmpty()) { + return Optional.of(new LinkedHashSet<>(enumValues)); + } + } + + return Optional.empty(); + } + private static Predicate getPropertyNameMatcher( String propertyName, boolean includeObjects) { return property -> @@ -784,6 +1016,13 @@ private boolean has3dOperand(List operands) { @Override public String visit(BinaryScalarOperation scalarOperation, List children) { + if (scalarOperation instanceof InResultSet) { + throw new IllegalArgumentException( + String.format( + "Filter is invalid. %s can only be used within a query expression.", + InResultSet.TYPE)); + } + String operator = SCALAR_OPERATORS.get(scalarOperation.getClass()); List expressions = processBinary(scalarOperation.getArgs(), children); @@ -1334,10 +1573,18 @@ public String visit(Property property, List children) { private class CqlToSql2 extends CqlToText { private final SqlQueryMapping mapping; + // non-null while encoding the producer filter of an enclosing inResultSet, so that nested + // result sets register their CTEs with the same collector instead of nesting inline + private final CteCollector collector; private CqlToSql2(SqlQueryMapping mapping) { + this(mapping, null); + } + + private CqlToSql2(SqlQueryMapping mapping, CteCollector collector) { super(coordinatesTransformer); this.mapping = mapping; + this.collector = collector; } protected FeatureSchema getSchema( @@ -1773,6 +2020,10 @@ private boolean has3dOperand(List operands) { @Override public String visit(BinaryScalarOperation scalarOperation, List children) { + if (scalarOperation instanceof InResultSet) { + return encodeInResultSet((InResultSet) scalarOperation, children.get(0)); + } + String operator = SCALAR_OPERATORS.get(scalarOperation.getClass()); List expressions = processBinary(scalarOperation.getArgs(), children); @@ -1781,6 +2032,89 @@ public String visit(BinaryScalarOperation scalarOperation, List children return String.format(expressions.get(0), "", operation); } + private String encodeInResultSet(InResultSet inResultSet, String mainExpression) { + if (!operandHasSelect(mainExpression)) { + throw new IllegalArgumentException( + String.format( + "Filter is invalid. The first argument of %s must be a queryable.", + InResultSet.TYPE)); + } + + String setName = inResultSet.getSetName(); + String producerType = + inResultSet + .getProducerType() + .orElseThrow( + () -> + new IllegalArgumentException( + String.format("Filter is invalid. Unknown result set: '%s'.", setName))); + + SqlQueryMapping producerMapping = + mappingResolver + .apply(producerType) + .orElseThrow( + () -> + new IllegalArgumentException( + String.format( + "Filter is invalid. Result set '%s' cannot be resolved for feature type '%s'.", + setName, producerType))); + + // type compatibility: the value type of the result set must be a valid target type of the + // consumed property. The target types of a property are known when it has a refType, a + // concat/coalesce, or a type sub-property with a constant or enum; otherwise they are + // unconstrained and the check is skipped (no false negatives). + String consumerProperty = + ((Property) inResultSet.getArgs().get(0)).getName().replaceAll("^\"|\"$", ""); + Optional> consumerTargets = targetTypes(mapping, consumerProperty); + Optional> setTypes = + inResultSet.getProducerValues().isPresent() + ? targetTypes(producerMapping, inResultSet.getProducerValues().get()) + : Optional.of(Set.of(producerType)); + if (consumerTargets.isPresent() + && setTypes.isPresent() + && Collections.disjoint(consumerTargets.get(), setTypes.get())) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn( + "inResultSet: the value type(s) {} of result set '{}' are not among the valid target" + + " types {} of property '{}'; the relation cannot match and the hop is skipped.", + setTypes.get(), + setName, + consumerTargets.get(), + consumerProperty); + } + return "1 = 0"; + } + + // if the result set has been materialized up front, inline its values as a literal IN list + if (inResultSet.getMaterializedValues().isPresent()) { + List values = inResultSet.getMaterializedValues().get(); + if (values.isEmpty()) { + return "1 = 0"; + } + String list = + values.stream() + .map(FilterEncoderSql::renderInlineLiteral) + .collect(Collectors.joining(", ")); + return String.format(mainExpression, "", String.format(" IN (%s)", list)); + } + + // otherwise re-derive the result set inline; hoist it (and any nested result sets referenced + // by its producer filter) into materialized CTEs so each is evaluated once within the + // statement. CTEs are emitted in dependency order; the outermost predicate prepends the WITH. + boolean outermost = collector == null; + CteCollector coll = outermost ? new CteCollector() : collector; + String cteName = + coll.register(setName, () -> resultSetProducerSelect(inResultSet, true, coll)); + + String reference = + outermost + ? String.format( + " IN (%s SELECT %s FROM %s)", coll.renderWith(sqlDialect), CTE_VALUE_COL, cteName) + : String.format(" IN (SELECT %s FROM %s)", CTE_VALUE_COL, cteName); + + return String.format(mainExpression, "", reference); + } + @Override public String visit(Like like, List children) { String operator = SCALAR_OPERATORS.get(like.getClass()); diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/ResultSetMaterializer.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/ResultSetMaterializer.java new file mode 100644 index 000000000..ee47050bd --- /dev/null +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/ResultSetMaterializer.java @@ -0,0 +1,272 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.app; + +import de.ii.xtraplatform.cql.domain.BinaryScalarOperation; +import de.ii.xtraplatform.cql.domain.Cql2Expression; +import de.ii.xtraplatform.cql.domain.CqlNode; +import de.ii.xtraplatform.cql.domain.CqlVisitorCopy; +import de.ii.xtraplatform.cql.domain.ImmutableInResultSet; +import de.ii.xtraplatform.cql.domain.InResultSet; +import de.ii.xtraplatform.features.domain.ImmutableMultiFeatureQuery; +import de.ii.xtraplatform.features.domain.ImmutableSubQuery; +import de.ii.xtraplatform.features.domain.MultiFeatureQuery; +import de.ii.xtraplatform.features.domain.MultiFeatureQuery.SubQuery; +import de.ii.xtraplatform.features.domain.SchemaBase; +import de.ii.xtraplatform.features.sql.domain.SqlClient; +import de.ii.xtraplatform.features.sql.domain.SqlQueryOptions; +import de.ii.xtraplatform.features.sql.domain.SqlRow; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Materializes the result sets of a multi-query once per response. Each result set's producer is + * run a single time (its dependencies already materialized as literal id lists), and the collected + * values are attached to the {@link InResultSet} nodes of the consuming filters so that they are + * encoded as a literal {@code IN} list instead of a per-statement nested subquery. A result set + * that exceeds the configured cap is left unmaterialized and falls back to the inline (CTE) + * encoding. + */ +public class ResultSetMaterializer { + + private static final Logger LOGGER = LoggerFactory.getLogger(ResultSetMaterializer.class); + + private final Supplier sqlClient; + private final FilterEncoderSql filterEncoder; + private final int maxSetSize; + + public ResultSetMaterializer( + Supplier sqlClient, FilterEncoderSql filterEncoder, int maxSetSize) { + this.sqlClient = sqlClient; + this.filterEncoder = filterEncoder; + this.maxSetSize = maxSetSize; + } + + /** + * Returns a copy of the query with every materializable result set computed and inlined. If the + * query uses no result sets, it is returned unchanged. + */ + public MultiFeatureQuery materialize(MultiFeatureQuery query) { + Map sets = new LinkedHashMap<>(); + for (SubQuery subQuery : query.getQueries()) { + for (Cql2Expression filter : subQuery.getFilters()) { + collect(filter, sets); + } + } + if (sets.isEmpty()) { + return query; + } + + Map> materialized = new HashMap<>(); + // materialize level by level: within a level the producers are independent and run concurrently + // (bounded by the connection pool). SQL is built single-threaded between levels, so the filter + // encoder is never invoked concurrently and the materialized map is only mutated on this + // thread. + for (List level : topologicalLevels(sets)) { + Map>> running = new LinkedHashMap<>(); + Map valueTypes = new HashMap<>(); + for (String name : level) { + InResultSet node = sets.get(name); + InResultSet prepared = + node.getProducerFilter().isPresent() + ? new ImmutableInResultSet.Builder() + .from(node) + .producerFilter(applyMaterialized(node.getProducerFilter().get(), materialized)) + .build() + : node; + + // bound the fetch to one past the cap so an oversized set is detected without loading it + // all + String producerQuery = + filterEncoder.encodeResultSetProducer(prepared) + " LIMIT " + (maxSetSize + 1); + valueTypes.put(name, filterEncoder.resultSetValueType(node)); + running.put(name, sqlClient.get().run(producerQuery, SqlQueryOptions.single())); + } + + for (Map.Entry>> entry : running.entrySet()) { + String name = entry.getKey(); + Collection rows = entry.getValue().join(); + if (rows.size() > maxSetSize) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn( + "Result set '{}' has more than the materialization cap of {} members; falling back" + + " to inline evaluation for this set.", + name, + maxSetSize); + } + continue; + } + List values = + rows.stream() + .map(row -> coerce(row.getValues().get(0), valueTypes.get(name))) + .distinct() + .collect(Collectors.toList()); + materialized.put(name, values); + } + } + + List rewritten = + query.getQueries().stream() + .map( + subQuery -> + (SubQuery) + ImmutableSubQuery.builder() + .from(subQuery) + .filters( + subQuery.getFilters().stream() + .map(filter -> applyMaterialized(filter, materialized)) + .collect(Collectors.toList())) + .build()) + .collect(Collectors.toList()); + + return ImmutableMultiFeatureQuery.builder().from(query).queries(rewritten).build(); + } + + /** True if the query contains at least one result-set reference. */ + public static boolean hasResultSets(MultiFeatureQuery query) { + Map sets = new LinkedHashMap<>(); + for (SubQuery subQuery : query.getQueries()) { + for (Cql2Expression filter : subQuery.getFilters()) { + collect(filter, sets); + if (!sets.isEmpty()) { + return true; + } + } + } + return false; + } + + private static void collect(Cql2Expression expression, Map sets) { + Collector collector = new Collector(); + expression.accept(collector); + for (InResultSet node : collector.found) { + if (!sets.containsKey(node.getSetName())) { + sets.put(node.getSetName(), node); + node.getProducerFilter().ifPresent(filter -> collect(filter, sets)); + } + } + } + + /** + * Groups the result sets into dependency levels: level 0 has no dependencies, level n depends + * only on sets in earlier levels. Sets within a level are independent and may be materialized + * concurrently. + */ + private List> topologicalLevels(Map sets) { + Map> deps = new HashMap<>(); + for (String name : sets.keySet()) { + deps.put(name, dependenciesOf(sets.get(name), sets)); + } + List> levels = new ArrayList<>(); + Set done = new HashSet<>(); + while (done.size() < sets.size()) { + List level = + sets.keySet().stream() + .filter(name -> !done.contains(name) && done.containsAll(deps.get(name))) + .collect(Collectors.toList()); + if (level.isEmpty()) { + // no progress (should not happen for forward-only references) — emit the rest as one level + level = + sets.keySet().stream() + .filter(name -> !done.contains(name)) + .collect(Collectors.toList()); + } + levels.add(level); + done.addAll(level); + } + return levels; + } + + private static Set dependenciesOf(InResultSet node, Map sets) { + if (node.getProducerFilter().isEmpty()) { + return Set.of(); + } + Collector collector = new Collector(); + node.getProducerFilter().get().accept(collector); + Set deps = new HashSet<>(); + for (InResultSet dependency : collector.found) { + if (sets.containsKey(dependency.getSetName())) { + deps.add(dependency.getSetName()); + } + } + return deps; + } + + private static Cql2Expression applyMaterialized( + Cql2Expression expression, Map> materialized) { + return (Cql2Expression) expression.accept(new ApplyMaterialized(materialized)); + } + + private static Object coerce(Object value, SchemaBase.Type type) { + if (!(value instanceof String)) { + return value; + } + String string = (String) value; + try { + switch (type) { + case INTEGER: + return Long.parseLong(string); + case FLOAT: + return Double.parseDouble(string); + case BOOLEAN: + return Boolean.parseBoolean(string); + default: + return string; + } + } catch (NumberFormatException e) { + return string; + } + } + + /** Records the {@link InResultSet} nodes encountered while traversing a filter. */ + private static class Collector extends CqlVisitorCopy { + private final List found = new ArrayList<>(); + + @Override + public CqlNode visit(BinaryScalarOperation scalarOperation, List children) { + CqlNode copy = super.visit(scalarOperation, children); + if (copy instanceof InResultSet) { + found.add((InResultSet) copy); + } + return copy; + } + } + + /** Attaches materialized values to the {@link InResultSet} nodes that have them. */ + private static class ApplyMaterialized extends CqlVisitorCopy { + private final Map> materialized; + + ApplyMaterialized(Map> materialized) { + this.materialized = materialized; + } + + @Override + public CqlNode visit(BinaryScalarOperation scalarOperation, List children) { + CqlNode copy = super.visit(scalarOperation, children); + if (copy instanceof InResultSet) { + InResultSet node = (InResultSet) copy; + List values = materialized.get(node.getSetName()); + if (values != null) { + return new ImmutableInResultSet.Builder().from(node).materializedValues(values).build(); + } + } + return copy; + } + } +} diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java index 83d6d0eac..6e7ba7402 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java @@ -94,6 +94,7 @@ import de.ii.xtraplatform.features.sql.app.MutationSchemaDeriver; import de.ii.xtraplatform.features.sql.app.PathParserSql; import de.ii.xtraplatform.features.sql.app.QuerySchemaDeriver; +import de.ii.xtraplatform.features.sql.app.ResultSetMaterializer; import de.ii.xtraplatform.features.sql.app.SqlInsertGenerator2; import de.ii.xtraplatform.features.sql.app.SqlMappingDeriver; import de.ii.xtraplatform.features.sql.app.SqlMutationSession; @@ -468,6 +469,7 @@ public class FeatureProviderSql private PathParserSql pathParser2; private SqlPathParser pathParser3; private FilterEncoderSql filterEncoder; + private ResultSetMaterializer resultSetMaterializer; private SourceSchemaValidator sourceSchemaValidator; private Map> tableSchemas; private Map> queryMappings; @@ -592,7 +594,11 @@ protected boolean onStartup() throws InterruptedException { crsInfo, cql, getCql2Functions(), - accentiCollation); + accentiCollation, + type -> + Optional.ofNullable(queryMappings.get(type)) + .filter(mappings -> mappings.size() == 1) + .map(mappings -> mappings.get(0))); AggregateStatsQueryGenerator queryGeneratorSql = new AggregateStatsQueryGenerator(sqlDialect, filterEncoder); @@ -619,6 +625,12 @@ protected boolean onStartup() throws InterruptedException { new FeatureQueryEncoderSql( allQueryTemplates, allQueryTemplates, getData().getQueryGeneration(), sqlDialect); + this.resultSetMaterializer = + new ResultSetMaterializer( + this::getSqlClient, + filterEncoder, + getData().getQueryGeneration().getResultSetMaterializationMaxSize()); + this.aggregateStatsReader = new AggregateStatsReaderSql( this::getSqlClient, @@ -1426,29 +1438,40 @@ protected Query preprocessQuery(Query query) { // - disable optimized paging as soon as a sort key is specified for at least one subquery // - fix a bug in SqlRowVals or transformations, it seems that the same number of columns is // expected for all queries - List queries = ((MultiFeatureQuery) query).getQueries(); + MultiFeatureQuery multiQuery = (MultiFeatureQuery) query; + List queries = multiQuery.getQueries(); OptionalInt maxSortKeys = queries.stream().mapToInt(subQuery -> subQuery.getSortKeys().size()).max(); if (maxSortKeys.orElse(0) > 0) { - return ImmutableMultiFeatureQuery.builder() - .from(query) - .queries( - queries.stream() - .map( - subQuery -> - ImmutableSubQuery.builder() - .from(subQuery) - .sortKeys( - subQuery.getSortKeys().size() < maxSortKeys.getAsInt() - ? IntStream.range(0, maxSortKeys.getAsInt()) - .mapToObj(i -> SortKey.of(ID_PLACEHOLDER)) - .collect(Collectors.toList()) - : subQuery.getSortKeys()) - .build()) - .collect(Collectors.toList())) - .build(); + multiQuery = + ImmutableMultiFeatureQuery.builder() + .from(multiQuery) + .queries( + queries.stream() + .map( + subQuery -> + ImmutableSubQuery.builder() + .from(subQuery) + .sortKeys( + subQuery.getSortKeys().size() < maxSortKeys.getAsInt() + ? IntStream.range(0, maxSortKeys.getAsInt()) + .mapToObj(i -> SortKey.of(ID_PLACEHOLDER)) + .collect(Collectors.toList()) + : subQuery.getSortKeys()) + .build()) + .collect(Collectors.toList())) + .build(); + } + + // materialize the result sets of a multi-query once and reuse them across its queries, + // instead + // of re-deriving each shared result set in every sub-query statement + if (ResultSetMaterializer.hasResultSets(multiQuery)) { + multiQuery = resultSetMaterializer.materialize(multiQuery); } + + return multiQuery; } return query; @@ -1513,6 +1536,11 @@ private List getDialectAwareCustomFunctions(SqlDialect sqlDialec return getData().getCql2Functions().stream() .filter( function -> { + if (function.getQueryExpressionOnly()) { + // encoded by a dedicated handler, not a SQL template; not template-renderable + return false; + } + if (Objects.nonNull(function.getExpression()) && !function.getExpression().isBlank()) { return true; diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java index b42b60e7b..04f6f1e24 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java @@ -165,6 +165,22 @@ default int getChunkSize() { return 10_000; } + /** + * @langEn Maximum number of members a result set of a multi-query may have to be materialized + * once and reused as a literal list across the queries of the request. Larger result sets + * fall back to inline (re-evaluated) subqueries. + * @langDe Maximale Anzahl von Elementen einer Ergebnismenge einer Multi-Query, bis zu der diese + * einmal materialisiert und als Literalliste über die Abfragen der Anfrage hinweg + * wiederverwendet wird. Größere Ergebnismengen werden auf eingebettete (neu ausgewertete) + * Unterabfragen zurückgesetzt. + * @default 100000 + */ + @DocIgnore + @Value.Default + default int getResultSetMaterializationMaxSize() { + return 100_000; + } + /** * @langEn Option to disable computation of the number of selected features for performance * reasons that are returned in `numberMatched`. As a general rule this should be disabled diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlConnector.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlConnector.java index 8de0dea3a..778c4d04d 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlConnector.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlConnector.java @@ -116,7 +116,12 @@ default Reactive.Source getSourceStream( .via( Transformer.flatMap( querySet -> { - String currentTable = querySet.getTableSchemas().get(0).getFullPathAsString(); + // multiple queries may use the same feature type, so the key includes the + // query index + String currentTable = + querySet.getQueryIndex() + + "_" + + querySet.getTableSchemas().get(0).getFullPathAsString(); Optional> maxLimitAndSkipped = paging.get(currentTable); @@ -226,6 +231,10 @@ default Reactive.Source getSourceStream( .getOptions() .getType()) .containerPriority(i[0]++) + .queryIndex( + querySets + .get(index) + .getQueryIndex()) .build())) .toArray( (IntFunction[]>) Source[]::new); @@ -257,11 +266,13 @@ default Reactive.Source getSourceStream( Transformer.flatMap( index -> { String currentTable = - querySets - .get(index) - .getTableSchemas() - .get(0) - .getFullPathAsString(); + querySets.get(index).getQueryIndex() + + "_" + + querySets + .get(index) + .getTableSchemas() + .get(0) + .getFullPathAsString(); int[] j = {0}; if (metaResults.get(index).getNumberReturned() <= 0) { @@ -298,6 +309,10 @@ default Reactive.Source getSourceStream( .getOptions() .getType()) .containerPriority(i[0]++) + .queryIndex( + querySets + .get(index) + .getQueryIndex()) .build())) .toArray((IntFunction[]>) Source[]::new); diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java index b28da310a..a09ee0674 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java @@ -86,6 +86,15 @@ default String applyToNoTable(String select) { return select; } + /** + * Render a common table expression that is guaranteed to be evaluated only once. The default is a + * plain {@code name AS (query)}; dialects that would otherwise inline a single-reference CTE + * (PostgreSQL 12+) must force materialization. + */ + default String materializedCte(String name, String query) { + return name + " AS (" + query + ")"; + } + String castToBigInt(int value); Optional parseExtent(String extent, EpsgCrs crs); diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectPgis.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectPgis.java index 0c39a8def..a8159237e 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectPgis.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectPgis.java @@ -40,6 +40,14 @@ public String getId() { return SqlDbmsPgis.ID; } + @Override + public String materializedCte(String name, String query) { + // PostgreSQL 12+ inlines a CTE that is referenced only once; MATERIALIZED forces a single + // evaluation, which is what lets a result set be computed once instead of re-evaluated per + // nesting level. + return name + " AS MATERIALIZED (" + query + ")"; + } + private static final Splitter BBOX_SPLITTER = Splitter.onPattern("[(), ]").omitEmptyStrings().trimResults(); private static final Map SPATIAL_OPERATORS_3D = diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryOptions.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryOptions.java index 46ae95378..30e949302 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryOptions.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQueryOptions.java @@ -68,6 +68,12 @@ default int getContainerPriority() { return 0; } + /** Index of the query in a multi-query that this query belongs to. */ + @Value.Default + default int getQueryIndex() { + return 0; + } + @Value.Default default int getChunkSize() { return 1000; diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQuerySet.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQuerySet.java index 23d3e2301..40b68ff84 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQuerySet.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlQuerySet.java @@ -24,4 +24,13 @@ public interface SqlQuerySet { List getTableSchemas(); SqlQueryOptions getOptions(); + + /** + * Index of the query in a multi-query that this query set belongs to. Multiple queries may use + * the same feature type, so the main table alone does not identify the query. + */ + @Value.Default + default int getQueryIndex() { + return 0; + } } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlRow.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlRow.java index bb52ae8a8..d010a3ae0 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlRow.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlRow.java @@ -43,6 +43,11 @@ default int getPriority() { Optional getType(); + /** Index of the query in a multi-query that produced this row. */ + default int getQueryIndex() { + return 0; + } + default List> getColumnPaths() { return ImmutableList.of(); } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlRowVals.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlRowVals.java index 46e81e5fb..a2ab58816 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlRowVals.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlRowVals.java @@ -44,6 +44,7 @@ class SqlRowVals implements SqlRow { private List sortKeyDirections; private final List values; private int priority; + private int queryIndex; private SqlQuerySchema tableSchema; private Optional type; @Nullable private final Collator collator; @@ -106,6 +107,11 @@ public Optional getType() { return type; } + @Override + public int getQueryIndex() { + return queryIndex; + } + @Override public List> getColumnPaths() { if (Objects.nonNull(tableSchema)) { @@ -156,6 +162,7 @@ public int getSchemaIndex(int i) { // TODO: use result.nextObject when column type info is supported SqlRow read(ResultSet result, SqlQueryOptions queryOptions) { this.priority = queryOptions.getContainerPriority(); + this.queryIndex = queryOptions.getQueryIndex(); List> columnTypes; int cursor = 1; diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlInResultSetSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlInResultSetSpec.groovy new file mode 100644 index 000000000..d88f7b04c --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlInResultSetSpec.groovy @@ -0,0 +1,214 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.app + +import de.ii.xtraplatform.cql.app.CqlImpl +import de.ii.xtraplatform.cql.domain.Eq +import de.ii.xtraplatform.cql.domain.ImmutableInResultSet +import de.ii.xtraplatform.cql.domain.InResultSet +import de.ii.xtraplatform.cql.domain.Property +import de.ii.xtraplatform.cql.domain.ScalarLiteral +import de.ii.xtraplatform.crs.domain.OgcCrs +import de.ii.xtraplatform.features.domain.FeatureSchemaFixtures +import de.ii.xtraplatform.features.domain.MappingOperationResolver +import de.ii.xtraplatform.features.domain.MappingRuleFixtures +import de.ii.xtraplatform.features.json.app.DecoderFactoryJson +import de.ii.xtraplatform.features.sql.domain.ImmutableQueryGeneratorSettings +import de.ii.xtraplatform.features.sql.domain.ImmutableSqlPathDefaults +import de.ii.xtraplatform.features.sql.domain.SqlDialectPgis +import de.ii.xtraplatform.features.sql.domain.SqlPathParser +import de.ii.xtraplatform.features.sql.domain.SqlQueryMapping +import spock.lang.Shared +import spock.lang.Specification + +import java.util.function.Function + +class FilterEncoderSqlInResultSetSpec extends Specification { + + @Shared + Map mappings = [:] + @Shared + FilterEncoderSql filterEncoder + + def setupSpec() { + def defaults = new ImmutableSqlPathDefaults.Builder().build() + def cql = new CqlImpl() + def pathParser = new SqlPathParser(defaults, cql, Map.of("JSON", new DecoderFactoryJson(), "EXPRESSION", new DecoderFactorySqlExpression())) + def mappingDeriver = new SqlMappingDeriver(pathParser, new ImmutableQueryGeneratorSettings.Builder().build()) + def mappingOperationResolver = new MappingOperationResolver() + + ["simple", "value_array", "simple_filter"].each { name -> + def schema = FeatureSchemaFixtures.fromYaml(name) + def resolved = schema.accept(mappingOperationResolver, List.of()) + def rules = MappingRuleFixtures.fromYaml(name) + mappings[name] = mappingDeriver.derive(rules, resolved).get(0) + } + + filterEncoder = new FilterEncoderSql(OgcCrs.CRS84, new SqlDialectPgis(), null, null, cql, List.of(), null, + { type -> Optional.ofNullable(mappings[type]) } as Function) + } + + static InResultSet resolved(InResultSet inResultSet, String producerType, de.ii.xtraplatform.cql.domain.Cql2Expression producerFilter) { + return resolved(inResultSet, producerType, producerFilter, null) + } + + static InResultSet resolved(InResultSet inResultSet, String producerType, de.ii.xtraplatform.cql.domain.Cql2Expression producerFilter, String values) { + def builder = new ImmutableInResultSet.Builder() + .from(inResultSet) + .producerType(producerType) + if (producerFilter != null) { + builder.producerFilter(producerFilter) + } + if (values != null) { + builder.producerValues(values) + } + return builder.build() + } + + def 'plain id set, consumer matches its id queryable'() { + given: 'a result set over type simple, consumed by a filter on the id queryable' + def filter = resolved(InResultSet.of("id", "s1"), "simple", + Eq.of(Property.of("id"), ScalarLiteral.of("foo"))) + + when: + def sql = filterEncoder.encode(filter, mappings["value_array"]) + + then: + sql == "A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id IN (WITH _rs_0_s1 AS MATERIALIZED (SELECT A.id AS rs_value FROM externalprovider A WHERE A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id = 'foo')) SELECT rs_value FROM _rs_0_s1))" + } + + def 'plain id set without a producer filter'() { + given: + def filter = resolved(InResultSet.of("id", "s1"), "simple", null) + + when: + def sql = filterEncoder.encode(filter, mappings["value_array"]) + + then: + sql == "A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id IN (WITH _rs_0_s1 AS MATERIALIZED (SELECT A.id AS rs_value FROM externalprovider A) SELECT rs_value FROM _rs_0_s1))" + } + + def 'plain id set, consumer matches an array property in a junction table'() { + given: 'the consumer property is multi-valued, the semantics are like A_OVERLAPS' + def filter = resolved(InResultSet.of("externalprovidername", "s1"), "simple", + Eq.of(Property.of("id"), ScalarLiteral.of("foo"))) + + when: + def sql = filterEncoder.encode(filter, mappings["value_array"]) + + then: + sql == "A.id IN (SELECT AA.id FROM externalprovider AA JOIN externalprovider_externalprovidername AB ON (AA.id=AB.externalprovider_fk) WHERE AB.externalprovidername IN (WITH _rs_0_s1 AS MATERIALIZED (SELECT A.id AS rs_value FROM externalprovider A WHERE A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id = 'foo')) SELECT rs_value FROM _rs_0_s1))" + } + + def 'chained result sets nest recursively'() { + given: 'the producer filter itself consumes another result set' + def inner = resolved(InResultSet.of("id", "s1"), "simple", + Eq.of(Property.of("id"), ScalarLiteral.of("foo"))) + def outer = resolved(InResultSet.of("id", "s2"), "value_array", inner) + + when: + def sql = filterEncoder.encode(outer, mappings["value_array"]) + + then: + sql == "A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id IN (WITH _rs_1_s1 AS MATERIALIZED (SELECT A.id AS rs_value FROM externalprovider A WHERE A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id = 'foo')), _rs_0_s2 AS MATERIALIZED (SELECT A.id AS rs_value FROM externalprovider A WHERE A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id IN (SELECT rs_value FROM _rs_1_s1))) SELECT rs_value FROM _rs_0_s2))" + } + + def 'an unresolved result set reference is rejected'() { + when: + filterEncoder.encode(InResultSet.of("id", "s1"), mappings["value_array"]) + + then: + def e = thrown IllegalArgumentException + e.message.contains("s1") + } + + def 'an unknown producer type is rejected'() { + when: + filterEncoder.encode(resolved(InResultSet.of("id", "s1"), "unknown", null), mappings["value_array"]) + + then: + def e = thrown IllegalArgumentException + e.message.contains("unknown") + } + + def 'projected result set over a junction table'() { + given: 'the set consists of the values referenced by an array property of the selected features' + def filter = resolved(InResultSet.of("id", "s1"), "value_array", + Eq.of(Property.of("id"), ScalarLiteral.of("foo")), "externalprovidername") + + when: + def sql = filterEncoder.encode(filter, mappings["simple"]) + + then: + sql == "A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id IN (WITH _rs_0_s1 AS MATERIALIZED (SELECT B.externalprovidername AS rs_value FROM externalprovider A JOIN externalprovider_externalprovidername B ON (A.id=B.externalprovider_fk) WHERE A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id = 'foo')) SELECT rs_value FROM _rs_0_s1))" + } + + def 'projected result set over a column of the main table'() { + given: + def filter = resolved(InResultSet.of("id", "s1"), "simple", null, "id") + + when: + def sql = filterEncoder.encode(filter, mappings["value_array"]) + + then: + sql == "A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id IN (WITH _rs_0_s1 AS MATERIALIZED (SELECT A.id AS rs_value FROM externalprovider A) SELECT rs_value FROM _rs_0_s1))" + } + + def 'the filter of the producer main table is applied to the result set'() { + given: + def filter = resolved(InResultSet.of("id", "s1"), "simple_filter", null) + + when: + def sql = filterEncoder.encode(filter, mappings["simple"]) + + then: + sql == "A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id IN (WITH _rs_0_s1 AS MATERIALIZED (SELECT A.id AS rs_value FROM externalprovider A WHERE A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.type = 1)) SELECT rs_value FROM _rs_0_s1))" + } + + def 'a materialized result set is inlined as a literal IN list'() { + given: + def filter = new ImmutableInResultSet.Builder() + .from(InResultSet.of("id", "s1")) + .producerType("simple") + .materializedValues(["foo", "bar"]) + .build() + + when: + def sql = filterEncoder.encode(filter, mappings["value_array"]) + + then: + sql == "A.id IN (SELECT AA.id FROM externalprovider AA WHERE AA.id IN ('foo', 'bar'))" + } + + def 'an empty materialized result set yields a false predicate'() { + given: + def filter = new ImmutableInResultSet.Builder() + .from(InResultSet.of("id", "s1")) + .producerType("simple") + .materializedValues([]) + .build() + + when: + def sql = filterEncoder.encode(filter, mappings["value_array"]) + + then: + sql == "1 = 0" + } + + def 'an unknown projected property is rejected'() { + given: + def filter = resolved(InResultSet.of("id", "s1"), "simple", null, "nosuchproperty") + + when: + filterEncoder.encode(filter, mappings["value_array"]) + + then: + def e = thrown IllegalArgumentException + e.message.contains("nosuchproperty") + } +} diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlTargetTypesSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlTargetTypesSpec.groovy new file mode 100644 index 000000000..801aa866d --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlTargetTypesSpec.groovy @@ -0,0 +1,113 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.app + +import de.ii.xtraplatform.cql.app.CqlImpl +import de.ii.xtraplatform.crs.domain.OgcCrs +import de.ii.xtraplatform.features.domain.ImmutableFeatureSchema +import de.ii.xtraplatform.features.domain.ImmutableSchemaConstraints +import de.ii.xtraplatform.features.domain.SchemaBase +import de.ii.xtraplatform.features.sql.domain.SqlDialectPgis +import spock.lang.Shared +import spock.lang.Specification + +class FilterEncoderSqlTargetTypesSpec extends Specification { + + @Shared + FilterEncoderSql encoder = new FilterEncoderSql(OgcCrs.CRS84, new SqlDialectPgis(), null, null, new CqlImpl(), null) + + static ImmutableFeatureSchema.Builder ref() { + new ImmutableFeatureSchema.Builder().name("ref").type(SchemaBase.Type.FEATURE_REF) + } + + def 'case 1: a single refType is the only valid target type'() { + given: + def schema = ref().refType("target_a").build() + + expect: + encoder.validTargetTypes(schema) == Optional.of(["target_a"] as Set) + } + + def 'case 1: refType DYNAMIC is treated as unconstrained'() { + given: + def schema = ref().refType("DYNAMIC").build() + + expect: + encoder.validTargetTypes(schema) == Optional.empty() + } + + def 'case 2: concat target types are the union of the members'() { + given: + def schema = new ImmutableFeatureSchema.Builder() + .name("ref") + .type(SchemaBase.Type.FEATURE_REF_ARRAY) + .concat([ + ref().name("a").refType("target_a").build(), + ref().name("b").refType("target_b").build() + ]) + .build() + + expect: + encoder.validTargetTypes(schema) == Optional.of(["target_a", "target_b"] as Set) + } + + def 'case 2: an open concat member leaves the whole reference unconstrained'() { + given: + def schema = new ImmutableFeatureSchema.Builder() + .name("ref") + .type(SchemaBase.Type.FEATURE_REF_ARRAY) + .concat([ + ref().name("a").refType("target_a").build(), + ref().name("b").refType("DYNAMIC").build() + ]) + .build() + + expect: + encoder.validTargetTypes(schema) == Optional.empty() + } + + def 'case 3: a constant on the type sub-property defines the target type'() { + given: + def schema = ref() + .putProperties2("type", new ImmutableFeatureSchema.Builder() + .type(SchemaBase.Type.STRING) + .role(SchemaBase.Role.TYPE) + .constantValue("target_c")) + .build() + + expect: + encoder.validTargetTypes(schema) == Optional.of(["target_c"] as Set) + } + + def 'case 3: an enum on the type sub-property defines the valid target types'() { + given: + def schema = ref() + .putProperties2("type", new ImmutableFeatureSchema.Builder() + .type(SchemaBase.Type.STRING) + .role(SchemaBase.Role.TYPE) + .constraints(new ImmutableSchemaConstraints.Builder() + .enumValues(["target_d", "target_e"]).build())) + .build() + + expect: + encoder.validTargetTypes(schema) == Optional.of(["target_d", "target_e"] as Set) + } + + def 'case 4: no refType, concat/coalesce or type constraint is unconstrained'() { + given: + def schema = ref().build() + + expect: + encoder.validTargetTypes(schema) == Optional.empty() + } + + def 'a null schema is unconstrained'() { + expect: + encoder.validTargetTypes(null) == Optional.empty() + } +} diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/ResultSetMaterializerSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/ResultSetMaterializerSpec.groovy new file mode 100644 index 000000000..701b6cbce --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/ResultSetMaterializerSpec.groovy @@ -0,0 +1,132 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.app + +import de.ii.xtraplatform.cql.app.CqlImpl +import de.ii.xtraplatform.cql.domain.Cql2Expression +import de.ii.xtraplatform.cql.domain.Eq +import de.ii.xtraplatform.cql.domain.ImmutableInResultSet +import de.ii.xtraplatform.cql.domain.InResultSet +import de.ii.xtraplatform.cql.domain.Property +import de.ii.xtraplatform.cql.domain.ScalarLiteral +import de.ii.xtraplatform.crs.domain.OgcCrs +import de.ii.xtraplatform.features.domain.FeatureSchemaFixtures +import de.ii.xtraplatform.features.domain.ImmutableMultiFeatureQuery +import de.ii.xtraplatform.features.domain.ImmutableSubQuery +import de.ii.xtraplatform.features.domain.MappingOperationResolver +import de.ii.xtraplatform.features.domain.MappingRuleFixtures +import de.ii.xtraplatform.features.domain.MultiFeatureQuery +import de.ii.xtraplatform.features.json.app.DecoderFactoryJson +import de.ii.xtraplatform.features.sql.domain.ImmutableQueryGeneratorSettings +import de.ii.xtraplatform.features.sql.domain.ImmutableSqlPathDefaults +import de.ii.xtraplatform.features.sql.domain.SqlClient +import de.ii.xtraplatform.features.sql.domain.SqlDialectPgis +import de.ii.xtraplatform.features.sql.domain.SqlPathParser +import de.ii.xtraplatform.features.sql.domain.SqlQueryMapping +import de.ii.xtraplatform.features.sql.domain.SqlRow +import spock.lang.Shared +import spock.lang.Specification + +import java.util.concurrent.CompletableFuture +import java.util.function.Function +import java.util.function.Supplier + +class ResultSetMaterializerSpec extends Specification { + + @Shared + Map mappings = [:] + @Shared + FilterEncoderSql filterEncoder + + def setupSpec() { + def defaults = new ImmutableSqlPathDefaults.Builder().build() + def cql = new CqlImpl() + def pathParser = new SqlPathParser(defaults, cql, Map.of("JSON", new DecoderFactoryJson(), "EXPRESSION", new DecoderFactorySqlExpression())) + def mappingDeriver = new SqlMappingDeriver(pathParser, new ImmutableQueryGeneratorSettings.Builder().build()) + def mappingOperationResolver = new MappingOperationResolver() + + ["simple", "value_array"].each { name -> + def schema = FeatureSchemaFixtures.fromYaml(name) + def resolved = schema.accept(mappingOperationResolver, List.of()) + def rules = MappingRuleFixtures.fromYaml(name) + mappings[name] = mappingDeriver.derive(rules, resolved).get(0) + } + + filterEncoder = new FilterEncoderSql(OgcCrs.CRS84, new SqlDialectPgis(), null, null, cql, List.of(), null, + { type -> Optional.ofNullable(mappings[type]) } as Function) + } + + static InResultSet resolved(String setName, String producerType, Cql2Expression producerFilter) { + return new ImmutableInResultSet.Builder() + .from(InResultSet.of("id", setName)) + .producerType(producerType) + .producerFilter(producerFilter) + .build() + } + + SqlRow row(Object value) { + return Stub(SqlRow) { + getValues() >> [value] + } + } + + static MultiFeatureQuery query(Cql2Expression filter) { + return ImmutableMultiFeatureQuery.builder() + .addQueries( + ImmutableSubQuery.builder() + .collectionId("c") + .type("value_array") + .addFilters(filter) + .build()) + .build() + } + + def 'a result set is materialized once and its values are attached to the consumer'() { + given: + def filter = resolved("s1", "simple", Eq.of(Property.of("id"), ScalarLiteral.of("foo"))) + def sqlClient = Mock(SqlClient) + def materializer = new ResultSetMaterializer({ -> sqlClient } as Supplier, filterEncoder, 100000) + + when: + def result = materializer.materialize(query(filter)) + + then: + 1 * sqlClient.run(_, _) >> CompletableFuture.completedFuture([row("x"), row("y")]) + def node = (InResultSet) result.getQueries().get(0).getFilters().get(0) + node.getMaterializedValues().get() == ["x", "y"] + } + + def 'a result set exceeding the cap is left unmaterialized'() { + given: + def filter = resolved("s1", "simple", Eq.of(Property.of("id"), ScalarLiteral.of("foo"))) + def sqlClient = Mock(SqlClient) + def materializer = new ResultSetMaterializer({ -> sqlClient } as Supplier, filterEncoder, 1) + + when: + def result = materializer.materialize(query(filter)) + + then: + 1 * sqlClient.run(_, _) >> CompletableFuture.completedFuture([row("x"), row("y")]) + def node = (InResultSet) result.getQueries().get(0).getFilters().get(0) + node.getMaterializedValues().isEmpty() + } + + def 'a query without result sets is returned unchanged'() { + given: + def filter = Eq.of(Property.of("id"), ScalarLiteral.of("foo")) + def sqlClient = Mock(SqlClient) + def materializer = new ResultSetMaterializer({ -> sqlClient } as Supplier, filterEncoder, 100000) + + when: + def result = materializer.materialize(query(filter)) + + then: + 0 * sqlClient.run(_, _) + result == query(filter) + } +} diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/AbstractFeatureProvider.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/AbstractFeatureProvider.java index 1745acf50..05c4a4547 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/AbstractFeatureProvider.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/AbstractFeatureProvider.java @@ -28,6 +28,7 @@ import de.ii.xtraplatform.features.domain.FeatureQueriesExtension.LIFECYCLE_HOOK; import de.ii.xtraplatform.features.domain.FeatureStream.ResultBase; import de.ii.xtraplatform.features.domain.ImmutableSchemaMapping.Builder; +import de.ii.xtraplatform.features.domain.MultiFeatureQuery.SubQuery; import de.ii.xtraplatform.features.domain.SchemaBase.Scope; import de.ii.xtraplatform.features.domain.transform.PropertyTransformations; import de.ii.xtraplatform.features.domain.transform.SchemaTransformerChain; @@ -41,6 +42,7 @@ import java.io.IOException; import java.util.EnumSet; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -605,19 +607,43 @@ private Map createMapping( } if (query instanceof MultiFeatureQuery) { - return ((MultiFeatureQuery) query) - .getQueries().stream() - .map( - typeQuery -> - Map.entry( - typeQuery.getType(), - createMapping(typeQuery, WITH_SCOPE_RETURNABLE, propertyTransformations))) - .collect(ImmutableMap.toImmutableMap(Entry::getKey, Entry::getValue)); + // multiple queries may use the same feature type; the mapping is per type, so the + // projections of such queries are merged + Map queriesByType = new LinkedHashMap<>(); + for (SubQuery subQuery : ((MultiFeatureQuery) query).getQueries()) { + queriesByType.merge( + subQuery.getType(), subQuery, AbstractFeatureProvider::mergeProjections); + } + + return queriesByType.entrySet().stream() + .map( + entry -> + Map.entry( + entry.getKey(), + createMapping( + entry.getValue(), WITH_SCOPE_RETURNABLE, propertyTransformations))) + .collect(ImmutableMap.toImmutableMap(Entry::getKey, Entry::getValue)); } return Map.of(); } + private static TypeQuery mergeProjections(TypeQuery query1, TypeQuery query2) { + List fields = + query1.getFields().contains("*") || query2.getFields().contains("*") + ? List.of("*") + : java.util.stream.Stream.concat( + query1.getFields().stream(), query2.getFields().stream()) + .distinct() + .toList(); + + return ImmutableSubQuery.builder() + .from((SubQuery) query1) + .fields(fields) + .skipGeometry(query1.skipGeometry() && query2.skipGeometry()) + .build(); + } + private SchemaMapping createMapping( TypeQuery query, WithScope withScope, diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureEventHandler.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureEventHandler.java index e5bd082a0..0343c3fb2 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureEventHandler.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureEventHandler.java @@ -261,30 +261,37 @@ default boolean shouldSkip() { private boolean shouldInclude(T schema, List parentSchemas, String path) { return schema.isId() - || (schema.isSpatial() && (Objects.isNull(typeQuery()) || !typeQuery().skipGeometry())) + || (schema.isSpatial() + && (typeQueries().isEmpty() + || typeQueries().stream().anyMatch(typeQuery -> !typeQuery.skipGeometry()))) // TODO: enable if projected output needs to be schema valid // || isRequired(schema, parentSchemas) || (!schema.isId() && propertyIsInFields(path)); } - private TypeQuery typeQuery() { + // multiple queries of a multi-query may use the same feature type, the projections of such + // queries are merged + private List typeQueries() { return query() instanceof FeatureQuery - ? (FeatureQuery) query() + ? List.of((FeatureQuery) query()) : query() instanceof MultiFeatureQuery ? ((MultiFeatureQuery) query()) .getQueries().stream() .filter(subQuery -> Objects.equals(subQuery.getType(), type())) - .findFirst() - .orElse(null) - : null; + .toList() + : List.of(); } default boolean propertyIsInFields(String property) { - TypeQuery typeQuery = typeQuery(); - return Objects.nonNull(typeQuery) - && (typeQuery.getFields().isEmpty() - || typeQuery.getFields().contains("*") - || typeQuery.getFields().stream().anyMatch(field -> field.startsWith(property))); + List typeQueries = typeQueries(); + return !typeQueries.isEmpty() + && typeQueries.stream() + .anyMatch( + typeQuery -> + typeQuery.getFields().isEmpty() + || typeQuery.getFields().contains("*") + || typeQuery.getFields().stream() + .anyMatch(field -> field.startsWith(property))); } default boolean isRequired(T schema, List parentSchemas) { diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureSchema.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureSchema.java index 611ba9790..85bbf55ad 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureSchema.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureSchema.java @@ -62,7 +62,6 @@ "excludedScopes", "transformations", "constraints", - "link", "properties" }) public interface FeatureSchema diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java index e24bb38db..13d6d6e36 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java @@ -51,6 +51,8 @@ public class FeatureStreamImpl implements FeatureStream { private final boolean stepEtag; private final boolean stepMetadata; private final boolean hasPropertyLinks; + private final boolean deduplicate; + private final boolean idsArePerType; public FeatureStreamImpl( Query query, @@ -89,6 +91,9 @@ public FeatureStreamImpl( !query.skipPipelineSteps().contains(PipelineSteps.METADATA) && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.hasPropertyLinks = hasPropertyLinks(query, data); + this.deduplicate = + query instanceof MultiFeatureQuery && ((MultiFeatureQuery) query).getDeduplicate(); + this.idsArePerType = !data.getGloballyUniqueFeatureIds(); } // For types without properties that are represented as links (an explicit `link` in the @@ -124,12 +129,17 @@ public CompletionStage runWith( BiFunction, Stream> stream = (tokenSource, virtualTables) -> { ImmutableResult.Builder resultBuilder = ImmutableResult.builder(); + // duplicates are dropped first so that no downstream step sees them + FeatureTokenSource deduplicated = + deduplicate + ? tokenSource.via(new FeatureTokenTransformerDeduplicate(idsArePerType)) + : tokenSource; // PropertyLinks must run before the per-format value-transformation step so it // captures the raw ISO timestamp, not a locale-formatted variant used in the body FeatureTokenSource source = hasPropertyLinks - ? tokenSource.via(new FeatureTokenTransformerPropertyLinks(resultBuilder)) - : tokenSource; + ? deduplicated.via(new FeatureTokenTransformerPropertyLinks(resultBuilder)) + : deduplicated; // FeatureTokenTransformerExtension query-extensions (e.g. composite-id rewrite) run in // the same pre-format slot so they see raw provider values and can mutate tokens before // any format-specific transformation @@ -207,12 +217,17 @@ public CompletionStage> runWith( BiFunction, Reactive.Stream>> stream = (tokenSource, virtualTables) -> { ImmutableResultReduced.Builder resultBuilder = ImmutableResultReduced.builder(); + // duplicates are dropped first so that no downstream step sees them + FeatureTokenSource deduplicated = + deduplicate + ? tokenSource.via(new FeatureTokenTransformerDeduplicate(idsArePerType)) + : tokenSource; // PropertyLinks must run before the per-format value-transformation step so it // captures the raw ISO timestamp, not a locale-formatted variant used in the body FeatureTokenSource source = hasPropertyLinks - ? tokenSource.via(new FeatureTokenTransformerPropertyLinks(resultBuilder)) - : tokenSource; + ? deduplicated.via(new FeatureTokenTransformerPropertyLinks(resultBuilder)) + : deduplicated; // FeatureTokenTransformerExtension query-extensions (e.g. composite-id rewrite) run in // the same pre-format slot so they see raw provider values and can mutate tokens before // any format-specific transformation @@ -349,6 +364,8 @@ static Map getMergedTransformations( } if (query instanceof MultiFeatureQuery multiFeatureQuery) { + // multiple queries may use the same feature type, the transformations only depend on the + // type return multiFeatureQuery.getQueries().stream() .map( typeQuery -> @@ -358,7 +375,8 @@ static Map getMergedTransformations( featureSchemas, typeQuery, Optional.ofNullable(propertyTransformations.get(typeQuery.getType()))))) - .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + .collect( + ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> a)); } return ImmutableMap.of(); diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerDeduplicate.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerDeduplicate.java new file mode 100644 index 000000000..16bf7e5f7 --- /dev/null +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerDeduplicate.java @@ -0,0 +1,262 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.domain; + +import de.ii.xtraplatform.features.domain.SchemaBase.Type; +import de.ii.xtraplatform.geometries.domain.Geometry; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.Queue; + +/** + * Drops features whose id has already been emitted. The tokens of a feature are buffered until its + * id property arrives; the feature is then either replayed or discarded as a whole. Duplicates can + * only arise when multiple queries of a multi-query select the same feature. + */ +public class FeatureTokenTransformerDeduplicate extends FeatureTokenTransformer { + + public static final int MAX_FEATURES = 16_777_216; + + private final PackedIdSet seen; + private final boolean idsArePerType; + + private final Queue tokenQueue; + private final Queue> pathQueue; + private final Queue schemaIndexQueue; + private final Queue> indexesQueue; + private final Queue valueQueue; + private final Queue valueTypeQueue; + private final Queue> geoQueue; + private final Queue inArrayQueue; + private final Queue inObjectQueue; + + private boolean buffering; + private boolean dropping; + private String currentType; + + public FeatureTokenTransformerDeduplicate(boolean idsArePerType) { + this.seen = new PackedIdSet(MAX_FEATURES); + this.idsArePerType = idsArePerType; + this.tokenQueue = new LinkedList<>(); + this.pathQueue = new LinkedList<>(); + this.schemaIndexQueue = new LinkedList<>(); + this.indexesQueue = new LinkedList<>(); + this.valueQueue = new LinkedList<>(); + this.valueTypeQueue = new LinkedList<>(); + this.geoQueue = new LinkedList<>(); + this.inArrayQueue = new LinkedList<>(); + this.inObjectQueue = new LinkedList<>(); + this.buffering = false; + this.dropping = false; + } + + @Override + public void onFeatureStart(ModifiableContext context) { + this.currentType = + Objects.nonNull(context.mapping()) ? context.mapping().getTargetSchema().getName() : null; + this.buffering = true; + this.dropping = false; + + buffer(context, FeatureTokenType.FEATURE); + } + + @Override + public void onFeatureEnd(ModifiableContext context) { + if (dropping) { + this.dropping = false; + return; + } + if (buffering) { + // a feature without an id is always emitted + flush(context); + } + + super.onFeatureEnd(context); + } + + @Override + public void onObjectStart(ModifiableContext context) { + if (dropping) { + return; + } + if (buffering) { + buffer(context, FeatureTokenType.OBJECT); + return; + } + super.onObjectStart(context); + } + + @Override + public void onObjectEnd(ModifiableContext context) { + if (dropping) { + return; + } + if (buffering) { + buffer(context, FeatureTokenType.OBJECT_END); + return; + } + super.onObjectEnd(context); + } + + @Override + public void onArrayStart(ModifiableContext context) { + if (dropping) { + return; + } + if (buffering) { + buffer(context, FeatureTokenType.ARRAY); + return; + } + super.onArrayStart(context); + } + + @Override + public void onArrayEnd(ModifiableContext context) { + if (dropping) { + return; + } + if (buffering) { + buffer(context, FeatureTokenType.ARRAY_END); + return; + } + super.onArrayEnd(context); + } + + @Override + public void onGeometry(ModifiableContext context) { + if (dropping) { + return; + } + if (buffering) { + buffer(context, FeatureTokenType.GEOMETRY); + return; + } + super.onGeometry(context); + } + + @Override + public void onValue(ModifiableContext context) { + if (dropping) { + return; + } + if (buffering) { + buffer(context, FeatureTokenType.VALUE); + + if (context.schema().filter(SchemaBase::isId).isPresent() + && Objects.nonNull(context.value())) { + String key = + idsArePerType && Objects.nonNull(currentType) + ? currentType + ":" + context.value() + : context.value(); + + if (seen.add(key)) { + flush(context); + } else { + clear(); + this.dropping = true; + this.buffering = false; + } + } + return; + } + super.onValue(context); + } + + private void buffer( + ModifiableContext context, FeatureTokenType token) { + tokenQueue.add(token); + pathQueue.add(List.copyOf(context.path())); + schemaIndexQueue.add(context.schemaIndex()); + indexesQueue.add(new ArrayList<>(context.indexes())); + valueQueue.add(context.value()); + valueTypeQueue.add(context.valueType()); + geoQueue.add(context.geometry()); + inArrayQueue.add(context.inArray()); + inObjectQueue.add(context.inObject()); + } + + private void clear() { + tokenQueue.clear(); + pathQueue.clear(); + schemaIndexQueue.clear(); + indexesQueue.clear(); + valueQueue.clear(); + valueTypeQueue.clear(); + geoQueue.clear(); + inArrayQueue.clear(); + inObjectQueue.clear(); + } + + private void flush(ModifiableContext context) { + this.buffering = false; + + List path = context.path(); + int schemaIndex = context.schemaIndex(); + List indexes = new ArrayList<>(context.indexes()); + String value = context.value(); + Type valueType = context.valueType(); + Geometry geometry = context.geometry(); + boolean inArray = context.inArray(); + boolean inObject = context.inObject(); + + while (!tokenQueue.isEmpty()) { + FeatureTokenType token = tokenQueue.remove(); + + context.pathTracker().track(pathQueue.remove()); + context.setSchemaIndex(schemaIndexQueue.remove()); + context.setIndexes(indexesQueue.remove()); + context.setValue(valueQueue.remove()); + context.setValueType(valueTypeQueue.remove()); + context.setGeometry(geoQueue.remove()); + context.setInArray(inArrayQueue.remove()); + context.setInObject(inObjectQueue.remove()); + + push(context, token); + } + + context.pathTracker().track(path); + context.setSchemaIndex(schemaIndex); + context.setIndexes(indexes); + context.setValue(value); + context.setValueType(valueType); + context.setGeometry(geometry); + context.setInArray(inArray); + context.setInObject(inObject); + } + + private void push( + ModifiableContext context, FeatureTokenType token) { + switch (token) { + case FEATURE: + super.onFeatureStart(context); + break; + case VALUE: + super.onValue(context); + break; + case GEOMETRY: + super.onGeometry(context); + break; + case OBJECT: + super.onObjectStart(context); + break; + case OBJECT_END: + super.onObjectEnd(context); + break; + case ARRAY: + super.onArrayStart(context); + break; + case ARRAY_END: + super.onArrayEnd(context); + break; + default: + break; + } + } +} diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java index 48f7085ad..7700c0de7 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java @@ -19,4 +19,22 @@ interface SubQuery extends TypeQuery { } List getQueries(); + + /** + * If enabled, a feature that is selected by more than one query is only included in the response + * once. + */ + @Value.Default + default boolean getDeduplicate() { + return false; + } + + /** + * If disabled, {@code numberMatched} is not computed. For a multi-query this avoids a count query + * per sub-query, each of which carries the full (possibly deeply nested) filter. + */ + @Value.Default + default boolean getComputeNumberMatched() { + return true; + } } diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/PackedIdSet.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/PackedIdSet.java new file mode 100644 index 000000000..c864fd114 --- /dev/null +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/PackedIdSet.java @@ -0,0 +1,190 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.domain; + +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; +import java.nio.charset.StandardCharsets; + +/** + * A memory-efficient set of feature ids. Ids are not stored as strings: an id with up to 20 + * characters from {@code [0-9A-Za-z_-]} is packed losslessly into 128 bits, any other id is + * represented by its 128-bit hash (collisions are negligible). The set is an open-addressing hash + * table of long pairs, so the memory footprint is 32 bytes per id at the default load factor. + * + *

The number of entries is bounded; adding an id beyond the bound throws an {@link + * IllegalStateException}. + */ +public class PackedIdSet { + + private static final long PACKED_MARKER = 0x8000_0000_0000_0000L; + private static final int MAX_PACKED_LENGTH = 20; + private static final int INITIAL_CAPACITY = 1024; + + private final int maxEntries; + private long[] table; + private int capacity; + private int size; + private boolean containsZero; + + public PackedIdSet(int maxEntries) { + this.maxEntries = maxEntries; + this.capacity = INITIAL_CAPACITY; + this.table = new long[2 * capacity]; + this.size = 0; + this.containsZero = false; + } + + /** + * Adds an id to the set. + * + * @param id the feature id + * @return {@code true} if the id was not in the set + */ + public boolean add(String id) { + long hi; + long lo; + + long[] packed = pack(id); + if (packed != null) { + hi = packed[0]; + lo = packed[1]; + } else { + HashCode hash = Hashing.murmur3_128().hashString(id, StandardCharsets.UTF_8); + byte[] bytes = hash.asBytes(); + hi = toLong(bytes, 0) & ~PACKED_MARKER; + lo = toLong(bytes, 8); + } + + if (hi == 0 && lo == 0) { + if (containsZero) { + return false; + } + checkBound(); + this.containsZero = true; + this.size++; + return true; + } + + int index = findSlot(table, capacity, hi, lo); + if (table[index] == hi && table[index + 1] == lo) { + return false; + } + + checkBound(); + table[index] = hi; + table[index + 1] = lo; + this.size++; + + if (size > capacity / 2 && capacity < Integer.MAX_VALUE / 4) { + grow(); + } + + return true; + } + + public int size() { + return size; + } + + private void checkBound() { + if (size >= maxEntries) { + throw new IllegalStateException( + String.format( + "The response exceeds the maximum number of features that can be deduplicated (%d).", + maxEntries)); + } + } + + private void grow() { + int newCapacity = capacity * 2; + long[] newTable = new long[2 * newCapacity]; + + for (int i = 0; i < table.length; i += 2) { + long hi = table[i]; + long lo = table[i + 1]; + if (hi != 0 || lo != 0) { + int index = findSlot(newTable, newCapacity, hi, lo); + newTable[index] = hi; + newTable[index + 1] = lo; + } + } + + this.table = newTable; + this.capacity = newCapacity; + } + + private static int findSlot(long[] table, int capacity, long hi, long lo) { + int mask = capacity - 1; + int slot = spread(hi, lo) & mask; + + while (true) { + int index = 2 * slot; + if ((table[index] == 0 && table[index + 1] == 0) + || (table[index] == hi && table[index + 1] == lo)) { + return index; + } + slot = (slot + 1) & mask; + } + } + + private static int spread(long hi, long lo) { + long mixed = (hi ^ (hi >>> 32)) * 0x9E3779B97F4A7C15L + (lo ^ (lo >>> 32)); + return (int) (mixed ^ (mixed >>> 32)) & Integer.MAX_VALUE; + } + + // 6 bits per character plus the length, so that ids with leading zero-characters stay distinct + private static long[] pack(String id) { + int length = id.length(); + if (length == 0 || length > MAX_PACKED_LENGTH) { + return null; + } + + long hi = 0; + long lo = 0; + for (int i = 0; i < length; i++) { + int bits = toBits(id.charAt(i)); + if (bits < 0) { + return null; + } + hi = (hi << 6) | (lo >>> 58); + lo = (lo << 6) | bits; + } + hi |= ((long) length) << 56; + hi |= PACKED_MARKER; + + return new long[] {hi, lo}; + } + + private static int toBits(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'A' && c <= 'Z') { + return c - 'A' + 10; + } + if (c >= 'a' && c <= 'z') { + return c - 'a' + 36; + } + if (c == '-') { + return 62; + } + if (c == '_') { + return 63; + } + return -1; + } + + private static long toLong(byte[] bytes, int offset) { + long result = 0; + for (int i = 0; i < 8; i++) { + result = (result << 8) | (bytes[offset + i] & 0xFF); + } + return result; + } +} diff --git a/xtraplatform-features/src/test/groovy/de/ii/xtraplatform/features/domain/FeatureTokenTransformerDeduplicateSpec.groovy b/xtraplatform-features/src/test/groovy/de/ii/xtraplatform/features/domain/FeatureTokenTransformerDeduplicateSpec.groovy new file mode 100644 index 000000000..a87c656b4 --- /dev/null +++ b/xtraplatform-features/src/test/groovy/de/ii/xtraplatform/features/domain/FeatureTokenTransformerDeduplicateSpec.groovy @@ -0,0 +1,115 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.domain + +import de.ii.xtraplatform.features.domain.SchemaBase.Type +import spock.lang.Specification + +class FeatureTokenTransformerDeduplicateSpec extends Specification { + + FeatureTokenReader tokenReader + List tokens + + def setup() { + FeatureTokenTransformerDeduplicate mapper = new FeatureTokenTransformerDeduplicate(false) + FeatureQuery query = ImmutableFeatureQuery.builder().type("test").build() + FeatureEventHandler.ModifiableContext context = mapper.createContext() + .setQuery(query) + .setMappings([test: FeatureSchemaFixtures.BIOTOP_MAPPING]) + .setType('test') + .setIsUseTargetPaths(true) + + tokenReader = new FeatureTokenReader(mapper, context) + tokens = [] + mapper.init(token -> tokens.add(token)) + } + + static List feature(String id, String kennung) { + return [ + FeatureTokenType.FEATURE, + FeatureTokenType.VALUE, + ["id"], + id, + Type.STRING, + FeatureTokenType.VALUE, + ["kennung"], + kennung, + Type.STRING, + FeatureTokenType.FEATURE_END + ] + } + + static List collection(List... features) { + List result = [FeatureTokenType.INPUT, true] + features.each { result.addAll(it) } + result.add(FeatureTokenType.INPUT_END) + return result + } + + def 'distinct features pass through unchanged'() { + given: + def input = collection(feature("24", "611320001-1"), feature("25", "611320001-2")) + + when: + input.forEach(token -> tokenReader.onToken(token)) + + then: + tokens == input + } + + def 'a feature with an already emitted id is dropped'() { + given: + def input = collection( + feature("24", "611320001-1"), + feature("25", "611320001-2"), + feature("24", "611320001-1")) + + when: + input.forEach(token -> tokenReader.onToken(token)) + + then: + tokens == collection(feature("24", "611320001-1"), feature("25", "611320001-2")) + } + + def 'consecutive duplicates collapse to one feature'() { + given: + def input = collection( + feature("24", "611320001-1"), + feature("24", "611320001-1"), + feature("24", "611320001-1")) + + when: + input.forEach(token -> tokenReader.onToken(token)) + + then: + tokens == collection(feature("24", "611320001-1")) + } + + def 'properties before the id are kept on the first occurrence'() { + given: 'kennung arrives before id' + def feature24 = [ + FeatureTokenType.FEATURE, + FeatureTokenType.VALUE, + ["kennung"], + "611320001-1", + Type.STRING, + FeatureTokenType.VALUE, + ["id"], + "24", + Type.STRING, + FeatureTokenType.FEATURE_END + ] + def input = collection(feature24, feature24) + + when: + input.forEach(token -> tokenReader.onToken(token)) + + then: + tokens == collection(feature24) + } +} diff --git a/xtraplatform-features/src/test/groovy/de/ii/xtraplatform/features/domain/PackedIdSetSpec.groovy b/xtraplatform-features/src/test/groovy/de/ii/xtraplatform/features/domain/PackedIdSetSpec.groovy new file mode 100644 index 000000000..3891d7268 --- /dev/null +++ b/xtraplatform-features/src/test/groovy/de/ii/xtraplatform/features/domain/PackedIdSetSpec.groovy @@ -0,0 +1,80 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.domain + +import spock.lang.Specification + +class PackedIdSetSpec extends Specification { + + def 'packable ids are deduplicated'() { + given: + def set = new PackedIdSet(1000) + + expect: + set.add("DEHE862010014MLB") + set.add("DEHE862010014MLC") + !set.add("DEHE862010014MLB") + !set.add("DEHE862010014MLC") + set.size() == 2 + } + + def 'ids with leading zero-characters stay distinct'() { + given: + def set = new PackedIdSet(1000) + + expect: + set.add("A") + set.add("0A") + set.add("00A") + set.size() == 3 + !set.add("0A") + } + + def 'ids that cannot be packed fall back to a hash'() { + given: + def set = new PackedIdSet(1000) + + expect: + set.add("urn:adv:oid:DEHE862010014MLB") + !set.add("urn:adv:oid:DEHE862010014MLB") + set.add("a-very-long-identifier-that-exceeds-the-packing-limit") + !set.add("a-very-long-identifier-that-exceeds-the-packing-limit") + set.size() == 2 + } + + def 'the set grows beyond the initial capacity'() { + given: + def set = new PackedIdSet(100000) + + when: + def added = (0..<50000).count { set.add("ID" + it) } + def readded = (0..<50000).count { set.add("ID" + it) } + + then: + added == 50000 + readded == 0 + set.size() == 50000 + } + + def 'exceeding the maximum number of entries fails'() { + given: + def set = new PackedIdSet(3) + set.add("A") + set.add("B") + set.add("C") + + when: + set.add("D") + + then: + thrown IllegalStateException + + and: 'duplicates are still detected' + !set.add("A") + } +}