diff --git a/docs/content/docs/sql/reference/data-types.md b/docs/content/docs/sql/reference/data-types.md index 8e43c2ec12ee27..fdc5487d1787eb 100644 --- a/docs/content/docs/sql/reference/data-types.md +++ b/docs/content/docs/sql/reference/data-types.md @@ -219,6 +219,7 @@ The default planner supports the following set of SQL types: | Structured types | Only exposed in user-defined functions yet. | | `VARIANT` | | | `BITMAP` | | +| `GEOGRAPHY` | Geography values in OGC:CRS84. | ### Character Strings @@ -1735,6 +1736,53 @@ DataTypes.BITMAP() {{< /tab >}} {{< /tabs >}} +#### `GEOGRAPHY` + +Data type of geography data. + +`GEOGRAPHY` represents geospatial values in the [OGC:CRS84](https://www.opengis.net/def/crs/OGC/1.3/CRS84) +coordinate reference system. Here, CRS means *Coordinate Reference System* and SRID means +*Spatial Reference Identifier*. OGC:CRS84 identifies the coordinate reference system and axis +order used by this type; CRS and SRID metadata are not stored in the WKB payload. + +Flink represents geography payloads as [ISO WKB](https://www.ogc.org/standards/sfa), where WKB +means *Well-Known Binary*. The v1 contract supports the standard 2D WKB geometry type codes. +[EWKB](https://postgis.net/docs/using_postgis_dbmanagement.html#EWKB_EWKT), or *Extended +Well-Known Binary*, is not part of this contract. CRS validation, CRS transformation, and +EWKB/SRID handling belong to constructors, functions, or connector-specific schema mapping. + +The type itself does not define SQL constructors, accessors, or spatial predicate functions. +Those functions are expected to be added separately. + +The geography type is an extension to the SQL standard. + +**Declaration** + +{{< tabs "9fef5895-3fa0-4b9b-9f92-9c94ba96ef1a" >}} +{{< tab "SQL" >}} +```text +GEOGRAPHY +``` + +{{< /tab >}} +{{< tab "Java/Scala" >}} +```java +DataTypes.GEOGRAPHY() +``` + +**Bridging to JVM Types** + +| Java Type | Input | Output | Remarks | +|:-----------------------------------------------|:-----:|:------:|:----------| +| `org.apache.flink.table.data.GeographyData` | X | X | *Default* | + +{{< /tab >}} +{{< /tabs >}} + +`GEOGRAPHY` values cannot be constructed with `CAST` from character or binary string +types. Likewise, `GEOGRAPHY` values cannot be cast to character or binary string types. +Use explicit geography functions for those conversions once such functions are available. + #### `RAW` Data type of an arbitrary serialized type. This type is a black box within the table ecosystem diff --git a/docs/static/generated/rest_v1_sql_gateway.yml b/docs/static/generated/rest_v1_sql_gateway.yml index 2d751a9711218e..772f48347e58ee 100644 --- a/docs/static/generated/rest_v1_sql_gateway.yml +++ b/docs/static/generated/rest_v1_sql_gateway.yml @@ -409,6 +409,7 @@ components: - VARIANT - BITMAP - UUID + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/docs/static/generated/rest_v2_sql_gateway.yml b/docs/static/generated/rest_v2_sql_gateway.yml index 5ca66181293fa9..f195a10e757e85 100644 --- a/docs/static/generated/rest_v2_sql_gateway.yml +++ b/docs/static/generated/rest_v2_sql_gateway.yml @@ -484,6 +484,7 @@ components: - VARIANT - BITMAP - UUID + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/docs/static/generated/rest_v3_sql_gateway.yml b/docs/static/generated/rest_v3_sql_gateway.yml index bef8b793a6480c..23a3b1129ef144 100644 --- a/docs/static/generated/rest_v3_sql_gateway.yml +++ b/docs/static/generated/rest_v3_sql_gateway.yml @@ -513,6 +513,7 @@ components: - VARIANT - BITMAP - UUID + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/docs/static/generated/rest_v4_sql_gateway.yml b/docs/static/generated/rest_v4_sql_gateway.yml index 0312a8e962dd8a..dfab504f7d9272 100644 --- a/docs/static/generated/rest_v4_sql_gateway.yml +++ b/docs/static/generated/rest_v4_sql_gateway.yml @@ -526,6 +526,7 @@ components: - VARIANT - BITMAP - UUID + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonDeserializer.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonDeserializer.java index 8c3a88b8fee7ea..036a7c8d0cca30 100644 --- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonDeserializer.java +++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonDeserializer.java @@ -32,6 +32,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -168,6 +169,8 @@ private LogicalType deserializeInternal(JsonNode logicalTypeNode) { return new BitmapType(isNullable); case UUID: return new UuidType(isNullable); + case GEOGRAPHY: + return new GeographyType(isNullable); default: throw new UnsupportedOperationException( String.format( diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonSerializer.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonSerializer.java index bb650453c4f344..f8c60b0d352919 100644 --- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonSerializer.java +++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonSerializer.java @@ -131,6 +131,7 @@ private void serializeInternal(LogicalType logicalType, JsonGenerator jsonGenera case VARIANT: case BITMAP: case UUID: + case GEOGRAPHY: break; case CHAR: jsonGenerator.writeNumberField( diff --git a/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonSerDeTest.java b/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonSerDeTest.java index dd1f71b255f363..460f260574ba35 100644 --- a/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonSerDeTest.java +++ b/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/rest/serde/LogicalTypeJsonSerDeTest.java @@ -34,6 +34,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -199,6 +200,7 @@ private static List generateTestData() { new MultisetType(VarBinaryType.ofEmptyLiteral()), new BitmapType(), new UuidType(), + new GeographyType(), RowType.of(new BigIntType(), new IntType(false), new VarCharType(200)), RowType.of( new LogicalType[] { @@ -207,6 +209,7 @@ private static List generateTestData() { new String[] {"f1", "f2", "f3"}), RowType.of( new TimestampType(false, 3), new LocalZonedTimestampType(false, 3)), + RowType.of(new GeographyType(false), new ArrayType(new GeographyType())), RowType.of( CharType.ofEmptyLiteral(), VarCharType.ofEmptyLiteral(), diff --git a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd index b7e6d7ab9ea3f4..db08bbfbf2c329 100644 --- a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd +++ b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd @@ -173,6 +173,7 @@ "org.apache.flink.sql.parser.type.ExtendedSqlCollectionTypeNameSpec" "org.apache.flink.sql.parser.type.ExtendedSqlRowTypeNameSpec" "org.apache.flink.sql.parser.type.SqlBitmapTypeNameSpec" + "org.apache.flink.sql.parser.type.SqlGeographyTypeNameSpec" "org.apache.flink.sql.parser.type.SqlRawTypeNameSpec" "org.apache.flink.sql.parser.type.SqlStructuredTypeNameSpec" "org.apache.flink.sql.parser.type.SqlTimestampLtzTypeNameSpec" @@ -225,6 +226,7 @@ "FROM_TIMESTAMP" "FUNCTIONS" "FRESHNESS" + "GEOGRAPHY" "HASH" "IF" "JSON_EXECUTION_PLAN" @@ -719,6 +721,7 @@ "ExtendedSqlRowTypeName()" "SqlStructuredTypeName()" "SqlBitmapTypeName()" + "SqlGeographyTypeName()" ] # List of methods for parsing builtin function calls. diff --git a/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl b/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl index 2b8250ea98ed9e..ede54c27ca86e0 100644 --- a/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl +++ b/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl @@ -2666,6 +2666,17 @@ SqlTypeNameSpec SqlBitmapTypeName() : } } +/** Parses GEOGRAPHY type. */ +SqlTypeNameSpec SqlGeographyTypeName() : +{ +} +{ + + { + return new SqlGeographyTypeNameSpec(getPos()); + } +} + /** * Parse a "name1 type1 [ NULL | NOT NULL] [ comment ] * [, name2 type2 [ NULL | NOT NULL] [ comment ] ]* ..." list. diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java new file mode 100644 index 00000000000000..55a3d7ff4fd7d6 --- /dev/null +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.sql.parser.type; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.calcite.ExtendedRelTypeFactory; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlTypeNameSpec; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.util.Litmus; + +/** Represents the GEOGRAPHY data type. */ +@Internal +public final class SqlGeographyTypeNameSpec extends SqlTypeNameSpec { + + private static final String GEOGRAPHY_TYPE_NAME = "GEOGRAPHY"; + + public SqlGeographyTypeNameSpec(SqlParserPos pos) { + super(new SqlIdentifier(GEOGRAPHY_TYPE_NAME, pos), pos); + } + + @Override + public RelDataType deriveType(SqlValidator validator) { + return ((ExtendedRelTypeFactory) validator.getTypeFactory()).createGeographyType(); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + writer.keyword(GEOGRAPHY_TYPE_NAME); + } + + @Override + public boolean equalsDeep(SqlTypeNameSpec spec, Litmus litmus) { + if (!(spec instanceof SqlGeographyTypeNameSpec)) { + return litmus.fail("{} != {}", this, spec); + } + return litmus.succeed(); + } +} diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java index 57b9bc615ec930..57be54e4cb6bd4 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java @@ -44,4 +44,7 @@ RelDataType createStructuredType( /** Creates a BITMAP type. */ RelDataType createBitmapType(); + + /** Creates a GEOGRAPHY type. */ + RelDataType createGeographyType(); } diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java index 01f8c61541d429..ec000f8e5ff9f5 100644 --- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java +++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java @@ -4214,4 +4214,23 @@ void testBitmapType() { sql("CREATE TABLE t (\n" + "^bitmap^ INT" + "\n)") .fails("(?s).*Encountered \"bitmap\" at line 2, column 1.\n.*"); } + + @Test + void testGeographyType() { + sql("CREATE TABLE t (\n" + "g geography" + "\n)") + .ok("CREATE TABLE `T` (\n" + " `G` GEOGRAPHY\n" + ")"); + + sql("CREATE TABLE t (\n" + "g geography NOT NULL" + "\n)") + .ok("CREATE TABLE `T` (\n" + " `G` GEOGRAPHY NOT NULL\n" + ")"); + + // GEOGRAPHY takes no parameters + sql("CREATE TABLE t (\n" + "g geography^(^1)" + "\n)") + .fails("(?s).*Encountered \"\\(\" at line 2, column 12.\n.*"); + sql("CREATE TABLE t (\n" + "g geography^(^4326)" + "\n)") + .fails("(?s).*Encountered \"\\(\" at line 2, column 12.\n.*"); + + // GEOGRAPHY is a reserved keyword and cannot be used as an identifier without escaping + sql("CREATE TABLE t (\n" + "^geography^ INT" + "\n)") + .fails("(?s).*Encountered \"geography\" at line 2, column 1.\n.*"); + } } diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java index 115ba8368eee1a..32e98304b7209c 100644 --- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java +++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java @@ -53,6 +53,11 @@ public RelDataType createBitmapType() { return canonize(new DummyBitmapType()); } + @Override + public RelDataType createGeographyType() { + return canonize(new DummyGeographyType()); + } + private static class DummyRawType extends RelDataTypeImpl { private final String className; @@ -118,4 +123,16 @@ protected void generateTypeString(StringBuilder sb, boolean withDetail) { sb.append("BITMAP"); } } + + private static class DummyGeographyType extends RelDataTypeImpl { + + DummyGeographyType() { + computeDigest(); + } + + @Override + protected void generateTypeString(StringBuilder sb, boolean withDetail) { + sb.append("GEOGRAPHY"); + } + } } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java index 9beae7d2f4ace4..d6ac2d497bb2cd 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java @@ -44,6 +44,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -1087,6 +1088,15 @@ public static DataType UUID() { return new AtomicDataType(new UuidType()); } + /** + * Data type of geography data. + * + * @see GeographyType + */ + public static DataType GEOGRAPHY() { + return new AtomicDataType(new GeographyType()); + } + // -------------------------------------------------------------------------------------------- // Helper functions // -------------------------------------------------------------------------------------------- diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java index c56955c4a5c2f7..6b647113336024 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java @@ -123,6 +123,12 @@ default Bitmap getBitmap(int pos) { "This ArrayData implementation does not support Bitmap type."); } + /** Returns the geography value at the given position. */ + default GeographyData getGeography(int pos) { + throw new UnsupportedOperationException( + "This ArrayData implementation does not support Geography type."); + } + // ------------------------------------------------------------------------------------------ // Conversion Utilities // ------------------------------------------------------------------------------------------ @@ -226,6 +232,9 @@ static ElementGetter createElementGetter(LogicalType elementType) { case BITMAP: elementGetter = ArrayData::getBitmap; break; + case GEOGRAPHY: + elementGetter = ArrayData::getGeography; + break; case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java index f609c2f22f1311..08856cdb0ceb1e 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java @@ -265,6 +265,11 @@ public Bitmap getBitmap(int pos) { return (Bitmap) getObject(pos); } + @Override + public GeographyData getGeography(int pos) { + return (GeographyData) getObject(pos); + } + private Object getObject(int pos) { return ((Object[]) array)[pos]; } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java index b234edef62d05b..98c77bca28f06d 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java @@ -192,6 +192,11 @@ public byte[] getBinary(int pos) { return (byte[]) this.fields[pos]; } + @Override + public GeographyData getGeography(int pos) { + return (GeographyData) this.fields[pos]; + } + @Override public ArrayData getArray(int pos) { return (ArrayData) this.fields[pos]; diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java new file mode 100644 index 00000000000000..b2b00c3c195b33 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.table.data.binary.BinaryGeographyData; +import org.apache.flink.table.types.logical.GeographyType; + +/** + * An internal data structure representing data of {@link GeographyType}. + * + *

The v1 runtime contract uses OGC:CRS84 and supports the seven standard 2D ISO WKB subtype + * codes defined by the OGC Simple Feature Access specification. Extended WKB variants such as Z, + * M, ZM, and EWKB are outside this contract. CRS and SRID metadata are not encoded in the ISO WKB + * payload and must be handled at the construction or connector boundary. + * + *

References: + * + *

+ */ +@PublicEvolving +public interface GeographyData { + + /** Supported 2D ISO WKB subtype ID for Point geometries. */ + int POINT = 1; + + /** Supported 2D ISO WKB subtype ID for LineString geometries. */ + int LINE_STRING = 2; + + /** Supported 2D ISO WKB subtype ID for Polygon geometries. */ + int POLYGON = 3; + + /** Supported 2D ISO WKB subtype ID for MultiPoint geometries. */ + int MULTI_POINT = 4; + + /** Supported 2D ISO WKB subtype ID for MultiLineString geometries. */ + int MULTI_LINE_STRING = 5; + + /** Supported 2D ISO WKB subtype ID for MultiPolygon geometries. */ + int MULTI_POLYGON = 6; + + /** Supported 2D ISO WKB subtype ID for GeometryCollection geometries. */ + int GEOMETRY_COLLECTION = 7; + + /** + * Converts this {@link GeographyData} object to an ISO WKB byte array. + * + *

Note: The returned byte array may be reused. + */ + byte[] toBytes(); + + /** Returns the supported 2D ISO WKB subtype ID. */ + int subtypeId(); + + /** Returns the size in bytes of the ISO WKB payload. */ + int sizeInBytes(); + + // ------------------------------------------------------------------------------------------ + // Construction Utilities + // ------------------------------------------------------------------------------------------ + + /** + * Creates an instance of {@link GeographyData} from the given ISO WKB byte array. Returns + * {@code null} if the input is {@code null}. + */ + static GeographyData fromBytes(byte[] bytes) { + return BinaryGeographyData.fromBytes(bytes); + } + + /** + * Creates an instance of {@link GeographyData} from the given ISO WKB byte range. Returns + * {@code null} if the input is {@code null}. + */ + static GeographyData fromBytes(byte[] bytes, int offset, int numBytes) { + return BinaryGeographyData.fromBytes(bytes, offset, numBytes); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java index 1a28ebfd8330be..f8699a57016f49 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java @@ -113,6 +113,8 @@ * +--------------------------------+-----------------------------------------+ * | UUID | byte[] (16 big-endian bytes) | * +--------------------------------+-----------------------------------------+ + * | GEOGRAPHY | {@link GeographyData} | + * +--------------------------------+-----------------------------------------+ * * *

Nullability is always handled by the container data structure. @@ -216,6 +218,12 @@ default Bitmap getBitmap(int pos) { "This RowData implementation does not support Bitmap type."); } + /** Returns the geography value at the given position. */ + default GeographyData getGeography(int pos) { + throw new UnsupportedOperationException( + "This RowData implementation does not support Geography type."); + } + // ------------------------------------------------------------------------------------------ // Access Utilities // ------------------------------------------------------------------------------------------ @@ -302,6 +310,9 @@ static FieldGetter createFieldGetter(LogicalType fieldType, int fieldPos) { case BITMAP: fieldGetter = row -> row.getBitmap(fieldPos); break; + case GEOGRAPHY: + fieldGetter = row -> row.getGeography(fieldPos); + break; case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java index a8e3087d72c69d..4dd46a6bf47f24 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java @@ -23,6 +23,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -96,6 +97,7 @@ public static int calculateFixLengthPartSize(LogicalType type) { case VARIANT: case BITMAP: case UUID: + case GEOGRAPHY: // long and double are 8 bytes; // otherwise it stores the length and offset of the variable-length part for types // such as is string, map, etc. @@ -270,6 +272,14 @@ public byte[] getBinary(int pos) { return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndSize); } + @Override + public GeographyData getGeography(int pos) { + assertIndexIsValid(pos); + int fieldOffset = getElementOffset(pos, 8); + final long offsetAndSize = BinarySegmentUtils.getLong(segments, fieldOffset); + return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndSize); + } + @Override public ArrayData getArray(int pos) { assertIndexIsValid(pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java new file mode 100644 index 00000000000000..08206f33d8e895 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data.binary; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GeographyData; + +import java.util.Arrays; + +/** + * A binary implementation of {@link GeographyData} backed by raw ISO WKB bytes. + * + *

GEOGRAPHY uses OGC:CRS84 by contract, but ISO WKB does not encode CRS or SRID metadata. This + * container stores the raw ISO WKB payload only; CRS validation, CRS transformation, and EWKB/SRID + * handling belong to constructors, functions, and connector schema mapping. + * + *

{@link #fromBytes(byte[], int, int)} is the validating, copying construction path. The + * internal {@link #fromAddress(MemorySegment[], int, int)} path creates a lazy view over trusted + * binary storage without validating the full WKB payload or taking ownership of the backing + * segments. Callers using the lazy path must preserve the backing storage for the lifetime of the + * view. + */ +@Internal +public final class BinaryGeographyData extends BinarySection implements GeographyData { + + private static final int MIN_WKB_HEADER_SIZE = 5; + private static final int WKB_COUNT_SIZE = 4; + private static final int WKB_POINT_COORDINATE_SIZE = 16; + private static final int BIG_ENDIAN = 0; + private static final int LITTLE_ENDIAN = 1; + private static final int UNKNOWN_SUBTYPE_ID = 0; + private static final long MAX_INT_ADDRESS_EXCLUSIVE = (long) Integer.MAX_VALUE + 1; + + private int subtypeId = UNKNOWN_SUBTYPE_ID; + + private BinaryGeographyData(MemorySegment[] segments, int offset, int sizeInBytes) { + super(segments, offset, sizeInBytes); + } + + /** + * Creates a {@link BinaryGeographyData} instance from the given address and length without + * validating the full ISO WKB payload. + */ + public static BinaryGeographyData fromAddress( + MemorySegment[] segments, int offset, int numBytes) { + requireAddressableRange(offset, numBytes, (long) offset + numBytes, "GEOGRAPHY section"); + return new BinaryGeographyData(segments, offset, numBytes); + } + + /** + * Creates a {@link BinaryGeographyData} view over a trusted internal ISO WKB byte range without + * validating the payload. + * + *

This method is zero-copy with respect to the WKB payload and aliases the provided backing + * array. It is intended only for trusted internal binary storage. Callers must keep the backing + * array unchanged for the lifetime of the returned value. + */ + public static BinaryGeographyData fromTrustedBytes(byte[] bytes, int offset, int numBytes) { + if (bytes == null) { + return null; + } + checkRange(bytes, offset, numBytes); + return fromAddress( + new MemorySegment[] {MemorySegmentFactory.wrap(bytes)}, offset, numBytes); + } + + /** Creates a {@link BinaryGeographyData} instance from the given ISO WKB bytes. */ + public static BinaryGeographyData fromBytes(byte[] bytes) { + return bytes == null ? null : fromBytes(bytes, 0, bytes.length); + } + + /** + * Creates a {@link BinaryGeographyData} instance from the given ISO WKB byte range after + * validating the full payload. + */ + public static BinaryGeographyData fromBytes(byte[] bytes, int offset, int numBytes) { + if (bytes == null) { + return null; + } + checkRange(bytes, offset, numBytes); + validatePayload(new MemorySegment[] {MemorySegmentFactory.wrap(bytes)}, offset, numBytes); + byte[] copy = Arrays.copyOfRange(bytes, offset, offset + numBytes); + return fromAddress(new MemorySegment[] {MemorySegmentFactory.wrap(copy)}, 0, copy.length); + } + + @Override + public byte[] toBytes() { + return BinarySegmentUtils.copyToBytes(segments, offset, sizeInBytes); + } + + @Override + public int subtypeId() { + if (subtypeId == UNKNOWN_SUBTYPE_ID) { + subtypeId = readSubtypeId(segments, offset, sizeInBytes); + } + return subtypeId; + } + + @Override + public int sizeInBytes() { + return sizeInBytes; + } + + private static void checkRange(byte[] bytes, int offset, int numBytes) { + if (offset < 0 || numBytes < 0 || offset > bytes.length - numBytes) { + throw new TableRuntimeException( + String.format( + "Invalid ISO WKB byte range: offset %d, length %d, array length %d.", + offset, numBytes, bytes.length)); + } + } + + private static void validatePayload(MemorySegment[] segments, int offset, int sizeInBytes) { + // WKB stores unsigned 32-bit counts, so parser cursors stay long to avoid overflow. Flink's + // binary memory model is still addressed by signed ints at the actual read boundary. + final long endOffset = (long) offset + sizeInBytes; + final GeometryHeader header = readHeader(segments, offset, endOffset); + final long consumedOffset = validateGeometry(segments, offset, endOffset, header); + if (consumedOffset != endOffset) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Found %d trailing byte(s).", + endOffset - consumedOffset)); + } + } + + private static int readSubtypeId(MemorySegment[] segments, int offset, int sizeInBytes) { + final long endOffset = (long) offset + sizeInBytes; + return readHeader(segments, offset, endOffset).subtypeId; + } + + private static long validateGeometry( + MemorySegment[] segments, long geometryOffset, long endOffset) { + final GeometryHeader header = readHeader(segments, geometryOffset, endOffset); + return validateGeometry(segments, geometryOffset, endOffset, header); + } + + private static long validateGeometry( + MemorySegment[] segments, long geometryOffset, long endOffset, GeometryHeader header) { + long cursor = geometryOffset + MIN_WKB_HEADER_SIZE; + + switch (header.subtypeId) { + case GeographyData.POINT: + return requireBytes( + cursor, WKB_POINT_COORDINATE_SIZE, endOffset, "POINT coordinates") + + WKB_POINT_COORDINATE_SIZE; + case GeographyData.LINE_STRING: + return skipCoordinateSequence(segments, cursor, endOffset, header.byteOrder); + case GeographyData.POLYGON: + return skipPolygon(segments, cursor, endOffset, header.byteOrder); + case GeographyData.MULTI_POINT: + return skipTypedGeometryCollection( + segments, cursor, endOffset, header.byteOrder, GeographyData.POINT); + case GeographyData.MULTI_LINE_STRING: + return skipTypedGeometryCollection( + segments, cursor, endOffset, header.byteOrder, GeographyData.LINE_STRING); + case GeographyData.MULTI_POLYGON: + return skipTypedGeometryCollection( + segments, cursor, endOffset, header.byteOrder, GeographyData.POLYGON); + case GeographyData.GEOMETRY_COLLECTION: + return skipGeometryCollection(segments, cursor, endOffset, header.byteOrder); + default: + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Unsupported geography subtype ID %d.", + header.subtypeId)); + } + } + + private static long skipCoordinateSequence( + MemorySegment[] segments, long offset, long endOffset, int byteOrder) { + final long numPoints = + readUnsignedInt(segments, offset, endOffset, byteOrder, "point count"); + long cursor = offset + WKB_COUNT_SIZE; + final long coordinateSequenceSize = + multiplyExact(numPoints, WKB_POINT_COORDINATE_SIZE, "coordinate sequence"); + return requireBytes(cursor, coordinateSequenceSize, endOffset, "coordinate sequence") + + coordinateSequenceSize; + } + + private static long skipPolygon( + MemorySegment[] segments, long offset, long endOffset, int byteOrder) { + final long numRings = readUnsignedInt(segments, offset, endOffset, byteOrder, "ring count"); + long cursor = offset + WKB_COUNT_SIZE; + for (long i = 0; i < numRings; i++) { + cursor = skipCoordinateSequence(segments, cursor, endOffset, byteOrder); + } + return cursor; + } + + private static long skipTypedGeometryCollection( + MemorySegment[] segments, + long offset, + long endOffset, + int byteOrder, + int expectedSubtypeId) { + final long numGeometries = + readUnsignedInt(segments, offset, endOffset, byteOrder, "geometry count"); + long cursor = offset + WKB_COUNT_SIZE; + for (long i = 0; i < numGeometries; i++) { + final GeometryHeader nestedHeader = readHeader(segments, cursor, endOffset); + if (nestedHeader.subtypeId != expectedSubtypeId) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Expected nested subtype ID %d but found %d.", + expectedSubtypeId, nestedHeader.subtypeId)); + } + cursor = validateGeometry(segments, cursor, endOffset); + } + return cursor; + } + + private static long skipGeometryCollection( + MemorySegment[] segments, long offset, long endOffset, int byteOrder) { + final long numGeometries = + readUnsignedInt(segments, offset, endOffset, byteOrder, "geometry count"); + long cursor = offset + WKB_COUNT_SIZE; + for (long i = 0; i < numGeometries; i++) { + cursor = validateGeometry(segments, cursor, endOffset); + } + return cursor; + } + + private static GeometryHeader readHeader( + MemorySegment[] segments, long offset, long endOffset) { + final int address = checkedAddress(offset, MIN_WKB_HEADER_SIZE, endOffset, "WKB header"); + + final int byteOrder = BinarySegmentUtils.getByte(segments, address) & 0xFF; + if (byteOrder != BIG_ENDIAN && byteOrder != LITTLE_ENDIAN) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Unsupported byte order %d.", byteOrder)); + } + + final long subtypeId = + readUnsignedInt(segments, offset + 1, endOffset, byteOrder, "subtype ID"); + + if (subtypeId < GeographyData.POINT || subtypeId > GeographyData.GEOMETRY_COLLECTION) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Unsupported geography subtype ID %d.", + subtypeId)); + } + return new GeometryHeader(byteOrder, (int) subtypeId); + } + + private static long readUnsignedInt( + MemorySegment[] segments, + long offset, + long endOffset, + int byteOrder, + String fieldName) { + final int address = checkedAddress(offset, WKB_COUNT_SIZE, endOffset, fieldName); + + if (byteOrder == LITTLE_ENDIAN) { + return (BinarySegmentUtils.getByte(segments, address) & 0xFFL) + | ((BinarySegmentUtils.getByte(segments, address + 1) & 0xFFL) << 8) + | ((BinarySegmentUtils.getByte(segments, address + 2) & 0xFFL) << 16) + | ((BinarySegmentUtils.getByte(segments, address + 3) & 0xFFL) << 24); + } + return ((BinarySegmentUtils.getByte(segments, address) & 0xFFL) << 24) + | ((BinarySegmentUtils.getByte(segments, address + 1) & 0xFFL) << 16) + | ((BinarySegmentUtils.getByte(segments, address + 2) & 0xFFL) << 8) + | (BinarySegmentUtils.getByte(segments, address + 3) & 0xFFL); + } + + private static int checkedAddress( + long offset, long numBytes, long endOffset, String componentName) { + requireAddressableRange(offset, numBytes, endOffset, componentName); + requireBytes(offset, numBytes, endOffset, componentName); + return (int) offset; + } + + private static void requireAddressableRange( + long offset, long numBytes, long endOffset, String componentName) { + if (offset < 0 + || numBytes < 0 + || endOffset < 0 + || offset > endOffset + || offset > Integer.MAX_VALUE + || endOffset > MAX_INT_ADDRESS_EXCLUSIVE) { + throw new TableRuntimeException( + String.format( + "Unsupported GEOGRAPHY binary address range for %s: offset %d, length %d, end %d exceeds Flink's signed int memory address space.", + componentName, offset, numBytes, endOffset)); + } + } + + private static long requireBytes( + long offset, long numBytes, long endOffset, String componentName) { + if (offset > endOffset || endOffset - offset < numBytes) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Incomplete %s: expected %d byte(s) but found %d.", + componentName, numBytes, Math.max(0, endOffset - offset))); + } + return offset; + } + + private static long multiplyExact(long value, int factor, String componentName) { + final long result = value * factor; + if (value != 0 && result / value != factor) { + throw new TableRuntimeException( + String.format("Malformed ISO WKB payload. %s size overflows.", componentName)); + } + return result; + } + + private static final class GeometryHeader { + private final int byteOrder; + private final int subtypeId; + + private GeometryHeader(int byteOrder, int subtypeId) { + this.byteOrder = byteOrder; + this.subtypeId = subtypeId; + } + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java index 45bcf7cd3f1128..04c9a0e50ed43d 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java @@ -22,6 +22,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -372,6 +373,14 @@ public byte[] getBinary(int pos) { return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen); } + @Override + public GeographyData getGeography(int pos) { + assertIndexIsValid(pos); + int fieldOffset = getFieldOffset(pos); + final long offsetAndLen = segments[0].getLong(fieldOffset); + return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndLen); + } + @Override public ArrayData getArray(int pos) { assertIndexIsValid(pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java index 653fc559df9bce..2dd12978d6bf5f 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java @@ -1056,6 +1056,34 @@ public static byte[] readBinary( } } + /** + * Get geography data, if len less than 8, it will be included in variablePartOffsetAndLen. + * + * @param baseOffset base offset of composite binary format. + * @param fieldOffset absolute start offset of 'variablePartOffsetAndLen'. + * @param variablePartOffsetAndLen a long value, real data or offset and len. + */ + public static BinaryGeographyData readGeographyData( + MemorySegment[] segments, + int baseOffset, + int fieldOffset, + long variablePartOffsetAndLen) { + long mark = variablePartOffsetAndLen & HIGHEST_FIRST_BIT; + if (mark == 0) { + final int subOffset = (int) (variablePartOffsetAndLen >> 32); + final int len = (int) variablePartOffsetAndLen; + return BinaryGeographyData.fromAddress(segments, baseOffset + subOffset, len); + } else { + int len = (int) ((variablePartOffsetAndLen & HIGHEST_SECOND_TO_EIGHTH_BIT) >>> 56); + if (BinarySegmentUtils.LITTLE_ENDIAN) { + return BinaryGeographyData.fromAddress(segments, fieldOffset, len); + } else { + // fieldOffset + 1 to skip header. + return BinaryGeographyData.fromAddress(segments, fieldOffset + 1, len); + } + } + } + /** * Get binary string, if len less than 8, will be include in variablePartOffsetAndLen. * diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java index 27085b487ae50d..225d3e103fe103 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java @@ -21,6 +21,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -294,6 +295,14 @@ public byte[] getBinary(int pos) { return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen); } + @Override + public GeographyData getGeography(int pos) { + assertIndexIsValid(pos); + int fieldOffset = getFieldOffset(pos); + final long offsetAndLen = BinarySegmentUtils.getLong(segments, fieldOffset); + return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndLen); + } + @Override public RowData getRow(int pos, int numFields) { assertIndexIsValid(pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java index a3b8e7dd56daf9..4d8ad838159990 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java @@ -21,11 +21,13 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.data.binary.TypedSetters; import org.apache.flink.table.data.columnar.vector.ArrayColumnVector; import org.apache.flink.table.data.columnar.vector.BooleanColumnVector; @@ -147,6 +149,13 @@ public byte[] getBinary(int pos) { } } + @Override + public GeographyData getGeography(int pos) { + BytesColumnVector.Bytes byteArray = getByteArray(pos); + return BinaryGeographyData.fromTrustedBytes( + byteArray.data, byteArray.offset, byteArray.len); + } + @Override public ArrayData getArray(int pos) { return ((ArrayColumnVector) data).getArray(offset + pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java index bd3dce1edf6bae..a2d55c2ec970d8 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java @@ -21,11 +21,13 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.data.binary.TypedSetters; import org.apache.flink.table.data.columnar.vector.BytesColumnVector.Bytes; import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch; @@ -152,6 +154,13 @@ public byte[] getBinary(int pos) { } } + @Override + public GeographyData getGeography(int pos) { + Bytes byteArray = vectorizedColumnBatch.getByteArray(rowId, pos); + return BinaryGeographyData.fromTrustedBytes( + byteArray.data, byteArray.offset, byteArray.len); + } + @Override public RowData getRow(int pos, int numFields) { return vectorizedColumnBatch.getRow(rowId, pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java index b9eff05a6c388f..d1420debfdb07a 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -269,6 +270,15 @@ public Bitmap getBitmap(int pos) { } } + @Override + public GeographyData getGeography(int pos) { + if (pos < row1.getArity()) { + return row1.getGeography(pos); + } else { + return row2.getGeography(pos - row1.getArity()); + } + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java index 7f29ca5e613236..e6a173bb4cd991 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java @@ -22,6 +22,7 @@ import org.apache.flink.table.connector.Projection; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -178,6 +179,11 @@ public Bitmap getBitmap(int pos) { return row.getBitmap(indexMapping[pos]); } + @Override + public GeographyData getGeography(int pos) { + return row.getGeography(indexMapping[pos]); + } + @Override public boolean equals(Object o) { throw new UnsupportedOperationException("Projected row data cannot be compared"); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java new file mode 100644 index 00000000000000..85f85641da1880 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.logical; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.binary.BinaryGeographyData; + +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * Data type of geography data. + * + *

The serializable string representation of this type is {@code GEOGRAPHY}. + */ +@PublicEvolving +public final class GeographyType extends LogicalType { + + private static final long serialVersionUID = 1L; + + private static final String FORMAT = "GEOGRAPHY"; + + private static final Class INPUT_OUTPUT_CONVERSION = GeographyData.class; + + private static final Set INPUT_OUTPUT_CONVERSIONS = + conversionSet(GeographyData.class.getName(), BinaryGeographyData.class.getName()); + + public GeographyType(boolean isNullable) { + super(isNullable, LogicalTypeRoot.GEOGRAPHY); + } + + public GeographyType() { + this(true); + } + + @Override + public LogicalType copy(boolean isNullable) { + return new GeographyType(isNullable); + } + + @Override + public String asSerializableString() { + return withNullability(FORMAT); + } + + @Override + public boolean supportsInputConversion(Class clazz) { + return INPUT_OUTPUT_CONVERSIONS.contains(clazz.getName()); + } + + @Override + public boolean supportsOutputConversion(Class clazz) { + return INPUT_OUTPUT_CONVERSIONS.contains(clazz.getName()); + } + + @Override + public Class getDefaultConversion() { + return INPUT_OUTPUT_CONVERSION; + } + + @Override + public List getChildren() { + return Collections.emptyList(); + } + + @Override + public R accept(LogicalTypeVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java index 1026070c710102..421c7b63c23e70 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java @@ -147,7 +147,9 @@ public enum LogicalTypeRoot { BITMAP(LogicalTypeFamily.EXTENSION), - UUID(LogicalTypeFamily.EXTENSION); + UUID(LogicalTypeFamily.EXTENSION), + + GEOGRAPHY(LogicalTypeFamily.EXTENSION); private final Set families; diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java index fe5548c8f65c04..c6d09bdd675207 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java @@ -105,5 +105,9 @@ default R visit(UuidType uuidType) { return visit((LogicalType) uuidType); } + default R visit(GeographyType geographyType) { + return visit((LogicalType) geographyType); + } + R visit(LogicalType other); } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java index 056ec5953ccdc3..43237c443f56e9 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java @@ -31,6 +31,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -203,6 +204,11 @@ public R visit(BitmapType bitmapType) { return defaultMethod(bitmapType); } + @Override + public R visit(GeographyType geographyType) { + return defaultMethod(geographyType); + } + @Override public R visit(LogicalType other) { return defaultMethod(other); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java index a8e859204338d7..9696110843f318 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java @@ -36,6 +36,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -338,7 +339,8 @@ private enum Keyword { STRUCTURED, VARIANT, BITMAP, - UUID + UUID, + GEOGRAPHY } private static final Set KEYWORDS = @@ -590,6 +592,8 @@ private LogicalType parseTypeByKeyword() { return new BitmapType(); case UUID: return new UuidType(); + case GEOGRAPHY: + return new GeographyType(); default: throw parsingError("Unsupported type: " + token().value); } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java index 849c2f22bbfd07..0ef5a23a6e630c 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -116,6 +117,8 @@ public static Class toInternalConversionClass(LogicalType type) { return Variant.class; case BITMAP: return Bitmap.class; + case GEOGRAPHY: + return GeographyData.class; case SYMBOL: case UNRESOLVED: default: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java index 9aa83f3cbee1d3..929aaec786f490 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java @@ -38,6 +38,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -265,6 +266,11 @@ public DataType visit(BitmapType bitmapType) { return new AtomicDataType(bitmapType); } + @Override + public DataType visit(GeographyType geographyType) { + return new AtomicDataType(geographyType); + } + @Override public DataType visit(LogicalType other) { if (other.is(LogicalTypeRoot.UNRESOLVED)) { diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java index 7b54e40f387935..7b4aef14cc3a80 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java @@ -51,5 +51,21 @@ void testFromResolvedSchema() { assertThat(newSchema.resolve(new TestSchemaResolver())).isEqualTo(originalSchema); } + + @Test + void testGeographyColumn() { + final Schema schema = + Schema.newBuilder() + .column("location", DataTypes.GEOGRAPHY()) + .column("required_location", DataTypes.GEOGRAPHY().notNull()) + .build(); + + assertThat(schema.resolve(new TestSchemaResolver())) + .isEqualTo( + ResolvedSchema.of( + Column.physical("location", DataTypes.GEOGRAPHY()), + Column.physical( + "required_location", DataTypes.GEOGRAPHY().notNull()))); + } } } diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java new file mode 100644 index 00000000000000..d440bd202baeff --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java @@ -0,0 +1,432 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data.binary; + +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.GeographyType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Arrays; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link BinaryGeographyData}. */ +class BinaryGeographyDataTest { + + private static final int BIG_ENDIAN = 0; + private static final int LITTLE_ENDIAN = 1; + private static final byte[] POINT_WKB = pointWkb(LITTLE_ENDIAN); + private static final long UNSIGNED_INT_MAX = 4_294_967_295L; + private static final long MAX_COORDINATE_SEQUENCE_SIZE = UNSIGNED_INT_MAX * 16; + + @Test + void testValidWkbRoundTrip() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + + assertThat(geography).isInstanceOf(BinaryGeographyData.class); + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(geography.sizeInBytes()).isEqualTo(POINT_WKB.length); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("supportedSubtypePayloads") + void testSupported2dSubtypePayloads(String name, int subtypeId, byte[] wkb) { + final BinaryGeographyData geography = BinaryGeographyData.fromBytes(wkb); + + assertThat(geography.subtypeId()).isEqualTo(subtypeId); + assertThat(geography.toBytes()).isEqualTo(wkb); + assertThat(geography.sizeInBytes()).isEqualTo(wkb.length); + } + + static Stream supportedSubtypePayloads() { + return Stream.of( + Arguments.of("Point", GeographyData.POINT, pointWkb(LITTLE_ENDIAN)), + Arguments.of("LineString", GeographyData.LINE_STRING, lineStringWkb(LITTLE_ENDIAN)), + Arguments.of("Polygon", GeographyData.POLYGON, polygonWkb(LITTLE_ENDIAN)), + Arguments.of("MultiPoint", GeographyData.MULTI_POINT, multiPointWkb(LITTLE_ENDIAN)), + Arguments.of( + "MultiLineString", + GeographyData.MULTI_LINE_STRING, + multiLineStringWkb(LITTLE_ENDIAN)), + Arguments.of( + "MultiPolygon", + GeographyData.MULTI_POLYGON, + multiPolygonWkb(LITTLE_ENDIAN)), + Arguments.of( + "GeometryCollection", + GeographyData.GEOMETRY_COLLECTION, + geometryCollectionWkb(LITTLE_ENDIAN))); + } + + @Test + void testBigEndianSubtypeExtraction() { + final BinaryGeographyData point = BinaryGeographyData.fromBytes(pointWkb(BIG_ENDIAN)); + final BinaryGeographyData lineString = + BinaryGeographyData.fromBytes(lineStringWkb(BIG_ENDIAN)); + + assertThat(point.subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(lineString.subtypeId()).isEqualTo(GeographyData.LINE_STRING); + } + + @Test + void testNullHandling() { + assertThat(GeographyData.fromBytes((byte[]) null)).isNull(); + assertThat(GeographyData.fromBytes(null, 0, 0)).isNull(); + assertThat(BinaryGeographyData.fromBytes((byte[]) null)).isNull(); + assertThat(BinaryGeographyData.fromBytes(null, 0, 0)).isNull(); + } + + @Test + void testGenericRowDataNullPath() { + final RowData.FieldGetter getter = RowData.createFieldGetter(new GeographyType(), 0); + final GenericRowData nullRow = GenericRowData.of((Object) null); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final GenericRowData valueRow = GenericRowData.of(geography); + + assertThat(getter.getFieldOrNull(nullRow)).isNull(); + assertThat(getter.getFieldOrNull(valueRow)).isSameAs(geography); + } + + @Test + void testBinaryRowDataNullPath() { + final RowData.FieldGetter getter = RowData.createFieldGetter(new GeographyType(), 0); + final int fixedLength = BinaryRowData.calculateFixPartSizeInBytes(1); + final BinaryRowData nullRow = new BinaryRowData(1); + nullRow.pointTo(MemorySegmentFactory.wrap(new byte[fixedLength]), 0, fixedLength); + nullRow.setNullAt(0); + + assertThat(getter.getFieldOrNull(nullRow)).isNull(); + } + + @Test + void testBinaryRowDataValuePath() { + final BinaryRowData row = binaryRowWithGeography(POINT_WKB); + + assertThat(row.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(row.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testBinaryRowDataDoesNotEagerlyValidateMalformedPayload() { + final byte[] malformedPointWkb = incompletePointWkb(); + final GeographyData geography = binaryRowWithGeography(malformedPointWkb).getGeography(0); + + assertThat(geography.toBytes()).isEqualTo(malformedPointWkb); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testBinaryArrayDataDoesNotEagerlyValidateMalformedPayload() { + final byte[] malformedPointWkb = incompletePointWkb(); + final GeographyData geography = binaryArrayWithGeography(malformedPointWkb).getGeography(0); + + assertThat(geography.toBytes()).isEqualTo(malformedPointWkb); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testExplicitConstructionStillRejectsMalformedPayload() { + assertThatThrownBy(() -> GeographyData.fromBytes(incompletePointWkb())) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete POINT coordinates"); + } + + @Test + void testByteRangeCopiesOnlySelectedPayload() { + final byte[] bytes = concat(bytes(42), POINT_WKB, bytes(99)); + final BinaryGeographyData geography = + BinaryGeographyData.fromBytes(bytes, 1, POINT_WKB.length); + + bytes[1] = 0; + + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testFromAddressSupportsSegmentBoundary() { + final byte[] first = concat(new byte[14], Arrays.copyOfRange(POINT_WKB, 0, 2)); + final byte[] second = Arrays.copyOfRange(POINT_WKB, 2, 18); + final byte[] third = + concat(Arrays.copyOfRange(POINT_WKB, 18, POINT_WKB.length), new byte[13]); + final MemorySegment[] segments = + new MemorySegment[] { + MemorySegmentFactory.wrap(first), + MemorySegmentFactory.wrap(second), + MemorySegmentFactory.wrap(third) + }; + + final BinaryGeographyData geography = + BinaryGeographyData.fromAddress(segments, 14, POINT_WKB.length); + + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testMalformedPayloadBoundaries() { + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes())) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete WKB header"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes(1, 1, 0, 0))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete WKB header"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes(2, 1, 0, 0, 0))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported byte order 2"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(header(LITTLE_ENDIAN, 0))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported geography subtype ID 0"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(header(LITTLE_ENDIAN, 8))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported geography subtype ID 8"); + } + + @Test + void testStructurallyIncompletePayloads() { + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + header(LITTLE_ENDIAN, GeographyData.POINT))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete POINT coordinates"); + + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat( + header(LITTLE_ENDIAN, GeographyData.LINE_STRING), + unsignedInt(LITTLE_ENDIAN, 1), + new byte[15]))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete coordinate sequence"); + + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat( + header(LITTLE_ENDIAN, GeographyData.MULTI_POINT), + unsignedInt(LITTLE_ENDIAN, 1), + lineStringWkb(LITTLE_ENDIAN)))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Expected nested subtype ID 1 but found 2"); + + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat(pointWkb(LITTLE_ENDIAN), bytes(42)))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("trailing byte"); + } + + @Test + void testUnsignedPointCountUsesLongArithmetic() { + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat( + header(LITTLE_ENDIAN, GeographyData.LINE_STRING), + unsignedInt(LITTLE_ENDIAN, -1)))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete coordinate sequence") + .hasMessageContaining(String.valueOf(MAX_COORDINATE_SEQUENCE_SIZE)); + } + + @Test + void testFromAddressRejectsUnsupportedPositiveAddressRange() { + assertThatThrownBy( + () -> + BinaryGeographyData.fromAddress( + new MemorySegment[] { + MemorySegmentFactory.wrap(new byte[8]) + }, + Integer.MAX_VALUE - 1, + 4)) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported GEOGRAPHY binary address range"); + } + + @Test + void testFromAddressRejectsNegativeOffset() { + assertThatThrownBy( + () -> + BinaryGeographyData.fromAddress( + new MemorySegment[] { + MemorySegmentFactory.wrap(new byte[8]) + }, + -1, + POINT_WKB.length)) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported GEOGRAPHY binary address range"); + } + + @Test + void testFromAddressRejectsNegativeSize() { + assertThatThrownBy( + () -> + BinaryGeographyData.fromAddress( + new MemorySegment[] { + MemorySegmentFactory.wrap(new byte[8]) + }, + 0, + -1)) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported GEOGRAPHY binary address range"); + } + + @Test + void testInvalidByteRanges() { + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(POINT_WKB, -1, POINT_WKB.length)) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid ISO WKB byte range"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(POINT_WKB, 0, POINT_WKB.length + 1)) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid ISO WKB byte range"); + } + + private static BinaryRowData binaryRowWithGeography(byte[] wkb) { + final int fixedLength = BinaryRowData.calculateFixPartSizeInBytes(1); + final byte[] rowBytes = new byte[fixedLength + wkb.length]; + final MemorySegment segment = MemorySegmentFactory.wrap(rowBytes); + segment.putLong(8, ((long) fixedLength << 32) | wkb.length); + segment.put(fixedLength, wkb, 0, wkb.length); + + final BinaryRowData row = new BinaryRowData(1); + row.pointTo(segment, 0, rowBytes.length); + return row; + } + + private static BinaryArrayData binaryArrayWithGeography(byte[] wkb) { + final int headerLength = BinaryArrayData.calculateHeaderInBytes(1); + final int variableOffset = headerLength + 8; + final int paddedLength = ((wkb.length + 7) / 8) * 8; + final byte[] arrayBytes = new byte[variableOffset + paddedLength]; + final MemorySegment segment = MemorySegmentFactory.wrap(arrayBytes); + segment.putInt(0, 1); + segment.putLong(headerLength, ((long) variableOffset << 32) | wkb.length); + segment.put(variableOffset, wkb, 0, wkb.length); + + final BinaryArrayData array = new BinaryArrayData(); + array.pointTo(segment, 0, arrayBytes.length); + return array; + } + + private static byte[] incompletePointWkb() { + return concat(header(LITTLE_ENDIAN, GeographyData.POINT), bytes(1, 2, 3)); + } + + private static byte[] pointWkb(int byteOrder) { + return concat(header(byteOrder, GeographyData.POINT), new byte[16]); + } + + private static byte[] lineStringWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.LINE_STRING), + unsignedInt(byteOrder, 2), + new byte[32]); + } + + private static byte[] polygonWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.POLYGON), + unsignedInt(byteOrder, 1), + unsignedInt(byteOrder, 4), + new byte[64]); + } + + private static byte[] multiPointWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.MULTI_POINT), + unsignedInt(byteOrder, 1), + pointWkb(byteOrder)); + } + + private static byte[] multiLineStringWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.MULTI_LINE_STRING), + unsignedInt(byteOrder, 1), + lineStringWkb(byteOrder)); + } + + private static byte[] multiPolygonWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.MULTI_POLYGON), + unsignedInt(byteOrder, 1), + polygonWkb(byteOrder)); + } + + private static byte[] geometryCollectionWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.GEOMETRY_COLLECTION), + unsignedInt(byteOrder, 2), + pointWkb(byteOrder), + lineStringWkb(byteOrder)); + } + + private static byte[] header(int byteOrder, int subtypeId) { + return concat(bytes(byteOrder), unsignedInt(byteOrder, subtypeId)); + } + + private static byte[] unsignedInt(int byteOrder, int value) { + if (byteOrder == LITTLE_ENDIAN) { + return bytes(value, value >>> 8, value >>> 16, value >>> 24); + } + return bytes(value >>> 24, value >>> 16, value >>> 8, value); + } + + private static byte[] concat(byte[]... values) { + int length = 0; + for (byte[] value : values) { + length += value.length; + } + + final byte[] result = new byte[length]; + int offset = 0; + for (byte[] value : values) { + System.arraycopy(value, 0, result, offset, value.length); + offset += value.length; + } + return result; + } + + private static byte[] bytes(int... values) { + final byte[] result = new byte[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = (byte) values[i]; + } + return result; + } +} diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/columnar/ColumnarArrayDataTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/columnar/ColumnarArrayDataTest.java index 0fddd5f5c25fd0..09d70b942b259a 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/columnar/ColumnarArrayDataTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/columnar/ColumnarArrayDataTest.java @@ -18,15 +18,53 @@ package org.apache.flink.table.data.columnar; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; class ColumnarArrayDataTest { + private static final byte[] LITTLE_ENDIAN_POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + private static final byte[] BIG_ENDIAN_POINT_WKB = + new byte[] { + 0, + 0, + 0, + 0, + 1, + 0x3F, + (byte) 0xF0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0x40, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }; + + private static final byte[] MALFORMED_POINT_HEADER = new byte[] {1, 1, 0, 0, 0}; + @Test @DisplayName("getBinary() should work correctly for slices with position 0") void testGetBinaryWhenOffsetIsZero() { @@ -48,21 +86,70 @@ void testGetBinaryWhenOffsetIsZero() { void testGetBinaryWhenPositionNonZero() { HeapBytesVector vector = new HeapBytesVector(3); - // Append a DUMMY element first byte[] dummyData = new byte[] {99, 99, 99, 99}; vector.appendBytes(0, dummyData, 0, 4); - // Append the REAL data byte[] sourceData1 = new byte[] {30, 40, 50, 60}; vector.appendBytes(1, sourceData1, 0, 4); byte[] sourceData2 = new byte[] {70, 80, 90, 100}; vector.appendBytes(2, sourceData2, 0, 4); - // Create a ColumnarArrayData that wraps the entire vector - ColumnarArrayData arrayData = new ColumnarArrayData(vector, 0, 1); + ColumnarArrayData arrayData = new ColumnarArrayData(vector, 0, 3); assertThat(arrayData.getBinary(0)).isEqualTo(dummyData); assertThat(arrayData.getBinary(1)).isEqualTo(sourceData1); assertThat(arrayData.getBinary(2)).isEqualTo(sourceData2); } + + @Test + void testGetGeographyReturnsBinaryViewForExactSlice() { + HeapBytesVector vector = new HeapBytesVector(3); + vector.appendBytes(0, new byte[] {99}, 0, 1); + vector.appendBytes(1, LITTLE_ENDIAN_POINT_WKB, 0, LITTLE_ENDIAN_POINT_WKB.length); + vector.appendBytes(2, BIG_ENDIAN_POINT_WKB, 0, BIG_ENDIAN_POINT_WKB.length); + + ColumnarArrayData arrayData = new ColumnarArrayData(vector, 0, 3); + + GeographyData littleEndian = arrayData.getGeography(1); + GeographyData bigEndian = arrayData.getGeography(2); + + assertThat(littleEndian).isInstanceOf(BinaryGeographyData.class); + assertThat(bigEndian).isInstanceOf(BinaryGeographyData.class); + assertThat(littleEndian.toBytes()).isEqualTo(LITTLE_ENDIAN_POINT_WKB); + assertThat(bigEndian.toBytes()).isEqualTo(BIG_ENDIAN_POINT_WKB); + assertThat(arrayData.getGeography(2).toBytes()).isEqualTo(BIG_ENDIAN_POINT_WKB); + } + + @Test + void testGetGeographyIsLazyAndAliasesBackingBuffer() { + HeapBytesVector vector = new HeapBytesVector(2); + vector.appendBytes(0, MALFORMED_POINT_HEADER, 0, MALFORMED_POINT_HEADER.length); + vector.appendBytes(1, LITTLE_ENDIAN_POINT_WKB, 0, LITTLE_ENDIAN_POINT_WKB.length); + + ColumnarArrayData arrayData = new ColumnarArrayData(vector, 0, 2); + + assertThatCode(() -> arrayData.getGeography(0)).doesNotThrowAnyException(); + GeographyData malformed = arrayData.getGeography(0); + assertThat(malformed).isInstanceOf(BinaryGeographyData.class); + assertThat(malformed.toBytes()).isEqualTo(MALFORMED_POINT_HEADER); + + GeographyData point = arrayData.getGeography(1); + byte expectedByte = 0x55; + vector.buffer[vector.start[1] + 5] = expectedByte; + + assertThat(point.toBytes()[5]).isEqualTo(expectedByte); + assertThat(arrayData.getGeography(1).toBytes()[5]).isEqualTo(expectedByte); + } + + @Test + void testGetGeographyNullHandlingUsesArrayContract() { + HeapBytesVector vector = new HeapBytesVector(2); + vector.appendBytes(0, LITTLE_ENDIAN_POINT_WKB, 0, LITTLE_ENDIAN_POINT_WKB.length); + vector.setNullAt(1); + + ColumnarArrayData arrayData = new ColumnarArrayData(vector, 0, 2); + + assertThat(arrayData.isNullAt(0)).isFalse(); + assertThat(arrayData.isNullAt(1)).isTrue(); + } } diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/columnar/ColumnarRowDataTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/columnar/ColumnarRowDataTest.java new file mode 100644 index 00000000000000..28cee789a15ac5 --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/columnar/ColumnarRowDataTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data.columnar; + +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.binary.BinaryGeographyData; +import org.apache.flink.table.data.columnar.vector.ColumnVector; +import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch; +import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector; +import org.apache.flink.table.types.logical.GeographyType; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +class ColumnarRowDataTest { + + private static final byte[] LITTLE_ENDIAN_POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + private static final byte[] BIG_ENDIAN_POINT_WKB = + new byte[] { + 0, + 0, + 0, + 0, + 1, + 0x3F, + (byte) 0xF0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0x40, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }; + + private static final byte[] MALFORMED_POINT_HEADER = new byte[] {1, 1, 0, 0, 0}; + + @Test + void testGetGeographyReturnsBinaryViewForRows() { + HeapBytesVector vector = new HeapBytesVector(3); + vector.appendBytes(0, new byte[] {42}, 0, 1); + vector.appendBytes(1, LITTLE_ENDIAN_POINT_WKB, 0, LITTLE_ENDIAN_POINT_WKB.length); + vector.appendBytes(2, BIG_ENDIAN_POINT_WKB, 0, BIG_ENDIAN_POINT_WKB.length); + + VectorizedColumnBatch batch = new VectorizedColumnBatch(new ColumnVector[] {vector}); + ColumnarRowData firstRow = new ColumnarRowData(batch, 1); + ColumnarRowData secondRow = new ColumnarRowData(batch, 2); + + GeographyData first = firstRow.getGeography(0); + GeographyData second = secondRow.getGeography(0); + + assertThat(first).isInstanceOf(BinaryGeographyData.class); + assertThat(second).isInstanceOf(BinaryGeographyData.class); + assertThat(first.toBytes()).isEqualTo(LITTLE_ENDIAN_POINT_WKB); + assertThat(second.toBytes()).isEqualTo(BIG_ENDIAN_POINT_WKB); + assertThat(secondRow.getGeography(0).toBytes()).isEqualTo(BIG_ENDIAN_POINT_WKB); + } + + @Test + void testGetGeographyIsLazyAndAliasesBackingBuffer() { + HeapBytesVector vector = new HeapBytesVector(2); + vector.appendBytes(0, MALFORMED_POINT_HEADER, 0, MALFORMED_POINT_HEADER.length); + vector.appendBytes(1, LITTLE_ENDIAN_POINT_WKB, 0, LITTLE_ENDIAN_POINT_WKB.length); + + VectorizedColumnBatch batch = new VectorizedColumnBatch(new ColumnVector[] {vector}); + ColumnarRowData malformedRow = new ColumnarRowData(batch, 0); + ColumnarRowData pointRow = new ColumnarRowData(batch, 1); + + assertThatCode(() -> malformedRow.getGeography(0)).doesNotThrowAnyException(); + GeographyData malformed = malformedRow.getGeography(0); + assertThat(malformed).isInstanceOf(BinaryGeographyData.class); + assertThat(malformed.toBytes()).isEqualTo(MALFORMED_POINT_HEADER); + + GeographyData point = pointRow.getGeography(0); + byte expectedByte = 0x33; + vector.buffer[vector.start[1] + 5] = expectedByte; + + assertThat(point.toBytes()[5]).isEqualTo(expectedByte); + assertThat(pointRow.getGeography(0).toBytes()[5]).isEqualTo(expectedByte); + } + + @Test + void testGetGeographyNullHandlingUsesFieldGetterContract() { + HeapBytesVector vector = new HeapBytesVector(2); + vector.appendBytes(0, LITTLE_ENDIAN_POINT_WKB, 0, LITTLE_ENDIAN_POINT_WKB.length); + vector.setNullAt(1); + + VectorizedColumnBatch batch = new VectorizedColumnBatch(new ColumnVector[] {vector}); + ColumnarRowData nullRow = new ColumnarRowData(batch, 1); + RowData.FieldGetter fieldGetter = RowData.createFieldGetter(new GeographyType(), 0); + + assertThat(nullRow.isNullAt(0)).isTrue(); + assertThat(fieldGetter.getFieldOrNull(nullRow)).isNull(); + } +} diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java index 7151f9b8c224d4..4a0a7abb1f2c04 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.common.typeutils.base.VoidSerializer; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.BinaryType; @@ -32,6 +33,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -84,6 +86,7 @@ import static org.apache.flink.table.api.DataTypes.DOUBLE; import static org.apache.flink.table.api.DataTypes.FIELD; import static org.apache.flink.table.api.DataTypes.FLOAT; +import static org.apache.flink.table.api.DataTypes.GEOGRAPHY; import static org.apache.flink.table.api.DataTypes.INT; import static org.apache.flink.table.api.DataTypes.INTERVAL; import static org.apache.flink.table.api.DataTypes.MAP; @@ -237,6 +240,9 @@ private static Stream testData() { TestSpec.forDataType(UUID()) .expectLogicalType(new UuidType()) .expectConversionClass(UUID.class), + TestSpec.forDataType(GEOGRAPHY()) + .expectLogicalType(new GeographyType()) + .expectConversionClass(GeographyData.class), TestSpec.forUnresolvedDataType(RAW(Types.VOID)) .expectUnresolvedString("[RAW('java.lang.Void', '?')]") .lookupReturns(dummyRaw(Void.class)) diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java index f6022d3f2cb20a..036714b744f24e 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java @@ -31,6 +31,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -391,7 +392,13 @@ private static Stream testData() { Arguments.of(new UuidType(), new IntType(), false, false), Arguments.of(new IntType(), new UuidType(), false, false), Arguments.of(new UuidType(), new VariantType(), false, false), - Arguments.of(new VariantType(), new UuidType(), false, false)); + Arguments.of(new VariantType(), new UuidType(), false, false), + + // GEOGRAPHY construction and serialization require explicit functions. + Arguments.of(new GeographyType(), VarCharType.STRING_TYPE, false, false), + Arguments.of(VarCharType.STRING_TYPE, new GeographyType(), false, false), + Arguments.of(new GeographyType(), new VarBinaryType(), false, false), + Arguments.of(new VarBinaryType(), new GeographyType(), false, false)); } @ParameterizedTest(name = "{index}: [From: {0}, To: {1}, Implicit: {2}, Explicit: {3}]") diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java index 8a87c4663f46b6..0fef6c13b95298 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java @@ -39,6 +39,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -312,6 +313,8 @@ private static Stream testData() { TestSpec.forString("BITMAP NOT NULL").expectType(new BitmapType(false)), TestSpec.forString("UUID").expectType(new UuidType()), TestSpec.forString("UUID NOT NULL").expectType(new UuidType(false)), + TestSpec.forString("GEOGRAPHY").expectType(new GeographyType()), + TestSpec.forString("GEOGRAPHY NOT NULL").expectType(new GeographyType(false)), // error message testing diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java index 03dacc344ee2ad..9eba8d63aeb874 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java @@ -26,6 +26,8 @@ import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.catalog.UnresolvedIdentifier; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.expressions.TimeIntervalUnit; import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.types.logical.ArrayType; @@ -40,6 +42,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -642,6 +645,20 @@ void testUuidType() { new UuidType(false))); } + @Test + void testGeographyType() { + assertThat(new GeographyType()) + .isJavaSerializable() + .satisfies( + baseAssertions( + "GEOGRAPHY", + "GEOGRAPHY", + new Class[] {GeographyData.class, BinaryGeographyData.class}, + new Class[] {GeographyData.class, BinaryGeographyData.class}, + new LogicalType[] {}, + new GeographyType(false))); + } + @Test void testTypeInformationRawType() { final TypeInformationRawType rawType = diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/logical/GeographyTypeTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/logical/GeographyTypeTest.java new file mode 100644 index 00000000000000..843872ace74d35 --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/logical/GeographyTypeTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.logical; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.binary.BinaryGeographyData; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class GeographyTypeTest { + + @Test + void testSupportsOnlyRegisteredBridgeClasses() { + GeographyType type = new GeographyType(); + + assertThat(type.supportsInputConversion(GeographyData.class)).isTrue(); + assertThat(type.supportsOutputConversion(GeographyData.class)).isTrue(); + assertThat(type.supportsInputConversion(BinaryGeographyData.class)).isTrue(); + assertThat(type.supportsOutputConversion(BinaryGeographyData.class)).isTrue(); + assertThat(type.supportsInputConversion(CustomGeographyData.class)).isFalse(); + assertThat(type.supportsOutputConversion(CustomGeographyData.class)).isFalse(); + } + + @Test + void testBridgedToRejectsUnsupportedGeographyImplementation() { + assertThatThrownBy(() -> DataTypes.GEOGRAPHY().bridgedTo(CustomGeographyData.class)) + .isInstanceOf(ValidationException.class); + } + + private static class CustomGeographyData implements GeographyData { + @Override + public int subtypeId() { + return GeographyData.POINT; + } + + @Override + public int sizeInBytes() { + return 0; + } + + @Override + public byte[] toBytes() { + return new byte[0]; + } + } +} diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml index 450dc204ec69ed..c02bd4dee0d0a9 100644 --- a/flink-table/flink-table-planner/pom.xml +++ b/flink-table/flink-table-planner/pom.xml @@ -189,6 +189,13 @@ under the License. test + + org.apache.flink + flink-table-code-splitter + ${project.version} + test + + org.apache.flink diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java index 478b57b3247a2e..e784f31240060f 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java @@ -287,6 +287,11 @@ private boolean canCastFrom(RelDataType toType, RelDataType fromType) { FlinkTypeFactory.toLogicalType(fromType), FlinkTypeFactory.toLogicalType(toType)); } + if (toTypeName == SqlTypeName.OTHER) { + return LogicalTypeCasts.supportsExplicitCast( + FlinkTypeFactory.toLogicalType(fromType), + FlinkTypeFactory.toLogicalType(toType)); + } switch (fromTypeName) { case ARRAY: case MAP: diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java index 4157e378da8c16..a018e621121c43 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java @@ -29,6 +29,7 @@ import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.planner.plan.schema.BitmapRelDataType; import org.apache.flink.table.planner.plan.schema.GenericRelDataType; +import org.apache.flink.table.planner.plan.schema.GeographyRelDataType; import org.apache.flink.table.planner.plan.schema.RawRelDataType; import org.apache.flink.table.planner.plan.schema.StructuredRelDataType; import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; @@ -47,6 +48,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -67,6 +69,7 @@ import org.apache.flink.table.types.logical.VarBinaryType; import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.table.types.logical.VariantType; +import org.apache.flink.table.types.logical.utils.LogicalTypeMerging; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.table.utils.TableSchemaUtils; import org.apache.flink.util.Preconditions; @@ -162,6 +165,11 @@ public RelDataType createBitmapType() { return canonize(new BitmapRelDataType(new BitmapType())); } + @Override + public RelDataType createGeographyType() { + return canonize(new GeographyRelDataType(new GeographyType())); + } + @Override public RelDataType createArrayType(RelDataType elementType, long maxCardinality) { // Just validate type, make sure there is a failure in validate phase. @@ -232,6 +240,8 @@ public RelDataType createTypeWithNullability(RelDataType relDataType, boolean is newType = ((StructuredRelDataType) relDataType).createWithNullability(isNullable); } else if (relDataType instanceof BitmapRelDataType) { newType = ((BitmapRelDataType) relDataType).createWithNullability(isNullable); + } else if (relDataType instanceof GeographyRelDataType) { + newType = ((GeographyRelDataType) relDataType).createWithNullability(isNullable); } else if (relDataType instanceof GenericRelDataType) { final GenericRelDataType generic = (GenericRelDataType) relDataType; newType = new GenericRelDataType(generic.genericType(), isNullable, getTypeSystem()); @@ -256,8 +266,19 @@ public RelDataType createTypeWithNullability(RelDataType relDataType, boolean is @Override public RelDataType leastRestrictive(List types) { final Optional resolved = resolveAllIdenticalTypes(types); - final RelDataType leastRestrictive = - resolved.orElseGet(() -> super.leastRestrictive(types)); + if (resolved.isPresent()) { + return normalizeLeastRestrictive(resolved.get()); + } + + if (containsGeographyType(types)) { + return normalizeLeastRestrictive(resolveCommonTypeForGeography(types).orElse(null)); + } + + final RelDataType leastRestrictive = super.leastRestrictive(types); + return normalizeLeastRestrictive(leastRestrictive); + } + + private RelDataType normalizeLeastRestrictive(RelDataType leastRestrictive) { // NULL is reserved for untyped literals only if (leastRestrictive == null || leastRestrictive.getSqlTypeName() == SqlTypeName.NULL) { return null; @@ -265,6 +286,18 @@ public RelDataType leastRestrictive(List types) { return leastRestrictive; } + private Optional resolveCommonTypeForGeography(List types) { + return LogicalTypeMerging.findCommonType( + types.stream() + .map(FlinkTypeFactory::toLogicalType) + .collect(Collectors.toList())) + .map(this::createFieldTypeFromLogicalType); + } + + private boolean containsGeographyType(List types) { + return types.stream().anyMatch(GeographyRelDataType.class::isInstance); + } + private Optional resolveAllIdenticalTypes(List types) { final RelDataType head = types.get(0); // check if all types are the same @@ -497,6 +530,9 @@ private RelDataType newRelDataType(LogicalType logicalType) { case BITMAP: return new BitmapRelDataType((BitmapType) logicalType); + case GEOGRAPHY: + return new GeographyRelDataType((GeographyType) logicalType); + default: throw new TableException("Type is not supported: " + logicalType); } @@ -873,6 +909,8 @@ private static LogicalType toLogicalTypeWithoutNullability(RelDataType relDataTy return ((RawRelDataType) relDataType).getRawType(); } else if (relDataType instanceof BitmapRelDataType) { return ((BitmapRelDataType) relDataType).getBitmapType(); + } else if (relDataType instanceof GeographyRelDataType) { + return ((GeographyRelDataType) relDataType).getGeographyType(); } else { throw new TableException("Type is not supported: " + relDataType); } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java index 27bdff6e398e1e..90fa23d2123fbe 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java @@ -548,6 +548,7 @@ protected Boolean defaultMethod(LogicalType logicalType) { case BITMAP: case VARIANT: case UUID: + case GEOGRAPHY: return true; default: // fall back to generic serialization diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java new file mode 100644 index 00000000000000..41b586f5863240 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.schema; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.types.logical.GeographyType; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.AbstractSqlType; +import org.apache.calcite.sql.type.SqlTypeName; + +/** The {@link RelDataType} representation of a {@link GeographyType}. */ +@Internal +public final class GeographyRelDataType extends AbstractSqlType { + + private final GeographyType geographyType; + + public GeographyRelDataType(GeographyType geographyType) { + super(SqlTypeName.OTHER, geographyType.isNullable(), null); + this.geographyType = geographyType; + computeDigest(); + } + + public GeographyType getGeographyType() { + return geographyType; + } + + public GeographyRelDataType createWithNullability(boolean nullable) { + if (nullable == isNullable()) { + return this; + } + return new GeographyRelDataType((GeographyType) geographyType.copy(nullable)); + } + + @Override + protected void generateTypeString(StringBuilder sb, boolean withDetail) { + sb.append(geographyType.asSummaryString()); + } + + @Override + protected void computeDigest() { + final StringBuilder sb = new StringBuilder(); + generateTypeString(sb, true); + digest = sb.toString(); + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java index 58c164b31ec92b..d0e73298f2488f 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java @@ -23,6 +23,7 @@ import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.plan.schema.BitmapRelDataType; +import org.apache.flink.table.planner.plan.schema.GeographyRelDataType; import org.apache.flink.table.planner.plan.schema.RawRelDataType; import org.apache.flink.table.planner.plan.schema.StructuredRelDataType; import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; @@ -40,6 +41,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -478,6 +480,11 @@ public RelDataType visit(BitmapType bitmapType) { return new BitmapRelDataType(bitmapType); } + @Override + public RelDataType visit(GeographyType geographyType) { + return new GeographyRelDataType(geographyType); + } + @Override public RelDataType visit(LogicalType other) { throw new TableException( @@ -612,6 +619,8 @@ private static LogicalType toLogicalTypeNotNull( return ((RawRelDataType) relDataType).getRawType(); } else if (relDataType instanceof BitmapRelDataType) { return ((BitmapRelDataType) relDataType).getBitmapType(); + } else if (relDataType instanceof GeographyRelDataType) { + return ((GeographyRelDataType) relDataType).getGeographyType(); } // fall through case REAL: diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala index 0dd59e18235ffd..5035384374b11a 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala @@ -283,6 +283,7 @@ object CodeGenUtils { case DESCRIPTOR => className[ColumnList] case VARIANT => className[Variant] case BITMAP => className[Bitmap] + case GEOGRAPHY => className[GeographyData] case SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t) } @@ -392,6 +393,8 @@ object CodeGenUtils { s"$term.hashCode()" case BITMAP => s"$term.hashCode()" + case GEOGRAPHY => + s"$term.hashCode()" case NULL | SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t) } @@ -544,6 +547,8 @@ object CodeGenUtils { s"$rowTerm.getVariant($indexTerm)" case BITMAP => s"$rowTerm.getBitmap($indexTerm)" + case GEOGRAPHY => + s"$rowTerm.getGeography($indexTerm)" case NULL | SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t) } @@ -841,6 +846,8 @@ object CodeGenUtils { s"$writerTerm.writeVariant($indexTerm, $fieldValTerm)" case BITMAP => s"$writerTerm.writeBitmap($indexTerm, $fieldValTerm)" + case GEOGRAPHY => + s"$writerTerm.writeGeography($indexTerm, $fieldValTerm)" case NULL | SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t); } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java index 0d5c13d48a538b..aa0039b7bd16bf 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java @@ -86,6 +86,33 @@ void testCompilePlanSql() throws IOException { .isEqualTo(getPreparedToCompareCompiledPlan(expected)); } + @Test + void testCompilePlanSqlWithGeographyColumns() { + tableEnv.executeSql( + "CREATE TABLE GeoSource (" + + "id INT, " + + "location GEOGRAPHY, " + + "required_location GEOGRAPHY NOT NULL" + + ") WITH (" + + "'connector' = 'values', " + + "'bounded' = 'false')"); + tableEnv.executeSql( + "CREATE TABLE GeoSink (" + + "id INT, " + + "location GEOGRAPHY, " + + "required_location GEOGRAPHY NOT NULL" + + ") WITH (" + + "'connector' = 'values', " + + "'table-sink-class' = 'DEFAULT')"); + + final CompiledPlan compiledPlan = + tableEnv.compilePlanSql("INSERT INTO GeoSink SELECT * FROM GeoSource"); + final String planJson = compiledPlan.asJsonString(); + + assertThat(planJson).contains("\"GEOGRAPHY\"", "\"GEOGRAPHY NOT NULL\""); + assertThat(tableEnv.loadPlan(PlanReference.fromJsonString(planJson))).isNotNull(); + } + @Test void testSourceTableWithHints() { CompiledPlan compiledPlan = diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java index 831b3b6ecc480a..ed2fef1ce876cd 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java @@ -26,7 +26,10 @@ import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.functions.ScalarFunction; import org.apache.flink.table.planner.utils.PlannerMocks; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.sql.SqlNode; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -158,6 +161,14 @@ void testMixedPositionalAndNamedArguments() { .hasMessageContaining("Cannot mix positional and named arguments"); } + private LogicalType projectedLogicalType(String expression) { + SqlNode parsed = plannerMocks.getPlanner().parser().parse("SELECT " + expression + " AS c"); + SqlNode validated = plannerMocks.getPlanner().validate(parsed); + RelRoot relRoot = plannerMocks.getPlanner().rel(validated); + return FlinkTypeFactory.toLogicalType( + relRoot.rel.getRowType().getFieldList().get(0).getType()); + } + /** Scalar function with named arguments for the mixed-argument validation test. */ public static class NamedArgsScalarFunction extends ScalarFunction { @FunctionHint( diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java index a5a85e11c3be5b..28f261eae4ea0c 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java @@ -27,6 +27,7 @@ import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.BitmapType; import org.apache.flink.table.types.logical.BooleanType; import org.apache.flink.table.types.logical.CharType; import org.apache.flink.table.types.logical.DateType; @@ -34,6 +35,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -63,6 +65,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.time.DayOfWeek; +import java.time.Month; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -94,17 +97,13 @@ static Stream testInternalToRelType() { new ArrayType(new DoubleType()), new MapType(new DoubleType(), VarCharType.STRING_TYPE), RowType.of(new DoubleType(), VarCharType.STRING_TYPE), - new RawType<>( - DayOfWeek.class, - new KryoSerializer<>(DayOfWeek.class, new SerializerConfigImpl()))); + rawType(DayOfWeek.class, true)); } @MethodSource("testInternalToRelType") @ParameterizedTest void testInternalToRelType(LogicalType logicalType) { - FlinkTypeFactory typeFactory = - new FlinkTypeFactory( - Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + FlinkTypeFactory typeFactory = createTypeFactory(); assertThat( FlinkTypeFactory.toLogicalType( @@ -129,9 +128,7 @@ void testInternalToRelType(LogicalType logicalType) { @Test void testInternalToRelTypeNull() { - FlinkTypeFactory typeFactory = - new FlinkTypeFactory( - Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + FlinkTypeFactory typeFactory = createTypeFactory(); LogicalType logicalType = new NullType(); @@ -148,9 +145,7 @@ void testInternalToRelTypeNull() { @Test void testDayTimeIntervalLeadingPrecisionUpToMaxIsSupported() { - FlinkTypeFactory typeFactory = - new FlinkTypeFactory( - Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + FlinkTypeFactory typeFactory = createTypeFactory(); RelDataType intervalType = typeFactory.createSqlIntervalType( @@ -167,9 +162,7 @@ void testDayTimeIntervalLeadingPrecisionUpToMaxIsSupported() { @Test void testDayTimeIntervalLeadingPrecisionAboveMaxIsRejected() { - FlinkTypeFactory typeFactory = - new FlinkTypeFactory( - Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + FlinkTypeFactory typeFactory = createTypeFactory(); RelDataType intervalType = typeFactory.createSqlIntervalType( @@ -197,9 +190,7 @@ void testDecimalInferType() { @Test void testCanonizeType() { - FlinkTypeFactory typeFactory = - new FlinkTypeFactory( - Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + FlinkTypeFactory typeFactory = createTypeFactory(); TypeInformation genericTypeInfo = Types.GENERIC(TestClass.class); TypeInformation genericTypeInfo2 = Types.GENERIC(TestClass2.class); @@ -227,7 +218,7 @@ static Stream testLeastRestrictive() { // Since the problem is actual for collection // then tests are for array, map, multiset // Also as https://issues.apache.org/jira/browse/CALCITE-4603 says - // before Calcite 1.27.0 it derived the type of nested collection based on the last + // before Calcite 1.27.0 it derived the type of nested collection based on the last // element, for that reason the type of the last element is narrower // than the type of element in the middle Arguments.of( @@ -259,9 +250,90 @@ static Stream testLeastRestrictive() { @MethodSource("testLeastRestrictive") @ParameterizedTest void testLeastRestrictive(List input, LogicalType expected) { - FlinkTypeFactory typeFactory = - new FlinkTypeFactory( - Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + assertLeastRestrictive(input, expected); + } + + static Stream testLeastRestrictiveGeographyCases() { + return Stream.of( + Arguments.of( + Arrays.asList(new GeographyType(false), new GeographyType(false)), + new GeographyType(false)), + Arguments.of( + Arrays.asList(new GeographyType(false), new GeographyType(true)), + new GeographyType(true)), + Arguments.of( + Arrays.asList(new GeographyType(false), new NullType()), + new GeographyType(true)), + Arguments.of( + Arrays.asList(new NullType(), new GeographyType(false)), + new GeographyType(true))); + } + + @MethodSource("testLeastRestrictiveGeographyCases") + @ParameterizedTest + void testLeastRestrictiveGeographyCases(List input, LogicalType expected) { + assertLeastRestrictive(input, expected); + } + + static Stream testLeastRestrictiveIncompatibleGeographyCases() { + return Stream.of( + Arguments.of(Arrays.asList(new GeographyType(), new BitmapType())), + Arguments.of(Arrays.asList(new GeographyType(), new IntType()))); + } + + @MethodSource("testLeastRestrictiveIncompatibleGeographyCases") + @ParameterizedTest + void testLeastRestrictiveIncompatibleGeographyCases(List input) { + assertLeastRestrictiveIsNull(input); + } + + static Stream testLeastRestrictiveBitmapRegressionCases() { + return Stream.of( + Arguments.of( + Arrays.asList(new BitmapType(false), new BitmapType(false)), + new BitmapType(false)), + Arguments.of(Arrays.asList(new BitmapType(false), new BitmapType(true)), null), + Arguments.of(Arrays.asList(new BitmapType(false), new NullType()), null)); + } + + @MethodSource("testLeastRestrictiveBitmapRegressionCases") + @ParameterizedTest + void testLeastRestrictiveBitmapRegressionCases(List input, LogicalType expected) { + if (expected == null) { + assertLeastRestrictiveIsNull(input); + } else { + assertLeastRestrictive(input, expected); + } + } + + static Stream testLeastRestrictiveRawRegressionCases() { + return Stream.of( + Arguments.of( + Arrays.asList( + rawType(DayOfWeek.class, false), rawType(DayOfWeek.class, false)), + rawType(DayOfWeek.class, false)), + Arguments.of( + Arrays.asList( + rawType(DayOfWeek.class, false), rawType(DayOfWeek.class, true)), + null), + Arguments.of(Arrays.asList(rawType(DayOfWeek.class, false), new NullType()), null), + Arguments.of( + Arrays.asList(rawType(DayOfWeek.class, false), rawType(Month.class, false)), + null)); + } + + @MethodSource("testLeastRestrictiveRawRegressionCases") + @ParameterizedTest + void testLeastRestrictiveRawRegressionCases(List input, LogicalType expected) { + if (expected == null) { + assertLeastRestrictiveIsNull(input); + } else { + assertLeastRestrictive(input, expected); + } + } + + private static void assertLeastRestrictive(List input, LogicalType expected) { + FlinkTypeFactory typeFactory = createTypeFactory(); assertThat( typeFactory.leastRestrictive( @@ -271,6 +343,28 @@ void testLeastRestrictive(List input, LogicalType expected) { .isEqualTo(typeFactory.createFieldTypeFromLogicalType(expected)); } + private static void assertLeastRestrictiveIsNull(List input) { + FlinkTypeFactory typeFactory = createTypeFactory(); + + assertThat( + typeFactory.leastRestrictive( + input.stream() + .map(typeFactory::createFieldTypeFromLogicalType) + .collect(Collectors.toList()))) + .isNull(); + } + + private static FlinkTypeFactory createTypeFactory() { + return new FlinkTypeFactory( + Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + } + + private static RawType rawType(Class clazz, boolean nullable) { + return (RawType) + new RawType<>(clazz, new KryoSerializer<>(clazz, new SerializerConfigImpl())) + .copy(nullable); + } + public static class TestClass { public int f0; public String f1; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java new file mode 100644 index 00000000000000..652d0af25d6427 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.codegen; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.runtime.generated.Projection; +import org.apache.flink.table.types.logical.GeographyType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for generated projections involving {@link GeographyData}. */ +class ProjectionCodeGeneratorGeographyTest { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Test + void testGeneratedProjectionForGeography() { + RowType inputType = RowType.of(new IntType(), new GeographyType(), new GeographyType()); + RowType outputType = RowType.of(new GeographyType(), new GeographyType(), new IntType()); + GenericRowData input = GenericRowData.of(7, GeographyData.fromBytes(POINT_WKB), null); + + Projection projection = + ProjectionCodeGenerator.generateProjection( + new CodeGeneratorContext( + new Configuration(), + Thread.currentThread().getContextClassLoader()), + "GeographyProjection", + inputType, + outputType, + new int[] {1, 2, 0}) + .newInstance(Thread.currentThread().getContextClassLoader()); + + BinaryRowData output = (BinaryRowData) projection.apply(input); + + assertThat(output.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(output.isNullAt(1)).isTrue(); + assertThat(output.getInt(2)).isEqualTo(7); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java index 218d911e881dba..f37a8acfa4ac47 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java @@ -39,6 +39,7 @@ import static org.apache.flink.table.api.DataTypes.BOOLEAN; import static org.apache.flink.table.api.DataTypes.BYTES; import static org.apache.flink.table.api.DataTypes.FIELD; +import static org.apache.flink.table.api.DataTypes.GEOGRAPHY; import static org.apache.flink.table.api.DataTypes.INT; import static org.apache.flink.table.api.DataTypes.MAP; import static org.apache.flink.table.api.DataTypes.ROW; @@ -260,6 +261,62 @@ Stream getTestSetSpecs() { .testSqlValidationError( "CAST(CreateMultiset(f0) AS BITMAP)", "Cast function cannot convert value of type VARCHAR(2147483647) MULTISET to type BITMAP"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast STRING to GEOGRAPHY") + .onFieldsWithData("POINT (0 0)") + .andDataTypes(STRING()) + .testTableApiValidationError( + $("f0").cast(GEOGRAPHY()), + "Unsupported cast from 'STRING' to 'GEOGRAPHY'") + .testSqlValidationError( + "CAST(f0 AS GEOGRAPHY)", + "Cast function cannot convert value of type VARCHAR(2147483647) to type GEOGRAPHY"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast BYTES to GEOGRAPHY") + .onFieldsWithData(new byte[] {1, 2, 3}) + .andDataTypes(BYTES()) + .testTableApiValidationError( + $("f0").cast(GEOGRAPHY()), + "Unsupported cast from 'BYTES' to 'GEOGRAPHY'") + .testSqlValidationError( + "CAST(f0 AS GEOGRAPHY)", + "Cast function cannot convert value of type VARBINARY(2147483647) to type GEOGRAPHY"), + TestSetSpec.forFunction( + BuiltInFunctionDefinitions.CAST, "cast VARBINARY to GEOGRAPHY") + .onFieldsWithData(new byte[] {1, 2, 3}) + .andDataTypes(VARBINARY(3)) + .testTableApiValidationError( + $("f0").cast(GEOGRAPHY()), + "Unsupported cast from 'VARBINARY(3)' to 'GEOGRAPHY'") + .testSqlValidationError( + "CAST(f0 AS GEOGRAPHY)", + "Cast function cannot convert value of type VARBINARY(3) to type GEOGRAPHY"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast GEOGRAPHY to STRING") + .onFieldsWithData((Object) null) + .andDataTypes(GEOGRAPHY()) + .testTableApiValidationError( + $("f0").cast(STRING()), + "Unsupported cast from 'GEOGRAPHY' to 'STRING'") + .testSqlValidationError( + "CAST(f0 AS STRING)", + "Cast function cannot convert value of type GEOGRAPHY to type VARCHAR(2147483647)"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast GEOGRAPHY to BYTES") + .onFieldsWithData((Object) null) + .andDataTypes(GEOGRAPHY()) + .testTableApiValidationError( + $("f0").cast(BYTES()), + "Unsupported cast from 'GEOGRAPHY' to 'BYTES'") + .testSqlValidationError( + "CAST(f0 AS BYTES)", + "Cast function cannot convert value of type GEOGRAPHY to type VARBINARY(2147483647)"), + TestSetSpec.forFunction( + BuiltInFunctionDefinitions.CAST, "cast GEOGRAPHY to VARBINARY") + .onFieldsWithData((Object) null) + .andDataTypes(GEOGRAPHY()) + .testTableApiValidationError( + $("f0").cast(VARBINARY(3)), + "Unsupported cast from 'GEOGRAPHY' to 'VARBINARY(3)'") + .testSqlValidationError( + "CAST(f0 AS VARBINARY(3))", + "Cast function cannot convert value of type GEOGRAPHY to type VARBINARY(3)"), TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast RAW to STRING") .onFieldsWithData("2020-11-11T18:08:01.123") .andDataTypes(STRING()) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java index bee8c249624171..0fbab22e154b77 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java @@ -62,6 +62,7 @@ private static Stream testDataTypeSerde() { DataTypes.VARIANT(), DataTypes.VARIANT().notNull(), DataTypes.ARRAY(DataTypes.VARIANT()), + DataTypes.GEOGRAPHY(), DataTypes.ROW( DataTypes.TIMESTAMP_LTZ(3).toInternal(), DataTypes.TIMESTAMP_LTZ(9).bridgedTo(long.class), diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java index 32554e7264e9c4..a2c3a01d85764f 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java @@ -43,6 +43,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -275,6 +276,7 @@ private static List testLogicalTypeSerde() { new MultisetType(new VariantType()), new MapType(new VarCharType(5), new VariantType()), RowType.of(new VariantType(), new VariantType(false)), + new GeographyType(), RowType.of(new BigIntType(), new IntType(false), new VarCharType(200)), RowType.of( new LogicalType[] { diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java index d5e7f361d7fa98..33030f2e35775c 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java @@ -154,6 +154,11 @@ public Bitmap getBitmap(int pos) { return (Bitmap) this.fields[pos]; } + @Override + public GeographyData getGeography(int pos) { + return (GeographyData) this.fields[pos]; + } + @Override public void setNullAt(int pos) { this.fields[pos] = null; diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java index 756ca6293c8676..6a9628aefa916b 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java @@ -35,12 +35,14 @@ import org.apache.flink.table.data.GenericArrayData; import org.apache.flink.table.data.GenericMapData; import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.data.binary.BinaryArrayData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.data.binary.BinaryMapData; import org.apache.flink.table.data.writer.BinaryArrayWriter; import org.apache.flink.table.data.writer.BinaryWriter; @@ -160,6 +162,10 @@ public class DataFormatConverters { t2C.put(DataTypes.BITMAP().bridgedTo(Bitmap.class), BitmapConverter.INSTANCE); t2C.put(DataTypes.BITMAP().bridgedTo(RoaringBitmapData.class), BitmapConverter.INSTANCE); + t2C.put(DataTypes.GEOGRAPHY().bridgedTo(GeographyData.class), GeographyConverter.INSTANCE); + t2C.put( + DataTypes.GEOGRAPHY().bridgedTo(BinaryGeographyData.class), + GeographyConverter.INSTANCE); TYPE_TO_CONVERTER = Collections.unmodifiableMap(t2C); } @@ -773,6 +779,20 @@ Bitmap toExternalImpl(RowData row, int column) { } } + public static final class GeographyConverter extends IdentityConverter { + + private static final long serialVersionUID = 1L; + + public static final GeographyConverter INSTANCE = new GeographyConverter(); + + private GeographyConverter() {} + + @Override + GeographyData toExternalImpl(RowData row, int column) { + return row.getGeography(column); + } + } + /** Converter for LocalDate. */ public static final class LocalDateConverter extends DataFormatConverter { diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/BinaryArrayDataTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/BinaryArrayDataTest.java index cb132cea5b51c2..7b8283ac77c99a 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/BinaryArrayDataTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/BinaryArrayDataTest.java @@ -23,6 +23,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.data.binary.BinaryArrayData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.data.binary.BinaryMapData; import org.apache.flink.table.data.binary.BinaryRowData; import org.apache.flink.table.data.binary.BinarySegmentUtils; @@ -52,6 +53,34 @@ /** Test of {@link BinaryArrayData} and {@link BinaryArrayWriter}. */ class BinaryArrayDataTest { + private static final byte[] BIG_ENDIAN_POINT_WKB = + new byte[] { + 0, + 0, + 0, + 0, + 1, + 0x3F, + (byte) 0xF0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0x40, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }; + @Test void testArray() { // 1.array test @@ -397,6 +426,63 @@ void testArrayTypes() { } } + @Test + void testGeographyFastPathForArrayWriter() { + BinaryGeographyData geography = + BinaryGeographyData.fromAddress( + splitBytes(BIG_ENDIAN_POINT_WKB, 2), 2, BIG_ENDIAN_POINT_WKB.length); + int expectedSubtype = geography.subtypeId(); + + BinaryArrayData array = new BinaryArrayData(); + BinaryArrayWriter writer = new BinaryArrayWriter(array, 2, 8); + writer.setNullAt(0); + writer.writeGeography(1, geography); + writer.complete(); + + geography.getSegments()[0].put(geography.getOffset(), (byte) 0x7F); + + assertThat(array.isNullAt(0)).isTrue(); + assertThat(array.getGeography(1).toBytes()).isEqualTo(BIG_ENDIAN_POINT_WKB); + assertThat(array.getGeography(1).subtypeId()).isEqualTo(expectedSubtype); + + BinaryArrayData splitArray = splitArray(array); + assertThat(splitArray.isNullAt(0)).isTrue(); + assertThat(splitArray.getGeography(1).toBytes()).isEqualTo(BIG_ENDIAN_POINT_WKB); + assertThat(splitArray.getGeography(1).subtypeId()).isEqualTo(expectedSubtype); + } + + @Test + void testGeographyFallbackForArrayWriter() { + int[] toBytesCalls = new int[1]; + GeographyData geography = + new GeographyData() { + @Override + public int subtypeId() { + return GeographyData.POINT; + } + + @Override + public int sizeInBytes() { + return BIG_ENDIAN_POINT_WKB.length; + } + + @Override + public byte[] toBytes() { + toBytesCalls[0]++; + return BIG_ENDIAN_POINT_WKB.clone(); + } + }; + + BinaryArrayData array = new BinaryArrayData(); + BinaryArrayWriter writer = new BinaryArrayWriter(array, 1, 8); + writer.writeGeography(0, geography); + writer.complete(); + + assertThat(toBytesCalls[0]).isEqualTo(1); + assertThat(array.getGeography(0).toBytes()).isEqualTo(BIG_ENDIAN_POINT_WKB); + assertThat(array.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + } + @Test void testMap() { BinaryArrayData array1 = new BinaryArrayData(); diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/BinaryRowDataTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/BinaryRowDataTest.java index 18b756f9ea9c2f..b3880b73974cc1 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/BinaryRowDataTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/BinaryRowDataTest.java @@ -35,6 +35,7 @@ import org.apache.flink.core.memory.RandomAccessOutputView; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.data.binary.BinaryArrayData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.data.binary.BinaryMapData; import org.apache.flink.table.data.binary.BinaryRowData; import org.apache.flink.table.data.binary.BinaryStringData; @@ -89,6 +90,39 @@ /** Test of {@link BinaryRowData} and {@link BinaryRowWriter}. */ class BinaryRowDataTest { + private static final byte[] LITTLE_ENDIAN_POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + private static final byte[] BIG_ENDIAN_POINT_WKB = + new byte[] { + 0, + 0, + 0, + 0, + 1, + 0x3F, + (byte) 0xF0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0x40, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }; + @Test void testBasic() { // consider header 1 byte. @@ -1150,6 +1184,70 @@ public void testVariant() { assertThat(row.getVariant(1)).isEqualTo(v2); } + @Test + void testGeographyFastPathSingleSegmentAndOffset() { + BinaryRowData row = new BinaryRowData(1); + BinaryRowWriter writer = new BinaryRowWriter(row); + + BinaryGeographyData singleSegment = wrapBinaryGeography(LITTLE_ENDIAN_POINT_WKB, 0); + int singleSegmentSubtype = singleSegment.subtypeId(); + writer.writeGeography(0, singleSegment); + writer.complete(); + + assertThat(row.getGeography(0).toBytes()).isEqualTo(LITTLE_ENDIAN_POINT_WKB); + assertThat(row.getGeography(0).subtypeId()).isEqualTo(singleSegmentSubtype); + + BinaryGeographyData offsetGeography = wrapBinaryGeography(BIG_ENDIAN_POINT_WKB, 3); + int offsetSubtype = offsetGeography.subtypeId(); + writer.reset(); + writer.writeGeography(0, offsetGeography); + writer.complete(); + + offsetGeography.getSegments()[0].put(offsetGeography.getOffset(), (byte) 0x7F); + + assertThat(row.getGeography(0).toBytes()).isEqualTo(BIG_ENDIAN_POINT_WKB); + assertThat(row.getGeography(0).subtypeId()).isEqualTo(offsetSubtype); + } + + @Test + void testGeographyFallbackUsesToBytes() { + int[] toBytesCalls = new int[1]; + GeographyData geography = + new GeographyData() { + @Override + public int subtypeId() { + return GeographyData.POINT; + } + + @Override + public int sizeInBytes() { + return BIG_ENDIAN_POINT_WKB.length; + } + + @Override + public byte[] toBytes() { + toBytesCalls[0]++; + return Arrays.copyOf(BIG_ENDIAN_POINT_WKB, BIG_ENDIAN_POINT_WKB.length); + } + }; + + BinaryRowData row = new BinaryRowData(1); + BinaryRowWriter writer = new BinaryRowWriter(row); + writer.writeGeography(0, geography); + writer.complete(); + + assertThat(toBytesCalls[0]).isEqualTo(1); + assertThat(row.getGeography(0).toBytes()).isEqualTo(BIG_ENDIAN_POINT_WKB); + assertThat(row.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + } + + private static BinaryGeographyData wrapBinaryGeography(byte[] bytes, int baseOffset) { + MemorySegment segment = MemorySegmentFactory.wrap(new byte[baseOffset + bytes.length]); + segment.put(baseOffset, bytes, 0, bytes.length); + return BinaryGeographyData.fromAddress( + new MemorySegment[] {segment}, baseOffset, bytes.length); + } + @Test public void testBitmap() { BinaryRowData row = new BinaryRowData(3); diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/DataFormatConvertersTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/DataFormatConvertersTest.java index 45390c54c08f71..115b41c5db33a4 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/DataFormatConvertersTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/DataFormatConvertersTest.java @@ -33,6 +33,7 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.data.util.DataFormatConverters; import org.apache.flink.table.data.util.DataFormatConverters.DataFormatConverter; import org.apache.flink.table.runtime.typeutils.DecimalDataTypeInfo; @@ -64,6 +65,11 @@ /** Test for {@link DataFormatConverters}. */ class DataFormatConvertersTest { + private static final byte[] POINT_WKB = + new byte[] { + 1, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + private TypeInformation[] simpleTypes = new TypeInformation[] { BasicTypeInfo.STRING_TYPE_INFO, @@ -179,6 +185,11 @@ private static void testDataType(DataType dataType, Object value) { assertThat(converter.toExternal(converter.toInternal(value))).isEqualTo(value); } + private static void assertGeographyBytes(GeographyData actual, byte[] expected) { + assertThat(actual).isNotNull(); + assertThat(actual.toBytes()).isEqualTo(expected); + } + @Test void testTypes() { for (int i = 0; i < simpleTypes.length; i++) { @@ -238,6 +249,65 @@ void testDataTypes() { } } + @Test + void testGeographyConverterLookup() { + assertThat(getConverter(DataTypes.GEOGRAPHY())) + .isInstanceOf(DataFormatConverters.GeographyConverter.class); + assertThat(getConverter(DataTypes.GEOGRAPHY().notNull())) + .isInstanceOf(DataFormatConverters.GeographyConverter.class); + } + + @Test + void testGeographyRoundTripPreservesWkbBytes() { + DataFormatConverter converter = getConverter(DataTypes.GEOGRAPHY()); + GeographyData geography = GeographyData.fromBytes(POINT_WKB); + + Object internal = converter.toInternal(geography); + GeographyData external = (GeographyData) converter.toExternal(internal); + + assertThat(internal).isSameAs(geography); + assertGeographyBytes(external, POINT_WKB); + } + + @Test + void testGeographyRowExtractionAndNullHandling() { + DataFormatConverter converter = getConverter(DataTypes.GEOGRAPHY()); + GeographyData geography = GeographyData.fromBytes(POINT_WKB); + GenericRowData row = GenericRowData.of(geography, null); + + assertThat(converter.toInternal(null)).isNull(); + assertThat(converter.toExternal((Object) null)).isNull(); + assertThat(converter.toExternal(row, 0)).isSameAs(geography); + assertThat(converter.toExternal(row, 1)).isNull(); + } + + @Test + void testGeographyBinaryBridgeRoundTrip() { + DataFormatConverter converter = + getConverter(DataTypes.GEOGRAPHY().bridgedTo(BinaryGeographyData.class)); + BinaryGeographyData geography = BinaryGeographyData.fromBytes(POINT_WKB); + + Object internal = converter.toInternal(geography); + Object external = converter.toExternal(internal); + + assertThat(internal).isSameAs(geography); + assertThat(external).isInstanceOf(BinaryGeographyData.class); + assertGeographyBytes((GeographyData) external, POINT_WKB); + } + + @Test + void testNestedGeographyArrayRoundTrip() { + DataFormatConverter converter = getConverter(DataTypes.ARRAY(DataTypes.GEOGRAPHY())); + GeographyData[] values = new GeographyData[] {GeographyData.fromBytes(POINT_WKB), null}; + + Object internal = converter.toInternal(values); + GeographyData[] external = (GeographyData[]) converter.toExternal(internal); + + assertThat(external).hasSize(2); + assertGeographyBytes(external[0], POINT_WKB); + assertThat(external[1]).isNull(); + } + /** Test pojo. */ public static class MyPojo { public int f1 = 0; diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java index 787e54957c7490..37e5d492a55774 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java @@ -33,11 +33,13 @@ import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.BinaryType; +import org.apache.flink.table.types.logical.BitmapType; import org.apache.flink.table.types.logical.BooleanType; import org.apache.flink.table.types.logical.CharType; import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.MultisetType; @@ -63,7 +65,11 @@ /** Test for {@link RowData}s. */ class RowDataTest { - private static final int NUM_FIELDS = 19; + private static final int NUM_FIELDS = 20; + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; private StringData str; private RawValueData generic; @@ -77,6 +83,7 @@ class RowDataTest { private TimestampData timestamp1; private TimestampData timestamp2; private Bitmap bitmap; + private GeographyData geography; @BeforeEach void before() { @@ -105,6 +112,7 @@ void before() { timestamp2 = TimestampData.fromLocalDateTime(LocalDateTime.of(1969, 1, 1, 0, 0, 0, 123456789)); bitmap = Bitmap.fromArray(new int[] {1, 2, 3}); + geography = GeographyData.fromBytes(POINT_WKB); } @Test @@ -152,6 +160,7 @@ private BinaryRowData getBinaryRow() { writer.writeTimestamp(16, timestamp1, 3); writer.writeTimestamp(17, timestamp2, 9); writer.writeBitmap(18, bitmap); + writer.writeGeography(19, geography); return row; } @@ -177,6 +186,7 @@ void testGenericRow() { row.setField(16, timestamp1); row.setField(17, timestamp2); row.setField(18, bitmap); + row.setField(19, geography); testGetters(row); } @@ -201,6 +211,7 @@ public void testBoxedWrapperRow() { row.setNonPrimitiveValue(16, timestamp1); row.setNonPrimitiveValue(17, timestamp2); row.setNonPrimitiveValue(18, bitmap); + row.setNonPrimitiveValue(19, geography); testGetters(row); testSetters(row); } @@ -230,6 +241,7 @@ public void testJoinedRow() { row2.setField(11, timestamp1); row2.setField(12, timestamp2); row2.setField(13, bitmap); + row2.setField(14, geography); testGetters(new JoinedRowData(row1, row2)); } @@ -277,6 +289,14 @@ void testFieldGetters() { .isEqualTo(timestamp1); assertThat(RowData.createFieldGetter(new TimestampType(9), 17).getFieldOrNull(row)) .isEqualTo(timestamp2); + assertThat(RowData.createFieldGetter(new BitmapType(), 18).getFieldOrNull(row)) + .isEqualTo(bitmap); + assertThat( + ((GeographyData) + RowData.createFieldGetter(new GeographyType(), 19) + .getFieldOrNull(row)) + .toBytes()) + .isEqualTo(geography.toBytes()); } @Test @@ -355,6 +375,10 @@ private void testFieldGettersWithNull(boolean nullable) { RowData.createFieldGetter(new TimestampType(nullable, 9), 17) .getFieldOrNull(row)) .isNull(); + assertThat(RowData.createFieldGetter(new BitmapType(nullable), 18).getFieldOrNull(row)) + .isNull(); + assertThat(RowData.createFieldGetter(new GeographyType(nullable), 19).getFieldOrNull(row)) + .isNull(); } private void testGetters(RowData row) { @@ -385,6 +409,7 @@ private void testGetters(RowData row) { assertThat(row.getTimestamp(16, 3)).isEqualTo(timestamp1); assertThat(row.getTimestamp(17, 9)).isEqualTo(timestamp2); assertThat(row.getBitmap(18)).isEqualTo(bitmap); + assertThat(row.getGeography(19).toBytes()).isEqualTo(geography.toBytes()); } private void testSetters(RowData row) { @@ -435,7 +460,7 @@ private void testSetters(RowData row) { } private static BinaryRowData getNullBinaryRow() { - BinaryRowData row = new BinaryRowData(18); + BinaryRowData row = new BinaryRowData(NUM_FIELDS); BinaryRowWriter binaryRowWriter = new BinaryRowWriter(row); for (int i = 0; i < row.getArity(); i++) { binaryRowWriter.setNullAt(i); diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java index 9d7ffdaea3d9df..e000c4c121ae6d 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java @@ -22,11 +22,13 @@ import org.apache.flink.table.api.TableException; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; @@ -203,6 +205,8 @@ public final class DataStructureConverters { putConverter(LogicalTypeRoot.VARIANT, Variant.class, identity()); putConverter(LogicalTypeRoot.BITMAP, Bitmap.class, constructor(BitmapBitmapConverter::new)); putConverter(LogicalTypeRoot.BITMAP, RoaringBitmapData.class, identity()); + putConverter(LogicalTypeRoot.GEOGRAPHY, GeographyData.class, identity()); + putConverter(LogicalTypeRoot.GEOGRAPHY, BinaryGeographyData.class, identity()); } /** Returns a converter for the given {@link DataType}. */ diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java index 73f817e64815eb..48745107ec68d0 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java @@ -24,6 +24,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -31,6 +32,7 @@ import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.data.binary.BinaryArrayData; import org.apache.flink.table.data.binary.BinaryFormat; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.data.binary.BinaryMapData; import org.apache.flink.table.data.binary.BinaryRawValueData; import org.apache.flink.table.data.binary.BinaryRowData; @@ -141,6 +143,17 @@ public void writeBitmap(int pos, Bitmap bitmap) { writeBytesToVarLenPart(pos, bytes, bytes.length); } + @Override + public void writeGeography(int pos, GeographyData geography) { + if (geography instanceof BinaryGeographyData) { + BinaryGeographyData binary = (BinaryGeographyData) geography; + writeSegmentsToVarLenPart( + pos, binary.getSegments(), binary.getOffset(), binary.getSizeInBytes()); + } else { + writeBinary(pos, geography.toBytes()); + } + } + private DataOutputViewStreamWrapper getOutputView() { if (outputView == null) { outputView = new DataOutputViewStreamWrapper(new BinaryRowWriterOutputView()); diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java index 0bfebd4cb99834..6ff5d8fe6c8268 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java @@ -248,6 +248,7 @@ public static NullSetter createNullSetter(LogicalType elementType) { case VARIANT: case BITMAP: case UUID: + case GEOGRAPHY: return BinaryArrayWriter::setNullLong; case BOOLEAN: return BinaryArrayWriter::setNullBoolean; diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java index 13efc76cf568f1..bb0e8c13a5f351 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -91,6 +92,8 @@ public interface BinaryWriter { void writeBitmap(int pos, Bitmap bitmap); + void writeGeography(int pos, GeographyData geography); + /** Finally, complete write to set real size to binary. */ void complete(); @@ -175,6 +178,9 @@ static void write( case BITMAP: writer.writeBitmap(pos, (Bitmap) o); break; + case GEOGRAPHY: + writer.writeGeography(pos, (GeographyData) o); + break; default: throw new UnsupportedOperationException("Not support type: " + type); } @@ -255,6 +261,8 @@ static ValueSetter createValueSetter(LogicalType elementType) { return (writer, pos, value) -> writer.writeVariant(pos, (Variant) value); case BITMAP: return (writer, pos, value) -> writer.writeBitmap(pos, (Bitmap) value); + case GEOGRAPHY: + return (writer, pos, value) -> writer.writeGeography(pos, (GeographyData) value); case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java new file mode 100644 index 00000000000000..3e54f24009eb82 --- /dev/null +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.typeutils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.table.data.GeographyData; + +import java.io.IOException; + +/** + * Serializer for {@link GeographyData} values of {@link + * org.apache.flink.table.types.logical.GeographyType}. + */ +@Internal +public final class GeographyTypeSerializer extends TypeSerializerSingleton { + + private static final long serialVersionUID = 1L; + + private static final int FORMAT_VERSION = 1; + + private static final byte[] EMPTY_GEOMETRY_COLLECTION = + new byte[] {1, GeographyData.GEOMETRY_COLLECTION, 0, 0, 0, 0, 0, 0, 0}; + + public static final GeographyTypeSerializer INSTANCE = new GeographyTypeSerializer(); + + private GeographyTypeSerializer() {} + + @Override + public boolean isImmutableType() { + return true; + } + + @Override + public GeographyData createInstance() { + return GeographyData.fromBytes(EMPTY_GEOMETRY_COLLECTION); + } + + @Override + public GeographyData copy(GeographyData from) { + return GeographyData.fromBytes(from.toBytes()); + } + + @Override + public GeographyData copy(GeographyData from, GeographyData reuse) { + return copy(from); + } + + @Override + public int getLength() { + return -1; + } + + @Override + public void serialize(GeographyData record, DataOutputView target) throws IOException { + final byte[] bytes = record.toBytes(); + target.writeByte(FORMAT_VERSION); + target.writeInt(bytes.length); + target.write(bytes); + } + + @Override + public GeographyData deserialize(DataInputView source) throws IOException { + readFormatVersion(source); + final int length = readPayloadLength(source); + final byte[] bytes = new byte[length]; + source.readFully(bytes); + return GeographyData.fromBytes(bytes); + } + + @Override + public GeographyData deserialize(GeographyData reuse, DataInputView source) throws IOException { + return deserialize(source); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + final int version = readFormatVersion(source); + final int length = readPayloadLength(source); + target.writeByte(version); + target.writeInt(length); + target.write(source, length); + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + return new GeographyTypeSerializerSnapshot(); + } + + private static int readFormatVersion(DataInputView source) throws IOException { + final int version = source.readUnsignedByte(); + if (version != FORMAT_VERSION) { + throw new IOException( + String.format( + "Unsupported GEOGRAPHY serializer format version %d. Expected %d.", + version, FORMAT_VERSION)); + } + return version; + } + + private static int readPayloadLength(DataInputView source) throws IOException { + final int length = source.readInt(); + if (length < 0) { + throw new IOException(String.format("Invalid GEOGRAPHY payload length %d.", length)); + } + return length; + } +} diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java new file mode 100644 index 00000000000000..a3159016479899 --- /dev/null +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.typeutils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.table.data.GeographyData; + +import java.io.IOException; + +/** Serializer snapshot for {@link GeographyTypeSerializer}. */ +@Internal +public final class GeographyTypeSerializerSnapshot + implements TypeSerializerSnapshot { + + private static final int CURRENT_VERSION = 1; + + @Override + public int getCurrentVersion() { + return CURRENT_VERSION; + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException {} + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) + throws IOException {} + + @Override + public TypeSerializer restoreSerializer() { + return GeographyTypeSerializer.INSTANCE; + } + + @Override + public TypeSerializerSchemaCompatibility resolveSchemaCompatibility( + TypeSerializerSnapshot oldSerializerSnapshot) { + if (oldSerializerSnapshot instanceof GeographyTypeSerializerSnapshot) { + return TypeSerializerSchemaCompatibility.compatibleAsIs(); + } + return TypeSerializerSchemaCompatibility.incompatible(); + } +} diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java index ef4d74776347ef..156c56dd8e9038 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java @@ -131,6 +131,8 @@ private static TypeSerializer createInternal(LogicalType type) { return VariantSerializer.INSTANCE; case BITMAP: return BitmapSerializer.INSTANCE; + case GEOGRAPHY: + return GeographyTypeSerializer.INSTANCE; case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java new file mode 100644 index 00000000000000..8a695c5c020f54 --- /dev/null +++ b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.typeutils; + +import org.apache.flink.api.common.typeutils.SerializerTestBase; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotSerializationUtil; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.binary.BinaryArrayData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.types.logical.GeographyType; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link GeographyTypeSerializer}. */ +class GeographyTypeSerializerTest extends SerializerTestBase { + + private static final int FORMAT_VERSION = 1; + + private static final byte[] POINT_WKB = + new byte[] { + 1, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + private static final byte[] BIG_ENDIAN_POINT_WKB = + new byte[] { + 0, 0, 0, 0, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + @Override + protected TypeSerializer createSerializer() { + return GeographyTypeSerializer.INSTANCE; + } + + @Override + protected int getLength() { + return -1; + } + + @Override + protected Class getTypeClass() { + return GeographyData.class; + } + + @Override + protected GeographyData[] getTestData() { + return new GeographyData[] { + GeographyData.fromBytes(POINT_WKB), + GeographyData.fromBytes(BIG_ENDIAN_POINT_WKB), + GeographyTypeSerializer.INSTANCE.createInstance() + }; + } + + @Override + protected void deepEquals(String message, GeographyData should, GeographyData is) { + assertThat(is.toBytes()).as(message).isEqualTo(should.toBytes()); + } + + @Test + void testInternalSerializerRoundTripsRawWkb() throws Exception { + final TypeSerializer serializer = + InternalSerializers.create(new GeographyType()); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + serializer.serialize(geography, new DataOutputViewStreamWrapper(bytes)); + final GeographyData deserialized = + serializer.deserialize( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(bytes.toByteArray()))); + + assertThat(deserialized.toBytes()).isEqualTo(POINT_WKB); + assertThat(deserialized.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testSerializedFormUsesVersionedLengthEnvelope() throws Exception { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + GeographyTypeSerializer.INSTANCE.serialize( + GeographyData.fromBytes(POINT_WKB), new DataOutputViewStreamWrapper(bytes)); + final DataInputViewStreamWrapper input = + new DataInputViewStreamWrapper(new ByteArrayInputStream(bytes.toByteArray())); + final byte[] payload = new byte[POINT_WKB.length]; + + assertThat(input.readUnsignedByte()).isEqualTo(FORMAT_VERSION); + assertThat(input.readInt()).isEqualTo(POINT_WKB.length); + input.readFully(payload); + assertThat(payload).isEqualTo(POINT_WKB); + } + + @Test + void testRejectsUnsupportedPayloadVersion() throws Exception { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + final DataOutputViewStreamWrapper output = new DataOutputViewStreamWrapper(bytes); + output.writeByte(2); + output.writeInt(POINT_WKB.length); + output.write(POINT_WKB); + + assertThatThrownBy( + () -> + GeographyTypeSerializer.INSTANCE.deserialize( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(bytes.toByteArray())))) + .isInstanceOf(IOException.class) + .hasMessageContaining("Unsupported GEOGRAPHY serializer format version 2"); + } + + @Test + void testSnapshotSelfCompatibility() throws Exception { + final TypeSerializerSnapshot snapshot = + GeographyTypeSerializer.INSTANCE.snapshotConfiguration(); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( + new DataOutputViewStreamWrapper(bytes), snapshot); + final TypeSerializerSnapshot restoredSnapshot = + TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(bytes.toByteArray())), + getClass().getClassLoader()); + + assertThat( + GeographyTypeSerializer.INSTANCE + .snapshotConfiguration() + .resolveSchemaCompatibility(restoredSnapshot) + .isCompatibleAsIs()) + .isTrue(); + assertThat(restoredSnapshot.restoreSerializer()).isSameAs(GeographyTypeSerializer.INSTANCE); + } + + @Test + void testCopyPreservesRawWkbBytes() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final GeographyData copied = GeographyTypeSerializer.INSTANCE.copy(geography); + + assertThat(copied).isNotSameAs(geography); + assertThat(copied.toBytes()).isEqualTo(POINT_WKB); + assertThat(copied.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testCreateInstanceReturnsValidGeographyData() { + final GeographyData instance = GeographyTypeSerializer.INSTANCE.createInstance(); + + assertThat(instance.subtypeId()).isEqualTo(GeographyData.GEOMETRY_COLLECTION); + assertThat(instance.sizeInBytes()).isEqualTo(9); + } + + @Test + void testRowDataSerializerConvertsGenericRowToBinaryRow() { + final RowDataSerializer serializer = + InternalTypeInfo.ofFields(new GeographyType()).toRowSerializer(); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final GenericRowData row = GenericRowData.of(geography); + + final BinaryRowData binaryRow = serializer.toBinaryRow(row, true); + + assertThat(binaryRow.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(binaryRow.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testRowDataSerializerPreservesNullGeography() { + final RowDataSerializer serializer = + InternalTypeInfo.ofFields(new GeographyType()).toRowSerializer(); + final GenericRowData row = GenericRowData.of((Object) null); + + final BinaryRowData binaryRow = serializer.toBinaryRow(row, true); + + assertThat(binaryRow.isNullAt(0)).isTrue(); + } + + @Test + void testArrayDataSerializerConvertsGenericArrayToBinaryArray() { + final ArrayDataSerializer serializer = new ArrayDataSerializer(new GeographyType()); + final GenericArrayData array = + new GenericArrayData(new Object[] {GeographyData.fromBytes(POINT_WKB), null}); + + final BinaryArrayData binaryArray = serializer.toBinaryArray(array); + + assertThat(binaryArray.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(binaryArray.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(binaryArray.isNullAt(1)).isTrue(); + } +} diff --git a/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java index 4035fc65d375f4..73b789ca6943a1 100644 --- a/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java +++ b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java @@ -24,6 +24,7 @@ import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; @@ -41,6 +42,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -62,6 +64,11 @@ /** Test for {@link RowDataSerializer}. */ abstract class RowDataSerializerTest extends SerializerTestInstance { + private static final byte[] POINT_WKB = + new byte[] { + 1, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + private final RowDataSerializer serializer; private final RowData[] testData; @@ -198,7 +205,28 @@ private static RowDataSerializer getRowSerializer() { return typeInfo.toRowSerializer(); } -} + + static final class RowDataSerializerWithGeographyTest extends RowDataSerializerTest { + public RowDataSerializerWithGeographyTest() { + super(getRowSerializer(), getData()); + } + + private static RowData[] getData() { + GenericRowData row1 = new GenericRowData(1); + row1.setField(0, GeographyData.fromBytes(POINT_WKB)); + + GenericRowData row2 = new GenericRowData(1); + row2.setField(0, null); + + return new RowData[] {row1, row2}; + } + + private static RowDataSerializer getRowSerializer() { + InternalTypeInfo typeInfo = InternalTypeInfo.ofFields(new GeographyType()); + + return typeInfo.toRowSerializer(); + } + } final class LargeRowDataSerializerTest extends RowDataSerializerTest { LargeRowDataSerializerTest() { diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml index 4ef3c5f273d53c..ee49aca935c6d1 100644 --- a/flink-tests/pom.xml +++ b/flink-tests/pom.xml @@ -69,6 +69,13 @@ under the License. test + + org.apache.flink + flink-connector-base + ${project.version} + test + + org.apache.flink flink-connector-files @@ -205,6 +212,13 @@ under the License. test + + org.apache.flink + flink-table-type-utils + ${project.version} + test + + org.apache.flink flink-runtime diff --git a/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java b/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java index 7f91776f841877..9c703ac489ced0 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java +++ b/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java @@ -85,6 +85,7 @@ import org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer; import org.apache.flink.table.runtime.typeutils.DecimalDataSerializer; import org.apache.flink.table.runtime.typeutils.ExternalSerializer; +import org.apache.flink.table.runtime.typeutils.GeographyTypeSerializer; import org.apache.flink.table.runtime.typeutils.LinkedListSerializer; import org.apache.flink.table.runtime.typeutils.MapDataSerializer; import org.apache.flink.table.runtime.typeutils.RawValueDataSerializer; @@ -199,6 +200,7 @@ void testTypeSerializerTestCoverage() { SharedBufferEdge.SharedBufferEdgeSerializer.class.getName(), RowDataSerializer.class.getName(), DecimalDataSerializer.class.getName(), + GeographyTypeSerializer.class.getName(), AvroSerializer.class.getName()); // type serializer whitelist for TypeSerializerUpgradeTestBase test coverage @@ -261,6 +263,7 @@ void testTypeSerializerTestCoverage() { SharedBufferEdge.SharedBufferEdgeSerializer.class.getName(), RowDataSerializer.class.getName(), DecimalDataSerializer.class.getName(), + GeographyTypeSerializer.class.getName(), AvroSerializer.class.getName(), // KeyAndValueSerializer shouldn't be used to serialize data to state and // doesn't need to ensure upgrade compatibility.